Print this page
8622 panic in PTE_set_all()
8623 IMMU_CONTIG_PADDR is broken for cookies with more than one page
8625 nvme causes bad free panic in IOMMU
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/i86pc/io/rootnex.c
+++ new/usr/src/uts/i86pc/io/rootnex.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
↓ open down ↓ |
17 lines elided |
↑ open up ↑ |
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24 /*
25 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
28 + * Copyright 2017 Joyent, Inc.
28 29 */
29 30
30 31 /*
31 32 * x86 root nexus driver
32 33 */
33 34
34 35 #include <sys/sysmacros.h>
35 36 #include <sys/conf.h>
36 37 #include <sys/autoconf.h>
37 38 #include <sys/sysmacros.h>
38 39 #include <sys/debug.h>
39 40 #include <sys/psw.h>
40 41 #include <sys/ddidmareq.h>
41 42 #include <sys/promif.h>
42 43 #include <sys/devops.h>
43 44 #include <sys/kmem.h>
44 45 #include <sys/cmn_err.h>
45 46 #include <vm/seg.h>
46 47 #include <vm/seg_kmem.h>
47 48 #include <vm/seg_dev.h>
48 49 #include <sys/vmem.h>
49 50 #include <sys/mman.h>
50 51 #include <vm/hat.h>
51 52 #include <vm/as.h>
52 53 #include <vm/page.h>
53 54 #include <sys/avintr.h>
54 55 #include <sys/errno.h>
55 56 #include <sys/modctl.h>
56 57 #include <sys/ddi_impldefs.h>
57 58 #include <sys/sunddi.h>
58 59 #include <sys/sunndi.h>
59 60 #include <sys/mach_intr.h>
60 61 #include <sys/psm.h>
61 62 #include <sys/ontrap.h>
62 63 #include <sys/atomic.h>
63 64 #include <sys/sdt.h>
64 65 #include <sys/rootnex.h>
65 66 #include <vm/hat_i86.h>
66 67 #include <sys/ddifm.h>
67 68 #include <sys/ddi_isa.h>
68 69 #include <sys/apic.h>
69 70
70 71 #ifdef __xpv
71 72 #include <sys/bootinfo.h>
72 73 #include <sys/hypervisor.h>
73 74 #include <sys/bootconf.h>
74 75 #include <vm/kboot_mmu.h>
75 76 #endif
76 77
77 78 #if defined(__amd64) && !defined(__xpv)
78 79 #include <sys/immu.h>
79 80 #endif
80 81
81 82
82 83 /*
83 84 * enable/disable extra checking of function parameters. Useful for debugging
84 85 * drivers.
85 86 */
86 87 #ifdef DEBUG
87 88 int rootnex_alloc_check_parms = 1;
88 89 int rootnex_bind_check_parms = 1;
89 90 int rootnex_bind_check_inuse = 1;
90 91 int rootnex_unbind_verify_buffer = 0;
91 92 int rootnex_sync_check_parms = 1;
92 93 #else
93 94 int rootnex_alloc_check_parms = 0;
94 95 int rootnex_bind_check_parms = 0;
95 96 int rootnex_bind_check_inuse = 0;
96 97 int rootnex_unbind_verify_buffer = 0;
97 98 int rootnex_sync_check_parms = 0;
98 99 #endif
99 100
100 101 boolean_t rootnex_dmar_not_setup;
101 102
102 103 /* Master Abort and Target Abort panic flag */
103 104 int rootnex_fm_ma_ta_panic_flag = 0;
104 105
105 106 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
106 107 int rootnex_bind_fail = 1;
107 108 int rootnex_bind_warn = 1;
108 109 uint8_t *rootnex_warn_list;
109 110 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
110 111 #define ROOTNEX_BIND_WARNING (0x1 << 0)
111 112
112 113 /*
113 114 * revert back to old broken behavior of always sync'ing entire copy buffer.
114 115 * This is useful if be have a buggy driver which doesn't correctly pass in
115 116 * the offset and size into ddi_dma_sync().
116 117 */
117 118 int rootnex_sync_ignore_params = 0;
118 119
119 120 /*
120 121 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
121 122 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
122 123 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
123 124 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
124 125 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
125 126 * (< 8K). We will still need to allocate the copy buffer during bind though
126 127 * (if we need one). These can only be modified in /etc/system before rootnex
127 128 * attach.
128 129 */
129 130 #if defined(__amd64)
130 131 int rootnex_prealloc_cookies = 65;
131 132 int rootnex_prealloc_windows = 4;
132 133 int rootnex_prealloc_copybuf = 2;
133 134 #else
134 135 int rootnex_prealloc_cookies = 33;
135 136 int rootnex_prealloc_windows = 4;
136 137 int rootnex_prealloc_copybuf = 2;
137 138 #endif
138 139
139 140 /* driver global state */
140 141 static rootnex_state_t *rootnex_state;
141 142
142 143 #ifdef DEBUG
143 144 /* shortcut to rootnex counters */
144 145 static uint64_t *rootnex_cnt;
145 146 #endif
146 147
147 148 /*
148 149 * XXX - does x86 even need these or are they left over from the SPARC days?
149 150 */
150 151 /* statically defined integer/boolean properties for the root node */
151 152 static rootnex_intprop_t rootnex_intprp[] = {
152 153 { "PAGESIZE", PAGESIZE },
153 154 { "MMU_PAGESIZE", MMU_PAGESIZE },
154 155 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
155 156 { DDI_RELATIVE_ADDRESSING, 1 },
156 157 };
157 158 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
158 159
159 160 /*
160 161 * If we're dom0, we're using a real device so we need to load
161 162 * the cookies with MFNs instead of PFNs.
162 163 */
163 164 #ifdef __xpv
164 165 typedef maddr_t rootnex_addr_t;
165 166 #define ROOTNEX_PADDR_TO_RBASE(pa) \
166 167 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
167 168 #else
168 169 typedef paddr_t rootnex_addr_t;
169 170 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
170 171 #endif
171 172
172 173 static struct cb_ops rootnex_cb_ops = {
173 174 nodev, /* open */
174 175 nodev, /* close */
175 176 nodev, /* strategy */
176 177 nodev, /* print */
177 178 nodev, /* dump */
178 179 nodev, /* read */
179 180 nodev, /* write */
180 181 nodev, /* ioctl */
181 182 nodev, /* devmap */
182 183 nodev, /* mmap */
183 184 nodev, /* segmap */
184 185 nochpoll, /* chpoll */
185 186 ddi_prop_op, /* cb_prop_op */
186 187 NULL, /* struct streamtab */
187 188 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
188 189 CB_REV, /* Rev */
189 190 nodev, /* cb_aread */
190 191 nodev /* cb_awrite */
191 192 };
192 193
193 194 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
194 195 off_t offset, off_t len, caddr_t *vaddrp);
195 196 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
196 197 struct hat *hat, struct seg *seg, caddr_t addr,
197 198 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
198 199 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
199 200 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
200 201 ddi_dma_handle_t *handlep);
201 202 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
202 203 ddi_dma_handle_t handle);
203 204 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
204 205 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
205 206 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
206 207 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
207 208 ddi_dma_handle_t handle);
208 209 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
209 210 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
210 211 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
211 212 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
212 213 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
213 214 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
214 215 ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
215 216 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
216 217 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
217 218 ddi_ctl_enum_t ctlop, void *arg, void *result);
218 219 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
219 220 ddi_iblock_cookie_t *ibc);
220 221 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
221 222 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
222 223 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
223 224 void *);
224 225 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
225 226
226 227 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
227 228 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
228 229 ddi_dma_handle_t *handlep);
229 230 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
230 231 ddi_dma_handle_t handle);
231 232 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
232 233 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
233 234 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
234 235 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
235 236 ddi_dma_handle_t handle);
236 237 #if defined(__amd64) && !defined(__xpv)
237 238 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
238 239 ddi_dma_handle_t handle);
239 240 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
240 241 ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
241 242 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
242 243 ddi_dma_cookie_t *cookiep, uint_t ccount);
243 244 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
244 245 ddi_dma_handle_t handle);
245 246 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
246 247 #endif
247 248 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
248 249 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
249 250 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
250 251 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
251 252 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
252 253
253 254 #if defined(__amd64) && !defined(__xpv)
254 255 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
255 256 ddi_dma_handle_t handle, void *v);
256 257 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
257 258 ddi_dma_handle_t handle);
258 259 #endif
259 260
260 261
261 262 static struct bus_ops rootnex_bus_ops = {
262 263 BUSO_REV,
263 264 rootnex_map,
264 265 NULL,
265 266 NULL,
266 267 NULL,
267 268 rootnex_map_fault,
268 269 0,
269 270 rootnex_dma_allochdl,
270 271 rootnex_dma_freehdl,
271 272 rootnex_dma_bindhdl,
272 273 rootnex_dma_unbindhdl,
273 274 rootnex_dma_sync,
274 275 rootnex_dma_win,
275 276 rootnex_dma_mctl,
276 277 rootnex_ctlops,
277 278 ddi_bus_prop_op,
278 279 i_ddi_rootnex_get_eventcookie,
279 280 i_ddi_rootnex_add_eventcall,
280 281 i_ddi_rootnex_remove_eventcall,
281 282 i_ddi_rootnex_post_event,
282 283 0, /* bus_intr_ctl */
283 284 0, /* bus_config */
284 285 0, /* bus_unconfig */
285 286 rootnex_fm_init, /* bus_fm_init */
286 287 NULL, /* bus_fm_fini */
287 288 NULL, /* bus_fm_access_enter */
288 289 NULL, /* bus_fm_access_exit */
289 290 NULL, /* bus_powr */
290 291 rootnex_intr_ops /* bus_intr_op */
291 292 };
292 293
293 294 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
294 295 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
295 296 static int rootnex_quiesce(dev_info_t *dip);
296 297
297 298 static struct dev_ops rootnex_ops = {
298 299 DEVO_REV,
299 300 0,
300 301 ddi_no_info,
301 302 nulldev,
302 303 nulldev,
303 304 rootnex_attach,
304 305 rootnex_detach,
305 306 nulldev,
306 307 &rootnex_cb_ops,
307 308 &rootnex_bus_ops,
308 309 NULL,
309 310 rootnex_quiesce, /* quiesce */
310 311 };
311 312
312 313 static struct modldrv rootnex_modldrv = {
313 314 &mod_driverops,
314 315 "i86pc root nexus",
315 316 &rootnex_ops
316 317 };
317 318
318 319 static struct modlinkage rootnex_modlinkage = {
319 320 MODREV_1,
320 321 (void *)&rootnex_modldrv,
321 322 NULL
322 323 };
323 324
324 325 #if defined(__amd64) && !defined(__xpv)
325 326 static iommulib_nexops_t iommulib_nexops = {
326 327 IOMMU_NEXOPS_VERSION,
327 328 "Rootnex IOMMU ops Vers 1.1",
328 329 NULL,
329 330 rootnex_coredma_allochdl,
330 331 rootnex_coredma_freehdl,
331 332 rootnex_coredma_bindhdl,
332 333 rootnex_coredma_unbindhdl,
333 334 rootnex_coredma_reset_cookies,
334 335 rootnex_coredma_get_cookies,
335 336 rootnex_coredma_set_cookies,
336 337 rootnex_coredma_clear_cookies,
337 338 rootnex_coredma_get_sleep_flags,
338 339 rootnex_coredma_sync,
339 340 rootnex_coredma_win,
340 341 rootnex_coredma_hdl_setprivate,
341 342 rootnex_coredma_hdl_getprivate
342 343 };
343 344 #endif
344 345
345 346 /*
346 347 * extern hacks
347 348 */
348 349 extern struct seg_ops segdev_ops;
349 350 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
350 351 #ifdef DDI_MAP_DEBUG
351 352 extern int ddi_map_debug_flag;
352 353 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
353 354 #endif
354 355 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
355 356 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
356 357 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
357 358 psm_intr_op_t, int *);
358 359 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
359 360 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
360 361
361 362 /*
362 363 * Use device arena to use for device control register mappings.
363 364 * Various kernel memory walkers (debugger, dtrace) need to know
364 365 * to avoid this address range to prevent undesired device activity.
365 366 */
366 367 extern void *device_arena_alloc(size_t size, int vm_flag);
367 368 extern void device_arena_free(void * vaddr, size_t size);
368 369
369 370
370 371 /*
371 372 * Internal functions
372 373 */
373 374 static int rootnex_dma_init();
374 375 static void rootnex_add_props(dev_info_t *);
375 376 static int rootnex_ctl_reportdev(dev_info_t *dip);
376 377 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
377 378 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
378 379 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
379 380 static int rootnex_map_handle(ddi_map_req_t *mp);
380 381 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
381 382 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
382 383 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
383 384 ddi_dma_attr_t *attr);
384 385 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
385 386 rootnex_sglinfo_t *sglinfo);
386 387 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
387 388 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
388 389 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
389 390 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
390 391 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
391 392 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
392 393 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
393 394 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
394 395 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
395 396 static void rootnex_teardown_windows(rootnex_dma_t *dma);
396 397 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
397 398 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
398 399 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
399 400 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
400 401 size_t *copybuf_used, page_t **cur_pp);
401 402 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
402 403 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
403 404 ddi_dma_attr_t *attr, off_t cur_offset);
404 405 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
405 406 rootnex_dma_t *dma, rootnex_window_t **windowp,
406 407 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
407 408 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
408 409 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
409 410 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
410 411 off_t offset, size_t size, uint_t cache_flags);
411 412 static int rootnex_verify_buffer(rootnex_dma_t *dma);
412 413 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
413 414 const void *comp_addr, const void *not_used);
414 415 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
415 416 rootnex_sglinfo_t *sglinfo);
416 417 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
417 418
418 419 /*
419 420 * _init()
420 421 *
421 422 */
422 423 int
423 424 _init(void)
424 425 {
425 426
426 427 rootnex_state = NULL;
427 428 return (mod_install(&rootnex_modlinkage));
428 429 }
429 430
430 431
431 432 /*
432 433 * _info()
433 434 *
434 435 */
435 436 int
436 437 _info(struct modinfo *modinfop)
437 438 {
438 439 return (mod_info(&rootnex_modlinkage, modinfop));
439 440 }
440 441
441 442
442 443 /*
443 444 * _fini()
444 445 *
445 446 */
446 447 int
447 448 _fini(void)
448 449 {
449 450 return (EBUSY);
450 451 }
451 452
452 453
453 454 /*
454 455 * rootnex_attach()
455 456 *
456 457 */
457 458 static int
458 459 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
459 460 {
460 461 int fmcap;
461 462 int e;
462 463
463 464 switch (cmd) {
464 465 case DDI_ATTACH:
465 466 break;
466 467 case DDI_RESUME:
467 468 #if defined(__amd64) && !defined(__xpv)
468 469 return (immu_unquiesce());
469 470 #else
470 471 return (DDI_SUCCESS);
471 472 #endif
472 473 default:
473 474 return (DDI_FAILURE);
474 475 }
475 476
476 477 /*
477 478 * We should only have one instance of rootnex. Save it away since we
478 479 * don't have an easy way to get it back later.
479 480 */
480 481 ASSERT(rootnex_state == NULL);
481 482 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
482 483
483 484 rootnex_state->r_dip = dip;
484 485 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
485 486 rootnex_state->r_reserved_msg_printed = B_FALSE;
486 487 #ifdef DEBUG
487 488 rootnex_cnt = &rootnex_state->r_counters[0];
488 489 #endif
489 490
490 491 /*
491 492 * Set minimum fm capability level for i86pc platforms and then
492 493 * initialize error handling. Since we're the rootnex, we don't
493 494 * care what's returned in the fmcap field.
494 495 */
495 496 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
496 497 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
497 498 fmcap = ddi_system_fmcap;
498 499 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
499 500
500 501 /* initialize DMA related state */
501 502 e = rootnex_dma_init();
502 503 if (e != DDI_SUCCESS) {
503 504 kmem_free(rootnex_state, sizeof (rootnex_state_t));
504 505 return (DDI_FAILURE);
505 506 }
506 507
507 508 /* Add static root node properties */
508 509 rootnex_add_props(dip);
509 510
510 511 /* since we can't call ddi_report_dev() */
511 512 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
512 513
513 514 /* Initialize rootnex event handle */
514 515 i_ddi_rootnex_init_events(dip);
515 516
516 517 #if defined(__amd64) && !defined(__xpv)
517 518 e = iommulib_nexus_register(dip, &iommulib_nexops,
518 519 &rootnex_state->r_iommulib_handle);
519 520
520 521 ASSERT(e == DDI_SUCCESS);
521 522 #endif
522 523
523 524 return (DDI_SUCCESS);
524 525 }
525 526
526 527
527 528 /*
528 529 * rootnex_detach()
529 530 *
530 531 */
531 532 /*ARGSUSED*/
532 533 static int
533 534 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
534 535 {
535 536 switch (cmd) {
536 537 case DDI_SUSPEND:
537 538 #if defined(__amd64) && !defined(__xpv)
538 539 return (immu_quiesce());
539 540 #else
540 541 return (DDI_SUCCESS);
541 542 #endif
542 543 default:
543 544 return (DDI_FAILURE);
544 545 }
545 546 /*NOTREACHED*/
546 547
547 548 }
548 549
549 550
550 551 /*
551 552 * rootnex_dma_init()
552 553 *
553 554 */
554 555 /*ARGSUSED*/
555 556 static int
556 557 rootnex_dma_init()
557 558 {
558 559 size_t bufsize;
559 560
560 561
561 562 /*
562 563 * size of our cookie/window/copybuf state needed in dma bind that we
563 564 * pre-alloc in dma_alloc_handle
564 565 */
565 566 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
566 567 rootnex_state->r_prealloc_size =
567 568 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
568 569 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
569 570 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
570 571
571 572 /*
572 573 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
573 574 * allocate 16 extra bytes for struct pointer alignment
574 575 * (p->dmai_private & dma->dp_prealloc_buffer)
575 576 */
576 577 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
577 578 rootnex_state->r_prealloc_size + 0x10;
578 579 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
579 580 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
580 581 if (rootnex_state->r_dmahdl_cache == NULL) {
581 582 return (DDI_FAILURE);
582 583 }
583 584
584 585 /*
585 586 * allocate array to track which major numbers we have printed warnings
586 587 * for.
587 588 */
588 589 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
589 590 KM_SLEEP);
590 591
591 592 return (DDI_SUCCESS);
592 593 }
593 594
594 595
595 596 /*
596 597 * rootnex_add_props()
597 598 *
598 599 */
599 600 static void
600 601 rootnex_add_props(dev_info_t *dip)
601 602 {
602 603 rootnex_intprop_t *rpp;
603 604 int i;
604 605
605 606 /* Add static integer/boolean properties to the root node */
606 607 rpp = rootnex_intprp;
607 608 for (i = 0; i < NROOT_INTPROPS; i++) {
608 609 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
609 610 rpp[i].prop_name, rpp[i].prop_value);
610 611 }
611 612 }
612 613
613 614
614 615
615 616 /*
616 617 * *************************
617 618 * ctlops related routines
618 619 * *************************
619 620 */
620 621
621 622 /*
622 623 * rootnex_ctlops()
623 624 *
624 625 */
625 626 /*ARGSUSED*/
626 627 static int
627 628 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
628 629 void *arg, void *result)
629 630 {
630 631 int n, *ptr;
631 632 struct ddi_parent_private_data *pdp;
632 633
633 634 switch (ctlop) {
634 635 case DDI_CTLOPS_DMAPMAPC:
635 636 /*
636 637 * Return 'partial' to indicate that dma mapping
637 638 * has to be done in the main MMU.
638 639 */
639 640 return (DDI_DMA_PARTIAL);
640 641
641 642 case DDI_CTLOPS_BTOP:
642 643 /*
643 644 * Convert byte count input to physical page units.
644 645 * (byte counts that are not a page-size multiple
645 646 * are rounded down)
646 647 */
647 648 *(ulong_t *)result = btop(*(ulong_t *)arg);
648 649 return (DDI_SUCCESS);
649 650
650 651 case DDI_CTLOPS_PTOB:
651 652 /*
652 653 * Convert size in physical pages to bytes
653 654 */
654 655 *(ulong_t *)result = ptob(*(ulong_t *)arg);
655 656 return (DDI_SUCCESS);
656 657
657 658 case DDI_CTLOPS_BTOPR:
658 659 /*
659 660 * Convert byte count input to physical page units
660 661 * (byte counts that are not a page-size multiple
661 662 * are rounded up)
662 663 */
663 664 *(ulong_t *)result = btopr(*(ulong_t *)arg);
664 665 return (DDI_SUCCESS);
665 666
666 667 case DDI_CTLOPS_INITCHILD:
667 668 return (impl_ddi_sunbus_initchild(arg));
668 669
669 670 case DDI_CTLOPS_UNINITCHILD:
670 671 impl_ddi_sunbus_removechild(arg);
671 672 return (DDI_SUCCESS);
672 673
673 674 case DDI_CTLOPS_REPORTDEV:
674 675 return (rootnex_ctl_reportdev(rdip));
675 676
676 677 case DDI_CTLOPS_IOMIN:
677 678 /*
678 679 * Nothing to do here but reflect back..
679 680 */
680 681 return (DDI_SUCCESS);
681 682
682 683 case DDI_CTLOPS_REGSIZE:
683 684 case DDI_CTLOPS_NREGS:
684 685 break;
685 686
686 687 case DDI_CTLOPS_SIDDEV:
687 688 if (ndi_dev_is_prom_node(rdip))
688 689 return (DDI_SUCCESS);
689 690 if (ndi_dev_is_persistent_node(rdip))
690 691 return (DDI_SUCCESS);
691 692 return (DDI_FAILURE);
692 693
693 694 case DDI_CTLOPS_POWER:
694 695 return ((*pm_platform_power)((power_req_t *)arg));
695 696
696 697 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
697 698 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
698 699 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
699 700 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
700 701 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
701 702 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
702 703 if (!rootnex_state->r_reserved_msg_printed) {
703 704 rootnex_state->r_reserved_msg_printed = B_TRUE;
704 705 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
705 706 "1 or more reserved/obsolete operations.");
706 707 }
707 708 return (DDI_FAILURE);
708 709
709 710 default:
710 711 return (DDI_FAILURE);
711 712 }
712 713 /*
713 714 * The rest are for "hardware" properties
714 715 */
715 716 if ((pdp = ddi_get_parent_data(rdip)) == NULL)
716 717 return (DDI_FAILURE);
717 718
718 719 if (ctlop == DDI_CTLOPS_NREGS) {
719 720 ptr = (int *)result;
720 721 *ptr = pdp->par_nreg;
721 722 } else {
722 723 off_t *size = (off_t *)result;
723 724
724 725 ptr = (int *)arg;
725 726 n = *ptr;
726 727 if (n >= pdp->par_nreg) {
727 728 return (DDI_FAILURE);
728 729 }
729 730 *size = (off_t)pdp->par_reg[n].regspec_size;
730 731 }
731 732 return (DDI_SUCCESS);
732 733 }
733 734
734 735
735 736 /*
736 737 * rootnex_ctl_reportdev()
737 738 *
738 739 */
739 740 static int
740 741 rootnex_ctl_reportdev(dev_info_t *dev)
741 742 {
742 743 int i, n, len, f_len = 0;
743 744 char *buf;
744 745
745 746 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
746 747 f_len += snprintf(buf, REPORTDEV_BUFSIZE,
747 748 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
748 749 len = strlen(buf);
749 750
750 751 for (i = 0; i < sparc_pd_getnreg(dev); i++) {
751 752
752 753 struct regspec *rp = sparc_pd_getreg(dev, i);
753 754
754 755 if (i == 0)
755 756 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
756 757 ": ");
757 758 else
758 759 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
759 760 " and ");
760 761 len = strlen(buf);
761 762
762 763 switch (rp->regspec_bustype) {
763 764
764 765 case BTEISA:
765 766 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
766 767 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
767 768 break;
768 769
769 770 case BTISA:
770 771 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
771 772 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
772 773 break;
773 774
774 775 default:
775 776 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
776 777 "space %x offset %x",
777 778 rp->regspec_bustype, rp->regspec_addr);
778 779 break;
779 780 }
780 781 len = strlen(buf);
781 782 }
782 783 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
783 784 int pri;
784 785
785 786 if (i != 0) {
786 787 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
787 788 ",");
788 789 len = strlen(buf);
789 790 }
790 791 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
791 792 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
792 793 " sparc ipl %d", pri);
793 794 len = strlen(buf);
794 795 }
795 796 #ifdef DEBUG
796 797 if (f_len + 1 >= REPORTDEV_BUFSIZE) {
797 798 cmn_err(CE_NOTE, "next message is truncated: "
798 799 "printed length 1024, real length %d", f_len);
799 800 }
800 801 #endif /* DEBUG */
801 802 cmn_err(CE_CONT, "?%s\n", buf);
802 803 kmem_free(buf, REPORTDEV_BUFSIZE);
803 804 return (DDI_SUCCESS);
804 805 }
805 806
806 807
807 808 /*
808 809 * ******************
809 810 * map related code
810 811 * ******************
811 812 */
812 813
813 814 /*
814 815 * rootnex_map()
815 816 *
816 817 */
817 818 static int
818 819 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
819 820 off_t len, caddr_t *vaddrp)
820 821 {
821 822 struct regspec *rp, tmp_reg;
822 823 ddi_map_req_t mr = *mp; /* Get private copy of request */
823 824 int error;
824 825
825 826 mp = &mr;
826 827
827 828 switch (mp->map_op) {
828 829 case DDI_MO_MAP_LOCKED:
829 830 case DDI_MO_UNMAP:
830 831 case DDI_MO_MAP_HANDLE:
831 832 break;
832 833 default:
833 834 #ifdef DDI_MAP_DEBUG
834 835 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
835 836 mp->map_op);
836 837 #endif /* DDI_MAP_DEBUG */
837 838 return (DDI_ME_UNIMPLEMENTED);
838 839 }
839 840
840 841 if (mp->map_flags & DDI_MF_USER_MAPPING) {
841 842 #ifdef DDI_MAP_DEBUG
842 843 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
843 844 #endif /* DDI_MAP_DEBUG */
844 845 return (DDI_ME_UNIMPLEMENTED);
845 846 }
846 847
847 848 /*
848 849 * First, if given an rnumber, convert it to a regspec...
849 850 * (Presumably, this is on behalf of a child of the root node?)
850 851 */
851 852
852 853 if (mp->map_type == DDI_MT_RNUMBER) {
853 854
854 855 int rnumber = mp->map_obj.rnumber;
855 856 #ifdef DDI_MAP_DEBUG
856 857 static char *out_of_range =
857 858 "rootnex_map: Out of range rnumber <%d>, device <%s>";
858 859 #endif /* DDI_MAP_DEBUG */
859 860
860 861 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
861 862 if (rp == NULL) {
862 863 #ifdef DDI_MAP_DEBUG
863 864 cmn_err(CE_WARN, out_of_range, rnumber,
864 865 ddi_get_name(rdip));
865 866 #endif /* DDI_MAP_DEBUG */
866 867 return (DDI_ME_RNUMBER_RANGE);
867 868 }
868 869
869 870 /*
870 871 * Convert the given ddi_map_req_t from rnumber to regspec...
871 872 */
872 873
873 874 mp->map_type = DDI_MT_REGSPEC;
874 875 mp->map_obj.rp = rp;
875 876 }
876 877
877 878 /*
878 879 * Adjust offset and length correspnding to called values...
879 880 * XXX: A non-zero length means override the one in the regspec
880 881 * XXX: (regardless of what's in the parent's range?)
881 882 */
882 883
883 884 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */
884 885 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */
885 886
886 887 #ifdef DDI_MAP_DEBUG
887 888 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
888 889 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
889 890 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
890 891 len, mp->map_handlep);
891 892 #endif /* DDI_MAP_DEBUG */
892 893
893 894 /*
894 895 * I/O or memory mapping:
895 896 *
896 897 * <bustype=0, addr=x, len=x>: memory
897 898 * <bustype=1, addr=x, len=x>: i/o
898 899 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
899 900 */
900 901
901 902 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
902 903 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
903 904 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
904 905 ddi_get_name(rdip), rp->regspec_bustype,
905 906 rp->regspec_addr, rp->regspec_size);
906 907 return (DDI_ME_INVAL);
907 908 }
908 909
909 910 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
910 911 /*
911 912 * compatibility i/o mapping
912 913 */
913 914 rp->regspec_bustype += (uint_t)offset;
914 915 } else {
915 916 /*
916 917 * Normal memory or i/o mapping
917 918 */
918 919 rp->regspec_addr += (uint_t)offset;
919 920 }
920 921
921 922 if (len != 0)
922 923 rp->regspec_size = (uint_t)len;
923 924
924 925 #ifdef DDI_MAP_DEBUG
925 926 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
926 927 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
927 928 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
928 929 offset, len, mp->map_handlep);
929 930 #endif /* DDI_MAP_DEBUG */
930 931
931 932 /*
932 933 * Apply any parent ranges at this level, if applicable.
933 934 * (This is where nexus specific regspec translation takes place.
934 935 * Use of this function is implicit agreement that translation is
935 936 * provided via ddi_apply_range.)
936 937 */
937 938
938 939 #ifdef DDI_MAP_DEBUG
939 940 ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
940 941 ddi_get_name(dip), ddi_get_name(rdip));
941 942 #endif /* DDI_MAP_DEBUG */
942 943
943 944 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
944 945 return (error);
945 946
946 947 switch (mp->map_op) {
947 948 case DDI_MO_MAP_LOCKED:
948 949
949 950 /*
950 951 * Set up the locked down kernel mapping to the regspec...
951 952 */
952 953
953 954 return (rootnex_map_regspec(mp, vaddrp));
954 955
955 956 case DDI_MO_UNMAP:
956 957
957 958 /*
958 959 * Release mapping...
959 960 */
960 961
961 962 return (rootnex_unmap_regspec(mp, vaddrp));
962 963
963 964 case DDI_MO_MAP_HANDLE:
964 965
965 966 return (rootnex_map_handle(mp));
966 967
967 968 default:
968 969 return (DDI_ME_UNIMPLEMENTED);
969 970 }
970 971 }
971 972
972 973
973 974 /*
974 975 * rootnex_map_fault()
975 976 *
976 977 * fault in mappings for requestors
977 978 */
978 979 /*ARGSUSED*/
979 980 static int
980 981 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
981 982 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
982 983 uint_t lock)
983 984 {
984 985
985 986 #ifdef DDI_MAP_DEBUG
986 987 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
987 988 ddi_map_debug(" Seg <%s>\n",
988 989 seg->s_ops == &segdev_ops ? "segdev" :
989 990 seg == &kvseg ? "segkmem" : "NONE!");
990 991 #endif /* DDI_MAP_DEBUG */
991 992
992 993 /*
993 994 * This is all terribly broken, but it is a start
994 995 *
995 996 * XXX Note that this test means that segdev_ops
996 997 * must be exported from seg_dev.c.
997 998 * XXX What about devices with their own segment drivers?
998 999 */
999 1000 if (seg->s_ops == &segdev_ops) {
1000 1001 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1001 1002
1002 1003 if (hat == NULL) {
1003 1004 /*
1004 1005 * This is one plausible interpretation of
1005 1006 * a null hat i.e. use the first hat on the
1006 1007 * address space hat list which by convention is
1007 1008 * the hat of the system MMU. At alternative
1008 1009 * would be to panic .. this might well be better ..
1009 1010 */
1010 1011 ASSERT(AS_READ_HELD(seg->s_as));
1011 1012 hat = seg->s_as->a_hat;
1012 1013 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1013 1014 }
1014 1015 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1015 1016 (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1016 1017 } else if (seg == &kvseg && dp == NULL) {
1017 1018 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1018 1019 HAT_LOAD_LOCK);
1019 1020 } else
1020 1021 return (DDI_FAILURE);
1021 1022 return (DDI_SUCCESS);
1022 1023 }
1023 1024
1024 1025
1025 1026 /*
1026 1027 * rootnex_map_regspec()
1027 1028 * we don't support mapping of I/O cards above 4Gb
1028 1029 */
1029 1030 static int
1030 1031 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1031 1032 {
1032 1033 rootnex_addr_t rbase;
1033 1034 void *cvaddr;
1034 1035 uint_t npages, pgoffset;
1035 1036 struct regspec *rp;
1036 1037 ddi_acc_hdl_t *hp;
1037 1038 ddi_acc_impl_t *ap;
1038 1039 uint_t hat_acc_flags;
1039 1040 paddr_t pbase;
1040 1041
1041 1042 rp = mp->map_obj.rp;
1042 1043 hp = mp->map_handlep;
1043 1044
1044 1045 #ifdef DDI_MAP_DEBUG
1045 1046 ddi_map_debug(
1046 1047 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1047 1048 rp->regspec_bustype, rp->regspec_addr,
1048 1049 rp->regspec_size, mp->map_handlep);
1049 1050 #endif /* DDI_MAP_DEBUG */
1050 1051
1051 1052 /*
1052 1053 * I/O or memory mapping
1053 1054 *
1054 1055 * <bustype=0, addr=x, len=x>: memory
1055 1056 * <bustype=1, addr=x, len=x>: i/o
1056 1057 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1057 1058 */
1058 1059
1059 1060 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1060 1061 cmn_err(CE_WARN, "rootnex: invalid register spec"
1061 1062 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1062 1063 rp->regspec_addr, rp->regspec_size);
1063 1064 return (DDI_FAILURE);
1064 1065 }
1065 1066
1066 1067 if (rp->regspec_bustype != 0) {
1067 1068 /*
1068 1069 * I/O space - needs a handle.
1069 1070 */
1070 1071 if (hp == NULL) {
1071 1072 return (DDI_FAILURE);
1072 1073 }
1073 1074 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1074 1075 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1075 1076 impl_acc_hdl_init(hp);
1076 1077
1077 1078 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1078 1079 #ifdef DDI_MAP_DEBUG
1079 1080 ddi_map_debug("rootnex_map_regspec: mmap() "
1080 1081 "to I/O space is not supported.\n");
1081 1082 #endif /* DDI_MAP_DEBUG */
1082 1083 return (DDI_ME_INVAL);
1083 1084 } else {
1084 1085 /*
1085 1086 * 1275-compliant vs. compatibility i/o mapping
1086 1087 */
1087 1088 *vaddrp =
1088 1089 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1089 1090 ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1090 1091 ((caddr_t)(uintptr_t)rp->regspec_addr);
1091 1092 #ifdef __xpv
1092 1093 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1093 1094 hp->ah_pfn = xen_assign_pfn(
1094 1095 mmu_btop((ulong_t)rp->regspec_addr &
1095 1096 MMU_PAGEMASK));
1096 1097 } else {
1097 1098 hp->ah_pfn = mmu_btop(
1098 1099 (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1099 1100 }
1100 1101 #else
1101 1102 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1102 1103 MMU_PAGEMASK);
1103 1104 #endif
1104 1105 hp->ah_pnum = mmu_btopr(rp->regspec_size +
1105 1106 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1106 1107 }
1107 1108
1108 1109 #ifdef DDI_MAP_DEBUG
1109 1110 ddi_map_debug(
1110 1111 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1111 1112 rp->regspec_size, *vaddrp);
1112 1113 #endif /* DDI_MAP_DEBUG */
1113 1114 return (DDI_SUCCESS);
1114 1115 }
1115 1116
1116 1117 /*
1117 1118 * Memory space
1118 1119 */
1119 1120
1120 1121 if (hp != NULL) {
1121 1122 /*
1122 1123 * hat layer ignores
1123 1124 * hp->ah_acc.devacc_attr_endian_flags.
1124 1125 */
1125 1126 switch (hp->ah_acc.devacc_attr_dataorder) {
1126 1127 case DDI_STRICTORDER_ACC:
1127 1128 hat_acc_flags = HAT_STRICTORDER;
1128 1129 break;
1129 1130 case DDI_UNORDERED_OK_ACC:
1130 1131 hat_acc_flags = HAT_UNORDERED_OK;
1131 1132 break;
1132 1133 case DDI_MERGING_OK_ACC:
1133 1134 hat_acc_flags = HAT_MERGING_OK;
1134 1135 break;
1135 1136 case DDI_LOADCACHING_OK_ACC:
1136 1137 hat_acc_flags = HAT_LOADCACHING_OK;
1137 1138 break;
1138 1139 case DDI_STORECACHING_OK_ACC:
1139 1140 hat_acc_flags = HAT_STORECACHING_OK;
1140 1141 break;
1141 1142 }
1142 1143 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1143 1144 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1144 1145 impl_acc_hdl_init(hp);
1145 1146 hp->ah_hat_flags = hat_acc_flags;
1146 1147 } else {
1147 1148 hat_acc_flags = HAT_STRICTORDER;
1148 1149 }
1149 1150
1150 1151 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1151 1152 #ifdef __xpv
1152 1153 /*
1153 1154 * If we're dom0, we're using a real device so we need to translate
1154 1155 * the MA to a PA.
1155 1156 */
1156 1157 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1157 1158 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1158 1159 } else {
1159 1160 pbase = rbase;
1160 1161 }
1161 1162 #else
1162 1163 pbase = rbase;
1163 1164 #endif
1164 1165 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1165 1166
1166 1167 if (rp->regspec_size == 0) {
1167 1168 #ifdef DDI_MAP_DEBUG
1168 1169 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1169 1170 #endif /* DDI_MAP_DEBUG */
1170 1171 return (DDI_ME_INVAL);
1171 1172 }
1172 1173
1173 1174 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1174 1175 /* extra cast to make gcc happy */
1175 1176 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1176 1177 } else {
1177 1178 npages = mmu_btopr(rp->regspec_size + pgoffset);
1178 1179
1179 1180 #ifdef DDI_MAP_DEBUG
1180 1181 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1181 1182 "physical %llx", npages, pbase);
1182 1183 #endif /* DDI_MAP_DEBUG */
1183 1184
1184 1185 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1185 1186 if (cvaddr == NULL)
1186 1187 return (DDI_ME_NORESOURCES);
1187 1188
1188 1189 /*
1189 1190 * Now map in the pages we've allocated...
1190 1191 */
1191 1192 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1192 1193 mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1193 1194 HAT_LOAD_LOCK);
1194 1195 *vaddrp = (caddr_t)cvaddr + pgoffset;
1195 1196
1196 1197 /* save away pfn and npages for FMA */
1197 1198 hp = mp->map_handlep;
1198 1199 if (hp) {
1199 1200 hp->ah_pfn = mmu_btop(pbase);
1200 1201 hp->ah_pnum = npages;
1201 1202 }
1202 1203 }
1203 1204
1204 1205 #ifdef DDI_MAP_DEBUG
1205 1206 ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1206 1207 #endif /* DDI_MAP_DEBUG */
1207 1208 return (DDI_SUCCESS);
1208 1209 }
1209 1210
1210 1211
1211 1212 /*
1212 1213 * rootnex_unmap_regspec()
1213 1214 *
1214 1215 */
1215 1216 static int
1216 1217 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1217 1218 {
1218 1219 caddr_t addr = (caddr_t)*vaddrp;
1219 1220 uint_t npages, pgoffset;
1220 1221 struct regspec *rp;
1221 1222
1222 1223 if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1223 1224 return (0);
1224 1225
1225 1226 rp = mp->map_obj.rp;
1226 1227
1227 1228 if (rp->regspec_size == 0) {
1228 1229 #ifdef DDI_MAP_DEBUG
1229 1230 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1230 1231 #endif /* DDI_MAP_DEBUG */
1231 1232 return (DDI_ME_INVAL);
1232 1233 }
1233 1234
1234 1235 /*
1235 1236 * I/O or memory mapping:
1236 1237 *
1237 1238 * <bustype=0, addr=x, len=x>: memory
1238 1239 * <bustype=1, addr=x, len=x>: i/o
1239 1240 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1240 1241 */
1241 1242 if (rp->regspec_bustype != 0) {
1242 1243 /*
1243 1244 * This is I/O space, which requires no particular
1244 1245 * processing on unmap since it isn't mapped in the
1245 1246 * first place.
1246 1247 */
1247 1248 return (DDI_SUCCESS);
1248 1249 }
1249 1250
1250 1251 /*
1251 1252 * Memory space
1252 1253 */
1253 1254 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1254 1255 npages = mmu_btopr(rp->regspec_size + pgoffset);
1255 1256 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1256 1257 device_arena_free(addr - pgoffset, ptob(npages));
1257 1258
1258 1259 /*
1259 1260 * Destroy the pointer - the mapping has logically gone
1260 1261 */
1261 1262 *vaddrp = NULL;
1262 1263
1263 1264 return (DDI_SUCCESS);
1264 1265 }
1265 1266
1266 1267
1267 1268 /*
1268 1269 * rootnex_map_handle()
1269 1270 *
1270 1271 */
1271 1272 static int
1272 1273 rootnex_map_handle(ddi_map_req_t *mp)
1273 1274 {
1274 1275 rootnex_addr_t rbase;
1275 1276 ddi_acc_hdl_t *hp;
1276 1277 uint_t pgoffset;
1277 1278 struct regspec *rp;
1278 1279 paddr_t pbase;
1279 1280
1280 1281 rp = mp->map_obj.rp;
1281 1282
1282 1283 #ifdef DDI_MAP_DEBUG
1283 1284 ddi_map_debug(
1284 1285 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1285 1286 rp->regspec_bustype, rp->regspec_addr,
1286 1287 rp->regspec_size, mp->map_handlep);
1287 1288 #endif /* DDI_MAP_DEBUG */
1288 1289
1289 1290 /*
1290 1291 * I/O or memory mapping:
1291 1292 *
1292 1293 * <bustype=0, addr=x, len=x>: memory
1293 1294 * <bustype=1, addr=x, len=x>: i/o
1294 1295 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1295 1296 */
1296 1297 if (rp->regspec_bustype != 0) {
1297 1298 /*
1298 1299 * This refers to I/O space, and we don't support "mapping"
1299 1300 * I/O space to a user.
1300 1301 */
1301 1302 return (DDI_FAILURE);
1302 1303 }
1303 1304
1304 1305 /*
1305 1306 * Set up the hat_flags for the mapping.
1306 1307 */
1307 1308 hp = mp->map_handlep;
1308 1309
1309 1310 switch (hp->ah_acc.devacc_attr_endian_flags) {
1310 1311 case DDI_NEVERSWAP_ACC:
1311 1312 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1312 1313 break;
1313 1314 case DDI_STRUCTURE_LE_ACC:
1314 1315 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1315 1316 break;
1316 1317 case DDI_STRUCTURE_BE_ACC:
1317 1318 return (DDI_FAILURE);
1318 1319 default:
1319 1320 return (DDI_REGS_ACC_CONFLICT);
1320 1321 }
1321 1322
1322 1323 switch (hp->ah_acc.devacc_attr_dataorder) {
1323 1324 case DDI_STRICTORDER_ACC:
1324 1325 break;
1325 1326 case DDI_UNORDERED_OK_ACC:
1326 1327 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1327 1328 break;
1328 1329 case DDI_MERGING_OK_ACC:
1329 1330 hp->ah_hat_flags |= HAT_MERGING_OK;
1330 1331 break;
1331 1332 case DDI_LOADCACHING_OK_ACC:
1332 1333 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1333 1334 break;
1334 1335 case DDI_STORECACHING_OK_ACC:
1335 1336 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1336 1337 break;
1337 1338 default:
1338 1339 return (DDI_FAILURE);
1339 1340 }
1340 1341
1341 1342 rbase = (rootnex_addr_t)rp->regspec_addr &
1342 1343 (~(rootnex_addr_t)MMU_PAGEOFFSET);
1343 1344 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1344 1345
1345 1346 if (rp->regspec_size == 0)
1346 1347 return (DDI_ME_INVAL);
1347 1348
1348 1349 #ifdef __xpv
1349 1350 /*
1350 1351 * If we're dom0, we're using a real device so we need to translate
1351 1352 * the MA to a PA.
1352 1353 */
1353 1354 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1354 1355 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1355 1356 (rbase & MMU_PAGEOFFSET);
1356 1357 } else {
1357 1358 pbase = rbase;
1358 1359 }
1359 1360 #else
1360 1361 pbase = rbase;
1361 1362 #endif
1362 1363
1363 1364 hp->ah_pfn = mmu_btop(pbase);
1364 1365 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1365 1366
1366 1367 return (DDI_SUCCESS);
1367 1368 }
1368 1369
1369 1370
1370 1371
1371 1372 /*
1372 1373 * ************************
1373 1374 * interrupt related code
1374 1375 * ************************
1375 1376 */
1376 1377
1377 1378 /*
1378 1379 * rootnex_intr_ops()
1379 1380 * bus_intr_op() function for interrupt support
1380 1381 */
1381 1382 /* ARGSUSED */
1382 1383 static int
1383 1384 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1384 1385 ddi_intr_handle_impl_t *hdlp, void *result)
1385 1386 {
1386 1387 struct intrspec *ispec;
1387 1388
1388 1389 DDI_INTR_NEXDBG((CE_CONT,
1389 1390 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1390 1391 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1391 1392
1392 1393 /* Process the interrupt operation */
1393 1394 switch (intr_op) {
1394 1395 case DDI_INTROP_GETCAP:
1395 1396 /* First check with pcplusmp */
1396 1397 if (psm_intr_ops == NULL)
1397 1398 return (DDI_FAILURE);
1398 1399
1399 1400 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1400 1401 *(int *)result = 0;
1401 1402 return (DDI_FAILURE);
1402 1403 }
1403 1404 break;
1404 1405 case DDI_INTROP_SETCAP:
1405 1406 if (psm_intr_ops == NULL)
1406 1407 return (DDI_FAILURE);
1407 1408
1408 1409 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1409 1410 return (DDI_FAILURE);
1410 1411 break;
1411 1412 case DDI_INTROP_ALLOC:
1412 1413 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1413 1414 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1414 1415 case DDI_INTROP_FREE:
1415 1416 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1416 1417 return (rootnex_free_intr_fixed(rdip, hdlp));
1417 1418 case DDI_INTROP_GETPRI:
1418 1419 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1419 1420 return (DDI_FAILURE);
1420 1421 *(int *)result = ispec->intrspec_pri;
1421 1422 break;
1422 1423 case DDI_INTROP_SETPRI:
1423 1424 /* Validate the interrupt priority passed to us */
1424 1425 if (*(int *)result > LOCK_LEVEL)
1425 1426 return (DDI_FAILURE);
1426 1427
1427 1428 /* Ensure that PSM is all initialized and ispec is ok */
1428 1429 if ((psm_intr_ops == NULL) ||
1429 1430 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1430 1431 return (DDI_FAILURE);
1431 1432
1432 1433 /* Change the priority */
1433 1434 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1434 1435 PSM_FAILURE)
1435 1436 return (DDI_FAILURE);
1436 1437
1437 1438 /* update the ispec with the new priority */
1438 1439 ispec->intrspec_pri = *(int *)result;
1439 1440 break;
1440 1441 case DDI_INTROP_ADDISR:
1441 1442 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1442 1443 return (DDI_FAILURE);
1443 1444 ispec->intrspec_func = hdlp->ih_cb_func;
1444 1445 break;
1445 1446 case DDI_INTROP_REMISR:
1446 1447 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1447 1448 return (DDI_FAILURE);
1448 1449 ispec->intrspec_func = (uint_t (*)()) 0;
1449 1450 break;
1450 1451 case DDI_INTROP_ENABLE:
1451 1452 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1452 1453 return (DDI_FAILURE);
1453 1454
1454 1455 /* Call psmi to translate irq with the dip */
1455 1456 if (psm_intr_ops == NULL)
1456 1457 return (DDI_FAILURE);
1457 1458
1458 1459 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1459 1460 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1460 1461 (int *)&hdlp->ih_vector) == PSM_FAILURE)
1461 1462 return (DDI_FAILURE);
1462 1463
1463 1464 /* Add the interrupt handler */
1464 1465 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1465 1466 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1466 1467 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1467 1468 return (DDI_FAILURE);
1468 1469 break;
1469 1470 case DDI_INTROP_DISABLE:
1470 1471 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1471 1472 return (DDI_FAILURE);
1472 1473
1473 1474 /* Call psm_ops() to translate irq with the dip */
1474 1475 if (psm_intr_ops == NULL)
1475 1476 return (DDI_FAILURE);
1476 1477
1477 1478 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1478 1479 (void) (*psm_intr_ops)(rdip, hdlp,
1479 1480 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1480 1481
1481 1482 /* Remove the interrupt handler */
1482 1483 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1483 1484 hdlp->ih_cb_func, hdlp->ih_vector);
1484 1485 break;
1485 1486 case DDI_INTROP_SETMASK:
1486 1487 if (psm_intr_ops == NULL)
1487 1488 return (DDI_FAILURE);
1488 1489
1489 1490 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1490 1491 return (DDI_FAILURE);
1491 1492 break;
1492 1493 case DDI_INTROP_CLRMASK:
1493 1494 if (psm_intr_ops == NULL)
1494 1495 return (DDI_FAILURE);
1495 1496
1496 1497 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1497 1498 return (DDI_FAILURE);
1498 1499 break;
1499 1500 case DDI_INTROP_GETPENDING:
1500 1501 if (psm_intr_ops == NULL)
1501 1502 return (DDI_FAILURE);
1502 1503
1503 1504 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1504 1505 result)) {
1505 1506 *(int *)result = 0;
1506 1507 return (DDI_FAILURE);
1507 1508 }
1508 1509 break;
1509 1510 case DDI_INTROP_NAVAIL:
1510 1511 case DDI_INTROP_NINTRS:
1511 1512 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1512 1513 if (*(int *)result == 0) {
1513 1514 /*
1514 1515 * Special case for 'pcic' driver' only. This driver
1515 1516 * driver is a child of 'isa' and 'rootnex' drivers.
1516 1517 *
1517 1518 * See detailed comments on this in the function
1518 1519 * rootnex_get_ispec().
1519 1520 *
1520 1521 * Children of 'pcic' send 'NINITR' request all the
1521 1522 * way to rootnex driver. But, the 'pdp->par_nintr'
1522 1523 * field may not initialized. So, we fake it here
1523 1524 * to return 1 (a la what PCMCIA nexus does).
1524 1525 */
1525 1526 if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1526 1527 *(int *)result = 1;
1527 1528 else
1528 1529 return (DDI_FAILURE);
1529 1530 }
1530 1531 break;
1531 1532 case DDI_INTROP_SUPPORTED_TYPES:
1532 1533 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
1533 1534 break;
1534 1535 default:
1535 1536 return (DDI_FAILURE);
1536 1537 }
1537 1538
1538 1539 return (DDI_SUCCESS);
1539 1540 }
1540 1541
1541 1542
1542 1543 /*
1543 1544 * rootnex_get_ispec()
1544 1545 * convert an interrupt number to an interrupt specification.
1545 1546 * The interrupt number determines which interrupt spec will be
1546 1547 * returned if more than one exists.
1547 1548 *
1548 1549 * Look into the parent private data area of the 'rdip' to find out
1549 1550 * the interrupt specification. First check to make sure there is
1550 1551 * one that matchs "inumber" and then return a pointer to it.
1551 1552 *
1552 1553 * Return NULL if one could not be found.
1553 1554 *
1554 1555 * NOTE: This is needed for rootnex_intr_ops()
1555 1556 */
1556 1557 static struct intrspec *
1557 1558 rootnex_get_ispec(dev_info_t *rdip, int inum)
1558 1559 {
1559 1560 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1560 1561
1561 1562 /*
1562 1563 * Special case handling for drivers that provide their own
1563 1564 * intrspec structures instead of relying on the DDI framework.
1564 1565 *
1565 1566 * A broken hardware driver in ON could potentially provide its
1566 1567 * own intrspec structure, instead of relying on the hardware.
1567 1568 * If these drivers are children of 'rootnex' then we need to
1568 1569 * continue to provide backward compatibility to them here.
1569 1570 *
1570 1571 * Following check is a special case for 'pcic' driver which
1571 1572 * was found to have broken hardwre andby provides its own intrspec.
1572 1573 *
1573 1574 * Verbatim comments from this driver are shown here:
1574 1575 * "Don't use the ddi_add_intr since we don't have a
1575 1576 * default intrspec in all cases."
1576 1577 *
1577 1578 * Since an 'ispec' may not be always created for it,
1578 1579 * check for that and create one if so.
1579 1580 *
1580 1581 * NOTE: Currently 'pcic' is the only driver found to do this.
1581 1582 */
1582 1583 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1583 1584 pdp->par_nintr = 1;
1584 1585 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1585 1586 pdp->par_nintr, KM_SLEEP);
1586 1587 }
1587 1588
1588 1589 /* Validate the interrupt number */
1589 1590 if (inum >= pdp->par_nintr)
1590 1591 return (NULL);
1591 1592
1592 1593 /* Get the interrupt structure pointer and return that */
1593 1594 return ((struct intrspec *)&pdp->par_intr[inum]);
1594 1595 }
1595 1596
1596 1597 /*
1597 1598 * Allocate interrupt vector for FIXED (legacy) type.
1598 1599 */
1599 1600 static int
1600 1601 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1601 1602 void *result)
1602 1603 {
1603 1604 struct intrspec *ispec;
1604 1605 ddi_intr_handle_impl_t info_hdl;
1605 1606 int ret;
1606 1607 int free_phdl = 0;
1607 1608 apic_get_type_t type_info;
1608 1609
1609 1610 if (psm_intr_ops == NULL)
1610 1611 return (DDI_FAILURE);
1611 1612
1612 1613 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1613 1614 return (DDI_FAILURE);
1614 1615
1615 1616 /*
1616 1617 * If the PSM module is "APIX" then pass the request for it
1617 1618 * to allocate the vector now.
1618 1619 */
1619 1620 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1620 1621 info_hdl.ih_private = &type_info;
1621 1622 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1622 1623 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1623 1624 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1624 1625 free_phdl = 1;
1625 1626 i_ddi_alloc_intr_phdl(hdlp);
1626 1627 }
1627 1628 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1628 1629 ret = (*psm_intr_ops)(rdip, hdlp,
1629 1630 PSM_INTR_OP_ALLOC_VECTORS, result);
1630 1631 if (free_phdl) { /* free up the phdl structure */
1631 1632 free_phdl = 0;
1632 1633 i_ddi_free_intr_phdl(hdlp);
1633 1634 hdlp->ih_private = NULL;
1634 1635 }
1635 1636 } else {
1636 1637 /*
1637 1638 * No APIX module; fall back to the old scheme where the
1638 1639 * interrupt vector is allocated during ddi_enable_intr() call.
1639 1640 */
1640 1641 hdlp->ih_pri = ispec->intrspec_pri;
1641 1642 *(int *)result = hdlp->ih_scratch1;
1642 1643 ret = DDI_SUCCESS;
1643 1644 }
1644 1645
1645 1646 return (ret);
1646 1647 }
1647 1648
1648 1649 /*
1649 1650 * Free up interrupt vector for FIXED (legacy) type.
1650 1651 */
1651 1652 static int
1652 1653 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1653 1654 {
1654 1655 struct intrspec *ispec;
1655 1656 struct ddi_parent_private_data *pdp;
1656 1657 ddi_intr_handle_impl_t info_hdl;
1657 1658 int ret;
1658 1659 apic_get_type_t type_info;
1659 1660
1660 1661 if (psm_intr_ops == NULL)
1661 1662 return (DDI_FAILURE);
1662 1663
1663 1664 /*
1664 1665 * If the PSM module is "APIX" then pass the request for it
1665 1666 * to free up the vector now.
1666 1667 */
1667 1668 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1668 1669 info_hdl.ih_private = &type_info;
1669 1670 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1670 1671 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1671 1672 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1672 1673 return (DDI_FAILURE);
1673 1674 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1674 1675 ret = (*psm_intr_ops)(rdip, hdlp,
1675 1676 PSM_INTR_OP_FREE_VECTORS, NULL);
1676 1677 } else {
1677 1678 /*
1678 1679 * No APIX module; fall back to the old scheme where
1679 1680 * the interrupt vector was already freed during
1680 1681 * ddi_disable_intr() call.
1681 1682 */
1682 1683 ret = DDI_SUCCESS;
1683 1684 }
1684 1685
1685 1686 pdp = ddi_get_parent_data(rdip);
1686 1687
1687 1688 /*
1688 1689 * Special case for 'pcic' driver' only.
1689 1690 * If an intrspec was created for it, clean it up here
1690 1691 * See detailed comments on this in the function
1691 1692 * rootnex_get_ispec().
1692 1693 */
1693 1694 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1694 1695 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1695 1696 pdp->par_nintr);
1696 1697 /*
1697 1698 * Set it to zero; so that
1698 1699 * DDI framework doesn't free it again
1699 1700 */
1700 1701 pdp->par_intr = NULL;
1701 1702 pdp->par_nintr = 0;
1702 1703 }
1703 1704
1704 1705 return (ret);
1705 1706 }
1706 1707
1707 1708
1708 1709 /*
1709 1710 * ******************
1710 1711 * dma related code
1711 1712 * ******************
1712 1713 */
1713 1714
1714 1715 /*ARGSUSED*/
1715 1716 static int
1716 1717 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1717 1718 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1718 1719 ddi_dma_handle_t *handlep)
1719 1720 {
1720 1721 uint64_t maxsegmentsize_ll;
1721 1722 uint_t maxsegmentsize;
1722 1723 ddi_dma_impl_t *hp;
1723 1724 rootnex_dma_t *dma;
1724 1725 uint64_t count_max;
1725 1726 uint64_t seg;
1726 1727 int kmflag;
1727 1728 int e;
1728 1729
1729 1730
1730 1731 /* convert our sleep flags */
1731 1732 if (waitfp == DDI_DMA_SLEEP) {
1732 1733 kmflag = KM_SLEEP;
1733 1734 } else {
1734 1735 kmflag = KM_NOSLEEP;
1735 1736 }
1736 1737
1737 1738 /*
1738 1739 * We try to do only one memory allocation here. We'll do a little
1739 1740 * pointer manipulation later. If the bind ends up taking more than
1740 1741 * our prealloc's space, we'll have to allocate more memory in the
1741 1742 * bind operation. Not great, but much better than before and the
1742 1743 * best we can do with the current bind interfaces.
1743 1744 */
1744 1745 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1745 1746 if (hp == NULL)
1746 1747 return (DDI_DMA_NORESOURCES);
1747 1748
1748 1749 /* Do our pointer manipulation now, align the structures */
1749 1750 hp->dmai_private = (void *)(((uintptr_t)hp +
1750 1751 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1751 1752 dma = (rootnex_dma_t *)hp->dmai_private;
1752 1753 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1753 1754 sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1754 1755
1755 1756 /* setup the handle */
1756 1757 rootnex_clean_dmahdl(hp);
1757 1758 hp->dmai_error.err_fep = NULL;
1758 1759 hp->dmai_error.err_cf = NULL;
1759 1760 dma->dp_dip = rdip;
1760 1761 dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1761 1762 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1762 1763
1763 1764 /*
1764 1765 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1765 1766 * is being used. Set the upper limit to the seg value.
1766 1767 * There will be enough DVMA space to always get addresses
1767 1768 * that will match the constraints.
1768 1769 */
1769 1770 if (IOMMU_USED(rdip) &&
1770 1771 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1771 1772 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1772 1773 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1773 1774 } else
1774 1775 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1775 1776
1776 1777 hp->dmai_minxfer = attr->dma_attr_minxfer;
1777 1778 hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1778 1779 hp->dmai_rdip = rdip;
1779 1780 hp->dmai_attr = *attr;
1780 1781
1781 1782 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1782 1783 dma->dp_sglinfo.si_cancross = B_FALSE;
1783 1784 else
1784 1785 dma->dp_sglinfo.si_cancross = B_TRUE;
1785 1786
1786 1787 /* we don't need to worry about the SPL since we do a tryenter */
1787 1788 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1788 1789
1789 1790 /*
1790 1791 * Figure out our maximum segment size. If the segment size is greater
1791 1792 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1792 1793 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1793 1794 * dma_attr_count_max are size-1 type values.
1794 1795 *
1795 1796 * Maximum segment size is the largest physically contiguous chunk of
1796 1797 * memory that we can return from a bind (i.e. the maximum size of a
1797 1798 * single cookie).
1798 1799 */
1799 1800
1800 1801 /* handle the rollover cases */
1801 1802 seg = attr->dma_attr_seg + 1;
1802 1803 if (seg < attr->dma_attr_seg) {
1803 1804 seg = attr->dma_attr_seg;
1804 1805 }
1805 1806 count_max = attr->dma_attr_count_max + 1;
1806 1807 if (count_max < attr->dma_attr_count_max) {
1807 1808 count_max = attr->dma_attr_count_max;
1808 1809 }
1809 1810
1810 1811 /*
1811 1812 * granularity may or may not be a power of two. If it isn't, we can't
1812 1813 * use a simple mask.
1813 1814 */
1814 1815 if (!ISP2(attr->dma_attr_granular)) {
1815 1816 dma->dp_granularity_power_2 = B_FALSE;
1816 1817 } else {
1817 1818 dma->dp_granularity_power_2 = B_TRUE;
1818 1819 }
1819 1820
1820 1821 /*
1821 1822 * maxxfer should be a whole multiple of granularity. If we're going to
1822 1823 * break up a window because we're greater than maxxfer, we might as
1823 1824 * well make sure it's maxxfer is a whole multiple so we don't have to
1824 1825 * worry about triming the window later on for this case.
1825 1826 */
1826 1827 if (attr->dma_attr_granular > 1) {
1827 1828 if (dma->dp_granularity_power_2) {
1828 1829 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1829 1830 (attr->dma_attr_maxxfer &
1830 1831 (attr->dma_attr_granular - 1));
1831 1832 } else {
1832 1833 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1833 1834 (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1834 1835 }
1835 1836 } else {
1836 1837 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1837 1838 }
1838 1839
1839 1840 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1840 1841 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1841 1842 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1842 1843 maxsegmentsize = 0xFFFFFFFF;
1843 1844 } else {
1844 1845 maxsegmentsize = maxsegmentsize_ll;
1845 1846 }
1846 1847 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1847 1848 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1848 1849
1849 1850 /* check the ddi_dma_attr arg to make sure it makes a little sense */
1850 1851 if (rootnex_alloc_check_parms) {
1851 1852 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1852 1853 if (e != DDI_SUCCESS) {
1853 1854 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1854 1855 (void) rootnex_dma_freehdl(dip, rdip,
1855 1856 (ddi_dma_handle_t)hp);
1856 1857 return (e);
1857 1858 }
1858 1859 }
1859 1860
1860 1861 *handlep = (ddi_dma_handle_t)hp;
1861 1862
1862 1863 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1863 1864 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1864 1865 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1865 1866
1866 1867 return (DDI_SUCCESS);
1867 1868 }
1868 1869
1869 1870
1870 1871 /*
1871 1872 * rootnex_dma_allochdl()
1872 1873 * called from ddi_dma_alloc_handle().
1873 1874 */
1874 1875 static int
1875 1876 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1876 1877 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1877 1878 {
1878 1879 int retval = DDI_SUCCESS;
1879 1880 #if defined(__amd64) && !defined(__xpv)
1880 1881
1881 1882 if (IOMMU_UNITIALIZED(rdip)) {
1882 1883 retval = iommulib_nex_open(dip, rdip);
1883 1884
1884 1885 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1885 1886 return (retval);
1886 1887 }
1887 1888
1888 1889 if (IOMMU_UNUSED(rdip)) {
1889 1890 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1890 1891 handlep);
1891 1892 } else {
1892 1893 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1893 1894 waitfp, arg, handlep);
1894 1895 }
1895 1896 #else
1896 1897 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1897 1898 handlep);
1898 1899 #endif
1899 1900 switch (retval) {
1900 1901 case DDI_DMA_NORESOURCES:
1901 1902 if (waitfp != DDI_DMA_DONTWAIT) {
1902 1903 ddi_set_callback(waitfp, arg,
1903 1904 &rootnex_state->r_dvma_call_list_id);
1904 1905 }
1905 1906 break;
1906 1907 case DDI_SUCCESS:
1907 1908 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1908 1909 break;
1909 1910 default:
1910 1911 break;
1911 1912 }
1912 1913 return (retval);
1913 1914 }
1914 1915
1915 1916 /*ARGSUSED*/
1916 1917 static int
1917 1918 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1918 1919 ddi_dma_handle_t handle)
1919 1920 {
1920 1921 ddi_dma_impl_t *hp;
1921 1922 rootnex_dma_t *dma;
1922 1923
1923 1924
1924 1925 hp = (ddi_dma_impl_t *)handle;
1925 1926 dma = (rootnex_dma_t *)hp->dmai_private;
1926 1927
1927 1928 /* unbind should have been called first */
1928 1929 ASSERT(!dma->dp_inuse);
1929 1930
1930 1931 mutex_destroy(&dma->dp_mutex);
1931 1932 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1932 1933
1933 1934 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1934 1935 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1935 1936 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1936 1937
1937 1938 return (DDI_SUCCESS);
1938 1939 }
1939 1940
1940 1941 /*
1941 1942 * rootnex_dma_freehdl()
1942 1943 * called from ddi_dma_free_handle().
1943 1944 */
1944 1945 static int
1945 1946 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1946 1947 {
1947 1948 int ret;
1948 1949
1949 1950 ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1950 1951 #if defined(__amd64) && !defined(__xpv)
1951 1952 if (IOMMU_USED(rdip))
1952 1953 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1953 1954 else
1954 1955 #endif
1955 1956 ret = rootnex_coredma_freehdl(dip, rdip, handle);
1956 1957
1957 1958 if (rootnex_state->r_dvma_call_list_id)
1958 1959 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1959 1960
1960 1961 return (ret);
1961 1962 }
1962 1963
1963 1964 /*ARGSUSED*/
1964 1965 static int
1965 1966 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1966 1967 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1967 1968 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1968 1969 {
1969 1970 rootnex_sglinfo_t *sinfo;
1970 1971 ddi_dma_obj_t *dmao;
1971 1972 #if defined(__amd64) && !defined(__xpv)
1972 1973 struct dvmaseg *dvs;
1973 1974 ddi_dma_cookie_t *cookie;
1974 1975 #endif
1975 1976 ddi_dma_attr_t *attr;
1976 1977 ddi_dma_impl_t *hp;
1977 1978 rootnex_dma_t *dma;
1978 1979 int kmflag;
1979 1980 int e;
1980 1981 uint_t ncookies;
1981 1982
1982 1983 hp = (ddi_dma_impl_t *)handle;
1983 1984 dma = (rootnex_dma_t *)hp->dmai_private;
1984 1985 dmao = &dma->dp_dma;
1985 1986 sinfo = &dma->dp_sglinfo;
1986 1987 attr = &hp->dmai_attr;
1987 1988
1988 1989 /* convert the sleep flags */
1989 1990 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1990 1991 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1991 1992 } else {
1992 1993 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1993 1994 }
1994 1995
1995 1996 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1996 1997
1997 1998 /*
1998 1999 * This is useful for debugging a driver. Not as useful in a production
1999 2000 * system. The only time this will fail is if you have a driver bug.
2000 2001 */
2001 2002 if (rootnex_bind_check_inuse) {
2002 2003 /*
2003 2004 * No one else should ever have this lock unless someone else
2004 2005 * is trying to use this handle. So contention on the lock
2005 2006 * is the same as inuse being set.
2006 2007 */
2007 2008 e = mutex_tryenter(&dma->dp_mutex);
2008 2009 if (e == 0) {
2009 2010 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2010 2011 return (DDI_DMA_INUSE);
2011 2012 }
2012 2013 if (dma->dp_inuse) {
2013 2014 mutex_exit(&dma->dp_mutex);
2014 2015 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2015 2016 return (DDI_DMA_INUSE);
2016 2017 }
2017 2018 dma->dp_inuse = B_TRUE;
2018 2019 mutex_exit(&dma->dp_mutex);
2019 2020 }
2020 2021
2021 2022 /* check the ddi_dma_attr arg to make sure it makes a little sense */
2022 2023 if (rootnex_bind_check_parms) {
2023 2024 e = rootnex_valid_bind_parms(dmareq, attr);
2024 2025 if (e != DDI_SUCCESS) {
2025 2026 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2026 2027 rootnex_clean_dmahdl(hp);
2027 2028 return (e);
2028 2029 }
2029 2030 }
2030 2031
2031 2032 /* save away the original bind info */
2032 2033 dma->dp_dma = dmareq->dmar_object;
2033 2034
2034 2035 #if defined(__amd64) && !defined(__xpv)
2035 2036 if (IOMMU_USED(rdip)) {
2036 2037 dmao = &dma->dp_dvma;
2037 2038 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2038 2039 switch (e) {
2039 2040 case DDI_SUCCESS:
2040 2041 if (sinfo->si_cancross ||
2041 2042 dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2042 2043 dmao->dmao_size > sinfo->si_max_cookie_size) {
2043 2044 dma->dp_dvma_used = B_TRUE;
2044 2045 break;
2045 2046 }
2046 2047 sinfo->si_sgl_size = 1;
2047 2048 hp->dmai_rflags |= DMP_NOSYNC;
2048 2049
2049 2050 dma->dp_dvma_used = B_TRUE;
2050 2051 dma->dp_need_to_free_cookie = B_FALSE;
2051 2052
2052 2053 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2053 2054 cookie = hp->dmai_cookie = dma->dp_cookies =
2054 2055 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2055 2056 cookie->dmac_laddress = dvs->dvs_start +
2056 2057 dmao->dmao_obj.dvma_obj.dv_off;
2057 2058 cookie->dmac_size = dvs->dvs_len;
2058 2059 cookie->dmac_type = 0;
2059 2060
2060 2061 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2061 2062 rdip);
2062 2063 goto fast;
2063 2064 case DDI_ENOTSUP:
2064 2065 break;
2065 2066 default:
2066 2067 rootnex_clean_dmahdl(hp);
2067 2068 return (e);
2068 2069 }
2069 2070 }
2070 2071 #endif
2071 2072
2072 2073 /*
2073 2074 * Figure out a rough estimate of what maximum number of pages
2074 2075 * this buffer could use (a high estimate of course).
2075 2076 */
2076 2077 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2077 2078
2078 2079 if (dma->dp_dvma_used) {
2079 2080 /*
2080 2081 * The number of physical pages is the worst case.
2081 2082 *
2082 2083 * For DVMA, the worst case is the length divided
2083 2084 * by the maximum cookie length, plus 1. Add to that
2084 2085 * the number of segment boundaries potentially crossed, and
2085 2086 * the additional number of DVMA segments that was returned.
2086 2087 *
2087 2088 * In the normal case, for modern devices, si_cancross will
2088 2089 * be false, and dv_nseg will be 1, and the fast path will
2089 2090 * have been taken above.
2090 2091 */
2091 2092 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2092 2093 + 1;
2093 2094 if (sinfo->si_cancross)
2094 2095 ncookies +=
2095 2096 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2096 2097 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2097 2098
2098 2099 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2099 2100 }
2100 2101
2101 2102 /*
2102 2103 * We'll use the pre-allocated cookies for any bind that will *always*
2103 2104 * fit (more important to be consistent, we don't want to create
2104 2105 * additional degenerate cases).
2105 2106 */
2106 2107 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2107 2108 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2108 2109 dma->dp_need_to_free_cookie = B_FALSE;
2109 2110 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2110 2111 uint_t, sinfo->si_max_pages);
2111 2112
2112 2113 /*
2113 2114 * For anything larger than that, we'll go ahead and allocate the
2114 2115 * maximum number of pages we expect to see. Hopefuly, we won't be
2115 2116 * seeing this path in the fast path for high performance devices very
2116 2117 * frequently.
2117 2118 *
2118 2119 * a ddi bind interface that allowed the driver to provide storage to
2119 2120 * the bind interface would speed this case up.
2120 2121 */
2121 2122 } else {
2122 2123 /*
2123 2124 * Save away how much memory we allocated. If we're doing a
2124 2125 * nosleep, the alloc could fail...
2125 2126 */
2126 2127 dma->dp_cookie_size = sinfo->si_max_pages *
2127 2128 sizeof (ddi_dma_cookie_t);
2128 2129 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2129 2130 if (dma->dp_cookies == NULL) {
2130 2131 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2131 2132 rootnex_clean_dmahdl(hp);
2132 2133 return (DDI_DMA_NORESOURCES);
2133 2134 }
2134 2135 dma->dp_need_to_free_cookie = B_TRUE;
2135 2136 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2136 2137 uint_t, sinfo->si_max_pages);
2137 2138 }
2138 2139 hp->dmai_cookie = dma->dp_cookies;
2139 2140
2140 2141 /*
2141 2142 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2142 2143 * looking at the constraints in the dma structure. It will then put
2143 2144 * some additional state about the sgl in the dma struct (i.e. is
2144 2145 * the sgl clean, or do we need to do some munging; how many pages
2145 2146 * need to be copied, etc.)
2146 2147 */
2147 2148 if (dma->dp_dvma_used)
2148 2149 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2149 2150 else
2150 2151 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2151 2152
2152 2153 out:
2153 2154 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2154 2155 /* if we don't need a copy buffer, we don't need to sync */
2155 2156 if (sinfo->si_copybuf_req == 0) {
2156 2157 hp->dmai_rflags |= DMP_NOSYNC;
2157 2158 }
2158 2159
2159 2160 /*
2160 2161 * if we don't need the copybuf and we don't need to do a partial, we
2161 2162 * hit the fast path. All the high performance devices should be trying
2162 2163 * to hit this path. To hit this path, a device should be able to reach
2163 2164 * all of memory, shouldn't try to bind more than it can transfer, and
2164 2165 * the buffer shouldn't require more cookies than the driver/device can
2165 2166 * handle [sgllen]).
2166 2167 *
2167 2168 * Note that negative values of dma_attr_sgllen are supposed
2168 2169 * to mean unlimited, but we just cast them to mean a
2169 2170 * "ridiculous large limit". This saves some extra checks on
2170 2171 * hot paths.
2171 2172 */
2172 2173 if ((sinfo->si_copybuf_req == 0) &&
2173 2174 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2174 2175 (dmao->dmao_size < dma->dp_maxxfer)) {
2175 2176 fast:
2176 2177 /*
2177 2178 * If the driver supports FMA, insert the handle in the FMA DMA
2178 2179 * handle cache.
2179 2180 */
2180 2181 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2181 2182 hp->dmai_error.err_cf = rootnex_dma_check;
2182 2183
2183 2184 /*
2184 2185 * copy out the first cookie and ccountp, set the cookie
2185 2186 * pointer to the second cookie. The first cookie is passed
2186 2187 * back on the stack. Additional cookies are accessed via
2187 2188 * ddi_dma_nextcookie()
2188 2189 */
2189 2190 *cookiep = dma->dp_cookies[0];
2190 2191 *ccountp = sinfo->si_sgl_size;
2191 2192 hp->dmai_cookie++;
2192 2193 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2193 2194 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2194 2195 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2195 2196 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2196 2197 uint_t, dmao->dmao_size, uint_t, *ccountp);
2197 2198
2198 2199
2199 2200 return (DDI_DMA_MAPPED);
2200 2201 }
2201 2202
2202 2203 /*
2203 2204 * go to the slow path, we may need to alloc more memory, create
2204 2205 * multiple windows, and munge up a sgl to make the device happy.
2205 2206 */
2206 2207
2207 2208 /*
2208 2209 * With the IOMMU mapobject method used, we should never hit
2209 2210 * the slow path. If we do, something is seriously wrong.
2210 2211 * Clean up and return an error.
2211 2212 */
2212 2213
2213 2214 #if defined(__amd64) && !defined(__xpv)
2214 2215
2215 2216 if (dma->dp_dvma_used) {
2216 2217 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2217 2218 &dma->dp_dvma);
2218 2219 e = DDI_DMA_NOMAPPING;
2219 2220 } else {
2220 2221 #endif
2221 2222 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2222 2223 kmflag);
2223 2224 #if defined(__amd64) && !defined(__xpv)
2224 2225 }
2225 2226 #endif
2226 2227 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2227 2228 if (dma->dp_need_to_free_cookie) {
2228 2229 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2229 2230 }
2230 2231 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2231 2232 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2232 2233 return (e);
2233 2234 }
2234 2235
2235 2236 /*
2236 2237 * If the driver supports FMA, insert the handle in the FMA DMA handle
2237 2238 * cache.
2238 2239 */
2239 2240 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2240 2241 hp->dmai_error.err_cf = rootnex_dma_check;
2241 2242
2242 2243 /* if the first window uses the copy buffer, sync it for the device */
2243 2244 if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2244 2245 (hp->dmai_rflags & DDI_DMA_WRITE)) {
2245 2246 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2246 2247 DDI_DMA_SYNC_FORDEV);
2247 2248 }
2248 2249
2249 2250 /*
2250 2251 * copy out the first cookie and ccountp, set the cookie pointer to the
2251 2252 * second cookie. Make sure the partial flag is set/cleared correctly.
2252 2253 * If we have a partial map (i.e. multiple windows), the number of
2253 2254 * cookies we return is the number of cookies in the first window.
2254 2255 */
2255 2256 if (e == DDI_DMA_MAPPED) {
2256 2257 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2257 2258 *ccountp = sinfo->si_sgl_size;
2258 2259 hp->dmai_nwin = 1;
2259 2260 } else {
2260 2261 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2261 2262 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2262 2263 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2263 2264 }
2264 2265 *cookiep = dma->dp_cookies[0];
2265 2266 hp->dmai_cookie++;
2266 2267
2267 2268 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2268 2269 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2269 2270 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2270 2271 dmao->dmao_size, uint_t, *ccountp);
2271 2272 return (e);
2272 2273 }
2273 2274
2274 2275 /*
2275 2276 * rootnex_dma_bindhdl()
2276 2277 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2277 2278 */
2278 2279 static int
2279 2280 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2280 2281 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2281 2282 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2282 2283 {
2283 2284 int ret;
2284 2285 #if defined(__amd64) && !defined(__xpv)
2285 2286 if (IOMMU_USED(rdip))
2286 2287 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2287 2288 cookiep, ccountp);
2288 2289 else
2289 2290 #endif
2290 2291 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2291 2292 cookiep, ccountp);
2292 2293
2293 2294 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2294 2295 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2295 2296 &rootnex_state->r_dvma_call_list_id);
2296 2297 }
2297 2298
2298 2299 return (ret);
2299 2300 }
2300 2301
2301 2302
2302 2303
2303 2304 /*ARGSUSED*/
2304 2305 static int
2305 2306 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2306 2307 ddi_dma_handle_t handle)
2307 2308 {
2308 2309 ddi_dma_impl_t *hp;
2309 2310 rootnex_dma_t *dma;
2310 2311 int e;
2311 2312
2312 2313 hp = (ddi_dma_impl_t *)handle;
2313 2314 dma = (rootnex_dma_t *)hp->dmai_private;
2314 2315
2315 2316 /* make sure the buffer wasn't free'd before calling unbind */
2316 2317 if (rootnex_unbind_verify_buffer) {
2317 2318 e = rootnex_verify_buffer(dma);
2318 2319 if (e != DDI_SUCCESS) {
2319 2320 ASSERT(0);
2320 2321 return (DDI_FAILURE);
2321 2322 }
2322 2323 }
2323 2324
2324 2325 /* sync the current window before unbinding the buffer */
2325 2326 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2326 2327 (hp->dmai_rflags & DDI_DMA_READ)) {
2327 2328 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2328 2329 DDI_DMA_SYNC_FORCPU);
↓ open down ↓ |
2291 lines elided |
↑ open up ↑ |
2329 2330 }
2330 2331
2331 2332 /*
2332 2333 * cleanup and copy buffer or window state. if we didn't use the copy
2333 2334 * buffer or windows, there won't be much to do :-)
2334 2335 */
2335 2336 rootnex_teardown_copybuf(dma);
2336 2337 rootnex_teardown_windows(dma);
2337 2338
2338 2339 #if defined(__amd64) && !defined(__xpv)
2339 - if (IOMMU_USED(rdip))
2340 + if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2340 2341 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2341 2342 &dma->dp_dvma);
2342 2343 #endif
2343 2344
2344 2345 /*
2345 2346 * If we had to allocate space to for the worse case sgl (it didn't
2346 2347 * fit into our pre-allocate buffer), free that up now
2347 2348 */
2348 2349 if (dma->dp_need_to_free_cookie) {
2349 2350 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2350 2351 }
2351 2352
2352 2353 /*
2353 2354 * clean up the handle so it's ready for the next bind (i.e. if the
2354 2355 * handle is reused).
2355 2356 */
2356 2357 rootnex_clean_dmahdl(hp);
2357 2358 hp->dmai_error.err_cf = NULL;
2358 2359
2359 2360 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2360 2361 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2361 2362 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2362 2363
2363 2364 return (DDI_SUCCESS);
2364 2365 }
2365 2366
2366 2367 /*
2367 2368 * rootnex_dma_unbindhdl()
2368 2369 * called from ddi_dma_unbind_handle()
2369 2370 */
2370 2371 /*ARGSUSED*/
2371 2372 static int
2372 2373 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2373 2374 ddi_dma_handle_t handle)
2374 2375 {
2375 2376 int ret;
2376 2377
2377 2378 #if defined(__amd64) && !defined(__xpv)
2378 2379 if (IOMMU_USED(rdip))
2379 2380 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2380 2381 else
2381 2382 #endif
2382 2383 ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2383 2384
2384 2385 if (rootnex_state->r_dvma_call_list_id)
2385 2386 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2386 2387
2387 2388 return (ret);
2388 2389 }
2389 2390
2390 2391 #if defined(__amd64) && !defined(__xpv)
2391 2392
2392 2393 static int
2393 2394 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2394 2395 {
2395 2396 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2396 2397 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2397 2398
2398 2399 if (dma->dp_sleep_flags != KM_SLEEP &&
2399 2400 dma->dp_sleep_flags != KM_NOSLEEP)
2400 2401 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2401 2402 return (dma->dp_sleep_flags);
2402 2403 }
2403 2404 /*ARGSUSED*/
2404 2405 static void
2405 2406 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2406 2407 {
2407 2408 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2408 2409 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2409 2410 rootnex_window_t *window;
2410 2411
2411 2412 if (dma->dp_window) {
2412 2413 window = &dma->dp_window[dma->dp_current_win];
2413 2414 hp->dmai_cookie = window->wd_first_cookie;
2414 2415 } else {
2415 2416 hp->dmai_cookie = dma->dp_cookies;
2416 2417 }
2417 2418 hp->dmai_cookie++;
2418 2419 }
2419 2420
2420 2421 /*ARGSUSED*/
2421 2422 static int
2422 2423 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2423 2424 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2424 2425 {
2425 2426 int i;
2426 2427 int km_flags;
2427 2428 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2428 2429 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2429 2430 rootnex_window_t *window;
2430 2431 ddi_dma_cookie_t *cp;
2431 2432 ddi_dma_cookie_t *cookie;
2432 2433
2433 2434 ASSERT(*cookiepp == NULL);
2434 2435 ASSERT(*ccountp == 0);
2435 2436
2436 2437 if (dma->dp_window) {
2437 2438 window = &dma->dp_window[dma->dp_current_win];
2438 2439 cp = window->wd_first_cookie;
2439 2440 *ccountp = window->wd_cookie_cnt;
2440 2441 } else {
2441 2442 cp = dma->dp_cookies;
2442 2443 *ccountp = dma->dp_sglinfo.si_sgl_size;
2443 2444 }
2444 2445
2445 2446 km_flags = rootnex_coredma_get_sleep_flags(handle);
2446 2447 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2447 2448 if (cookie == NULL) {
2448 2449 return (DDI_DMA_NORESOURCES);
2449 2450 }
2450 2451
2451 2452 for (i = 0; i < *ccountp; i++) {
2452 2453 cookie[i].dmac_notused = cp[i].dmac_notused;
2453 2454 cookie[i].dmac_type = cp[i].dmac_type;
2454 2455 cookie[i].dmac_address = cp[i].dmac_address;
2455 2456 cookie[i].dmac_size = cp[i].dmac_size;
2456 2457 }
2457 2458
2458 2459 *cookiepp = cookie;
2459 2460
2460 2461 return (DDI_SUCCESS);
2461 2462 }
2462 2463
2463 2464 /*ARGSUSED*/
2464 2465 static int
2465 2466 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2466 2467 ddi_dma_cookie_t *cookiep, uint_t ccount)
2467 2468 {
2468 2469 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2469 2470 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2470 2471 rootnex_window_t *window;
2471 2472 ddi_dma_cookie_t *cur_cookiep;
2472 2473
2473 2474 ASSERT(cookiep);
2474 2475 ASSERT(ccount != 0);
2475 2476 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2476 2477
2477 2478 if (dma->dp_window) {
2478 2479 window = &dma->dp_window[dma->dp_current_win];
2479 2480 dma->dp_saved_cookies = window->wd_first_cookie;
2480 2481 window->wd_first_cookie = cookiep;
2481 2482 ASSERT(ccount == window->wd_cookie_cnt);
2482 2483 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2483 2484 + window->wd_first_cookie;
2484 2485 } else {
2485 2486 dma->dp_saved_cookies = dma->dp_cookies;
2486 2487 dma->dp_cookies = cookiep;
2487 2488 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2488 2489 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2489 2490 + dma->dp_cookies;
2490 2491 }
2491 2492
2492 2493 dma->dp_need_to_switch_cookies = B_TRUE;
2493 2494 hp->dmai_cookie = cur_cookiep;
2494 2495
2495 2496 return (DDI_SUCCESS);
2496 2497 }
2497 2498
2498 2499 /*ARGSUSED*/
2499 2500 static int
2500 2501 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2501 2502 {
2502 2503 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2503 2504 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2504 2505 rootnex_window_t *window;
2505 2506 ddi_dma_cookie_t *cur_cookiep;
2506 2507 ddi_dma_cookie_t *cookie_array;
2507 2508 uint_t ccount;
2508 2509
2509 2510 /* check if cookies have not been switched */
2510 2511 if (dma->dp_need_to_switch_cookies == B_FALSE)
2511 2512 return (DDI_SUCCESS);
2512 2513
2513 2514 ASSERT(dma->dp_saved_cookies);
2514 2515
2515 2516 if (dma->dp_window) {
2516 2517 window = &dma->dp_window[dma->dp_current_win];
2517 2518 cookie_array = window->wd_first_cookie;
2518 2519 window->wd_first_cookie = dma->dp_saved_cookies;
2519 2520 dma->dp_saved_cookies = NULL;
2520 2521 ccount = window->wd_cookie_cnt;
2521 2522 cur_cookiep = (hp->dmai_cookie - cookie_array)
2522 2523 + window->wd_first_cookie;
2523 2524 } else {
2524 2525 cookie_array = dma->dp_cookies;
2525 2526 dma->dp_cookies = dma->dp_saved_cookies;
2526 2527 dma->dp_saved_cookies = NULL;
2527 2528 ccount = dma->dp_sglinfo.si_sgl_size;
2528 2529 cur_cookiep = (hp->dmai_cookie - cookie_array)
2529 2530 + dma->dp_cookies;
2530 2531 }
2531 2532
2532 2533 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2533 2534
2534 2535 hp->dmai_cookie = cur_cookiep;
2535 2536
2536 2537 dma->dp_need_to_switch_cookies = B_FALSE;
2537 2538
2538 2539 return (DDI_SUCCESS);
2539 2540 }
2540 2541
2541 2542 #endif
2542 2543
2543 2544 static struct as *
2544 2545 rootnex_get_as(ddi_dma_obj_t *dmao)
2545 2546 {
2546 2547 struct as *asp;
2547 2548
2548 2549 switch (dmao->dmao_type) {
2549 2550 case DMA_OTYP_VADDR:
2550 2551 case DMA_OTYP_BUFVADDR:
2551 2552 asp = dmao->dmao_obj.virt_obj.v_as;
2552 2553 if (asp == NULL)
2553 2554 asp = &kas;
2554 2555 break;
2555 2556 default:
2556 2557 asp = NULL;
2557 2558 break;
2558 2559 }
2559 2560 return (asp);
2560 2561 }
2561 2562
2562 2563 /*
2563 2564 * rootnex_verify_buffer()
2564 2565 * verify buffer wasn't free'd
2565 2566 */
2566 2567 static int
2567 2568 rootnex_verify_buffer(rootnex_dma_t *dma)
2568 2569 {
2569 2570 page_t **pplist;
2570 2571 caddr_t vaddr;
2571 2572 uint_t pcnt;
2572 2573 uint_t poff;
2573 2574 page_t *pp;
2574 2575 char b;
2575 2576 int i;
2576 2577
2577 2578 /* Figure out how many pages this buffer occupies */
2578 2579 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2579 2580 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2580 2581 } else {
2581 2582 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2582 2583 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2583 2584 }
2584 2585 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2585 2586
2586 2587 switch (dma->dp_dma.dmao_type) {
2587 2588 case DMA_OTYP_PAGES:
2588 2589 /*
2589 2590 * for a linked list of pp's walk through them to make sure
2590 2591 * they're locked and not free.
2591 2592 */
2592 2593 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2593 2594 for (i = 0; i < pcnt; i++) {
2594 2595 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2595 2596 return (DDI_FAILURE);
2596 2597 }
2597 2598 pp = pp->p_next;
2598 2599 }
2599 2600 break;
2600 2601
2601 2602 case DMA_OTYP_VADDR:
2602 2603 case DMA_OTYP_BUFVADDR:
2603 2604 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2604 2605 /*
2605 2606 * for an array of pp's walk through them to make sure they're
2606 2607 * not free. It's possible that they may not be locked.
2607 2608 */
2608 2609 if (pplist) {
2609 2610 for (i = 0; i < pcnt; i++) {
2610 2611 if (PP_ISFREE(pplist[i])) {
2611 2612 return (DDI_FAILURE);
2612 2613 }
2613 2614 }
2614 2615
2615 2616 /* For a virtual address, try to peek at each page */
2616 2617 } else {
2617 2618 if (rootnex_get_as(&dma->dp_dma) == &kas) {
2618 2619 for (i = 0; i < pcnt; i++) {
2619 2620 if (ddi_peek8(NULL, vaddr, &b) ==
2620 2621 DDI_FAILURE)
2621 2622 return (DDI_FAILURE);
2622 2623 vaddr += MMU_PAGESIZE;
2623 2624 }
2624 2625 }
2625 2626 }
2626 2627 break;
2627 2628
2628 2629 default:
2629 2630 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2630 2631 break;
2631 2632 }
2632 2633
2633 2634 return (DDI_SUCCESS);
2634 2635 }
2635 2636
2636 2637
2637 2638 /*
2638 2639 * rootnex_clean_dmahdl()
2639 2640 * Clean the dma handle. This should be called on a handle alloc and an
2640 2641 * unbind handle. Set the handle state to the default settings.
2641 2642 */
2642 2643 static void
2643 2644 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2644 2645 {
2645 2646 rootnex_dma_t *dma;
2646 2647
2647 2648
2648 2649 dma = (rootnex_dma_t *)hp->dmai_private;
2649 2650
2650 2651 hp->dmai_nwin = 0;
2651 2652 dma->dp_current_cookie = 0;
2652 2653 dma->dp_copybuf_size = 0;
2653 2654 dma->dp_window = NULL;
2654 2655 dma->dp_cbaddr = NULL;
2655 2656 dma->dp_inuse = B_FALSE;
2656 2657 dma->dp_dvma_used = B_FALSE;
2657 2658 dma->dp_need_to_free_cookie = B_FALSE;
2658 2659 dma->dp_need_to_switch_cookies = B_FALSE;
2659 2660 dma->dp_saved_cookies = NULL;
2660 2661 dma->dp_sleep_flags = KM_PANIC;
2661 2662 dma->dp_need_to_free_window = B_FALSE;
2662 2663 dma->dp_partial_required = B_FALSE;
2663 2664 dma->dp_trim_required = B_FALSE;
2664 2665 dma->dp_sglinfo.si_copybuf_req = 0;
2665 2666 #if !defined(__amd64)
2666 2667 dma->dp_cb_remaping = B_FALSE;
2667 2668 dma->dp_kva = NULL;
2668 2669 #endif
2669 2670
2670 2671 /* FMA related initialization */
2671 2672 hp->dmai_fault = 0;
2672 2673 hp->dmai_fault_check = NULL;
2673 2674 hp->dmai_fault_notify = NULL;
2674 2675 hp->dmai_error.err_ena = 0;
2675 2676 hp->dmai_error.err_status = DDI_FM_OK;
2676 2677 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2677 2678 hp->dmai_error.err_ontrap = NULL;
2678 2679 }
2679 2680
2680 2681
2681 2682 /*
2682 2683 * rootnex_valid_alloc_parms()
2683 2684 * Called in ddi_dma_alloc_handle path to validate its parameters.
2684 2685 */
2685 2686 static int
2686 2687 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2687 2688 {
2688 2689 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2689 2690 (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2690 2691 (attr->dma_attr_granular > MMU_PAGESIZE) ||
2691 2692 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2692 2693 return (DDI_DMA_BADATTR);
2693 2694 }
2694 2695
2695 2696 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2696 2697 return (DDI_DMA_BADATTR);
2697 2698 }
2698 2699
2699 2700 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2700 2701 MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2701 2702 attr->dma_attr_sgllen == 0) {
2702 2703 return (DDI_DMA_BADATTR);
2703 2704 }
2704 2705
2705 2706 /* We should be able to DMA into every byte offset in a page */
2706 2707 if (maxsegmentsize < MMU_PAGESIZE) {
2707 2708 return (DDI_DMA_BADATTR);
2708 2709 }
2709 2710
2710 2711 /* if we're bouncing on seg, seg must be <= addr_hi */
2711 2712 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2712 2713 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2713 2714 return (DDI_DMA_BADATTR);
2714 2715 }
2715 2716 return (DDI_SUCCESS);
2716 2717 }
2717 2718
2718 2719 /*
2719 2720 * rootnex_valid_bind_parms()
2720 2721 * Called in ddi_dma_*_bind_handle path to validate its parameters.
2721 2722 */
2722 2723 /* ARGSUSED */
2723 2724 static int
2724 2725 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2725 2726 {
2726 2727 #if !defined(__amd64)
2727 2728 /*
2728 2729 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2729 2730 * we can track the offset for the obsoleted interfaces.
2730 2731 */
2731 2732 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2732 2733 return (DDI_DMA_TOOBIG);
2733 2734 }
2734 2735 #endif
2735 2736
2736 2737 return (DDI_SUCCESS);
2737 2738 }
2738 2739
2739 2740
2740 2741 /*
2741 2742 * rootnex_need_bounce_seg()
2742 2743 * check to see if the buffer lives on both side of the seg.
2743 2744 */
2744 2745 static boolean_t
2745 2746 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2746 2747 {
2747 2748 ddi_dma_atyp_t buftype;
2748 2749 rootnex_addr_t raddr;
2749 2750 boolean_t lower_addr;
2750 2751 boolean_t upper_addr;
2751 2752 uint64_t offset;
2752 2753 page_t **pplist;
2753 2754 uint64_t paddr;
2754 2755 uint32_t psize;
2755 2756 uint32_t size;
2756 2757 caddr_t vaddr;
2757 2758 uint_t pcnt;
2758 2759 page_t *pp;
2759 2760
2760 2761
2761 2762 /* shortcuts */
2762 2763 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2763 2764 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2764 2765 buftype = dmar_object->dmao_type;
2765 2766 size = dmar_object->dmao_size;
2766 2767
2767 2768 lower_addr = B_FALSE;
2768 2769 upper_addr = B_FALSE;
2769 2770 pcnt = 0;
2770 2771
2771 2772 /*
2772 2773 * Process the first page to handle the initial offset of the buffer.
2773 2774 * We'll use the base address we get later when we loop through all
2774 2775 * the pages.
2775 2776 */
2776 2777 if (buftype == DMA_OTYP_PAGES) {
2777 2778 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2778 2779 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2779 2780 MMU_PAGEOFFSET;
2780 2781 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2781 2782 psize = MIN(size, (MMU_PAGESIZE - offset));
2782 2783 pp = pp->p_next;
2783 2784 sglinfo->si_asp = NULL;
2784 2785 } else if (pplist != NULL) {
2785 2786 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2786 2787 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2787 2788 if (sglinfo->si_asp == NULL) {
2788 2789 sglinfo->si_asp = &kas;
2789 2790 }
2790 2791 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2791 2792 paddr += offset;
2792 2793 psize = MIN(size, (MMU_PAGESIZE - offset));
2793 2794 pcnt++;
2794 2795 } else {
2795 2796 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2796 2797 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2797 2798 if (sglinfo->si_asp == NULL) {
2798 2799 sglinfo->si_asp = &kas;
2799 2800 }
2800 2801 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2801 2802 paddr += offset;
2802 2803 psize = MIN(size, (MMU_PAGESIZE - offset));
2803 2804 vaddr += psize;
2804 2805 }
2805 2806
2806 2807 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2807 2808
2808 2809 if ((raddr + psize) > sglinfo->si_segmask) {
2809 2810 upper_addr = B_TRUE;
2810 2811 } else {
2811 2812 lower_addr = B_TRUE;
2812 2813 }
2813 2814 size -= psize;
2814 2815
2815 2816 /*
2816 2817 * Walk through the rest of the pages in the buffer. Track to see
2817 2818 * if we have pages on both sides of the segment boundary.
2818 2819 */
2819 2820 while (size > 0) {
2820 2821 /* partial or full page */
2821 2822 psize = MIN(size, MMU_PAGESIZE);
2822 2823
2823 2824 if (buftype == DMA_OTYP_PAGES) {
2824 2825 /* get the paddr from the page_t */
2825 2826 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2826 2827 paddr = pfn_to_pa(pp->p_pagenum);
2827 2828 pp = pp->p_next;
2828 2829 } else if (pplist != NULL) {
2829 2830 /* index into the array of page_t's to get the paddr */
2830 2831 ASSERT(!PP_ISFREE(pplist[pcnt]));
2831 2832 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2832 2833 pcnt++;
2833 2834 } else {
2834 2835 /* call into the VM to get the paddr */
2835 2836 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2836 2837 vaddr));
2837 2838 vaddr += psize;
2838 2839 }
2839 2840
2840 2841 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2841 2842
2842 2843 if ((raddr + psize) > sglinfo->si_segmask) {
2843 2844 upper_addr = B_TRUE;
2844 2845 } else {
2845 2846 lower_addr = B_TRUE;
2846 2847 }
2847 2848 /*
2848 2849 * if the buffer lives both above and below the segment
2849 2850 * boundary, or the current page is the page immediately
2850 2851 * after the segment, we will use a copy/bounce buffer for
2851 2852 * all pages > seg.
2852 2853 */
2853 2854 if ((lower_addr && upper_addr) ||
2854 2855 (raddr == (sglinfo->si_segmask + 1))) {
2855 2856 return (B_TRUE);
2856 2857 }
2857 2858
2858 2859 size -= psize;
2859 2860 }
2860 2861
2861 2862 return (B_FALSE);
2862 2863 }
2863 2864
2864 2865 /*
2865 2866 * rootnex_get_sgl()
2866 2867 * Called in bind fastpath to get the sgl. Most of this will be replaced
2867 2868 * with a call to the vm layer when vm2.0 comes around...
2868 2869 */
2869 2870 static void
2870 2871 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2871 2872 rootnex_sglinfo_t *sglinfo)
2872 2873 {
2873 2874 ddi_dma_atyp_t buftype;
2874 2875 rootnex_addr_t raddr;
2875 2876 uint64_t last_page;
2876 2877 uint64_t offset;
2877 2878 uint64_t addrhi;
2878 2879 uint64_t addrlo;
2879 2880 uint64_t maxseg;
2880 2881 page_t **pplist;
2881 2882 uint64_t paddr;
2882 2883 uint32_t psize;
2883 2884 uint32_t size;
2884 2885 caddr_t vaddr;
2885 2886 uint_t pcnt;
2886 2887 page_t *pp;
2887 2888 uint_t cnt;
2888 2889
2889 2890
2890 2891 /* shortcuts */
2891 2892 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2892 2893 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2893 2894 maxseg = sglinfo->si_max_cookie_size;
2894 2895 buftype = dmar_object->dmao_type;
2895 2896 addrhi = sglinfo->si_max_addr;
2896 2897 addrlo = sglinfo->si_min_addr;
2897 2898 size = dmar_object->dmao_size;
2898 2899
2899 2900 pcnt = 0;
2900 2901 cnt = 0;
2901 2902
2902 2903
2903 2904 /*
2904 2905 * check to see if we need to use the copy buffer for pages over
2905 2906 * the segment attr.
2906 2907 */
2907 2908 sglinfo->si_bounce_on_seg = B_FALSE;
2908 2909 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2909 2910 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2910 2911 dmar_object, sglinfo);
2911 2912 }
2912 2913
2913 2914 /*
2914 2915 * if we were passed down a linked list of pages, i.e. pointer to
2915 2916 * page_t, use this to get our physical address and buf offset.
2916 2917 */
2917 2918 if (buftype == DMA_OTYP_PAGES) {
2918 2919 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2919 2920 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2920 2921 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2921 2922 MMU_PAGEOFFSET;
2922 2923 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2923 2924 psize = MIN(size, (MMU_PAGESIZE - offset));
2924 2925 pp = pp->p_next;
2925 2926 sglinfo->si_asp = NULL;
2926 2927
2927 2928 /*
2928 2929 * We weren't passed down a linked list of pages, but if we were passed
2929 2930 * down an array of pages, use this to get our physical address and buf
2930 2931 * offset.
2931 2932 */
2932 2933 } else if (pplist != NULL) {
2933 2934 ASSERT((buftype == DMA_OTYP_VADDR) ||
2934 2935 (buftype == DMA_OTYP_BUFVADDR));
2935 2936
2936 2937 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2937 2938 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2938 2939 if (sglinfo->si_asp == NULL) {
2939 2940 sglinfo->si_asp = &kas;
2940 2941 }
2941 2942
2942 2943 ASSERT(!PP_ISFREE(pplist[pcnt]));
2943 2944 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2944 2945 paddr += offset;
2945 2946 psize = MIN(size, (MMU_PAGESIZE - offset));
2946 2947 pcnt++;
2947 2948
2948 2949 /*
2949 2950 * All we have is a virtual address, we'll need to call into the VM
2950 2951 * to get the physical address.
2951 2952 */
2952 2953 } else {
2953 2954 ASSERT((buftype == DMA_OTYP_VADDR) ||
2954 2955 (buftype == DMA_OTYP_BUFVADDR));
2955 2956
2956 2957 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2957 2958 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2958 2959 if (sglinfo->si_asp == NULL) {
2959 2960 sglinfo->si_asp = &kas;
2960 2961 }
2961 2962
2962 2963 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2963 2964 paddr += offset;
2964 2965 psize = MIN(size, (MMU_PAGESIZE - offset));
2965 2966 vaddr += psize;
2966 2967 }
2967 2968
2968 2969 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2969 2970
2970 2971 /*
2971 2972 * Setup the first cookie with the physical address of the page and the
2972 2973 * size of the page (which takes into account the initial offset into
2973 2974 * the page.
2974 2975 */
2975 2976 sgl[cnt].dmac_laddress = raddr;
2976 2977 sgl[cnt].dmac_size = psize;
2977 2978 sgl[cnt].dmac_type = 0;
2978 2979
2979 2980 /*
2980 2981 * Save away the buffer offset into the page. We'll need this later in
2981 2982 * the copy buffer code to help figure out the page index within the
2982 2983 * buffer and the offset into the current page.
2983 2984 */
2984 2985 sglinfo->si_buf_offset = offset;
2985 2986
2986 2987 /*
2987 2988 * If we are using the copy buffer for anything over the segment
2988 2989 * boundary, and this page is over the segment boundary.
2989 2990 * OR
2990 2991 * if the DMA engine can't reach the physical address.
2991 2992 */
2992 2993 if (((sglinfo->si_bounce_on_seg) &&
2993 2994 ((raddr + psize) > sglinfo->si_segmask)) ||
2994 2995 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2995 2996 /*
2996 2997 * Increase how much copy buffer we use. We always increase by
2997 2998 * pagesize so we don't have to worry about converting offsets.
2998 2999 * Set a flag in the cookies dmac_type to indicate that it uses
2999 3000 * the copy buffer. If this isn't the last cookie, go to the
3000 3001 * next cookie (since we separate each page which uses the copy
3001 3002 * buffer in case the copy buffer is not physically contiguous.
3002 3003 */
3003 3004 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3004 3005 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3005 3006 if ((cnt + 1) < sglinfo->si_max_pages) {
3006 3007 cnt++;
3007 3008 sgl[cnt].dmac_laddress = 0;
3008 3009 sgl[cnt].dmac_size = 0;
3009 3010 sgl[cnt].dmac_type = 0;
3010 3011 }
3011 3012 }
3012 3013
3013 3014 /*
3014 3015 * save this page's physical address so we can figure out if the next
3015 3016 * page is physically contiguous. Keep decrementing size until we are
3016 3017 * done with the buffer.
3017 3018 */
3018 3019 last_page = raddr & MMU_PAGEMASK;
3019 3020 size -= psize;
3020 3021
3021 3022 while (size > 0) {
3022 3023 /* Get the size for this page (i.e. partial or full page) */
3023 3024 psize = MIN(size, MMU_PAGESIZE);
3024 3025
3025 3026 if (buftype == DMA_OTYP_PAGES) {
3026 3027 /* get the paddr from the page_t */
3027 3028 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3028 3029 paddr = pfn_to_pa(pp->p_pagenum);
3029 3030 pp = pp->p_next;
3030 3031 } else if (pplist != NULL) {
3031 3032 /* index into the array of page_t's to get the paddr */
3032 3033 ASSERT(!PP_ISFREE(pplist[pcnt]));
3033 3034 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3034 3035 pcnt++;
3035 3036 } else {
3036 3037 /* call into the VM to get the paddr */
3037 3038 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3038 3039 vaddr));
3039 3040 vaddr += psize;
3040 3041 }
3041 3042
3042 3043 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3043 3044
3044 3045 /*
3045 3046 * If we are using the copy buffer for anything over the
3046 3047 * segment boundary, and this page is over the segment
3047 3048 * boundary.
3048 3049 * OR
3049 3050 * if the DMA engine can't reach the physical address.
3050 3051 */
3051 3052 if (((sglinfo->si_bounce_on_seg) &&
3052 3053 ((raddr + psize) > sglinfo->si_segmask)) ||
3053 3054 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3054 3055
3055 3056 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3056 3057
3057 3058 /*
3058 3059 * if there is something in the current cookie, go to
3059 3060 * the next one. We only want one page in a cookie which
3060 3061 * uses the copybuf since the copybuf doesn't have to
3061 3062 * be physically contiguous.
3062 3063 */
3063 3064 if (sgl[cnt].dmac_size != 0) {
3064 3065 cnt++;
3065 3066 }
3066 3067 sgl[cnt].dmac_laddress = raddr;
3067 3068 sgl[cnt].dmac_size = psize;
3068 3069 #if defined(__amd64)
3069 3070 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3070 3071 #else
3071 3072 /*
3072 3073 * save the buf offset for 32-bit kernel. used in the
3073 3074 * obsoleted interfaces.
3074 3075 */
3075 3076 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3076 3077 (dmar_object->dmao_size - size);
3077 3078 #endif
3078 3079 /* if this isn't the last cookie, go to the next one */
3079 3080 if ((cnt + 1) < sglinfo->si_max_pages) {
3080 3081 cnt++;
3081 3082 sgl[cnt].dmac_laddress = 0;
3082 3083 sgl[cnt].dmac_size = 0;
3083 3084 sgl[cnt].dmac_type = 0;
3084 3085 }
3085 3086
3086 3087 /*
3087 3088 * this page didn't need the copy buffer, if it's not physically
3088 3089 * contiguous, or it would put us over a segment boundary, or it
3089 3090 * puts us over the max cookie size, or the current sgl doesn't
3090 3091 * have anything in it.
3091 3092 */
3092 3093 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3093 3094 !(raddr & sglinfo->si_segmask) ||
3094 3095 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3095 3096 (sgl[cnt].dmac_size == 0)) {
3096 3097 /*
3097 3098 * if we're not already in a new cookie, go to the next
3098 3099 * cookie.
3099 3100 */
3100 3101 if (sgl[cnt].dmac_size != 0) {
3101 3102 cnt++;
3102 3103 }
3103 3104
3104 3105 /* save the cookie information */
3105 3106 sgl[cnt].dmac_laddress = raddr;
3106 3107 sgl[cnt].dmac_size = psize;
3107 3108 #if defined(__amd64)
3108 3109 sgl[cnt].dmac_type = 0;
3109 3110 #else
3110 3111 /*
3111 3112 * save the buf offset for 32-bit kernel. used in the
3112 3113 * obsoleted interfaces.
3113 3114 */
3114 3115 sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3115 3116 #endif
3116 3117
3117 3118 /*
3118 3119 * this page didn't need the copy buffer, it is physically
3119 3120 * contiguous with the last page, and it's <= the max cookie
3120 3121 * size.
3121 3122 */
3122 3123 } else {
3123 3124 sgl[cnt].dmac_size += psize;
3124 3125
3125 3126 /*
3126 3127 * if this exactly == the maximum cookie size, and
3127 3128 * it isn't the last cookie, go to the next cookie.
3128 3129 */
3129 3130 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3130 3131 ((cnt + 1) < sglinfo->si_max_pages)) {
3131 3132 cnt++;
3132 3133 sgl[cnt].dmac_laddress = 0;
3133 3134 sgl[cnt].dmac_size = 0;
3134 3135 sgl[cnt].dmac_type = 0;
3135 3136 }
3136 3137 }
3137 3138
3138 3139 /*
3139 3140 * save this page's physical address so we can figure out if the
3140 3141 * next page is physically contiguous. Keep decrementing size
3141 3142 * until we are done with the buffer.
3142 3143 */
3143 3144 last_page = raddr;
3144 3145 size -= psize;
3145 3146 }
3146 3147
3147 3148 /* we're done, save away how many cookies the sgl has */
3148 3149 if (sgl[cnt].dmac_size == 0) {
3149 3150 ASSERT(cnt < sglinfo->si_max_pages);
3150 3151 sglinfo->si_sgl_size = cnt;
3151 3152 } else {
3152 3153 sglinfo->si_sgl_size = cnt + 1;
3153 3154 }
3154 3155 }
3155 3156
3156 3157 static void
3157 3158 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3158 3159 rootnex_sglinfo_t *sglinfo)
3159 3160 {
3160 3161 uint64_t offset;
3161 3162 uint64_t maxseg;
3162 3163 uint64_t dvaddr;
3163 3164 struct dvmaseg *dvs;
3164 3165 uint64_t paddr;
3165 3166 uint32_t psize, ssize;
3166 3167 uint32_t size;
3167 3168 uint_t cnt;
3168 3169 int physcontig;
3169 3170
3170 3171 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3171 3172
3172 3173 /* shortcuts */
3173 3174 maxseg = sglinfo->si_max_cookie_size;
3174 3175 size = dmar_object->dmao_size;
3175 3176
3176 3177 cnt = 0;
3177 3178 sglinfo->si_bounce_on_seg = B_FALSE;
3178 3179
3179 3180 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3180 3181 offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3181 3182 ssize = dvs->dvs_len;
3182 3183 paddr = dvs->dvs_start;
3183 3184 paddr += offset;
3184 3185 psize = MIN(ssize, (maxseg - offset));
3185 3186 dvaddr = paddr + psize;
3186 3187 ssize -= psize;
3187 3188
3188 3189 sgl[cnt].dmac_laddress = paddr;
3189 3190 sgl[cnt].dmac_size = psize;
3190 3191 sgl[cnt].dmac_type = 0;
3191 3192
3192 3193 size -= psize;
3193 3194 while (size > 0) {
3194 3195 if (ssize == 0) {
3195 3196 dvs++;
3196 3197 ssize = dvs->dvs_len;
3197 3198 dvaddr = dvs->dvs_start;
3198 3199 physcontig = 0;
3199 3200 } else
3200 3201 physcontig = 1;
3201 3202
3202 3203 paddr = dvaddr;
3203 3204 psize = MIN(ssize, maxseg);
3204 3205 dvaddr += psize;
3205 3206 ssize -= psize;
3206 3207
3207 3208 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3208 3209 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3209 3210 (sgl[cnt].dmac_size == 0)) {
3210 3211 /*
3211 3212 * if we're not already in a new cookie, go to the next
3212 3213 * cookie.
3213 3214 */
3214 3215 if (sgl[cnt].dmac_size != 0) {
3215 3216 cnt++;
3216 3217 }
3217 3218
3218 3219 /* save the cookie information */
3219 3220 sgl[cnt].dmac_laddress = paddr;
3220 3221 sgl[cnt].dmac_size = psize;
3221 3222 sgl[cnt].dmac_type = 0;
3222 3223 } else {
3223 3224 sgl[cnt].dmac_size += psize;
3224 3225
3225 3226 /*
3226 3227 * if this exactly == the maximum cookie size, and
3227 3228 * it isn't the last cookie, go to the next cookie.
3228 3229 */
3229 3230 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3230 3231 ((cnt + 1) < sglinfo->si_max_pages)) {
3231 3232 cnt++;
3232 3233 sgl[cnt].dmac_laddress = 0;
3233 3234 sgl[cnt].dmac_size = 0;
3234 3235 sgl[cnt].dmac_type = 0;
3235 3236 }
3236 3237 }
3237 3238 size -= psize;
3238 3239 }
3239 3240
3240 3241 /* we're done, save away how many cookies the sgl has */
3241 3242 if (sgl[cnt].dmac_size == 0) {
3242 3243 sglinfo->si_sgl_size = cnt;
3243 3244 } else {
3244 3245 sglinfo->si_sgl_size = cnt + 1;
3245 3246 }
3246 3247 }
3247 3248
3248 3249 /*
3249 3250 * rootnex_bind_slowpath()
3250 3251 * Call in the bind path if the calling driver can't use the sgl without
3251 3252 * modifying it. We either need to use the copy buffer and/or we will end up
3252 3253 * with a partial bind.
3253 3254 */
3254 3255 static int
3255 3256 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3256 3257 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3257 3258 {
3258 3259 rootnex_sglinfo_t *sinfo;
3259 3260 rootnex_window_t *window;
3260 3261 ddi_dma_cookie_t *cookie;
3261 3262 size_t copybuf_used;
3262 3263 size_t dmac_size;
3263 3264 boolean_t partial;
3264 3265 off_t cur_offset;
3265 3266 page_t *cur_pp;
3266 3267 major_t mnum;
3267 3268 int e;
3268 3269 int i;
3269 3270
3270 3271
3271 3272 sinfo = &dma->dp_sglinfo;
3272 3273 copybuf_used = 0;
3273 3274 partial = B_FALSE;
3274 3275
3275 3276 /*
3276 3277 * If we're using the copybuf, set the copybuf state in dma struct.
3277 3278 * Needs to be first since it sets the copy buffer size.
3278 3279 */
3279 3280 if (sinfo->si_copybuf_req != 0) {
3280 3281 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3281 3282 if (e != DDI_SUCCESS) {
3282 3283 return (e);
3283 3284 }
3284 3285 } else {
3285 3286 dma->dp_copybuf_size = 0;
3286 3287 }
3287 3288
3288 3289 /*
3289 3290 * Figure out if we need to do a partial mapping. If so, figure out
3290 3291 * if we need to trim the buffers when we munge the sgl.
3291 3292 */
3292 3293 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3293 3294 (dmao->dmao_size > dma->dp_maxxfer) ||
3294 3295 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3295 3296 dma->dp_partial_required = B_TRUE;
3296 3297 if (attr->dma_attr_granular != 1) {
3297 3298 dma->dp_trim_required = B_TRUE;
3298 3299 }
3299 3300 } else {
3300 3301 dma->dp_partial_required = B_FALSE;
3301 3302 dma->dp_trim_required = B_FALSE;
3302 3303 }
3303 3304
3304 3305 /* If we need to do a partial bind, make sure the driver supports it */
3305 3306 if (dma->dp_partial_required &&
3306 3307 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3307 3308
3308 3309 mnum = ddi_driver_major(dma->dp_dip);
3309 3310 /*
3310 3311 * patchable which allows us to print one warning per major
3311 3312 * number.
3312 3313 */
3313 3314 if ((rootnex_bind_warn) &&
3314 3315 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3315 3316 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3316 3317 cmn_err(CE_WARN, "!%s: coding error detected, the "
3317 3318 "driver is using ddi_dma_attr(9S) incorrectly. "
3318 3319 "There is a small risk of data corruption in "
3319 3320 "particular with large I/Os. The driver should be "
3320 3321 "replaced with a corrected version for proper "
3321 3322 "system operation. To disable this warning, add "
3322 3323 "'set rootnex:rootnex_bind_warn=0' to "
3323 3324 "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3324 3325 }
3325 3326 return (DDI_DMA_TOOBIG);
3326 3327 }
3327 3328
3328 3329 /*
3329 3330 * we might need multiple windows, setup state to handle them. In this
3330 3331 * code path, we will have at least one window.
3331 3332 */
3332 3333 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3333 3334 if (e != DDI_SUCCESS) {
3334 3335 rootnex_teardown_copybuf(dma);
3335 3336 return (e);
3336 3337 }
3337 3338
3338 3339 window = &dma->dp_window[0];
3339 3340 cookie = &dma->dp_cookies[0];
3340 3341 cur_offset = 0;
3341 3342 rootnex_init_win(hp, dma, window, cookie, cur_offset);
3342 3343 if (dmao->dmao_type == DMA_OTYP_PAGES) {
3343 3344 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3344 3345 }
3345 3346
3346 3347 /* loop though all the cookies we got back from get_sgl() */
3347 3348 for (i = 0; i < sinfo->si_sgl_size; i++) {
3348 3349 /*
3349 3350 * If we're using the copy buffer, check this cookie and setup
3350 3351 * its associated copy buffer state. If this cookie uses the
3351 3352 * copy buffer, make sure we sync this window during dma_sync.
3352 3353 */
3353 3354 if (dma->dp_copybuf_size > 0) {
3354 3355 rootnex_setup_cookie(dmao, dma, cookie,
3355 3356 cur_offset, ©buf_used, &cur_pp);
3356 3357 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3357 3358 window->wd_dosync = B_TRUE;
3358 3359 }
3359 3360 }
3360 3361
3361 3362 /*
3362 3363 * save away the cookie size, since it could be modified in
3363 3364 * the windowing code.
3364 3365 */
3365 3366 dmac_size = cookie->dmac_size;
3366 3367
3367 3368 /* if we went over max copybuf size */
3368 3369 if (dma->dp_copybuf_size &&
3369 3370 (copybuf_used > dma->dp_copybuf_size)) {
3370 3371 partial = B_TRUE;
3371 3372 e = rootnex_copybuf_window_boundary(hp, dma, &window,
3372 3373 cookie, cur_offset, ©buf_used);
3373 3374 if (e != DDI_SUCCESS) {
3374 3375 rootnex_teardown_copybuf(dma);
3375 3376 rootnex_teardown_windows(dma);
3376 3377 return (e);
3377 3378 }
3378 3379
3379 3380 /*
3380 3381 * if the coookie uses the copy buffer, make sure the
3381 3382 * new window we just moved to is set to sync.
3382 3383 */
3383 3384 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3384 3385 window->wd_dosync = B_TRUE;
3385 3386 }
3386 3387 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3387 3388 dma->dp_dip);
3388 3389
3389 3390 /* if the cookie cnt == max sgllen, move to the next window */
3390 3391 } else if (window->wd_cookie_cnt >=
3391 3392 (unsigned)attr->dma_attr_sgllen) {
3392 3393 partial = B_TRUE;
3393 3394 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3394 3395 e = rootnex_sgllen_window_boundary(hp, dma, &window,
3395 3396 cookie, attr, cur_offset);
3396 3397 if (e != DDI_SUCCESS) {
3397 3398 rootnex_teardown_copybuf(dma);
3398 3399 rootnex_teardown_windows(dma);
3399 3400 return (e);
3400 3401 }
3401 3402
3402 3403 /*
3403 3404 * if the coookie uses the copy buffer, make sure the
3404 3405 * new window we just moved to is set to sync.
3405 3406 */
3406 3407 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3407 3408 window->wd_dosync = B_TRUE;
3408 3409 }
3409 3410 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3410 3411 dma->dp_dip);
3411 3412
3412 3413 /* else if we will be over maxxfer */
3413 3414 } else if ((window->wd_size + dmac_size) >
3414 3415 dma->dp_maxxfer) {
3415 3416 partial = B_TRUE;
3416 3417 e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3417 3418 cookie);
3418 3419 if (e != DDI_SUCCESS) {
3419 3420 rootnex_teardown_copybuf(dma);
3420 3421 rootnex_teardown_windows(dma);
3421 3422 return (e);
3422 3423 }
3423 3424
3424 3425 /*
3425 3426 * if the coookie uses the copy buffer, make sure the
3426 3427 * new window we just moved to is set to sync.
3427 3428 */
3428 3429 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3429 3430 window->wd_dosync = B_TRUE;
3430 3431 }
3431 3432 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3432 3433 dma->dp_dip);
3433 3434
3434 3435 /* else this cookie fits in the current window */
3435 3436 } else {
3436 3437 window->wd_cookie_cnt++;
3437 3438 window->wd_size += dmac_size;
3438 3439 }
3439 3440
3440 3441 /* track our offset into the buffer, go to the next cookie */
3441 3442 ASSERT(dmac_size <= dmao->dmao_size);
3442 3443 ASSERT(cookie->dmac_size <= dmac_size);
3443 3444 cur_offset += dmac_size;
3444 3445 cookie++;
3445 3446 }
3446 3447
3447 3448 /* if we ended up with a zero sized window in the end, clean it up */
3448 3449 if (window->wd_size == 0) {
3449 3450 hp->dmai_nwin--;
3450 3451 window--;
3451 3452 }
3452 3453
3453 3454 ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3454 3455
3455 3456 if (!partial) {
3456 3457 return (DDI_DMA_MAPPED);
3457 3458 }
3458 3459
3459 3460 ASSERT(dma->dp_partial_required);
3460 3461 return (DDI_DMA_PARTIAL_MAP);
3461 3462 }
3462 3463
3463 3464 /*
3464 3465 * rootnex_setup_copybuf()
3465 3466 * Called in bind slowpath. Figures out if we're going to use the copy
3466 3467 * buffer, and if we do, sets up the basic state to handle it.
3467 3468 */
3468 3469 static int
3469 3470 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3470 3471 rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3471 3472 {
3472 3473 rootnex_sglinfo_t *sinfo;
3473 3474 ddi_dma_attr_t lattr;
3474 3475 size_t max_copybuf;
3475 3476 int cansleep;
3476 3477 int e;
3477 3478 #if !defined(__amd64)
3478 3479 int vmflag;
3479 3480 #endif
3480 3481
3481 3482 ASSERT(!dma->dp_dvma_used);
3482 3483
3483 3484 sinfo = &dma->dp_sglinfo;
3484 3485
3485 3486 /* read this first so it's consistent through the routine */
3486 3487 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3487 3488
3488 3489 /* We need to call into the rootnex on ddi_dma_sync() */
3489 3490 hp->dmai_rflags &= ~DMP_NOSYNC;
3490 3491
3491 3492 /* make sure the copybuf size <= the max size */
3492 3493 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3493 3494 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3494 3495
3495 3496 #if !defined(__amd64)
3496 3497 /*
3497 3498 * if we don't have kva space to copy to/from, allocate the KVA space
3498 3499 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3499 3500 * the 64-bit kernel.
3500 3501 */
3501 3502 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3502 3503 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3503 3504
3504 3505 /* convert the sleep flags */
3505 3506 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3506 3507 vmflag = VM_SLEEP;
3507 3508 } else {
3508 3509 vmflag = VM_NOSLEEP;
3509 3510 }
3510 3511
3511 3512 /* allocate Kernel VA space that we can bcopy to/from */
3512 3513 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3513 3514 vmflag);
3514 3515 if (dma->dp_kva == NULL) {
3515 3516 return (DDI_DMA_NORESOURCES);
3516 3517 }
3517 3518 }
3518 3519 #endif
3519 3520
3520 3521 /* convert the sleep flags */
3521 3522 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3522 3523 cansleep = 1;
3523 3524 } else {
3524 3525 cansleep = 0;
3525 3526 }
3526 3527
3527 3528 /*
3528 3529 * Allocate the actual copy buffer. This needs to fit within the DMA
3529 3530 * engine limits, so we can't use kmem_alloc... We don't need
3530 3531 * contiguous memory (sgllen) since we will be forcing windows on
3531 3532 * sgllen anyway.
3532 3533 */
3533 3534 lattr = *attr;
3534 3535 lattr.dma_attr_align = MMU_PAGESIZE;
3535 3536 lattr.dma_attr_sgllen = -1; /* no limit */
3536 3537 /*
3537 3538 * if we're using the copy buffer because of seg, use that for our
3538 3539 * upper address limit.
3539 3540 */
3540 3541 if (sinfo->si_bounce_on_seg) {
3541 3542 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3542 3543 }
3543 3544 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3544 3545 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3545 3546 if (e != DDI_SUCCESS) {
3546 3547 #if !defined(__amd64)
3547 3548 if (dma->dp_kva != NULL) {
3548 3549 vmem_free(heap_arena, dma->dp_kva,
3549 3550 dma->dp_copybuf_size);
3550 3551 }
3551 3552 #endif
3552 3553 return (DDI_DMA_NORESOURCES);
3553 3554 }
3554 3555
3555 3556 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3556 3557 size_t, dma->dp_copybuf_size);
3557 3558
3558 3559 return (DDI_SUCCESS);
3559 3560 }
3560 3561
3561 3562
3562 3563 /*
3563 3564 * rootnex_setup_windows()
3564 3565 * Called in bind slowpath to setup the window state. We always have windows
3565 3566 * in the slowpath. Even if the window count = 1.
3566 3567 */
3567 3568 static int
3568 3569 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3569 3570 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3570 3571 {
3571 3572 rootnex_window_t *windowp;
3572 3573 rootnex_sglinfo_t *sinfo;
3573 3574 size_t copy_state_size;
3574 3575 size_t win_state_size;
3575 3576 size_t state_available;
3576 3577 size_t space_needed;
3577 3578 uint_t copybuf_win;
3578 3579 uint_t maxxfer_win;
3579 3580 size_t space_used;
3580 3581 uint_t sglwin;
3581 3582
3582 3583
3583 3584 sinfo = &dma->dp_sglinfo;
3584 3585
3585 3586 dma->dp_current_win = 0;
3586 3587 hp->dmai_nwin = 0;
3587 3588
3588 3589 /* If we don't need to do a partial, we only have one window */
3589 3590 if (!dma->dp_partial_required) {
3590 3591 dma->dp_max_win = 1;
3591 3592
3592 3593 /*
3593 3594 * we need multiple windows, need to figure out the worse case number
3594 3595 * of windows.
3595 3596 */
3596 3597 } else {
3597 3598 /*
3598 3599 * if we need windows because we need more copy buffer that
3599 3600 * we allow, the worse case number of windows we could need
3600 3601 * here would be (copybuf space required / copybuf space that
3601 3602 * we have) plus one for remainder, and plus 2 to handle the
3602 3603 * extra pages on the trim for the first and last pages of the
3603 3604 * buffer (a page is the minimum window size so under the right
3604 3605 * attr settings, you could have a window for each page).
3605 3606 * The last page will only be hit here if the size is not a
3606 3607 * multiple of the granularity (which theoretically shouldn't
3607 3608 * be the case but never has been enforced, so we could have
3608 3609 * broken things without it).
3609 3610 */
3610 3611 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3611 3612 ASSERT(dma->dp_copybuf_size > 0);
3612 3613 copybuf_win = (sinfo->si_copybuf_req /
3613 3614 dma->dp_copybuf_size) + 1 + 2;
3614 3615 } else {
3615 3616 copybuf_win = 0;
3616 3617 }
3617 3618
3618 3619 /*
3619 3620 * if we need windows because we have more cookies than the H/W
3620 3621 * can handle, the number of windows we would need here would
3621 3622 * be (cookie count / cookies count H/W supports minus 1[for
3622 3623 * trim]) plus one for remainder.
3623 3624 */
3624 3625 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3625 3626 sglwin = (sinfo->si_sgl_size /
3626 3627 (attr->dma_attr_sgllen - 1)) + 1;
3627 3628 } else {
3628 3629 sglwin = 0;
3629 3630 }
3630 3631
3631 3632 /*
3632 3633 * if we need windows because we're binding more memory than the
3633 3634 * H/W can transfer at once, the number of windows we would need
3634 3635 * here would be (xfer count / max xfer H/W supports) plus one
3635 3636 * for remainder, and plus 2 to handle the extra pages on the
3636 3637 * trim (see above comment about trim)
3637 3638 */
3638 3639 if (dmao->dmao_size > dma->dp_maxxfer) {
3639 3640 maxxfer_win = (dmao->dmao_size /
3640 3641 dma->dp_maxxfer) + 1 + 2;
3641 3642 } else {
3642 3643 maxxfer_win = 0;
3643 3644 }
3644 3645 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
3645 3646 ASSERT(dma->dp_max_win > 0);
3646 3647 }
3647 3648 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3648 3649
3649 3650 /*
3650 3651 * Get space for window and potential copy buffer state. Before we
3651 3652 * go and allocate memory, see if we can get away with using what's
3652 3653 * left in the pre-allocted state or the dynamically allocated sgl.
3653 3654 */
3654 3655 space_used = (uintptr_t)(sinfo->si_sgl_size *
3655 3656 sizeof (ddi_dma_cookie_t));
3656 3657
3657 3658 /* if we dynamically allocated space for the cookies */
3658 3659 if (dma->dp_need_to_free_cookie) {
3659 3660 /* if we have more space in the pre-allocted buffer, use it */
3660 3661 ASSERT(space_used <= dma->dp_cookie_size);
3661 3662 if ((dma->dp_cookie_size - space_used) <=
3662 3663 rootnex_state->r_prealloc_size) {
3663 3664 state_available = rootnex_state->r_prealloc_size;
3664 3665 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3665 3666
3666 3667 /*
3667 3668 * else, we have more free space in the dynamically allocated
3668 3669 * buffer, i.e. the buffer wasn't worse case fragmented so we
3669 3670 * didn't need a lot of cookies.
3670 3671 */
3671 3672 } else {
3672 3673 state_available = dma->dp_cookie_size - space_used;
3673 3674 windowp = (rootnex_window_t *)
3674 3675 &dma->dp_cookies[sinfo->si_sgl_size];
3675 3676 }
3676 3677
3677 3678 /* we used the pre-alloced buffer */
3678 3679 } else {
3679 3680 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3680 3681 state_available = rootnex_state->r_prealloc_size - space_used;
3681 3682 windowp = (rootnex_window_t *)
3682 3683 &dma->dp_cookies[sinfo->si_sgl_size];
3683 3684 }
3684 3685
3685 3686 /*
3686 3687 * figure out how much state we need to track the copy buffer. Add an
3687 3688 * addition 8 bytes for pointer alignemnt later.
3688 3689 */
3689 3690 if (dma->dp_copybuf_size > 0) {
3690 3691 copy_state_size = sinfo->si_max_pages *
3691 3692 sizeof (rootnex_pgmap_t);
3692 3693 } else {
3693 3694 copy_state_size = 0;
3694 3695 }
3695 3696 /* add an additional 8 bytes for pointer alignment */
3696 3697 space_needed = win_state_size + copy_state_size + 0x8;
3697 3698
3698 3699 /* if we have enough space already, use it */
3699 3700 if (state_available >= space_needed) {
3700 3701 dma->dp_window = windowp;
3701 3702 dma->dp_need_to_free_window = B_FALSE;
3702 3703
3703 3704 /* not enough space, need to allocate more. */
3704 3705 } else {
3705 3706 dma->dp_window = kmem_alloc(space_needed, kmflag);
3706 3707 if (dma->dp_window == NULL) {
3707 3708 return (DDI_DMA_NORESOURCES);
3708 3709 }
3709 3710 dma->dp_need_to_free_window = B_TRUE;
3710 3711 dma->dp_window_size = space_needed;
3711 3712 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3712 3713 dma->dp_dip, size_t, space_needed);
3713 3714 }
3714 3715
3715 3716 /*
3716 3717 * we allocate copy buffer state and window state at the same time.
3717 3718 * setup our copy buffer state pointers. Make sure it's aligned.
3718 3719 */
3719 3720 if (dma->dp_copybuf_size > 0) {
3720 3721 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3721 3722 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3722 3723
3723 3724 #if !defined(__amd64)
3724 3725 /*
3725 3726 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3726 3727 * false/NULL. Should be quicker to bzero vs loop and set.
3727 3728 */
3728 3729 bzero(dma->dp_pgmap, copy_state_size);
3729 3730 #endif
3730 3731 } else {
3731 3732 dma->dp_pgmap = NULL;
3732 3733 }
3733 3734
3734 3735 return (DDI_SUCCESS);
3735 3736 }
3736 3737
3737 3738
3738 3739 /*
3739 3740 * rootnex_teardown_copybuf()
3740 3741 * cleans up after rootnex_setup_copybuf()
3741 3742 */
3742 3743 static void
3743 3744 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3744 3745 {
3745 3746 #if !defined(__amd64)
3746 3747 int i;
3747 3748
3748 3749 /*
3749 3750 * if we allocated kernel heap VMEM space, go through all the pages and
3750 3751 * map out any of the ones that we're mapped into the kernel heap VMEM
3751 3752 * arena. Then free the VMEM space.
3752 3753 */
3753 3754 if (dma->dp_kva != NULL) {
3754 3755 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3755 3756 if (dma->dp_pgmap[i].pm_mapped) {
3756 3757 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3757 3758 MMU_PAGESIZE, HAT_UNLOAD);
3758 3759 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3759 3760 }
3760 3761 }
3761 3762
3762 3763 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3763 3764 }
3764 3765
3765 3766 #endif
3766 3767
3767 3768 /* if we allocated a copy buffer, free it */
3768 3769 if (dma->dp_cbaddr != NULL) {
3769 3770 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3770 3771 }
3771 3772 }
3772 3773
3773 3774
3774 3775 /*
3775 3776 * rootnex_teardown_windows()
3776 3777 * cleans up after rootnex_setup_windows()
3777 3778 */
3778 3779 static void
3779 3780 rootnex_teardown_windows(rootnex_dma_t *dma)
3780 3781 {
3781 3782 /*
3782 3783 * if we had to allocate window state on the last bind (because we
3783 3784 * didn't have enough pre-allocated space in the handle), free it.
3784 3785 */
3785 3786 if (dma->dp_need_to_free_window) {
3786 3787 kmem_free(dma->dp_window, dma->dp_window_size);
3787 3788 }
3788 3789 }
3789 3790
3790 3791
3791 3792 /*
3792 3793 * rootnex_init_win()
3793 3794 * Called in bind slow path during creation of a new window. Initializes
3794 3795 * window state to default values.
3795 3796 */
3796 3797 /*ARGSUSED*/
3797 3798 static void
3798 3799 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3799 3800 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3800 3801 {
3801 3802 hp->dmai_nwin++;
3802 3803 window->wd_dosync = B_FALSE;
3803 3804 window->wd_offset = cur_offset;
3804 3805 window->wd_size = 0;
3805 3806 window->wd_first_cookie = cookie;
3806 3807 window->wd_cookie_cnt = 0;
3807 3808 window->wd_trim.tr_trim_first = B_FALSE;
3808 3809 window->wd_trim.tr_trim_last = B_FALSE;
3809 3810 window->wd_trim.tr_first_copybuf_win = B_FALSE;
3810 3811 window->wd_trim.tr_last_copybuf_win = B_FALSE;
3811 3812 #if !defined(__amd64)
3812 3813 window->wd_remap_copybuf = dma->dp_cb_remaping;
3813 3814 #endif
3814 3815 }
3815 3816
3816 3817
3817 3818 /*
3818 3819 * rootnex_setup_cookie()
3819 3820 * Called in the bind slow path when the sgl uses the copy buffer. If any of
3820 3821 * the sgl uses the copy buffer, we need to go through each cookie, figure
3821 3822 * out if it uses the copy buffer, and if it does, save away everything we'll
3822 3823 * need during sync.
3823 3824 */
3824 3825 static void
3825 3826 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3826 3827 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3827 3828 page_t **cur_pp)
3828 3829 {
3829 3830 boolean_t copybuf_sz_power_2;
3830 3831 rootnex_sglinfo_t *sinfo;
3831 3832 paddr_t paddr;
3832 3833 uint_t pidx;
3833 3834 uint_t pcnt;
3834 3835 off_t poff;
3835 3836 #if defined(__amd64)
3836 3837 pfn_t pfn;
3837 3838 #else
3838 3839 page_t **pplist;
3839 3840 #endif
3840 3841
3841 3842 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3842 3843
3843 3844 sinfo = &dma->dp_sglinfo;
3844 3845
3845 3846 /*
3846 3847 * Calculate the page index relative to the start of the buffer. The
3847 3848 * index to the current page for our buffer is the offset into the
3848 3849 * first page of the buffer plus our current offset into the buffer
3849 3850 * itself, shifted of course...
3850 3851 */
3851 3852 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3852 3853 ASSERT(pidx < sinfo->si_max_pages);
3853 3854
3854 3855 /* if this cookie uses the copy buffer */
3855 3856 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3856 3857 /*
3857 3858 * NOTE: we know that since this cookie uses the copy buffer, it
3858 3859 * is <= MMU_PAGESIZE.
3859 3860 */
3860 3861
3861 3862 /*
3862 3863 * get the offset into the page. For the 64-bit kernel, get the
3863 3864 * pfn which we'll use with seg kpm.
3864 3865 */
3865 3866 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3866 3867 #if defined(__amd64)
3867 3868 /* mfn_to_pfn() is a NOP on i86pc */
3868 3869 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3869 3870 #endif /* __amd64 */
3870 3871
3871 3872 /* figure out if the copybuf size is a power of 2 */
3872 3873 if (!ISP2(dma->dp_copybuf_size)) {
3873 3874 copybuf_sz_power_2 = B_FALSE;
3874 3875 } else {
3875 3876 copybuf_sz_power_2 = B_TRUE;
3876 3877 }
3877 3878
3878 3879 /* This page uses the copy buffer */
3879 3880 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3880 3881
3881 3882 /*
3882 3883 * save the copy buffer KVA that we'll use with this page.
3883 3884 * if we still fit within the copybuf, it's a simple add.
3884 3885 * otherwise, we need to wrap over using & or % accordingly.
3885 3886 */
3886 3887 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3887 3888 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3888 3889 *copybuf_used;
3889 3890 } else {
3890 3891 if (copybuf_sz_power_2) {
3891 3892 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3892 3893 (uintptr_t)dma->dp_cbaddr +
3893 3894 (*copybuf_used &
3894 3895 (dma->dp_copybuf_size - 1)));
3895 3896 } else {
3896 3897 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3897 3898 (uintptr_t)dma->dp_cbaddr +
3898 3899 (*copybuf_used % dma->dp_copybuf_size));
3899 3900 }
3900 3901 }
3901 3902
3902 3903 /*
3903 3904 * over write the cookie physical address with the address of
3904 3905 * the physical address of the copy buffer page that we will
3905 3906 * use.
3906 3907 */
3907 3908 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3908 3909 dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3909 3910
3910 3911 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3911 3912
3912 3913 /* if we have a kernel VA, it's easy, just save that address */
3913 3914 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3914 3915 (sinfo->si_asp == &kas)) {
3915 3916 /*
3916 3917 * save away the page aligned virtual address of the
3917 3918 * driver buffer. Offsets are handled in the sync code.
3918 3919 */
3919 3920 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3920 3921 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3921 3922 & MMU_PAGEMASK);
3922 3923 #if !defined(__amd64)
3923 3924 /*
3924 3925 * we didn't need to, and will never need to map this
3925 3926 * page.
3926 3927 */
3927 3928 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3928 3929 #endif
3929 3930
3930 3931 /* we don't have a kernel VA. We need one for the bcopy. */
3931 3932 } else {
3932 3933 #if defined(__amd64)
3933 3934 /*
3934 3935 * for the 64-bit kernel, it's easy. We use seg kpm to
3935 3936 * get a Kernel VA for the corresponding pfn.
3936 3937 */
3937 3938 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3938 3939 #else
3939 3940 /*
3940 3941 * for the 32-bit kernel, this is a pain. First we'll
3941 3942 * save away the page_t or user VA for this page. This
3942 3943 * is needed in rootnex_dma_win() when we switch to a
3943 3944 * new window which requires us to re-map the copy
3944 3945 * buffer.
3945 3946 */
3946 3947 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3947 3948 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3948 3949 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3949 3950 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3950 3951 } else if (pplist != NULL) {
3951 3952 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3952 3953 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3953 3954 } else {
3954 3955 dma->dp_pgmap[pidx].pm_pp = NULL;
3955 3956 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3956 3957 (((uintptr_t)
3957 3958 dmar_object->dmao_obj.virt_obj.v_addr +
3958 3959 cur_offset) & MMU_PAGEMASK);
3959 3960 }
3960 3961
3961 3962 /*
3962 3963 * save away the page aligned virtual address which was
3963 3964 * allocated from the kernel heap arena (taking into
3964 3965 * account if we need more copy buffer than we alloced
3965 3966 * and use multiple windows to handle this, i.e. &,%).
3966 3967 * NOTE: there isn't and physical memory backing up this
3967 3968 * virtual address space currently.
3968 3969 */
3969 3970 if ((*copybuf_used + MMU_PAGESIZE) <=
3970 3971 dma->dp_copybuf_size) {
3971 3972 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3972 3973 (((uintptr_t)dma->dp_kva + *copybuf_used) &
3973 3974 MMU_PAGEMASK);
3974 3975 } else {
3975 3976 if (copybuf_sz_power_2) {
3976 3977 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3977 3978 (((uintptr_t)dma->dp_kva +
3978 3979 (*copybuf_used &
3979 3980 (dma->dp_copybuf_size - 1))) &
3980 3981 MMU_PAGEMASK);
3981 3982 } else {
3982 3983 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3983 3984 (((uintptr_t)dma->dp_kva +
3984 3985 (*copybuf_used %
3985 3986 dma->dp_copybuf_size)) &
3986 3987 MMU_PAGEMASK);
3987 3988 }
3988 3989 }
3989 3990
3990 3991 /*
3991 3992 * if we haven't used up the available copy buffer yet,
3992 3993 * map the kva to the physical page.
3993 3994 */
3994 3995 if (!dma->dp_cb_remaping && ((*copybuf_used +
3995 3996 MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3996 3997 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3997 3998 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3998 3999 i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3999 4000 dma->dp_pgmap[pidx].pm_kaddr);
4000 4001 } else {
4001 4002 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4002 4003 sinfo->si_asp,
4003 4004 dma->dp_pgmap[pidx].pm_kaddr);
4004 4005 }
4005 4006
4006 4007 /*
4007 4008 * we've used up the available copy buffer, this page
4008 4009 * will have to be mapped during rootnex_dma_win() when
4009 4010 * we switch to a new window which requires a re-map
4010 4011 * the copy buffer. (32-bit kernel only)
4011 4012 */
4012 4013 } else {
4013 4014 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4014 4015 }
4015 4016 #endif
4016 4017 /* go to the next page_t */
4017 4018 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4018 4019 *cur_pp = (*cur_pp)->p_next;
4019 4020 }
4020 4021 }
4021 4022
4022 4023 /* add to the copy buffer count */
4023 4024 *copybuf_used += MMU_PAGESIZE;
4024 4025
4025 4026 /*
4026 4027 * This cookie doesn't use the copy buffer. Walk through the pages this
4027 4028 * cookie occupies to reflect this.
4028 4029 */
4029 4030 } else {
4030 4031 /*
4031 4032 * figure out how many pages the cookie occupies. We need to
4032 4033 * use the original page offset of the buffer and the cookies
4033 4034 * offset in the buffer to do this.
4034 4035 */
4035 4036 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4036 4037 pcnt = mmu_btopr(cookie->dmac_size + poff);
4037 4038
4038 4039 while (pcnt > 0) {
4039 4040 #if !defined(__amd64)
4040 4041 /*
4041 4042 * the 32-bit kernel doesn't have seg kpm, so we need
4042 4043 * to map in the driver buffer (if it didn't come down
4043 4044 * with a kernel VA) on the fly. Since this page doesn't
4044 4045 * use the copy buffer, it's not, or will it ever, have
4045 4046 * to be mapped in.
4046 4047 */
4047 4048 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4048 4049 #endif
4049 4050 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4050 4051
4051 4052 /*
4052 4053 * we need to update pidx and cur_pp or we'll loose
4053 4054 * track of where we are.
4054 4055 */
4055 4056 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4056 4057 *cur_pp = (*cur_pp)->p_next;
4057 4058 }
4058 4059 pidx++;
4059 4060 pcnt--;
4060 4061 }
4061 4062 }
4062 4063 }
4063 4064
4064 4065
4065 4066 /*
4066 4067 * rootnex_sgllen_window_boundary()
4067 4068 * Called in the bind slow path when the next cookie causes us to exceed (in
4068 4069 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4069 4070 * length supported by the DMA H/W.
4070 4071 */
4071 4072 static int
4072 4073 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4073 4074 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4074 4075 off_t cur_offset)
4075 4076 {
4076 4077 off_t new_offset;
4077 4078 size_t trim_sz;
4078 4079 off_t coffset;
4079 4080
4080 4081
4081 4082 /*
4082 4083 * if we know we'll never have to trim, it's pretty easy. Just move to
4083 4084 * the next window and init it. We're done.
4084 4085 */
4085 4086 if (!dma->dp_trim_required) {
4086 4087 (*windowp)++;
4087 4088 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4088 4089 (*windowp)->wd_cookie_cnt++;
4089 4090 (*windowp)->wd_size = cookie->dmac_size;
4090 4091 return (DDI_SUCCESS);
4091 4092 }
4092 4093
4093 4094 /* figure out how much we need to trim from the window */
4094 4095 ASSERT(attr->dma_attr_granular != 0);
4095 4096 if (dma->dp_granularity_power_2) {
4096 4097 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4097 4098 } else {
4098 4099 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4099 4100 }
4100 4101
4101 4102 /* The window's a whole multiple of granularity. We're done */
4102 4103 if (trim_sz == 0) {
4103 4104 (*windowp)++;
4104 4105 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4105 4106 (*windowp)->wd_cookie_cnt++;
4106 4107 (*windowp)->wd_size = cookie->dmac_size;
4107 4108 return (DDI_SUCCESS);
4108 4109 }
4109 4110
4110 4111 /*
4111 4112 * The window's not a whole multiple of granularity, since we know this
4112 4113 * is due to the sgllen, we need to go back to the last cookie and trim
4113 4114 * that one, add the left over part of the old cookie into the new
4114 4115 * window, and then add in the new cookie into the new window.
4115 4116 */
4116 4117
4117 4118 /*
4118 4119 * make sure the driver isn't making us do something bad... Trimming and
4119 4120 * sgllen == 1 don't go together.
4120 4121 */
4121 4122 if (attr->dma_attr_sgllen == 1) {
4122 4123 return (DDI_DMA_NOMAPPING);
4123 4124 }
4124 4125
4125 4126 /*
4126 4127 * first, setup the current window to account for the trim. Need to go
4127 4128 * back to the last cookie for this.
4128 4129 */
4129 4130 cookie--;
4130 4131 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4131 4132 (*windowp)->wd_trim.tr_last_cookie = cookie;
4132 4133 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4133 4134 ASSERT(cookie->dmac_size > trim_sz);
4134 4135 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4135 4136 (*windowp)->wd_size -= trim_sz;
4136 4137
4137 4138 /* save the buffer offsets for the next window */
4138 4139 coffset = cookie->dmac_size - trim_sz;
4139 4140 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4140 4141
4141 4142 /*
4142 4143 * set this now in case this is the first window. all other cases are
4143 4144 * set in dma_win()
4144 4145 */
4145 4146 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4146 4147
4147 4148 /*
4148 4149 * initialize the next window using what's left over in the previous
4149 4150 * cookie.
4150 4151 */
4151 4152 (*windowp)++;
4152 4153 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4153 4154 (*windowp)->wd_cookie_cnt++;
4154 4155 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4155 4156 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4156 4157 (*windowp)->wd_trim.tr_first_size = trim_sz;
4157 4158 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4158 4159 (*windowp)->wd_dosync = B_TRUE;
4159 4160 }
4160 4161
4161 4162 /*
4162 4163 * now go back to the current cookie and add it to the new window. set
4163 4164 * the new window size to the what was left over from the previous
4164 4165 * cookie and what's in the current cookie.
4165 4166 */
4166 4167 cookie++;
4167 4168 (*windowp)->wd_cookie_cnt++;
4168 4169 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4169 4170
4170 4171 /*
4171 4172 * trim plus the next cookie could put us over maxxfer (a cookie can be
4172 4173 * a max size of maxxfer). Handle that case.
4173 4174 */
4174 4175 if ((*windowp)->wd_size > dma->dp_maxxfer) {
4175 4176 /*
4176 4177 * maxxfer is already a whole multiple of granularity, and this
4177 4178 * trim will be <= the previous trim (since a cookie can't be
4178 4179 * larger than maxxfer). Make things simple here.
4179 4180 */
4180 4181 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4181 4182 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4182 4183 (*windowp)->wd_trim.tr_last_cookie = cookie;
4183 4184 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4184 4185 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4185 4186 (*windowp)->wd_size -= trim_sz;
4186 4187 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4187 4188
4188 4189 /* save the buffer offsets for the next window */
4189 4190 coffset = cookie->dmac_size - trim_sz;
4190 4191 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4191 4192
4192 4193 /* setup the next window */
4193 4194 (*windowp)++;
4194 4195 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4195 4196 (*windowp)->wd_cookie_cnt++;
4196 4197 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4197 4198 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4198 4199 coffset;
4199 4200 (*windowp)->wd_trim.tr_first_size = trim_sz;
4200 4201 }
4201 4202
4202 4203 return (DDI_SUCCESS);
4203 4204 }
4204 4205
4205 4206
4206 4207 /*
4207 4208 * rootnex_copybuf_window_boundary()
4208 4209 * Called in bind slowpath when we get to a window boundary because we used
4209 4210 * up all the copy buffer that we have.
4210 4211 */
4211 4212 static int
4212 4213 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4213 4214 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4214 4215 size_t *copybuf_used)
4215 4216 {
4216 4217 rootnex_sglinfo_t *sinfo;
4217 4218 off_t new_offset;
4218 4219 size_t trim_sz;
4219 4220 paddr_t paddr;
4220 4221 off_t coffset;
4221 4222 uint_t pidx;
4222 4223 off_t poff;
4223 4224
4224 4225
4225 4226 sinfo = &dma->dp_sglinfo;
4226 4227
4227 4228 /*
4228 4229 * the copy buffer should be a whole multiple of page size. We know that
4229 4230 * this cookie is <= MMU_PAGESIZE.
4230 4231 */
4231 4232 ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4232 4233
4233 4234 /*
4234 4235 * from now on, all new windows in this bind need to be re-mapped during
4235 4236 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4236 4237 * space...
4237 4238 */
4238 4239 #if !defined(__amd64)
4239 4240 dma->dp_cb_remaping = B_TRUE;
4240 4241 #endif
4241 4242
4242 4243 /* reset copybuf used */
4243 4244 *copybuf_used = 0;
4244 4245
4245 4246 /*
4246 4247 * if we don't have to trim (since granularity is set to 1), go to the
4247 4248 * next window and add the current cookie to it. We know the current
4248 4249 * cookie uses the copy buffer since we're in this code path.
4249 4250 */
4250 4251 if (!dma->dp_trim_required) {
4251 4252 (*windowp)++;
4252 4253 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4253 4254
4254 4255 /* Add this cookie to the new window */
4255 4256 (*windowp)->wd_cookie_cnt++;
4256 4257 (*windowp)->wd_size += cookie->dmac_size;
4257 4258 *copybuf_used += MMU_PAGESIZE;
4258 4259 return (DDI_SUCCESS);
4259 4260 }
4260 4261
4261 4262 /*
4262 4263 * *** may need to trim, figure it out.
4263 4264 */
4264 4265
4265 4266 /* figure out how much we need to trim from the window */
4266 4267 if (dma->dp_granularity_power_2) {
4267 4268 trim_sz = (*windowp)->wd_size &
4268 4269 (hp->dmai_attr.dma_attr_granular - 1);
4269 4270 } else {
4270 4271 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4271 4272 }
4272 4273
4273 4274 /*
4274 4275 * if the window's a whole multiple of granularity, go to the next
4275 4276 * window, init it, then add in the current cookie. We know the current
4276 4277 * cookie uses the copy buffer since we're in this code path.
4277 4278 */
4278 4279 if (trim_sz == 0) {
4279 4280 (*windowp)++;
4280 4281 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4281 4282
4282 4283 /* Add this cookie to the new window */
4283 4284 (*windowp)->wd_cookie_cnt++;
4284 4285 (*windowp)->wd_size += cookie->dmac_size;
4285 4286 *copybuf_used += MMU_PAGESIZE;
4286 4287 return (DDI_SUCCESS);
4287 4288 }
4288 4289
4289 4290 /*
4290 4291 * *** We figured it out, we definitly need to trim
4291 4292 */
4292 4293
4293 4294 /*
4294 4295 * make sure the driver isn't making us do something bad...
4295 4296 * Trimming and sgllen == 1 don't go together.
4296 4297 */
4297 4298 if (hp->dmai_attr.dma_attr_sgllen == 1) {
4298 4299 return (DDI_DMA_NOMAPPING);
4299 4300 }
4300 4301
4301 4302 /*
4302 4303 * first, setup the current window to account for the trim. Need to go
4303 4304 * back to the last cookie for this. Some of the last cookie will be in
4304 4305 * the current window, and some of the last cookie will be in the new
4305 4306 * window. All of the current cookie will be in the new window.
4306 4307 */
4307 4308 cookie--;
4308 4309 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4309 4310 (*windowp)->wd_trim.tr_last_cookie = cookie;
4310 4311 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4311 4312 ASSERT(cookie->dmac_size > trim_sz);
4312 4313 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4313 4314 (*windowp)->wd_size -= trim_sz;
4314 4315
4315 4316 /*
4316 4317 * we're trimming the last cookie (not the current cookie). So that
4317 4318 * last cookie may have or may not have been using the copy buffer (
4318 4319 * we know the cookie passed in uses the copy buffer since we're in
4319 4320 * this code path).
4320 4321 *
4321 4322 * If the last cookie doesn't use the copy buffer, nothing special to
4322 4323 * do. However, if it does uses the copy buffer, it will be both the
4323 4324 * last page in the current window and the first page in the next
4324 4325 * window. Since we are reusing the copy buffer (and KVA space on the
4325 4326 * 32-bit kernel), this page will use the end of the copy buffer in the
4326 4327 * current window, and the start of the copy buffer in the next window.
4327 4328 * Track that info... The cookie physical address was already set to
4328 4329 * the copy buffer physical address in setup_cookie..
4329 4330 */
4330 4331 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4331 4332 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4332 4333 (*windowp)->wd_size) >> MMU_PAGESHIFT;
4333 4334 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4334 4335 (*windowp)->wd_trim.tr_last_pidx = pidx;
4335 4336 (*windowp)->wd_trim.tr_last_cbaddr =
4336 4337 dma->dp_pgmap[pidx].pm_cbaddr;
4337 4338 #if !defined(__amd64)
4338 4339 (*windowp)->wd_trim.tr_last_kaddr =
4339 4340 dma->dp_pgmap[pidx].pm_kaddr;
4340 4341 #endif
4341 4342 }
4342 4343
4343 4344 /* save the buffer offsets for the next window */
4344 4345 coffset = cookie->dmac_size - trim_sz;
4345 4346 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4346 4347
4347 4348 /*
4348 4349 * set this now in case this is the first window. all other cases are
4349 4350 * set in dma_win()
4350 4351 */
4351 4352 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4352 4353
4353 4354 /*
4354 4355 * initialize the next window using what's left over in the previous
4355 4356 * cookie.
4356 4357 */
4357 4358 (*windowp)++;
4358 4359 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4359 4360 (*windowp)->wd_cookie_cnt++;
4360 4361 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4361 4362 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4362 4363 (*windowp)->wd_trim.tr_first_size = trim_sz;
4363 4364
4364 4365 /*
4365 4366 * again, we're tracking if the last cookie uses the copy buffer.
4366 4367 * read the comment above for more info on why we need to track
4367 4368 * additional state.
4368 4369 *
4369 4370 * For the first cookie in the new window, we need reset the physical
4370 4371 * address to DMA into to the start of the copy buffer plus any
4371 4372 * initial page offset which may be present.
4372 4373 */
4373 4374 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4374 4375 (*windowp)->wd_dosync = B_TRUE;
4375 4376 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4376 4377 (*windowp)->wd_trim.tr_first_pidx = pidx;
4377 4378 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4378 4379 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4379 4380
4380 4381 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4381 4382 poff;
4382 4383 (*windowp)->wd_trim.tr_first_paddr =
4383 4384 ROOTNEX_PADDR_TO_RBASE(paddr);
4384 4385
4385 4386 #if !defined(__amd64)
4386 4387 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4387 4388 #endif
4388 4389 /* account for the cookie copybuf usage in the new window */
4389 4390 *copybuf_used += MMU_PAGESIZE;
4390 4391
4391 4392 /*
4392 4393 * every piece of code has to have a hack, and here is this
4393 4394 * ones :-)
4394 4395 *
4395 4396 * There is a complex interaction between setup_cookie and the
4396 4397 * copybuf window boundary. The complexity had to be in either
4397 4398 * the maxxfer window, or the copybuf window, and I chose the
4398 4399 * copybuf code.
4399 4400 *
4400 4401 * So in this code path, we have taken the last cookie,
4401 4402 * virtually broken it in half due to the trim, and it happens
4402 4403 * to use the copybuf which further complicates life. At the
4403 4404 * same time, we have already setup the current cookie, which
4404 4405 * is now wrong. More background info: the current cookie uses
4405 4406 * the copybuf, so it is only a page long max. So we need to
4406 4407 * fix the current cookies copy buffer address, physical
4407 4408 * address, and kva for the 32-bit kernel. We due this by
4408 4409 * bumping them by page size (of course, we can't due this on
4409 4410 * the physical address since the copy buffer may not be
4410 4411 * physically contiguous).
4411 4412 */
4412 4413 cookie++;
4413 4414 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4414 4415 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4415 4416
4416 4417 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4417 4418 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4418 4419 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4419 4420
4420 4421 #if !defined(__amd64)
4421 4422 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4422 4423 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4423 4424 #endif
4424 4425 } else {
4425 4426 /* go back to the current cookie */
4426 4427 cookie++;
4427 4428 }
4428 4429
4429 4430 /*
4430 4431 * add the current cookie to the new window. set the new window size to
4431 4432 * the what was left over from the previous cookie and what's in the
4432 4433 * current cookie.
4433 4434 */
4434 4435 (*windowp)->wd_cookie_cnt++;
4435 4436 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4436 4437 ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4437 4438
4438 4439 /*
4439 4440 * we know that the cookie passed in always uses the copy buffer. We
4440 4441 * wouldn't be here if it didn't.
4441 4442 */
4442 4443 *copybuf_used += MMU_PAGESIZE;
4443 4444
4444 4445 return (DDI_SUCCESS);
4445 4446 }
4446 4447
4447 4448
4448 4449 /*
4449 4450 * rootnex_maxxfer_window_boundary()
4450 4451 * Called in bind slowpath when we get to a window boundary because we will
4451 4452 * go over maxxfer.
4452 4453 */
4453 4454 static int
4454 4455 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4455 4456 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4456 4457 {
4457 4458 size_t dmac_size;
4458 4459 off_t new_offset;
4459 4460 size_t trim_sz;
4460 4461 off_t coffset;
4461 4462
4462 4463
4463 4464 /*
4464 4465 * calculate how much we have to trim off of the current cookie to equal
4465 4466 * maxxfer. We don't have to account for granularity here since our
4466 4467 * maxxfer already takes that into account.
4467 4468 */
4468 4469 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4469 4470 ASSERT(trim_sz <= cookie->dmac_size);
4470 4471 ASSERT(trim_sz <= dma->dp_maxxfer);
4471 4472
4472 4473 /* save cookie size since we need it later and we might change it */
4473 4474 dmac_size = cookie->dmac_size;
4474 4475
4475 4476 /*
4476 4477 * if we're not trimming the entire cookie, setup the current window to
4477 4478 * account for the trim.
4478 4479 */
4479 4480 if (trim_sz < cookie->dmac_size) {
4480 4481 (*windowp)->wd_cookie_cnt++;
4481 4482 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4482 4483 (*windowp)->wd_trim.tr_last_cookie = cookie;
4483 4484 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4484 4485 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4485 4486 (*windowp)->wd_size = dma->dp_maxxfer;
4486 4487
4487 4488 /*
4488 4489 * set the adjusted cookie size now in case this is the first
4489 4490 * window. All other windows are taken care of in get win
4490 4491 */
4491 4492 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4492 4493 }
4493 4494
4494 4495 /*
4495 4496 * coffset is the current offset within the cookie, new_offset is the
4496 4497 * current offset with the entire buffer.
4497 4498 */
4498 4499 coffset = dmac_size - trim_sz;
4499 4500 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4500 4501
4501 4502 /* initialize the next window */
4502 4503 (*windowp)++;
4503 4504 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4504 4505 (*windowp)->wd_cookie_cnt++;
4505 4506 (*windowp)->wd_size = trim_sz;
4506 4507 if (trim_sz < dmac_size) {
4507 4508 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4508 4509 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4509 4510 coffset;
4510 4511 (*windowp)->wd_trim.tr_first_size = trim_sz;
4511 4512 }
4512 4513
4513 4514 return (DDI_SUCCESS);
4514 4515 }
4515 4516
4516 4517
4517 4518 /*ARGSUSED*/
4518 4519 static int
4519 4520 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4520 4521 off_t off, size_t len, uint_t cache_flags)
4521 4522 {
4522 4523 rootnex_sglinfo_t *sinfo;
4523 4524 rootnex_pgmap_t *cbpage;
4524 4525 rootnex_window_t *win;
4525 4526 ddi_dma_impl_t *hp;
4526 4527 rootnex_dma_t *dma;
4527 4528 caddr_t fromaddr;
4528 4529 caddr_t toaddr;
4529 4530 uint_t psize;
4530 4531 off_t offset;
4531 4532 uint_t pidx;
4532 4533 size_t size;
4533 4534 off_t poff;
4534 4535 int e;
4535 4536
4536 4537
4537 4538 hp = (ddi_dma_impl_t *)handle;
4538 4539 dma = (rootnex_dma_t *)hp->dmai_private;
4539 4540 sinfo = &dma->dp_sglinfo;
4540 4541
4541 4542 /*
4542 4543 * if we don't have any windows, we don't need to sync. A copybuf
4543 4544 * will cause us to have at least one window.
4544 4545 */
4545 4546 if (dma->dp_window == NULL) {
4546 4547 return (DDI_SUCCESS);
4547 4548 }
4548 4549
4549 4550 /* This window may not need to be sync'd */
4550 4551 win = &dma->dp_window[dma->dp_current_win];
4551 4552 if (!win->wd_dosync) {
4552 4553 return (DDI_SUCCESS);
4553 4554 }
4554 4555
4555 4556 /* handle off and len special cases */
4556 4557 if ((off == 0) || (rootnex_sync_ignore_params)) {
4557 4558 offset = win->wd_offset;
4558 4559 } else {
4559 4560 offset = off;
4560 4561 }
4561 4562 if ((len == 0) || (rootnex_sync_ignore_params)) {
4562 4563 size = win->wd_size;
4563 4564 } else {
4564 4565 size = len;
4565 4566 }
4566 4567
4567 4568 /* check the sync args to make sure they make a little sense */
4568 4569 if (rootnex_sync_check_parms) {
4569 4570 e = rootnex_valid_sync_parms(hp, win, offset, size,
4570 4571 cache_flags);
4571 4572 if (e != DDI_SUCCESS) {
4572 4573 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4573 4574 return (DDI_FAILURE);
4574 4575 }
4575 4576 }
4576 4577
4577 4578 /*
4578 4579 * special case the first page to handle the offset into the page. The
4579 4580 * offset to the current page for our buffer is the offset into the
4580 4581 * first page of the buffer plus our current offset into the buffer
4581 4582 * itself, masked of course.
4582 4583 */
4583 4584 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4584 4585 psize = MIN((MMU_PAGESIZE - poff), size);
4585 4586
4586 4587 /* go through all the pages that we want to sync */
4587 4588 while (size > 0) {
4588 4589 /*
4589 4590 * Calculate the page index relative to the start of the buffer.
4590 4591 * The index to the current page for our buffer is the offset
4591 4592 * into the first page of the buffer plus our current offset
4592 4593 * into the buffer itself, shifted of course...
4593 4594 */
4594 4595 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4595 4596 ASSERT(pidx < sinfo->si_max_pages);
4596 4597
4597 4598 /*
4598 4599 * if this page uses the copy buffer, we need to sync it,
4599 4600 * otherwise, go on to the next page.
4600 4601 */
4601 4602 cbpage = &dma->dp_pgmap[pidx];
4602 4603 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4603 4604 (cbpage->pm_uses_copybuf == B_FALSE));
4604 4605 if (cbpage->pm_uses_copybuf) {
4605 4606 /* cbaddr and kaddr should be page aligned */
4606 4607 ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4607 4608 MMU_PAGEOFFSET) == 0);
4608 4609 ASSERT(((uintptr_t)cbpage->pm_kaddr &
4609 4610 MMU_PAGEOFFSET) == 0);
4610 4611
4611 4612 /*
4612 4613 * if we're copying for the device, we are going to
4613 4614 * copy from the drivers buffer and to the rootnex
4614 4615 * allocated copy buffer.
4615 4616 */
4616 4617 if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4617 4618 fromaddr = cbpage->pm_kaddr + poff;
4618 4619 toaddr = cbpage->pm_cbaddr + poff;
4619 4620 ROOTNEX_DPROBE2(rootnex__sync__dev,
4620 4621 dev_info_t *, dma->dp_dip, size_t, psize);
4621 4622
4622 4623 /*
4623 4624 * if we're copying for the cpu/kernel, we are going to
4624 4625 * copy from the rootnex allocated copy buffer to the
4625 4626 * drivers buffer.
4626 4627 */
4627 4628 } else {
4628 4629 fromaddr = cbpage->pm_cbaddr + poff;
4629 4630 toaddr = cbpage->pm_kaddr + poff;
4630 4631 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4631 4632 dev_info_t *, dma->dp_dip, size_t, psize);
4632 4633 }
4633 4634
4634 4635 bcopy(fromaddr, toaddr, psize);
4635 4636 }
4636 4637
4637 4638 /*
4638 4639 * decrement size until we're done, update our offset into the
4639 4640 * buffer, and get the next page size.
4640 4641 */
4641 4642 size -= psize;
4642 4643 offset += psize;
4643 4644 psize = MIN(MMU_PAGESIZE, size);
4644 4645
4645 4646 /* page offset is zero for the rest of this loop */
4646 4647 poff = 0;
4647 4648 }
4648 4649
4649 4650 return (DDI_SUCCESS);
4650 4651 }
4651 4652
4652 4653 /*
4653 4654 * rootnex_dma_sync()
4654 4655 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4655 4656 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4656 4657 * is set, ddi_dma_sync() returns immediately passing back success.
4657 4658 */
4658 4659 /*ARGSUSED*/
4659 4660 static int
4660 4661 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4661 4662 off_t off, size_t len, uint_t cache_flags)
4662 4663 {
4663 4664 #if defined(__amd64) && !defined(__xpv)
4664 4665 if (IOMMU_USED(rdip)) {
4665 4666 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4666 4667 cache_flags));
4667 4668 }
4668 4669 #endif
4669 4670 return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4670 4671 cache_flags));
4671 4672 }
4672 4673
4673 4674 /*
4674 4675 * rootnex_valid_sync_parms()
4675 4676 * checks the parameters passed to sync to verify they are correct.
4676 4677 */
4677 4678 static int
4678 4679 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4679 4680 off_t offset, size_t size, uint_t cache_flags)
4680 4681 {
4681 4682 off_t woffset;
4682 4683
4683 4684
4684 4685 /*
4685 4686 * the first part of the test to make sure the offset passed in is
4686 4687 * within the window.
4687 4688 */
4688 4689 if (offset < win->wd_offset) {
4689 4690 return (DDI_FAILURE);
4690 4691 }
4691 4692
4692 4693 /*
4693 4694 * second and last part of the test to make sure the offset and length
4694 4695 * passed in is within the window.
4695 4696 */
4696 4697 woffset = offset - win->wd_offset;
4697 4698 if ((woffset + size) > win->wd_size) {
4698 4699 return (DDI_FAILURE);
4699 4700 }
4700 4701
4701 4702 /*
4702 4703 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4703 4704 * be set too.
4704 4705 */
4705 4706 if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4706 4707 (hp->dmai_rflags & DDI_DMA_WRITE)) {
4707 4708 return (DDI_SUCCESS);
4708 4709 }
4709 4710
4710 4711 /*
4711 4712 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4712 4713 * should be set. Also DDI_DMA_READ should be set in the flags.
4713 4714 */
4714 4715 if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4715 4716 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4716 4717 (hp->dmai_rflags & DDI_DMA_READ)) {
4717 4718 return (DDI_SUCCESS);
4718 4719 }
4719 4720
4720 4721 return (DDI_FAILURE);
4721 4722 }
4722 4723
4723 4724
4724 4725 /*ARGSUSED*/
4725 4726 static int
4726 4727 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4727 4728 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4728 4729 uint_t *ccountp)
4729 4730 {
4730 4731 rootnex_window_t *window;
4731 4732 rootnex_trim_t *trim;
4732 4733 ddi_dma_impl_t *hp;
4733 4734 rootnex_dma_t *dma;
4734 4735 ddi_dma_obj_t *dmao;
4735 4736 #if !defined(__amd64)
4736 4737 rootnex_sglinfo_t *sinfo;
4737 4738 rootnex_pgmap_t *pmap;
4738 4739 uint_t pidx;
4739 4740 uint_t pcnt;
4740 4741 off_t poff;
4741 4742 int i;
4742 4743 #endif
4743 4744
4744 4745
4745 4746 hp = (ddi_dma_impl_t *)handle;
4746 4747 dma = (rootnex_dma_t *)hp->dmai_private;
4747 4748 #if !defined(__amd64)
4748 4749 sinfo = &dma->dp_sglinfo;
4749 4750 #endif
4750 4751
4751 4752 /* If we try and get a window which doesn't exist, return failure */
4752 4753 if (win >= hp->dmai_nwin) {
4753 4754 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4754 4755 return (DDI_FAILURE);
4755 4756 }
4756 4757
4757 4758 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4758 4759
4759 4760 /*
4760 4761 * if we don't have any windows, and they're asking for the first
4761 4762 * window, setup the cookie pointer to the first cookie in the bind.
4762 4763 * setup our return values, then increment the cookie since we return
4763 4764 * the first cookie on the stack.
4764 4765 */
4765 4766 if (dma->dp_window == NULL) {
4766 4767 if (win != 0) {
4767 4768 ROOTNEX_DPROF_INC(
4768 4769 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4769 4770 return (DDI_FAILURE);
4770 4771 }
4771 4772 hp->dmai_cookie = dma->dp_cookies;
4772 4773 *offp = 0;
4773 4774 *lenp = dmao->dmao_size;
4774 4775 *ccountp = dma->dp_sglinfo.si_sgl_size;
4775 4776 *cookiep = hp->dmai_cookie[0];
4776 4777 hp->dmai_cookie++;
4777 4778 return (DDI_SUCCESS);
4778 4779 }
4779 4780
4780 4781 /* sync the old window before moving on to the new one */
4781 4782 window = &dma->dp_window[dma->dp_current_win];
4782 4783 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4783 4784 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4784 4785 DDI_DMA_SYNC_FORCPU);
4785 4786 }
4786 4787
4787 4788 #if !defined(__amd64)
4788 4789 /*
4789 4790 * before we move to the next window, if we need to re-map, unmap all
4790 4791 * the pages in this window.
4791 4792 */
4792 4793 if (dma->dp_cb_remaping) {
4793 4794 /*
4794 4795 * If we switch to this window again, we'll need to map in
4795 4796 * on the fly next time.
4796 4797 */
4797 4798 window->wd_remap_copybuf = B_TRUE;
4798 4799
4799 4800 /*
4800 4801 * calculate the page index into the buffer where this window
4801 4802 * starts, and the number of pages this window takes up.
4802 4803 */
4803 4804 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4804 4805 MMU_PAGESHIFT;
4805 4806 poff = (sinfo->si_buf_offset + window->wd_offset) &
4806 4807 MMU_PAGEOFFSET;
4807 4808 pcnt = mmu_btopr(window->wd_size + poff);
4808 4809 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4809 4810
4810 4811 /* unmap pages which are currently mapped in this window */
4811 4812 for (i = 0; i < pcnt; i++) {
4812 4813 if (dma->dp_pgmap[pidx].pm_mapped) {
4813 4814 hat_unload(kas.a_hat,
4814 4815 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4815 4816 HAT_UNLOAD);
4816 4817 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4817 4818 }
4818 4819 pidx++;
4819 4820 }
4820 4821 }
4821 4822 #endif
4822 4823
4823 4824 /*
4824 4825 * Move to the new window.
4825 4826 * NOTE: current_win must be set for sync to work right
4826 4827 */
4827 4828 dma->dp_current_win = win;
4828 4829 window = &dma->dp_window[win];
4829 4830
4830 4831 /* if needed, adjust the first and/or last cookies for trim */
4831 4832 trim = &window->wd_trim;
4832 4833 if (trim->tr_trim_first) {
4833 4834 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4834 4835 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4835 4836 #if !defined(__amd64)
4836 4837 window->wd_first_cookie->dmac_type =
4837 4838 (window->wd_first_cookie->dmac_type &
4838 4839 ROOTNEX_USES_COPYBUF) + window->wd_offset;
4839 4840 #endif
4840 4841 if (trim->tr_first_copybuf_win) {
4841 4842 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4842 4843 trim->tr_first_cbaddr;
4843 4844 #if !defined(__amd64)
4844 4845 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4845 4846 trim->tr_first_kaddr;
4846 4847 #endif
4847 4848 }
4848 4849 }
4849 4850 if (trim->tr_trim_last) {
4850 4851 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4851 4852 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4852 4853 if (trim->tr_last_copybuf_win) {
4853 4854 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4854 4855 trim->tr_last_cbaddr;
4855 4856 #if !defined(__amd64)
4856 4857 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4857 4858 trim->tr_last_kaddr;
4858 4859 #endif
4859 4860 }
4860 4861 }
4861 4862
4862 4863 /*
4863 4864 * setup the cookie pointer to the first cookie in the window. setup
4864 4865 * our return values, then increment the cookie since we return the
4865 4866 * first cookie on the stack.
4866 4867 */
4867 4868 hp->dmai_cookie = window->wd_first_cookie;
4868 4869 *offp = window->wd_offset;
4869 4870 *lenp = window->wd_size;
4870 4871 *ccountp = window->wd_cookie_cnt;
4871 4872 *cookiep = hp->dmai_cookie[0];
4872 4873 hp->dmai_cookie++;
4873 4874
4874 4875 #if !defined(__amd64)
4875 4876 /* re-map copybuf if required for this window */
4876 4877 if (dma->dp_cb_remaping) {
4877 4878 /*
4878 4879 * calculate the page index into the buffer where this
4879 4880 * window starts.
4880 4881 */
4881 4882 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4882 4883 MMU_PAGESHIFT;
4883 4884 ASSERT(pidx < sinfo->si_max_pages);
4884 4885
4885 4886 /*
4886 4887 * the first page can get unmapped if it's shared with the
4887 4888 * previous window. Even if the rest of this window is already
4888 4889 * mapped in, we need to still check this one.
4889 4890 */
4890 4891 pmap = &dma->dp_pgmap[pidx];
4891 4892 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4892 4893 if (pmap->pm_pp != NULL) {
4893 4894 pmap->pm_mapped = B_TRUE;
4894 4895 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4895 4896 } else if (pmap->pm_vaddr != NULL) {
4896 4897 pmap->pm_mapped = B_TRUE;
4897 4898 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4898 4899 pmap->pm_kaddr);
4899 4900 }
4900 4901 }
4901 4902 pidx++;
4902 4903
4903 4904 /* map in the rest of the pages if required */
4904 4905 if (window->wd_remap_copybuf) {
4905 4906 window->wd_remap_copybuf = B_FALSE;
4906 4907
4907 4908 /* figure out many pages this window takes up */
4908 4909 poff = (sinfo->si_buf_offset + window->wd_offset) &
4909 4910 MMU_PAGEOFFSET;
4910 4911 pcnt = mmu_btopr(window->wd_size + poff);
4911 4912 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4912 4913
4913 4914 /* map pages which require it */
4914 4915 for (i = 1; i < pcnt; i++) {
4915 4916 pmap = &dma->dp_pgmap[pidx];
4916 4917 if (pmap->pm_uses_copybuf) {
4917 4918 ASSERT(pmap->pm_mapped == B_FALSE);
4918 4919 if (pmap->pm_pp != NULL) {
4919 4920 pmap->pm_mapped = B_TRUE;
4920 4921 i86_pp_map(pmap->pm_pp,
4921 4922 pmap->pm_kaddr);
4922 4923 } else if (pmap->pm_vaddr != NULL) {
4923 4924 pmap->pm_mapped = B_TRUE;
4924 4925 i86_va_map(pmap->pm_vaddr,
4925 4926 sinfo->si_asp,
4926 4927 pmap->pm_kaddr);
4927 4928 }
4928 4929 }
4929 4930 pidx++;
4930 4931 }
4931 4932 }
4932 4933 }
4933 4934 #endif
4934 4935
4935 4936 /* if the new window uses the copy buffer, sync it for the device */
4936 4937 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4937 4938 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4938 4939 DDI_DMA_SYNC_FORDEV);
4939 4940 }
4940 4941
4941 4942 return (DDI_SUCCESS);
4942 4943 }
4943 4944
4944 4945 /*
4945 4946 * rootnex_dma_win()
4946 4947 * called from ddi_dma_getwin()
4947 4948 */
4948 4949 /*ARGSUSED*/
4949 4950 static int
4950 4951 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4951 4952 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4952 4953 uint_t *ccountp)
4953 4954 {
4954 4955 #if defined(__amd64) && !defined(__xpv)
4955 4956 if (IOMMU_USED(rdip)) {
4956 4957 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4957 4958 cookiep, ccountp));
4958 4959 }
4959 4960 #endif
4960 4961
4961 4962 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4962 4963 cookiep, ccountp));
4963 4964 }
4964 4965
4965 4966 #if defined(__amd64) && !defined(__xpv)
4966 4967 /*ARGSUSED*/
4967 4968 static int
4968 4969 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4969 4970 ddi_dma_handle_t handle, void *v)
4970 4971 {
4971 4972 ddi_dma_impl_t *hp;
4972 4973 rootnex_dma_t *dma;
4973 4974
4974 4975 hp = (ddi_dma_impl_t *)handle;
4975 4976 dma = (rootnex_dma_t *)hp->dmai_private;
4976 4977 dma->dp_iommu_private = v;
4977 4978
4978 4979 return (DDI_SUCCESS);
4979 4980 }
4980 4981
4981 4982 /*ARGSUSED*/
4982 4983 static void *
4983 4984 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4984 4985 ddi_dma_handle_t handle)
4985 4986 {
4986 4987 ddi_dma_impl_t *hp;
4987 4988 rootnex_dma_t *dma;
4988 4989
4989 4990 hp = (ddi_dma_impl_t *)handle;
4990 4991 dma = (rootnex_dma_t *)hp->dmai_private;
4991 4992
4992 4993 return (dma->dp_iommu_private);
4993 4994 }
4994 4995 #endif
4995 4996
4996 4997 /*
4997 4998 * ************************
4998 4999 * obsoleted dma routines
4999 5000 * ************************
5000 5001 */
5001 5002
5002 5003 /*
5003 5004 * rootnex_dma_mctl()
5004 5005 *
5005 5006 * We don't support this legacy interface any more on x86.
5006 5007 */
5007 5008 /* ARGSUSED */
5008 5009 static int
5009 5010 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5010 5011 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5011 5012 uint_t cache_flags)
5012 5013 {
5013 5014 /*
5014 5015 * The only thing dma_mctl is usef for anymore is legacy SPARC
5015 5016 * dvma and sbus-specific routines.
5016 5017 */
5017 5018 return (DDI_FAILURE);
5018 5019 }
5019 5020
5020 5021 /*
5021 5022 * *********
5022 5023 * FMA Code
5023 5024 * *********
5024 5025 */
5025 5026
5026 5027 /*
5027 5028 * rootnex_fm_init()
5028 5029 * FMA init busop
5029 5030 */
5030 5031 /* ARGSUSED */
5031 5032 static int
5032 5033 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5033 5034 ddi_iblock_cookie_t *ibc)
5034 5035 {
5035 5036 *ibc = rootnex_state->r_err_ibc;
5036 5037
5037 5038 return (ddi_system_fmcap);
5038 5039 }
5039 5040
5040 5041 /*
5041 5042 * rootnex_dma_check()
5042 5043 * Function called after a dma fault occurred to find out whether the
5043 5044 * fault address is associated with a driver that is able to handle faults
5044 5045 * and recover from faults.
5045 5046 */
5046 5047 /* ARGSUSED */
5047 5048 static int
5048 5049 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5049 5050 const void *not_used)
5050 5051 {
5051 5052 rootnex_window_t *window;
5052 5053 uint64_t start_addr;
5053 5054 uint64_t fault_addr;
5054 5055 ddi_dma_impl_t *hp;
5055 5056 rootnex_dma_t *dma;
5056 5057 uint64_t end_addr;
5057 5058 size_t csize;
5058 5059 int i;
5059 5060 int j;
5060 5061
5061 5062
5062 5063 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5063 5064 hp = (ddi_dma_impl_t *)handle;
5064 5065 ASSERT(hp);
5065 5066
5066 5067 dma = (rootnex_dma_t *)hp->dmai_private;
5067 5068
5068 5069 /* Get the address that we need to search for */
5069 5070 fault_addr = *(uint64_t *)addr;
5070 5071
5071 5072 /*
5072 5073 * if we don't have any windows, we can just walk through all the
5073 5074 * cookies.
5074 5075 */
5075 5076 if (dma->dp_window == NULL) {
5076 5077 /* for each cookie */
5077 5078 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5078 5079 /*
5079 5080 * if the faulted address is within the physical address
5080 5081 * range of the cookie, return DDI_FM_NONFATAL.
5081 5082 */
5082 5083 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5083 5084 (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5084 5085 dma->dp_cookies[i].dmac_size))) {
5085 5086 return (DDI_FM_NONFATAL);
5086 5087 }
5087 5088 }
5088 5089
5089 5090 /* fault_addr not within this DMA handle */
5090 5091 return (DDI_FM_UNKNOWN);
5091 5092 }
5092 5093
5093 5094 /* we have mutiple windows, walk through each window */
5094 5095 for (i = 0; i < hp->dmai_nwin; i++) {
5095 5096 window = &dma->dp_window[i];
5096 5097
5097 5098 /* Go through all the cookies in the window */
5098 5099 for (j = 0; j < window->wd_cookie_cnt; j++) {
5099 5100
5100 5101 start_addr = window->wd_first_cookie[j].dmac_laddress;
5101 5102 csize = window->wd_first_cookie[j].dmac_size;
5102 5103
5103 5104 /*
5104 5105 * if we are trimming the first cookie in the window,
5105 5106 * and this is the first cookie, adjust the start
5106 5107 * address and size of the cookie to account for the
5107 5108 * trim.
5108 5109 */
5109 5110 if (window->wd_trim.tr_trim_first && (j == 0)) {
5110 5111 start_addr = window->wd_trim.tr_first_paddr;
5111 5112 csize = window->wd_trim.tr_first_size;
5112 5113 }
5113 5114
5114 5115 /*
5115 5116 * if we are trimming the last cookie in the window,
5116 5117 * and this is the last cookie, adjust the start
5117 5118 * address and size of the cookie to account for the
5118 5119 * trim.
5119 5120 */
5120 5121 if (window->wd_trim.tr_trim_last &&
5121 5122 (j == (window->wd_cookie_cnt - 1))) {
5122 5123 start_addr = window->wd_trim.tr_last_paddr;
5123 5124 csize = window->wd_trim.tr_last_size;
5124 5125 }
5125 5126
5126 5127 end_addr = start_addr + csize;
5127 5128
5128 5129 /*
5129 5130 * if the faulted address is within the physical
5130 5131 * address of the cookie, return DDI_FM_NONFATAL.
5131 5132 */
5132 5133 if ((fault_addr >= start_addr) &&
5133 5134 (fault_addr <= end_addr)) {
5134 5135 return (DDI_FM_NONFATAL);
5135 5136 }
5136 5137 }
5137 5138 }
5138 5139
5139 5140 /* fault_addr not within this DMA handle */
5140 5141 return (DDI_FM_UNKNOWN);
5141 5142 }
5142 5143
5143 5144 /*ARGSUSED*/
5144 5145 static int
5145 5146 rootnex_quiesce(dev_info_t *dip)
5146 5147 {
5147 5148 #if defined(__amd64) && !defined(__xpv)
5148 5149 return (immu_quiesce());
5149 5150 #else
5150 5151 return (DDI_SUCCESS);
5151 5152 #endif
5152 5153 }
5153 5154
5154 5155 #if defined(__xpv)
5155 5156 void
5156 5157 immu_init(void)
5157 5158 {
5158 5159 ;
5159 5160 }
5160 5161
5161 5162 void
5162 5163 immu_startup(void)
5163 5164 {
5164 5165 ;
5165 5166 }
5166 5167 /*ARGSUSED*/
5167 5168 void
5168 5169 immu_physmem_update(uint64_t addr, uint64_t size)
5169 5170 {
5170 5171 ;
5171 5172 }
5172 5173 #endif
↓ open down ↓ |
2823 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX