1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
28 * Copyright 2017 Joyent, Inc.
29 */
30
31 /*
32 * x86 root nexus driver
33 */
34
35 #include <sys/sysmacros.h>
36 #include <sys/conf.h>
37 #include <sys/autoconf.h>
38 #include <sys/sysmacros.h>
39 #include <sys/debug.h>
40 #include <sys/psw.h>
41 #include <sys/ddidmareq.h>
42 #include <sys/promif.h>
43 #include <sys/devops.h>
44 #include <sys/kmem.h>
45 #include <sys/cmn_err.h>
46 #include <vm/seg.h>
47 #include <vm/seg_kmem.h>
48 #include <vm/seg_dev.h>
49 #include <sys/vmem.h>
50 #include <sys/mman.h>
51 #include <vm/hat.h>
52 #include <vm/as.h>
53 #include <vm/page.h>
54 #include <sys/avintr.h>
55 #include <sys/errno.h>
56 #include <sys/modctl.h>
57 #include <sys/ddi_impldefs.h>
58 #include <sys/sunddi.h>
59 #include <sys/sunndi.h>
60 #include <sys/mach_intr.h>
61 #include <sys/psm.h>
62 #include <sys/ontrap.h>
63 #include <sys/atomic.h>
64 #include <sys/sdt.h>
65 #include <sys/rootnex.h>
66 #include <vm/hat_i86.h>
67 #include <sys/ddifm.h>
68 #include <sys/ddi_isa.h>
69 #include <sys/apic.h>
70
71 #ifdef __xpv
72 #include <sys/bootinfo.h>
73 #include <sys/hypervisor.h>
74 #include <sys/bootconf.h>
75 #include <vm/kboot_mmu.h>
76 #endif
77
78 #if defined(__amd64) && !defined(__xpv)
79 #include <sys/immu.h>
80 #endif
81
82
83 /*
84 * enable/disable extra checking of function parameters. Useful for debugging
85 * drivers.
86 */
87 #ifdef DEBUG
88 int rootnex_alloc_check_parms = 1;
89 int rootnex_bind_check_parms = 1;
90 int rootnex_bind_check_inuse = 1;
91 int rootnex_unbind_verify_buffer = 0;
92 int rootnex_sync_check_parms = 1;
93 #else
94 int rootnex_alloc_check_parms = 0;
95 int rootnex_bind_check_parms = 0;
96 int rootnex_bind_check_inuse = 0;
97 int rootnex_unbind_verify_buffer = 0;
98 int rootnex_sync_check_parms = 0;
99 #endif
100
101 boolean_t rootnex_dmar_not_setup;
102
103 /* Master Abort and Target Abort panic flag */
104 int rootnex_fm_ma_ta_panic_flag = 0;
105
106 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
107 int rootnex_bind_fail = 1;
108 int rootnex_bind_warn = 1;
109 uint8_t *rootnex_warn_list;
110 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
111 #define ROOTNEX_BIND_WARNING (0x1 << 0)
112
113 /*
114 * revert back to old broken behavior of always sync'ing entire copy buffer.
115 * This is useful if be have a buggy driver which doesn't correctly pass in
116 * the offset and size into ddi_dma_sync().
117 */
118 int rootnex_sync_ignore_params = 0;
119
120 /*
121 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
122 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
123 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
124 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
125 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
126 * (< 8K). We will still need to allocate the copy buffer during bind though
127 * (if we need one). These can only be modified in /etc/system before rootnex
128 * attach.
129 */
130 #if defined(__amd64)
131 int rootnex_prealloc_cookies = 65;
132 int rootnex_prealloc_windows = 4;
133 int rootnex_prealloc_copybuf = 2;
134 #else
135 int rootnex_prealloc_cookies = 33;
136 int rootnex_prealloc_windows = 4;
137 int rootnex_prealloc_copybuf = 2;
138 #endif
139
140 /* driver global state */
141 static rootnex_state_t *rootnex_state;
142
143 #ifdef DEBUG
144 /* shortcut to rootnex counters */
145 static uint64_t *rootnex_cnt;
146 #endif
147
148 /*
149 * XXX - does x86 even need these or are they left over from the SPARC days?
150 */
151 /* statically defined integer/boolean properties for the root node */
152 static rootnex_intprop_t rootnex_intprp[] = {
153 { "PAGESIZE", PAGESIZE },
154 { "MMU_PAGESIZE", MMU_PAGESIZE },
155 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
156 { DDI_RELATIVE_ADDRESSING, 1 },
157 };
158 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
159
160 /*
161 * If we're dom0, we're using a real device so we need to load
162 * the cookies with MFNs instead of PFNs.
163 */
164 #ifdef __xpv
165 typedef maddr_t rootnex_addr_t;
166 #define ROOTNEX_PADDR_TO_RBASE(pa) \
167 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
168 #else
169 typedef paddr_t rootnex_addr_t;
170 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
171 #endif
172
173 static struct cb_ops rootnex_cb_ops = {
174 nodev, /* open */
175 nodev, /* close */
176 nodev, /* strategy */
177 nodev, /* print */
178 nodev, /* dump */
179 nodev, /* read */
180 nodev, /* write */
181 nodev, /* ioctl */
182 nodev, /* devmap */
183 nodev, /* mmap */
184 nodev, /* segmap */
185 nochpoll, /* chpoll */
186 ddi_prop_op, /* cb_prop_op */
187 NULL, /* struct streamtab */
188 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
189 CB_REV, /* Rev */
190 nodev, /* cb_aread */
191 nodev /* cb_awrite */
192 };
193
194 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
195 off_t offset, off_t len, caddr_t *vaddrp);
196 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
197 struct hat *hat, struct seg *seg, caddr_t addr,
198 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
199 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
200 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
201 ddi_dma_handle_t *handlep);
202 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
203 ddi_dma_handle_t handle);
204 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
205 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
206 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
207 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
208 ddi_dma_handle_t handle);
209 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
210 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
211 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
212 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
213 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
214 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
215 ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
216 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
217 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
218 ddi_ctl_enum_t ctlop, void *arg, void *result);
219 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
220 ddi_iblock_cookie_t *ibc);
221 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
222 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
223 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
224 void *);
225 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
226
227 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
228 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
229 ddi_dma_handle_t *handlep);
230 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
231 ddi_dma_handle_t handle);
232 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
233 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
234 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
235 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
236 ddi_dma_handle_t handle);
237 #if defined(__amd64) && !defined(__xpv)
238 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
239 ddi_dma_handle_t handle);
240 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
241 ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
242 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
243 ddi_dma_cookie_t *cookiep, uint_t ccount);
244 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
245 ddi_dma_handle_t handle);
246 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
247 #endif
248 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
249 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
250 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
251 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
252 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
253
254 #if defined(__amd64) && !defined(__xpv)
255 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
256 ddi_dma_handle_t handle, void *v);
257 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
258 ddi_dma_handle_t handle);
259 #endif
260
261
262 static struct bus_ops rootnex_bus_ops = {
263 BUSO_REV,
264 rootnex_map,
265 NULL,
266 NULL,
267 NULL,
268 rootnex_map_fault,
269 0,
270 rootnex_dma_allochdl,
271 rootnex_dma_freehdl,
272 rootnex_dma_bindhdl,
273 rootnex_dma_unbindhdl,
274 rootnex_dma_sync,
275 rootnex_dma_win,
276 rootnex_dma_mctl,
277 rootnex_ctlops,
278 ddi_bus_prop_op,
279 i_ddi_rootnex_get_eventcookie,
280 i_ddi_rootnex_add_eventcall,
281 i_ddi_rootnex_remove_eventcall,
282 i_ddi_rootnex_post_event,
283 0, /* bus_intr_ctl */
284 0, /* bus_config */
285 0, /* bus_unconfig */
286 rootnex_fm_init, /* bus_fm_init */
287 NULL, /* bus_fm_fini */
288 NULL, /* bus_fm_access_enter */
289 NULL, /* bus_fm_access_exit */
290 NULL, /* bus_powr */
291 rootnex_intr_ops /* bus_intr_op */
292 };
293
294 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
295 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
296 static int rootnex_quiesce(dev_info_t *dip);
297
298 static struct dev_ops rootnex_ops = {
299 DEVO_REV,
300 0,
301 ddi_no_info,
302 nulldev,
303 nulldev,
304 rootnex_attach,
305 rootnex_detach,
306 nulldev,
307 &rootnex_cb_ops,
308 &rootnex_bus_ops,
309 NULL,
310 rootnex_quiesce, /* quiesce */
311 };
312
313 static struct modldrv rootnex_modldrv = {
314 &mod_driverops,
315 "i86pc root nexus",
316 &rootnex_ops
317 };
318
319 static struct modlinkage rootnex_modlinkage = {
320 MODREV_1,
321 (void *)&rootnex_modldrv,
322 NULL
323 };
324
325 #if defined(__amd64) && !defined(__xpv)
326 static iommulib_nexops_t iommulib_nexops = {
327 IOMMU_NEXOPS_VERSION,
328 "Rootnex IOMMU ops Vers 1.1",
329 NULL,
330 rootnex_coredma_allochdl,
331 rootnex_coredma_freehdl,
332 rootnex_coredma_bindhdl,
333 rootnex_coredma_unbindhdl,
334 rootnex_coredma_reset_cookies,
335 rootnex_coredma_get_cookies,
336 rootnex_coredma_set_cookies,
337 rootnex_coredma_clear_cookies,
338 rootnex_coredma_get_sleep_flags,
339 rootnex_coredma_sync,
340 rootnex_coredma_win,
341 rootnex_coredma_hdl_setprivate,
342 rootnex_coredma_hdl_getprivate
343 };
344 #endif
345
346 /*
347 * extern hacks
348 */
349 extern struct seg_ops segdev_ops;
350 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
351 #ifdef DDI_MAP_DEBUG
352 extern int ddi_map_debug_flag;
353 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
354 #endif
355 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
356 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
357 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
358 psm_intr_op_t, int *);
359 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
360 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
361
362 /*
363 * Use device arena to use for device control register mappings.
364 * Various kernel memory walkers (debugger, dtrace) need to know
365 * to avoid this address range to prevent undesired device activity.
366 */
367 extern void *device_arena_alloc(size_t size, int vm_flag);
368 extern void device_arena_free(void * vaddr, size_t size);
369
370
371 /*
372 * Internal functions
373 */
374 static int rootnex_dma_init();
375 static void rootnex_add_props(dev_info_t *);
376 static int rootnex_ctl_reportdev(dev_info_t *dip);
377 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
378 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
379 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
380 static int rootnex_map_handle(ddi_map_req_t *mp);
381 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
382 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
383 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
384 ddi_dma_attr_t *attr);
385 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
386 rootnex_sglinfo_t *sglinfo);
387 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
388 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
389 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
390 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
391 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
392 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
393 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
394 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
395 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
396 static void rootnex_teardown_windows(rootnex_dma_t *dma);
397 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
398 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
399 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
400 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
401 size_t *copybuf_used, page_t **cur_pp);
402 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
403 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
404 ddi_dma_attr_t *attr, off_t cur_offset);
405 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
406 rootnex_dma_t *dma, rootnex_window_t **windowp,
407 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
408 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
409 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
410 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
411 off_t offset, size_t size, uint_t cache_flags);
412 static int rootnex_verify_buffer(rootnex_dma_t *dma);
413 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
414 const void *comp_addr, const void *not_used);
415 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
416 rootnex_sglinfo_t *sglinfo);
417 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
418
419 /*
420 * _init()
421 *
422 */
423 int
424 _init(void)
425 {
426
427 rootnex_state = NULL;
428 return (mod_install(&rootnex_modlinkage));
429 }
430
431
432 /*
433 * _info()
434 *
435 */
436 int
437 _info(struct modinfo *modinfop)
438 {
439 return (mod_info(&rootnex_modlinkage, modinfop));
440 }
441
442
443 /*
444 * _fini()
445 *
446 */
447 int
448 _fini(void)
449 {
450 return (EBUSY);
451 }
452
453
454 /*
455 * rootnex_attach()
456 *
457 */
458 static int
459 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
460 {
461 int fmcap;
462 int e;
463
464 switch (cmd) {
465 case DDI_ATTACH:
466 break;
467 case DDI_RESUME:
468 #if defined(__amd64) && !defined(__xpv)
469 return (immu_unquiesce());
470 #else
471 return (DDI_SUCCESS);
472 #endif
473 default:
474 return (DDI_FAILURE);
475 }
476
477 /*
478 * We should only have one instance of rootnex. Save it away since we
479 * don't have an easy way to get it back later.
480 */
481 ASSERT(rootnex_state == NULL);
482 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
483
484 rootnex_state->r_dip = dip;
485 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
486 rootnex_state->r_reserved_msg_printed = B_FALSE;
487 #ifdef DEBUG
488 rootnex_cnt = &rootnex_state->r_counters[0];
489 #endif
490
491 /*
492 * Set minimum fm capability level for i86pc platforms and then
493 * initialize error handling. Since we're the rootnex, we don't
494 * care what's returned in the fmcap field.
495 */
496 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
497 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
498 fmcap = ddi_system_fmcap;
499 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
500
501 /* initialize DMA related state */
502 e = rootnex_dma_init();
503 if (e != DDI_SUCCESS) {
504 kmem_free(rootnex_state, sizeof (rootnex_state_t));
505 return (DDI_FAILURE);
506 }
507
508 /* Add static root node properties */
509 rootnex_add_props(dip);
510
511 /* since we can't call ddi_report_dev() */
512 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
513
514 /* Initialize rootnex event handle */
515 i_ddi_rootnex_init_events(dip);
516
517 #if defined(__amd64) && !defined(__xpv)
518 e = iommulib_nexus_register(dip, &iommulib_nexops,
519 &rootnex_state->r_iommulib_handle);
520
521 ASSERT(e == DDI_SUCCESS);
522 #endif
523
524 return (DDI_SUCCESS);
525 }
526
527
528 /*
529 * rootnex_detach()
530 *
531 */
532 /*ARGSUSED*/
533 static int
534 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
535 {
536 switch (cmd) {
537 case DDI_SUSPEND:
538 #if defined(__amd64) && !defined(__xpv)
539 return (immu_quiesce());
540 #else
541 return (DDI_SUCCESS);
542 #endif
543 default:
544 return (DDI_FAILURE);
545 }
546 /*NOTREACHED*/
547
548 }
549
550
551 /*
552 * rootnex_dma_init()
553 *
554 */
555 /*ARGSUSED*/
556 static int
557 rootnex_dma_init()
558 {
559 size_t bufsize;
560
561
562 /*
563 * size of our cookie/window/copybuf state needed in dma bind that we
564 * pre-alloc in dma_alloc_handle
565 */
566 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
567 rootnex_state->r_prealloc_size =
568 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
569 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
570 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
571
572 /*
573 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
574 * allocate 16 extra bytes for struct pointer alignment
575 * (p->dmai_private & dma->dp_prealloc_buffer)
576 */
577 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
578 rootnex_state->r_prealloc_size + 0x10;
579 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
580 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
581 if (rootnex_state->r_dmahdl_cache == NULL) {
582 return (DDI_FAILURE);
583 }
584
585 /*
586 * allocate array to track which major numbers we have printed warnings
587 * for.
588 */
589 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
590 KM_SLEEP);
591
592 return (DDI_SUCCESS);
593 }
594
595
596 /*
597 * rootnex_add_props()
598 *
599 */
600 static void
601 rootnex_add_props(dev_info_t *dip)
602 {
603 rootnex_intprop_t *rpp;
604 int i;
605
606 /* Add static integer/boolean properties to the root node */
607 rpp = rootnex_intprp;
608 for (i = 0; i < NROOT_INTPROPS; i++) {
609 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
610 rpp[i].prop_name, rpp[i].prop_value);
611 }
612 }
613
614
615
616 /*
617 * *************************
618 * ctlops related routines
619 * *************************
620 */
621
622 /*
623 * rootnex_ctlops()
624 *
625 */
626 /*ARGSUSED*/
627 static int
628 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
629 void *arg, void *result)
630 {
631 int n, *ptr;
632 struct ddi_parent_private_data *pdp;
633
634 switch (ctlop) {
635 case DDI_CTLOPS_DMAPMAPC:
636 /*
637 * Return 'partial' to indicate that dma mapping
638 * has to be done in the main MMU.
639 */
640 return (DDI_DMA_PARTIAL);
641
642 case DDI_CTLOPS_BTOP:
643 /*
644 * Convert byte count input to physical page units.
645 * (byte counts that are not a page-size multiple
646 * are rounded down)
647 */
648 *(ulong_t *)result = btop(*(ulong_t *)arg);
649 return (DDI_SUCCESS);
650
651 case DDI_CTLOPS_PTOB:
652 /*
653 * Convert size in physical pages to bytes
654 */
655 *(ulong_t *)result = ptob(*(ulong_t *)arg);
656 return (DDI_SUCCESS);
657
658 case DDI_CTLOPS_BTOPR:
659 /*
660 * Convert byte count input to physical page units
661 * (byte counts that are not a page-size multiple
662 * are rounded up)
663 */
664 *(ulong_t *)result = btopr(*(ulong_t *)arg);
665 return (DDI_SUCCESS);
666
667 case DDI_CTLOPS_INITCHILD:
668 return (impl_ddi_sunbus_initchild(arg));
669
670 case DDI_CTLOPS_UNINITCHILD:
671 impl_ddi_sunbus_removechild(arg);
672 return (DDI_SUCCESS);
673
674 case DDI_CTLOPS_REPORTDEV:
675 return (rootnex_ctl_reportdev(rdip));
676
677 case DDI_CTLOPS_IOMIN:
678 /*
679 * Nothing to do here but reflect back..
680 */
681 return (DDI_SUCCESS);
682
683 case DDI_CTLOPS_REGSIZE:
684 case DDI_CTLOPS_NREGS:
685 break;
686
687 case DDI_CTLOPS_SIDDEV:
688 if (ndi_dev_is_prom_node(rdip))
689 return (DDI_SUCCESS);
690 if (ndi_dev_is_persistent_node(rdip))
691 return (DDI_SUCCESS);
692 return (DDI_FAILURE);
693
694 case DDI_CTLOPS_POWER:
695 return ((*pm_platform_power)((power_req_t *)arg));
696
697 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
698 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
699 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
700 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
701 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
702 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
703 if (!rootnex_state->r_reserved_msg_printed) {
704 rootnex_state->r_reserved_msg_printed = B_TRUE;
705 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
706 "1 or more reserved/obsolete operations.");
707 }
708 return (DDI_FAILURE);
709
710 default:
711 return (DDI_FAILURE);
712 }
713 /*
714 * The rest are for "hardware" properties
715 */
716 if ((pdp = ddi_get_parent_data(rdip)) == NULL)
717 return (DDI_FAILURE);
718
719 if (ctlop == DDI_CTLOPS_NREGS) {
720 ptr = (int *)result;
721 *ptr = pdp->par_nreg;
722 } else {
723 off_t *size = (off_t *)result;
724
725 ptr = (int *)arg;
726 n = *ptr;
727 if (n >= pdp->par_nreg) {
728 return (DDI_FAILURE);
729 }
730 *size = (off_t)pdp->par_reg[n].regspec_size;
731 }
732 return (DDI_SUCCESS);
733 }
734
735
736 /*
737 * rootnex_ctl_reportdev()
738 *
739 */
740 static int
741 rootnex_ctl_reportdev(dev_info_t *dev)
742 {
743 int i, n, len, f_len = 0;
744 char *buf;
745
746 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
747 f_len += snprintf(buf, REPORTDEV_BUFSIZE,
748 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
749 len = strlen(buf);
750
751 for (i = 0; i < sparc_pd_getnreg(dev); i++) {
752
753 struct regspec *rp = sparc_pd_getreg(dev, i);
754
755 if (i == 0)
756 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
757 ": ");
758 else
759 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
760 " and ");
761 len = strlen(buf);
762
763 switch (rp->regspec_bustype) {
764
765 case BTEISA:
766 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
767 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
768 break;
769
770 case BTISA:
771 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
772 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
773 break;
774
775 default:
776 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
777 "space %x offset %x",
778 rp->regspec_bustype, rp->regspec_addr);
779 break;
780 }
781 len = strlen(buf);
782 }
783 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
784 int pri;
785
786 if (i != 0) {
787 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
788 ",");
789 len = strlen(buf);
790 }
791 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
792 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
793 " sparc ipl %d", pri);
794 len = strlen(buf);
795 }
796 #ifdef DEBUG
797 if (f_len + 1 >= REPORTDEV_BUFSIZE) {
798 cmn_err(CE_NOTE, "next message is truncated: "
799 "printed length 1024, real length %d", f_len);
800 }
801 #endif /* DEBUG */
802 cmn_err(CE_CONT, "?%s\n", buf);
803 kmem_free(buf, REPORTDEV_BUFSIZE);
804 return (DDI_SUCCESS);
805 }
806
807
808 /*
809 * ******************
810 * map related code
811 * ******************
812 */
813
814 /*
815 * rootnex_map()
816 *
817 */
818 static int
819 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
820 off_t len, caddr_t *vaddrp)
821 {
822 struct regspec *rp, tmp_reg;
823 ddi_map_req_t mr = *mp; /* Get private copy of request */
824 int error;
825
826 mp = &mr;
827
828 switch (mp->map_op) {
829 case DDI_MO_MAP_LOCKED:
830 case DDI_MO_UNMAP:
831 case DDI_MO_MAP_HANDLE:
832 break;
833 default:
834 #ifdef DDI_MAP_DEBUG
835 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
836 mp->map_op);
837 #endif /* DDI_MAP_DEBUG */
838 return (DDI_ME_UNIMPLEMENTED);
839 }
840
841 if (mp->map_flags & DDI_MF_USER_MAPPING) {
842 #ifdef DDI_MAP_DEBUG
843 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
844 #endif /* DDI_MAP_DEBUG */
845 return (DDI_ME_UNIMPLEMENTED);
846 }
847
848 /*
849 * First, if given an rnumber, convert it to a regspec...
850 * (Presumably, this is on behalf of a child of the root node?)
851 */
852
853 if (mp->map_type == DDI_MT_RNUMBER) {
854
855 int rnumber = mp->map_obj.rnumber;
856 #ifdef DDI_MAP_DEBUG
857 static char *out_of_range =
858 "rootnex_map: Out of range rnumber <%d>, device <%s>";
859 #endif /* DDI_MAP_DEBUG */
860
861 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
862 if (rp == NULL) {
863 #ifdef DDI_MAP_DEBUG
864 cmn_err(CE_WARN, out_of_range, rnumber,
865 ddi_get_name(rdip));
866 #endif /* DDI_MAP_DEBUG */
867 return (DDI_ME_RNUMBER_RANGE);
868 }
869
870 /*
871 * Convert the given ddi_map_req_t from rnumber to regspec...
872 */
873
874 mp->map_type = DDI_MT_REGSPEC;
875 mp->map_obj.rp = rp;
876 }
877
878 /*
879 * Adjust offset and length correspnding to called values...
880 * XXX: A non-zero length means override the one in the regspec
881 * XXX: (regardless of what's in the parent's range?)
882 */
883
884 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */
885 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */
886
887 #ifdef DDI_MAP_DEBUG
888 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
889 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
890 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
891 len, mp->map_handlep);
892 #endif /* DDI_MAP_DEBUG */
893
894 /*
895 * I/O or memory mapping:
896 *
897 * <bustype=0, addr=x, len=x>: memory
898 * <bustype=1, addr=x, len=x>: i/o
899 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
900 */
901
902 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
903 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
904 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
905 ddi_get_name(rdip), rp->regspec_bustype,
906 rp->regspec_addr, rp->regspec_size);
907 return (DDI_ME_INVAL);
908 }
909
910 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
911 /*
912 * compatibility i/o mapping
913 */
914 rp->regspec_bustype += (uint_t)offset;
915 } else {
916 /*
917 * Normal memory or i/o mapping
918 */
919 rp->regspec_addr += (uint_t)offset;
920 }
921
922 if (len != 0)
923 rp->regspec_size = (uint_t)len;
924
925 #ifdef DDI_MAP_DEBUG
926 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
927 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
928 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
929 offset, len, mp->map_handlep);
930 #endif /* DDI_MAP_DEBUG */
931
932 /*
933 * Apply any parent ranges at this level, if applicable.
934 * (This is where nexus specific regspec translation takes place.
935 * Use of this function is implicit agreement that translation is
936 * provided via ddi_apply_range.)
937 */
938
939 #ifdef DDI_MAP_DEBUG
940 ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
941 ddi_get_name(dip), ddi_get_name(rdip));
942 #endif /* DDI_MAP_DEBUG */
943
944 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
945 return (error);
946
947 switch (mp->map_op) {
948 case DDI_MO_MAP_LOCKED:
949
950 /*
951 * Set up the locked down kernel mapping to the regspec...
952 */
953
954 return (rootnex_map_regspec(mp, vaddrp));
955
956 case DDI_MO_UNMAP:
957
958 /*
959 * Release mapping...
960 */
961
962 return (rootnex_unmap_regspec(mp, vaddrp));
963
964 case DDI_MO_MAP_HANDLE:
965
966 return (rootnex_map_handle(mp));
967
968 default:
969 return (DDI_ME_UNIMPLEMENTED);
970 }
971 }
972
973
974 /*
975 * rootnex_map_fault()
976 *
977 * fault in mappings for requestors
978 */
979 /*ARGSUSED*/
980 static int
981 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
982 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
983 uint_t lock)
984 {
985
986 #ifdef DDI_MAP_DEBUG
987 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
988 ddi_map_debug(" Seg <%s>\n",
989 seg->s_ops == &segdev_ops ? "segdev" :
990 seg == &kvseg ? "segkmem" : "NONE!");
991 #endif /* DDI_MAP_DEBUG */
992
993 /*
994 * This is all terribly broken, but it is a start
995 *
996 * XXX Note that this test means that segdev_ops
997 * must be exported from seg_dev.c.
998 * XXX What about devices with their own segment drivers?
999 */
1000 if (seg->s_ops == &segdev_ops) {
1001 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1002
1003 if (hat == NULL) {
1004 /*
1005 * This is one plausible interpretation of
1006 * a null hat i.e. use the first hat on the
1007 * address space hat list which by convention is
1008 * the hat of the system MMU. At alternative
1009 * would be to panic .. this might well be better ..
1010 */
1011 ASSERT(AS_READ_HELD(seg->s_as));
1012 hat = seg->s_as->a_hat;
1013 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1014 }
1015 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1016 (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1017 } else if (seg == &kvseg && dp == NULL) {
1018 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1019 HAT_LOAD_LOCK);
1020 } else
1021 return (DDI_FAILURE);
1022 return (DDI_SUCCESS);
1023 }
1024
1025
1026 /*
1027 * rootnex_map_regspec()
1028 * we don't support mapping of I/O cards above 4Gb
1029 */
1030 static int
1031 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1032 {
1033 rootnex_addr_t rbase;
1034 void *cvaddr;
1035 uint_t npages, pgoffset;
1036 struct regspec *rp;
1037 ddi_acc_hdl_t *hp;
1038 ddi_acc_impl_t *ap;
1039 uint_t hat_acc_flags;
1040 paddr_t pbase;
1041
1042 rp = mp->map_obj.rp;
1043 hp = mp->map_handlep;
1044
1045 #ifdef DDI_MAP_DEBUG
1046 ddi_map_debug(
1047 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1048 rp->regspec_bustype, rp->regspec_addr,
1049 rp->regspec_size, mp->map_handlep);
1050 #endif /* DDI_MAP_DEBUG */
1051
1052 /*
1053 * I/O or memory mapping
1054 *
1055 * <bustype=0, addr=x, len=x>: memory
1056 * <bustype=1, addr=x, len=x>: i/o
1057 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1058 */
1059
1060 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1061 cmn_err(CE_WARN, "rootnex: invalid register spec"
1062 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1063 rp->regspec_addr, rp->regspec_size);
1064 return (DDI_FAILURE);
1065 }
1066
1067 if (rp->regspec_bustype != 0) {
1068 /*
1069 * I/O space - needs a handle.
1070 */
1071 if (hp == NULL) {
1072 return (DDI_FAILURE);
1073 }
1074 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1075 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1076 impl_acc_hdl_init(hp);
1077
1078 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1079 #ifdef DDI_MAP_DEBUG
1080 ddi_map_debug("rootnex_map_regspec: mmap() "
1081 "to I/O space is not supported.\n");
1082 #endif /* DDI_MAP_DEBUG */
1083 return (DDI_ME_INVAL);
1084 } else {
1085 /*
1086 * 1275-compliant vs. compatibility i/o mapping
1087 */
1088 *vaddrp =
1089 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1090 ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1091 ((caddr_t)(uintptr_t)rp->regspec_addr);
1092 #ifdef __xpv
1093 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1094 hp->ah_pfn = xen_assign_pfn(
1095 mmu_btop((ulong_t)rp->regspec_addr &
1096 MMU_PAGEMASK));
1097 } else {
1098 hp->ah_pfn = mmu_btop(
1099 (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1100 }
1101 #else
1102 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1103 MMU_PAGEMASK);
1104 #endif
1105 hp->ah_pnum = mmu_btopr(rp->regspec_size +
1106 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1107 }
1108
1109 #ifdef DDI_MAP_DEBUG
1110 ddi_map_debug(
1111 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1112 rp->regspec_size, *vaddrp);
1113 #endif /* DDI_MAP_DEBUG */
1114 return (DDI_SUCCESS);
1115 }
1116
1117 /*
1118 * Memory space
1119 */
1120
1121 if (hp != NULL) {
1122 /*
1123 * hat layer ignores
1124 * hp->ah_acc.devacc_attr_endian_flags.
1125 */
1126 switch (hp->ah_acc.devacc_attr_dataorder) {
1127 case DDI_STRICTORDER_ACC:
1128 hat_acc_flags = HAT_STRICTORDER;
1129 break;
1130 case DDI_UNORDERED_OK_ACC:
1131 hat_acc_flags = HAT_UNORDERED_OK;
1132 break;
1133 case DDI_MERGING_OK_ACC:
1134 hat_acc_flags = HAT_MERGING_OK;
1135 break;
1136 case DDI_LOADCACHING_OK_ACC:
1137 hat_acc_flags = HAT_LOADCACHING_OK;
1138 break;
1139 case DDI_STORECACHING_OK_ACC:
1140 hat_acc_flags = HAT_STORECACHING_OK;
1141 break;
1142 }
1143 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1144 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1145 impl_acc_hdl_init(hp);
1146 hp->ah_hat_flags = hat_acc_flags;
1147 } else {
1148 hat_acc_flags = HAT_STRICTORDER;
1149 }
1150
1151 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1152 #ifdef __xpv
1153 /*
1154 * If we're dom0, we're using a real device so we need to translate
1155 * the MA to a PA.
1156 */
1157 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1158 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1159 } else {
1160 pbase = rbase;
1161 }
1162 #else
1163 pbase = rbase;
1164 #endif
1165 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1166
1167 if (rp->regspec_size == 0) {
1168 #ifdef DDI_MAP_DEBUG
1169 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1170 #endif /* DDI_MAP_DEBUG */
1171 return (DDI_ME_INVAL);
1172 }
1173
1174 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1175 /* extra cast to make gcc happy */
1176 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1177 } else {
1178 npages = mmu_btopr(rp->regspec_size + pgoffset);
1179
1180 #ifdef DDI_MAP_DEBUG
1181 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1182 "physical %llx", npages, pbase);
1183 #endif /* DDI_MAP_DEBUG */
1184
1185 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1186 if (cvaddr == NULL)
1187 return (DDI_ME_NORESOURCES);
1188
1189 /*
1190 * Now map in the pages we've allocated...
1191 */
1192 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1193 mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1194 HAT_LOAD_LOCK);
1195 *vaddrp = (caddr_t)cvaddr + pgoffset;
1196
1197 /* save away pfn and npages for FMA */
1198 hp = mp->map_handlep;
1199 if (hp) {
1200 hp->ah_pfn = mmu_btop(pbase);
1201 hp->ah_pnum = npages;
1202 }
1203 }
1204
1205 #ifdef DDI_MAP_DEBUG
1206 ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1207 #endif /* DDI_MAP_DEBUG */
1208 return (DDI_SUCCESS);
1209 }
1210
1211
1212 /*
1213 * rootnex_unmap_regspec()
1214 *
1215 */
1216 static int
1217 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1218 {
1219 caddr_t addr = (caddr_t)*vaddrp;
1220 uint_t npages, pgoffset;
1221 struct regspec *rp;
1222
1223 if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1224 return (0);
1225
1226 rp = mp->map_obj.rp;
1227
1228 if (rp->regspec_size == 0) {
1229 #ifdef DDI_MAP_DEBUG
1230 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1231 #endif /* DDI_MAP_DEBUG */
1232 return (DDI_ME_INVAL);
1233 }
1234
1235 /*
1236 * I/O or memory mapping:
1237 *
1238 * <bustype=0, addr=x, len=x>: memory
1239 * <bustype=1, addr=x, len=x>: i/o
1240 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1241 */
1242 if (rp->regspec_bustype != 0) {
1243 /*
1244 * This is I/O space, which requires no particular
1245 * processing on unmap since it isn't mapped in the
1246 * first place.
1247 */
1248 return (DDI_SUCCESS);
1249 }
1250
1251 /*
1252 * Memory space
1253 */
1254 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1255 npages = mmu_btopr(rp->regspec_size + pgoffset);
1256 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1257 device_arena_free(addr - pgoffset, ptob(npages));
1258
1259 /*
1260 * Destroy the pointer - the mapping has logically gone
1261 */
1262 *vaddrp = NULL;
1263
1264 return (DDI_SUCCESS);
1265 }
1266
1267
1268 /*
1269 * rootnex_map_handle()
1270 *
1271 */
1272 static int
1273 rootnex_map_handle(ddi_map_req_t *mp)
1274 {
1275 rootnex_addr_t rbase;
1276 ddi_acc_hdl_t *hp;
1277 uint_t pgoffset;
1278 struct regspec *rp;
1279 paddr_t pbase;
1280
1281 rp = mp->map_obj.rp;
1282
1283 #ifdef DDI_MAP_DEBUG
1284 ddi_map_debug(
1285 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1286 rp->regspec_bustype, rp->regspec_addr,
1287 rp->regspec_size, mp->map_handlep);
1288 #endif /* DDI_MAP_DEBUG */
1289
1290 /*
1291 * I/O or memory mapping:
1292 *
1293 * <bustype=0, addr=x, len=x>: memory
1294 * <bustype=1, addr=x, len=x>: i/o
1295 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1296 */
1297 if (rp->regspec_bustype != 0) {
1298 /*
1299 * This refers to I/O space, and we don't support "mapping"
1300 * I/O space to a user.
1301 */
1302 return (DDI_FAILURE);
1303 }
1304
1305 /*
1306 * Set up the hat_flags for the mapping.
1307 */
1308 hp = mp->map_handlep;
1309
1310 switch (hp->ah_acc.devacc_attr_endian_flags) {
1311 case DDI_NEVERSWAP_ACC:
1312 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1313 break;
1314 case DDI_STRUCTURE_LE_ACC:
1315 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1316 break;
1317 case DDI_STRUCTURE_BE_ACC:
1318 return (DDI_FAILURE);
1319 default:
1320 return (DDI_REGS_ACC_CONFLICT);
1321 }
1322
1323 switch (hp->ah_acc.devacc_attr_dataorder) {
1324 case DDI_STRICTORDER_ACC:
1325 break;
1326 case DDI_UNORDERED_OK_ACC:
1327 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1328 break;
1329 case DDI_MERGING_OK_ACC:
1330 hp->ah_hat_flags |= HAT_MERGING_OK;
1331 break;
1332 case DDI_LOADCACHING_OK_ACC:
1333 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1334 break;
1335 case DDI_STORECACHING_OK_ACC:
1336 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1337 break;
1338 default:
1339 return (DDI_FAILURE);
1340 }
1341
1342 rbase = (rootnex_addr_t)rp->regspec_addr &
1343 (~(rootnex_addr_t)MMU_PAGEOFFSET);
1344 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1345
1346 if (rp->regspec_size == 0)
1347 return (DDI_ME_INVAL);
1348
1349 #ifdef __xpv
1350 /*
1351 * If we're dom0, we're using a real device so we need to translate
1352 * the MA to a PA.
1353 */
1354 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1355 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1356 (rbase & MMU_PAGEOFFSET);
1357 } else {
1358 pbase = rbase;
1359 }
1360 #else
1361 pbase = rbase;
1362 #endif
1363
1364 hp->ah_pfn = mmu_btop(pbase);
1365 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1366
1367 return (DDI_SUCCESS);
1368 }
1369
1370
1371
1372 /*
1373 * ************************
1374 * interrupt related code
1375 * ************************
1376 */
1377
1378 /*
1379 * rootnex_intr_ops()
1380 * bus_intr_op() function for interrupt support
1381 */
1382 /* ARGSUSED */
1383 static int
1384 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1385 ddi_intr_handle_impl_t *hdlp, void *result)
1386 {
1387 struct intrspec *ispec;
1388
1389 DDI_INTR_NEXDBG((CE_CONT,
1390 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1391 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1392
1393 /* Process the interrupt operation */
1394 switch (intr_op) {
1395 case DDI_INTROP_GETCAP:
1396 /* First check with pcplusmp */
1397 if (psm_intr_ops == NULL)
1398 return (DDI_FAILURE);
1399
1400 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1401 *(int *)result = 0;
1402 return (DDI_FAILURE);
1403 }
1404 break;
1405 case DDI_INTROP_SETCAP:
1406 if (psm_intr_ops == NULL)
1407 return (DDI_FAILURE);
1408
1409 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1410 return (DDI_FAILURE);
1411 break;
1412 case DDI_INTROP_ALLOC:
1413 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1414 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1415 case DDI_INTROP_FREE:
1416 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1417 return (rootnex_free_intr_fixed(rdip, hdlp));
1418 case DDI_INTROP_GETPRI:
1419 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1420 return (DDI_FAILURE);
1421 *(int *)result = ispec->intrspec_pri;
1422 break;
1423 case DDI_INTROP_SETPRI:
1424 /* Validate the interrupt priority passed to us */
1425 if (*(int *)result > LOCK_LEVEL)
1426 return (DDI_FAILURE);
1427
1428 /* Ensure that PSM is all initialized and ispec is ok */
1429 if ((psm_intr_ops == NULL) ||
1430 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1431 return (DDI_FAILURE);
1432
1433 /* Change the priority */
1434 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1435 PSM_FAILURE)
1436 return (DDI_FAILURE);
1437
1438 /* update the ispec with the new priority */
1439 ispec->intrspec_pri = *(int *)result;
1440 break;
1441 case DDI_INTROP_ADDISR:
1442 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1443 return (DDI_FAILURE);
1444 ispec->intrspec_func = hdlp->ih_cb_func;
1445 break;
1446 case DDI_INTROP_REMISR:
1447 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1448 return (DDI_FAILURE);
1449 ispec->intrspec_func = (uint_t (*)()) 0;
1450 break;
1451 case DDI_INTROP_ENABLE:
1452 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1453 return (DDI_FAILURE);
1454
1455 /* Call psmi to translate irq with the dip */
1456 if (psm_intr_ops == NULL)
1457 return (DDI_FAILURE);
1458
1459 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1460 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1461 (int *)&hdlp->ih_vector) == PSM_FAILURE)
1462 return (DDI_FAILURE);
1463
1464 /* Add the interrupt handler */
1465 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1466 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1467 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1468 return (DDI_FAILURE);
1469 break;
1470 case DDI_INTROP_DISABLE:
1471 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1472 return (DDI_FAILURE);
1473
1474 /* Call psm_ops() to translate irq with the dip */
1475 if (psm_intr_ops == NULL)
1476 return (DDI_FAILURE);
1477
1478 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1479 (void) (*psm_intr_ops)(rdip, hdlp,
1480 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1481
1482 /* Remove the interrupt handler */
1483 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1484 hdlp->ih_cb_func, hdlp->ih_vector);
1485 break;
1486 case DDI_INTROP_SETMASK:
1487 if (psm_intr_ops == NULL)
1488 return (DDI_FAILURE);
1489
1490 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1491 return (DDI_FAILURE);
1492 break;
1493 case DDI_INTROP_CLRMASK:
1494 if (psm_intr_ops == NULL)
1495 return (DDI_FAILURE);
1496
1497 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1498 return (DDI_FAILURE);
1499 break;
1500 case DDI_INTROP_GETPENDING:
1501 if (psm_intr_ops == NULL)
1502 return (DDI_FAILURE);
1503
1504 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1505 result)) {
1506 *(int *)result = 0;
1507 return (DDI_FAILURE);
1508 }
1509 break;
1510 case DDI_INTROP_NAVAIL:
1511 case DDI_INTROP_NINTRS:
1512 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1513 if (*(int *)result == 0) {
1514 /*
1515 * Special case for 'pcic' driver' only. This driver
1516 * driver is a child of 'isa' and 'rootnex' drivers.
1517 *
1518 * See detailed comments on this in the function
1519 * rootnex_get_ispec().
1520 *
1521 * Children of 'pcic' send 'NINITR' request all the
1522 * way to rootnex driver. But, the 'pdp->par_nintr'
1523 * field may not initialized. So, we fake it here
1524 * to return 1 (a la what PCMCIA nexus does).
1525 */
1526 if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1527 *(int *)result = 1;
1528 else
1529 return (DDI_FAILURE);
1530 }
1531 break;
1532 case DDI_INTROP_SUPPORTED_TYPES:
1533 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
1534 break;
1535 default:
1536 return (DDI_FAILURE);
1537 }
1538
1539 return (DDI_SUCCESS);
1540 }
1541
1542
1543 /*
1544 * rootnex_get_ispec()
1545 * convert an interrupt number to an interrupt specification.
1546 * The interrupt number determines which interrupt spec will be
1547 * returned if more than one exists.
1548 *
1549 * Look into the parent private data area of the 'rdip' to find out
1550 * the interrupt specification. First check to make sure there is
1551 * one that matchs "inumber" and then return a pointer to it.
1552 *
1553 * Return NULL if one could not be found.
1554 *
1555 * NOTE: This is needed for rootnex_intr_ops()
1556 */
1557 static struct intrspec *
1558 rootnex_get_ispec(dev_info_t *rdip, int inum)
1559 {
1560 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1561
1562 /*
1563 * Special case handling for drivers that provide their own
1564 * intrspec structures instead of relying on the DDI framework.
1565 *
1566 * A broken hardware driver in ON could potentially provide its
1567 * own intrspec structure, instead of relying on the hardware.
1568 * If these drivers are children of 'rootnex' then we need to
1569 * continue to provide backward compatibility to them here.
1570 *
1571 * Following check is a special case for 'pcic' driver which
1572 * was found to have broken hardwre andby provides its own intrspec.
1573 *
1574 * Verbatim comments from this driver are shown here:
1575 * "Don't use the ddi_add_intr since we don't have a
1576 * default intrspec in all cases."
1577 *
1578 * Since an 'ispec' may not be always created for it,
1579 * check for that and create one if so.
1580 *
1581 * NOTE: Currently 'pcic' is the only driver found to do this.
1582 */
1583 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1584 pdp->par_nintr = 1;
1585 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1586 pdp->par_nintr, KM_SLEEP);
1587 }
1588
1589 /* Validate the interrupt number */
1590 if (inum >= pdp->par_nintr)
1591 return (NULL);
1592
1593 /* Get the interrupt structure pointer and return that */
1594 return ((struct intrspec *)&pdp->par_intr[inum]);
1595 }
1596
1597 /*
1598 * Allocate interrupt vector for FIXED (legacy) type.
1599 */
1600 static int
1601 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1602 void *result)
1603 {
1604 struct intrspec *ispec;
1605 ddi_intr_handle_impl_t info_hdl;
1606 int ret;
1607 int free_phdl = 0;
1608 apic_get_type_t type_info;
1609
1610 if (psm_intr_ops == NULL)
1611 return (DDI_FAILURE);
1612
1613 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1614 return (DDI_FAILURE);
1615
1616 /*
1617 * If the PSM module is "APIX" then pass the request for it
1618 * to allocate the vector now.
1619 */
1620 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1621 info_hdl.ih_private = &type_info;
1622 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1623 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1624 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1625 free_phdl = 1;
1626 i_ddi_alloc_intr_phdl(hdlp);
1627 }
1628 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1629 ret = (*psm_intr_ops)(rdip, hdlp,
1630 PSM_INTR_OP_ALLOC_VECTORS, result);
1631 if (free_phdl) { /* free up the phdl structure */
1632 free_phdl = 0;
1633 i_ddi_free_intr_phdl(hdlp);
1634 hdlp->ih_private = NULL;
1635 }
1636 } else {
1637 /*
1638 * No APIX module; fall back to the old scheme where the
1639 * interrupt vector is allocated during ddi_enable_intr() call.
1640 */
1641 hdlp->ih_pri = ispec->intrspec_pri;
1642 *(int *)result = hdlp->ih_scratch1;
1643 ret = DDI_SUCCESS;
1644 }
1645
1646 return (ret);
1647 }
1648
1649 /*
1650 * Free up interrupt vector for FIXED (legacy) type.
1651 */
1652 static int
1653 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1654 {
1655 struct intrspec *ispec;
1656 struct ddi_parent_private_data *pdp;
1657 ddi_intr_handle_impl_t info_hdl;
1658 int ret;
1659 apic_get_type_t type_info;
1660
1661 if (psm_intr_ops == NULL)
1662 return (DDI_FAILURE);
1663
1664 /*
1665 * If the PSM module is "APIX" then pass the request for it
1666 * to free up the vector now.
1667 */
1668 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1669 info_hdl.ih_private = &type_info;
1670 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1671 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1672 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1673 return (DDI_FAILURE);
1674 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1675 ret = (*psm_intr_ops)(rdip, hdlp,
1676 PSM_INTR_OP_FREE_VECTORS, NULL);
1677 } else {
1678 /*
1679 * No APIX module; fall back to the old scheme where
1680 * the interrupt vector was already freed during
1681 * ddi_disable_intr() call.
1682 */
1683 ret = DDI_SUCCESS;
1684 }
1685
1686 pdp = ddi_get_parent_data(rdip);
1687
1688 /*
1689 * Special case for 'pcic' driver' only.
1690 * If an intrspec was created for it, clean it up here
1691 * See detailed comments on this in the function
1692 * rootnex_get_ispec().
1693 */
1694 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1695 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1696 pdp->par_nintr);
1697 /*
1698 * Set it to zero; so that
1699 * DDI framework doesn't free it again
1700 */
1701 pdp->par_intr = NULL;
1702 pdp->par_nintr = 0;
1703 }
1704
1705 return (ret);
1706 }
1707
1708
1709 /*
1710 * ******************
1711 * dma related code
1712 * ******************
1713 */
1714
1715 /*ARGSUSED*/
1716 static int
1717 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1718 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1719 ddi_dma_handle_t *handlep)
1720 {
1721 uint64_t maxsegmentsize_ll;
1722 uint_t maxsegmentsize;
1723 ddi_dma_impl_t *hp;
1724 rootnex_dma_t *dma;
1725 uint64_t count_max;
1726 uint64_t seg;
1727 int kmflag;
1728 int e;
1729
1730
1731 /* convert our sleep flags */
1732 if (waitfp == DDI_DMA_SLEEP) {
1733 kmflag = KM_SLEEP;
1734 } else {
1735 kmflag = KM_NOSLEEP;
1736 }
1737
1738 /*
1739 * We try to do only one memory allocation here. We'll do a little
1740 * pointer manipulation later. If the bind ends up taking more than
1741 * our prealloc's space, we'll have to allocate more memory in the
1742 * bind operation. Not great, but much better than before and the
1743 * best we can do with the current bind interfaces.
1744 */
1745 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1746 if (hp == NULL)
1747 return (DDI_DMA_NORESOURCES);
1748
1749 /* Do our pointer manipulation now, align the structures */
1750 hp->dmai_private = (void *)(((uintptr_t)hp +
1751 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1752 dma = (rootnex_dma_t *)hp->dmai_private;
1753 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1754 sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1755
1756 /* setup the handle */
1757 rootnex_clean_dmahdl(hp);
1758 hp->dmai_error.err_fep = NULL;
1759 hp->dmai_error.err_cf = NULL;
1760 dma->dp_dip = rdip;
1761 dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1762 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1763
1764 /*
1765 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1766 * is being used. Set the upper limit to the seg value.
1767 * There will be enough DVMA space to always get addresses
1768 * that will match the constraints.
1769 */
1770 if (IOMMU_USED(rdip) &&
1771 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1772 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1773 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1774 } else
1775 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1776
1777 hp->dmai_minxfer = attr->dma_attr_minxfer;
1778 hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1779 hp->dmai_rdip = rdip;
1780 hp->dmai_attr = *attr;
1781
1782 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1783 dma->dp_sglinfo.si_cancross = B_FALSE;
1784 else
1785 dma->dp_sglinfo.si_cancross = B_TRUE;
1786
1787 /* we don't need to worry about the SPL since we do a tryenter */
1788 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1789
1790 /*
1791 * Figure out our maximum segment size. If the segment size is greater
1792 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1793 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1794 * dma_attr_count_max are size-1 type values.
1795 *
1796 * Maximum segment size is the largest physically contiguous chunk of
1797 * memory that we can return from a bind (i.e. the maximum size of a
1798 * single cookie).
1799 */
1800
1801 /* handle the rollover cases */
1802 seg = attr->dma_attr_seg + 1;
1803 if (seg < attr->dma_attr_seg) {
1804 seg = attr->dma_attr_seg;
1805 }
1806 count_max = attr->dma_attr_count_max + 1;
1807 if (count_max < attr->dma_attr_count_max) {
1808 count_max = attr->dma_attr_count_max;
1809 }
1810
1811 /*
1812 * granularity may or may not be a power of two. If it isn't, we can't
1813 * use a simple mask.
1814 */
1815 if (!ISP2(attr->dma_attr_granular)) {
1816 dma->dp_granularity_power_2 = B_FALSE;
1817 } else {
1818 dma->dp_granularity_power_2 = B_TRUE;
1819 }
1820
1821 /*
1822 * maxxfer should be a whole multiple of granularity. If we're going to
1823 * break up a window because we're greater than maxxfer, we might as
1824 * well make sure it's maxxfer is a whole multiple so we don't have to
1825 * worry about triming the window later on for this case.
1826 */
1827 if (attr->dma_attr_granular > 1) {
1828 if (dma->dp_granularity_power_2) {
1829 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1830 (attr->dma_attr_maxxfer &
1831 (attr->dma_attr_granular - 1));
1832 } else {
1833 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1834 (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1835 }
1836 } else {
1837 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1838 }
1839
1840 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1841 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1842 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1843 maxsegmentsize = 0xFFFFFFFF;
1844 } else {
1845 maxsegmentsize = maxsegmentsize_ll;
1846 }
1847 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1848 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1849
1850 /* check the ddi_dma_attr arg to make sure it makes a little sense */
1851 if (rootnex_alloc_check_parms) {
1852 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1853 if (e != DDI_SUCCESS) {
1854 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1855 (void) rootnex_dma_freehdl(dip, rdip,
1856 (ddi_dma_handle_t)hp);
1857 return (e);
1858 }
1859 }
1860
1861 *handlep = (ddi_dma_handle_t)hp;
1862
1863 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1864 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1865 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1866
1867 return (DDI_SUCCESS);
1868 }
1869
1870
1871 /*
1872 * rootnex_dma_allochdl()
1873 * called from ddi_dma_alloc_handle().
1874 */
1875 static int
1876 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1877 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1878 {
1879 int retval = DDI_SUCCESS;
1880 #if defined(__amd64) && !defined(__xpv)
1881
1882 if (IOMMU_UNITIALIZED(rdip)) {
1883 retval = iommulib_nex_open(dip, rdip);
1884
1885 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1886 return (retval);
1887 }
1888
1889 if (IOMMU_UNUSED(rdip)) {
1890 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1891 handlep);
1892 } else {
1893 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1894 waitfp, arg, handlep);
1895 }
1896 #else
1897 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1898 handlep);
1899 #endif
1900 switch (retval) {
1901 case DDI_DMA_NORESOURCES:
1902 if (waitfp != DDI_DMA_DONTWAIT) {
1903 ddi_set_callback(waitfp, arg,
1904 &rootnex_state->r_dvma_call_list_id);
1905 }
1906 break;
1907 case DDI_SUCCESS:
1908 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1909 break;
1910 default:
1911 break;
1912 }
1913 return (retval);
1914 }
1915
1916 /*ARGSUSED*/
1917 static int
1918 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1919 ddi_dma_handle_t handle)
1920 {
1921 ddi_dma_impl_t *hp;
1922 rootnex_dma_t *dma;
1923
1924
1925 hp = (ddi_dma_impl_t *)handle;
1926 dma = (rootnex_dma_t *)hp->dmai_private;
1927
1928 /* unbind should have been called first */
1929 ASSERT(!dma->dp_inuse);
1930
1931 mutex_destroy(&dma->dp_mutex);
1932 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1933
1934 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1935 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1936 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1937
1938 return (DDI_SUCCESS);
1939 }
1940
1941 /*
1942 * rootnex_dma_freehdl()
1943 * called from ddi_dma_free_handle().
1944 */
1945 static int
1946 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1947 {
1948 int ret;
1949
1950 ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1951 #if defined(__amd64) && !defined(__xpv)
1952 if (IOMMU_USED(rdip))
1953 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1954 else
1955 #endif
1956 ret = rootnex_coredma_freehdl(dip, rdip, handle);
1957
1958 if (rootnex_state->r_dvma_call_list_id)
1959 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1960
1961 return (ret);
1962 }
1963
1964 /*ARGSUSED*/
1965 static int
1966 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1967 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1968 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1969 {
1970 rootnex_sglinfo_t *sinfo;
1971 ddi_dma_obj_t *dmao;
1972 #if defined(__amd64) && !defined(__xpv)
1973 struct dvmaseg *dvs;
1974 ddi_dma_cookie_t *cookie;
1975 #endif
1976 ddi_dma_attr_t *attr;
1977 ddi_dma_impl_t *hp;
1978 rootnex_dma_t *dma;
1979 int kmflag;
1980 int e;
1981 uint_t ncookies;
1982
1983 hp = (ddi_dma_impl_t *)handle;
1984 dma = (rootnex_dma_t *)hp->dmai_private;
1985 dmao = &dma->dp_dma;
1986 sinfo = &dma->dp_sglinfo;
1987 attr = &hp->dmai_attr;
1988
1989 /* convert the sleep flags */
1990 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1991 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1992 } else {
1993 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1994 }
1995
1996 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1997
1998 /*
1999 * This is useful for debugging a driver. Not as useful in a production
2000 * system. The only time this will fail is if you have a driver bug.
2001 */
2002 if (rootnex_bind_check_inuse) {
2003 /*
2004 * No one else should ever have this lock unless someone else
2005 * is trying to use this handle. So contention on the lock
2006 * is the same as inuse being set.
2007 */
2008 e = mutex_tryenter(&dma->dp_mutex);
2009 if (e == 0) {
2010 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2011 return (DDI_DMA_INUSE);
2012 }
2013 if (dma->dp_inuse) {
2014 mutex_exit(&dma->dp_mutex);
2015 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2016 return (DDI_DMA_INUSE);
2017 }
2018 dma->dp_inuse = B_TRUE;
2019 mutex_exit(&dma->dp_mutex);
2020 }
2021
2022 /* check the ddi_dma_attr arg to make sure it makes a little sense */
2023 if (rootnex_bind_check_parms) {
2024 e = rootnex_valid_bind_parms(dmareq, attr);
2025 if (e != DDI_SUCCESS) {
2026 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2027 rootnex_clean_dmahdl(hp);
2028 return (e);
2029 }
2030 }
2031
2032 /* save away the original bind info */
2033 dma->dp_dma = dmareq->dmar_object;
2034
2035 #if defined(__amd64) && !defined(__xpv)
2036 if (IOMMU_USED(rdip)) {
2037 dmao = &dma->dp_dvma;
2038 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2039 switch (e) {
2040 case DDI_SUCCESS:
2041 if (sinfo->si_cancross ||
2042 dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2043 dmao->dmao_size > sinfo->si_max_cookie_size) {
2044 dma->dp_dvma_used = B_TRUE;
2045 break;
2046 }
2047 sinfo->si_sgl_size = 1;
2048 hp->dmai_rflags |= DMP_NOSYNC;
2049
2050 dma->dp_dvma_used = B_TRUE;
2051 dma->dp_need_to_free_cookie = B_FALSE;
2052
2053 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2054 cookie = hp->dmai_cookie = dma->dp_cookies =
2055 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2056 cookie->dmac_laddress = dvs->dvs_start +
2057 dmao->dmao_obj.dvma_obj.dv_off;
2058 cookie->dmac_size = dvs->dvs_len;
2059 cookie->dmac_type = 0;
2060
2061 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2062 rdip);
2063 goto fast;
2064 case DDI_ENOTSUP:
2065 break;
2066 default:
2067 rootnex_clean_dmahdl(hp);
2068 return (e);
2069 }
2070 }
2071 #endif
2072
2073 /*
2074 * Figure out a rough estimate of what maximum number of pages
2075 * this buffer could use (a high estimate of course).
2076 */
2077 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2078
2079 if (dma->dp_dvma_used) {
2080 /*
2081 * The number of physical pages is the worst case.
2082 *
2083 * For DVMA, the worst case is the length divided
2084 * by the maximum cookie length, plus 1. Add to that
2085 * the number of segment boundaries potentially crossed, and
2086 * the additional number of DVMA segments that was returned.
2087 *
2088 * In the normal case, for modern devices, si_cancross will
2089 * be false, and dv_nseg will be 1, and the fast path will
2090 * have been taken above.
2091 */
2092 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2093 + 1;
2094 if (sinfo->si_cancross)
2095 ncookies +=
2096 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2097 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2098
2099 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2100 }
2101
2102 /*
2103 * We'll use the pre-allocated cookies for any bind that will *always*
2104 * fit (more important to be consistent, we don't want to create
2105 * additional degenerate cases).
2106 */
2107 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2108 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2109 dma->dp_need_to_free_cookie = B_FALSE;
2110 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2111 uint_t, sinfo->si_max_pages);
2112
2113 /*
2114 * For anything larger than that, we'll go ahead and allocate the
2115 * maximum number of pages we expect to see. Hopefuly, we won't be
2116 * seeing this path in the fast path for high performance devices very
2117 * frequently.
2118 *
2119 * a ddi bind interface that allowed the driver to provide storage to
2120 * the bind interface would speed this case up.
2121 */
2122 } else {
2123 /*
2124 * Save away how much memory we allocated. If we're doing a
2125 * nosleep, the alloc could fail...
2126 */
2127 dma->dp_cookie_size = sinfo->si_max_pages *
2128 sizeof (ddi_dma_cookie_t);
2129 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2130 if (dma->dp_cookies == NULL) {
2131 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2132 rootnex_clean_dmahdl(hp);
2133 return (DDI_DMA_NORESOURCES);
2134 }
2135 dma->dp_need_to_free_cookie = B_TRUE;
2136 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2137 uint_t, sinfo->si_max_pages);
2138 }
2139 hp->dmai_cookie = dma->dp_cookies;
2140
2141 /*
2142 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2143 * looking at the constraints in the dma structure. It will then put
2144 * some additional state about the sgl in the dma struct (i.e. is
2145 * the sgl clean, or do we need to do some munging; how many pages
2146 * need to be copied, etc.)
2147 */
2148 if (dma->dp_dvma_used)
2149 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2150 else
2151 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2152
2153 out:
2154 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2155 /* if we don't need a copy buffer, we don't need to sync */
2156 if (sinfo->si_copybuf_req == 0) {
2157 hp->dmai_rflags |= DMP_NOSYNC;
2158 }
2159
2160 /*
2161 * if we don't need the copybuf and we don't need to do a partial, we
2162 * hit the fast path. All the high performance devices should be trying
2163 * to hit this path. To hit this path, a device should be able to reach
2164 * all of memory, shouldn't try to bind more than it can transfer, and
2165 * the buffer shouldn't require more cookies than the driver/device can
2166 * handle [sgllen]).
2167 *
2168 * Note that negative values of dma_attr_sgllen are supposed
2169 * to mean unlimited, but we just cast them to mean a
2170 * "ridiculous large limit". This saves some extra checks on
2171 * hot paths.
2172 */
2173 if ((sinfo->si_copybuf_req == 0) &&
2174 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2175 (dmao->dmao_size < dma->dp_maxxfer)) {
2176 fast:
2177 /*
2178 * If the driver supports FMA, insert the handle in the FMA DMA
2179 * handle cache.
2180 */
2181 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2182 hp->dmai_error.err_cf = rootnex_dma_check;
2183
2184 /*
2185 * copy out the first cookie and ccountp, set the cookie
2186 * pointer to the second cookie. The first cookie is passed
2187 * back on the stack. Additional cookies are accessed via
2188 * ddi_dma_nextcookie()
2189 */
2190 *cookiep = dma->dp_cookies[0];
2191 *ccountp = sinfo->si_sgl_size;
2192 hp->dmai_cookie++;
2193 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2194 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2195 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2196 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2197 uint_t, dmao->dmao_size, uint_t, *ccountp);
2198
2199
2200 return (DDI_DMA_MAPPED);
2201 }
2202
2203 /*
2204 * go to the slow path, we may need to alloc more memory, create
2205 * multiple windows, and munge up a sgl to make the device happy.
2206 */
2207
2208 /*
2209 * With the IOMMU mapobject method used, we should never hit
2210 * the slow path. If we do, something is seriously wrong.
2211 * Clean up and return an error.
2212 */
2213
2214 #if defined(__amd64) && !defined(__xpv)
2215
2216 if (dma->dp_dvma_used) {
2217 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2218 &dma->dp_dvma);
2219 e = DDI_DMA_NOMAPPING;
2220 } else {
2221 #endif
2222 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2223 kmflag);
2224 #if defined(__amd64) && !defined(__xpv)
2225 }
2226 #endif
2227 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2228 if (dma->dp_need_to_free_cookie) {
2229 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2230 }
2231 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2232 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2233 return (e);
2234 }
2235
2236 /*
2237 * If the driver supports FMA, insert the handle in the FMA DMA handle
2238 * cache.
2239 */
2240 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2241 hp->dmai_error.err_cf = rootnex_dma_check;
2242
2243 /* if the first window uses the copy buffer, sync it for the device */
2244 if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2245 (hp->dmai_rflags & DDI_DMA_WRITE)) {
2246 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2247 DDI_DMA_SYNC_FORDEV);
2248 }
2249
2250 /*
2251 * copy out the first cookie and ccountp, set the cookie pointer to the
2252 * second cookie. Make sure the partial flag is set/cleared correctly.
2253 * If we have a partial map (i.e. multiple windows), the number of
2254 * cookies we return is the number of cookies in the first window.
2255 */
2256 if (e == DDI_DMA_MAPPED) {
2257 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2258 *ccountp = sinfo->si_sgl_size;
2259 hp->dmai_nwin = 1;
2260 } else {
2261 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2262 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2263 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2264 }
2265 *cookiep = dma->dp_cookies[0];
2266 hp->dmai_cookie++;
2267
2268 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2269 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2270 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2271 dmao->dmao_size, uint_t, *ccountp);
2272 return (e);
2273 }
2274
2275 /*
2276 * rootnex_dma_bindhdl()
2277 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2278 */
2279 static int
2280 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2281 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2282 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2283 {
2284 int ret;
2285 #if defined(__amd64) && !defined(__xpv)
2286 if (IOMMU_USED(rdip))
2287 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2288 cookiep, ccountp);
2289 else
2290 #endif
2291 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2292 cookiep, ccountp);
2293
2294 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2295 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2296 &rootnex_state->r_dvma_call_list_id);
2297 }
2298
2299 return (ret);
2300 }
2301
2302
2303
2304 /*ARGSUSED*/
2305 static int
2306 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2307 ddi_dma_handle_t handle)
2308 {
2309 ddi_dma_impl_t *hp;
2310 rootnex_dma_t *dma;
2311 int e;
2312
2313 hp = (ddi_dma_impl_t *)handle;
2314 dma = (rootnex_dma_t *)hp->dmai_private;
2315
2316 /* make sure the buffer wasn't free'd before calling unbind */
2317 if (rootnex_unbind_verify_buffer) {
2318 e = rootnex_verify_buffer(dma);
2319 if (e != DDI_SUCCESS) {
2320 ASSERT(0);
2321 return (DDI_FAILURE);
2322 }
2323 }
2324
2325 /* sync the current window before unbinding the buffer */
2326 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2327 (hp->dmai_rflags & DDI_DMA_READ)) {
2328 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2329 DDI_DMA_SYNC_FORCPU);
2330 }
2331
2332 /*
2333 * cleanup and copy buffer or window state. if we didn't use the copy
2334 * buffer or windows, there won't be much to do :-)
2335 */
2336 rootnex_teardown_copybuf(dma);
2337 rootnex_teardown_windows(dma);
2338
2339 #if defined(__amd64) && !defined(__xpv)
2340 if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2341 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2342 &dma->dp_dvma);
2343 #endif
2344
2345 /*
2346 * If we had to allocate space to for the worse case sgl (it didn't
2347 * fit into our pre-allocate buffer), free that up now
2348 */
2349 if (dma->dp_need_to_free_cookie) {
2350 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2351 }
2352
2353 /*
2354 * clean up the handle so it's ready for the next bind (i.e. if the
2355 * handle is reused).
2356 */
2357 rootnex_clean_dmahdl(hp);
2358 hp->dmai_error.err_cf = NULL;
2359
2360 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2361 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2362 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2363
2364 return (DDI_SUCCESS);
2365 }
2366
2367 /*
2368 * rootnex_dma_unbindhdl()
2369 * called from ddi_dma_unbind_handle()
2370 */
2371 /*ARGSUSED*/
2372 static int
2373 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2374 ddi_dma_handle_t handle)
2375 {
2376 int ret;
2377
2378 #if defined(__amd64) && !defined(__xpv)
2379 if (IOMMU_USED(rdip))
2380 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2381 else
2382 #endif
2383 ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2384
2385 if (rootnex_state->r_dvma_call_list_id)
2386 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2387
2388 return (ret);
2389 }
2390
2391 #if defined(__amd64) && !defined(__xpv)
2392
2393 static int
2394 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2395 {
2396 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2397 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2398
2399 if (dma->dp_sleep_flags != KM_SLEEP &&
2400 dma->dp_sleep_flags != KM_NOSLEEP)
2401 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2402 return (dma->dp_sleep_flags);
2403 }
2404 /*ARGSUSED*/
2405 static void
2406 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2407 {
2408 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2409 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2410 rootnex_window_t *window;
2411
2412 if (dma->dp_window) {
2413 window = &dma->dp_window[dma->dp_current_win];
2414 hp->dmai_cookie = window->wd_first_cookie;
2415 } else {
2416 hp->dmai_cookie = dma->dp_cookies;
2417 }
2418 hp->dmai_cookie++;
2419 }
2420
2421 /*ARGSUSED*/
2422 static int
2423 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2424 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2425 {
2426 int i;
2427 int km_flags;
2428 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2429 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2430 rootnex_window_t *window;
2431 ddi_dma_cookie_t *cp;
2432 ddi_dma_cookie_t *cookie;
2433
2434 ASSERT(*cookiepp == NULL);
2435 ASSERT(*ccountp == 0);
2436
2437 if (dma->dp_window) {
2438 window = &dma->dp_window[dma->dp_current_win];
2439 cp = window->wd_first_cookie;
2440 *ccountp = window->wd_cookie_cnt;
2441 } else {
2442 cp = dma->dp_cookies;
2443 *ccountp = dma->dp_sglinfo.si_sgl_size;
2444 }
2445
2446 km_flags = rootnex_coredma_get_sleep_flags(handle);
2447 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2448 if (cookie == NULL) {
2449 return (DDI_DMA_NORESOURCES);
2450 }
2451
2452 for (i = 0; i < *ccountp; i++) {
2453 cookie[i].dmac_notused = cp[i].dmac_notused;
2454 cookie[i].dmac_type = cp[i].dmac_type;
2455 cookie[i].dmac_address = cp[i].dmac_address;
2456 cookie[i].dmac_size = cp[i].dmac_size;
2457 }
2458
2459 *cookiepp = cookie;
2460
2461 return (DDI_SUCCESS);
2462 }
2463
2464 /*ARGSUSED*/
2465 static int
2466 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2467 ddi_dma_cookie_t *cookiep, uint_t ccount)
2468 {
2469 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2470 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2471 rootnex_window_t *window;
2472 ddi_dma_cookie_t *cur_cookiep;
2473
2474 ASSERT(cookiep);
2475 ASSERT(ccount != 0);
2476 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2477
2478 if (dma->dp_window) {
2479 window = &dma->dp_window[dma->dp_current_win];
2480 dma->dp_saved_cookies = window->wd_first_cookie;
2481 window->wd_first_cookie = cookiep;
2482 ASSERT(ccount == window->wd_cookie_cnt);
2483 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2484 + window->wd_first_cookie;
2485 } else {
2486 dma->dp_saved_cookies = dma->dp_cookies;
2487 dma->dp_cookies = cookiep;
2488 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2489 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2490 + dma->dp_cookies;
2491 }
2492
2493 dma->dp_need_to_switch_cookies = B_TRUE;
2494 hp->dmai_cookie = cur_cookiep;
2495
2496 return (DDI_SUCCESS);
2497 }
2498
2499 /*ARGSUSED*/
2500 static int
2501 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2502 {
2503 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2504 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2505 rootnex_window_t *window;
2506 ddi_dma_cookie_t *cur_cookiep;
2507 ddi_dma_cookie_t *cookie_array;
2508 uint_t ccount;
2509
2510 /* check if cookies have not been switched */
2511 if (dma->dp_need_to_switch_cookies == B_FALSE)
2512 return (DDI_SUCCESS);
2513
2514 ASSERT(dma->dp_saved_cookies);
2515
2516 if (dma->dp_window) {
2517 window = &dma->dp_window[dma->dp_current_win];
2518 cookie_array = window->wd_first_cookie;
2519 window->wd_first_cookie = dma->dp_saved_cookies;
2520 dma->dp_saved_cookies = NULL;
2521 ccount = window->wd_cookie_cnt;
2522 cur_cookiep = (hp->dmai_cookie - cookie_array)
2523 + window->wd_first_cookie;
2524 } else {
2525 cookie_array = dma->dp_cookies;
2526 dma->dp_cookies = dma->dp_saved_cookies;
2527 dma->dp_saved_cookies = NULL;
2528 ccount = dma->dp_sglinfo.si_sgl_size;
2529 cur_cookiep = (hp->dmai_cookie - cookie_array)
2530 + dma->dp_cookies;
2531 }
2532
2533 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2534
2535 hp->dmai_cookie = cur_cookiep;
2536
2537 dma->dp_need_to_switch_cookies = B_FALSE;
2538
2539 return (DDI_SUCCESS);
2540 }
2541
2542 #endif
2543
2544 static struct as *
2545 rootnex_get_as(ddi_dma_obj_t *dmao)
2546 {
2547 struct as *asp;
2548
2549 switch (dmao->dmao_type) {
2550 case DMA_OTYP_VADDR:
2551 case DMA_OTYP_BUFVADDR:
2552 asp = dmao->dmao_obj.virt_obj.v_as;
2553 if (asp == NULL)
2554 asp = &kas;
2555 break;
2556 default:
2557 asp = NULL;
2558 break;
2559 }
2560 return (asp);
2561 }
2562
2563 /*
2564 * rootnex_verify_buffer()
2565 * verify buffer wasn't free'd
2566 */
2567 static int
2568 rootnex_verify_buffer(rootnex_dma_t *dma)
2569 {
2570 page_t **pplist;
2571 caddr_t vaddr;
2572 uint_t pcnt;
2573 uint_t poff;
2574 page_t *pp;
2575 char b;
2576 int i;
2577
2578 /* Figure out how many pages this buffer occupies */
2579 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2580 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2581 } else {
2582 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2583 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2584 }
2585 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2586
2587 switch (dma->dp_dma.dmao_type) {
2588 case DMA_OTYP_PAGES:
2589 /*
2590 * for a linked list of pp's walk through them to make sure
2591 * they're locked and not free.
2592 */
2593 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2594 for (i = 0; i < pcnt; i++) {
2595 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2596 return (DDI_FAILURE);
2597 }
2598 pp = pp->p_next;
2599 }
2600 break;
2601
2602 case DMA_OTYP_VADDR:
2603 case DMA_OTYP_BUFVADDR:
2604 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2605 /*
2606 * for an array of pp's walk through them to make sure they're
2607 * not free. It's possible that they may not be locked.
2608 */
2609 if (pplist) {
2610 for (i = 0; i < pcnt; i++) {
2611 if (PP_ISFREE(pplist[i])) {
2612 return (DDI_FAILURE);
2613 }
2614 }
2615
2616 /* For a virtual address, try to peek at each page */
2617 } else {
2618 if (rootnex_get_as(&dma->dp_dma) == &kas) {
2619 for (i = 0; i < pcnt; i++) {
2620 if (ddi_peek8(NULL, vaddr, &b) ==
2621 DDI_FAILURE)
2622 return (DDI_FAILURE);
2623 vaddr += MMU_PAGESIZE;
2624 }
2625 }
2626 }
2627 break;
2628
2629 default:
2630 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2631 break;
2632 }
2633
2634 return (DDI_SUCCESS);
2635 }
2636
2637
2638 /*
2639 * rootnex_clean_dmahdl()
2640 * Clean the dma handle. This should be called on a handle alloc and an
2641 * unbind handle. Set the handle state to the default settings.
2642 */
2643 static void
2644 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2645 {
2646 rootnex_dma_t *dma;
2647
2648
2649 dma = (rootnex_dma_t *)hp->dmai_private;
2650
2651 hp->dmai_nwin = 0;
2652 dma->dp_current_cookie = 0;
2653 dma->dp_copybuf_size = 0;
2654 dma->dp_window = NULL;
2655 dma->dp_cbaddr = NULL;
2656 dma->dp_inuse = B_FALSE;
2657 dma->dp_dvma_used = B_FALSE;
2658 dma->dp_need_to_free_cookie = B_FALSE;
2659 dma->dp_need_to_switch_cookies = B_FALSE;
2660 dma->dp_saved_cookies = NULL;
2661 dma->dp_sleep_flags = KM_PANIC;
2662 dma->dp_need_to_free_window = B_FALSE;
2663 dma->dp_partial_required = B_FALSE;
2664 dma->dp_trim_required = B_FALSE;
2665 dma->dp_sglinfo.si_copybuf_req = 0;
2666 #if !defined(__amd64)
2667 dma->dp_cb_remaping = B_FALSE;
2668 dma->dp_kva = NULL;
2669 #endif
2670
2671 /* FMA related initialization */
2672 hp->dmai_fault = 0;
2673 hp->dmai_fault_check = NULL;
2674 hp->dmai_fault_notify = NULL;
2675 hp->dmai_error.err_ena = 0;
2676 hp->dmai_error.err_status = DDI_FM_OK;
2677 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2678 hp->dmai_error.err_ontrap = NULL;
2679 }
2680
2681
2682 /*
2683 * rootnex_valid_alloc_parms()
2684 * Called in ddi_dma_alloc_handle path to validate its parameters.
2685 */
2686 static int
2687 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2688 {
2689 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2690 (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2691 (attr->dma_attr_granular > MMU_PAGESIZE) ||
2692 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2693 return (DDI_DMA_BADATTR);
2694 }
2695
2696 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2697 return (DDI_DMA_BADATTR);
2698 }
2699
2700 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2701 MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2702 attr->dma_attr_sgllen == 0) {
2703 return (DDI_DMA_BADATTR);
2704 }
2705
2706 /* We should be able to DMA into every byte offset in a page */
2707 if (maxsegmentsize < MMU_PAGESIZE) {
2708 return (DDI_DMA_BADATTR);
2709 }
2710
2711 /* if we're bouncing on seg, seg must be <= addr_hi */
2712 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2713 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2714 return (DDI_DMA_BADATTR);
2715 }
2716 return (DDI_SUCCESS);
2717 }
2718
2719 /*
2720 * rootnex_valid_bind_parms()
2721 * Called in ddi_dma_*_bind_handle path to validate its parameters.
2722 */
2723 /* ARGSUSED */
2724 static int
2725 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2726 {
2727 #if !defined(__amd64)
2728 /*
2729 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2730 * we can track the offset for the obsoleted interfaces.
2731 */
2732 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2733 return (DDI_DMA_TOOBIG);
2734 }
2735 #endif
2736
2737 return (DDI_SUCCESS);
2738 }
2739
2740
2741 /*
2742 * rootnex_need_bounce_seg()
2743 * check to see if the buffer lives on both side of the seg.
2744 */
2745 static boolean_t
2746 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2747 {
2748 ddi_dma_atyp_t buftype;
2749 rootnex_addr_t raddr;
2750 boolean_t lower_addr;
2751 boolean_t upper_addr;
2752 uint64_t offset;
2753 page_t **pplist;
2754 uint64_t paddr;
2755 uint32_t psize;
2756 uint32_t size;
2757 caddr_t vaddr;
2758 uint_t pcnt;
2759 page_t *pp;
2760
2761
2762 /* shortcuts */
2763 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2764 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2765 buftype = dmar_object->dmao_type;
2766 size = dmar_object->dmao_size;
2767
2768 lower_addr = B_FALSE;
2769 upper_addr = B_FALSE;
2770 pcnt = 0;
2771
2772 /*
2773 * Process the first page to handle the initial offset of the buffer.
2774 * We'll use the base address we get later when we loop through all
2775 * the pages.
2776 */
2777 if (buftype == DMA_OTYP_PAGES) {
2778 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2779 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2780 MMU_PAGEOFFSET;
2781 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2782 psize = MIN(size, (MMU_PAGESIZE - offset));
2783 pp = pp->p_next;
2784 sglinfo->si_asp = NULL;
2785 } else if (pplist != NULL) {
2786 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2787 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2788 if (sglinfo->si_asp == NULL) {
2789 sglinfo->si_asp = &kas;
2790 }
2791 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2792 paddr += offset;
2793 psize = MIN(size, (MMU_PAGESIZE - offset));
2794 pcnt++;
2795 } else {
2796 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2797 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2798 if (sglinfo->si_asp == NULL) {
2799 sglinfo->si_asp = &kas;
2800 }
2801 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2802 paddr += offset;
2803 psize = MIN(size, (MMU_PAGESIZE - offset));
2804 vaddr += psize;
2805 }
2806
2807 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2808
2809 if ((raddr + psize) > sglinfo->si_segmask) {
2810 upper_addr = B_TRUE;
2811 } else {
2812 lower_addr = B_TRUE;
2813 }
2814 size -= psize;
2815
2816 /*
2817 * Walk through the rest of the pages in the buffer. Track to see
2818 * if we have pages on both sides of the segment boundary.
2819 */
2820 while (size > 0) {
2821 /* partial or full page */
2822 psize = MIN(size, MMU_PAGESIZE);
2823
2824 if (buftype == DMA_OTYP_PAGES) {
2825 /* get the paddr from the page_t */
2826 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2827 paddr = pfn_to_pa(pp->p_pagenum);
2828 pp = pp->p_next;
2829 } else if (pplist != NULL) {
2830 /* index into the array of page_t's to get the paddr */
2831 ASSERT(!PP_ISFREE(pplist[pcnt]));
2832 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2833 pcnt++;
2834 } else {
2835 /* call into the VM to get the paddr */
2836 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2837 vaddr));
2838 vaddr += psize;
2839 }
2840
2841 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2842
2843 if ((raddr + psize) > sglinfo->si_segmask) {
2844 upper_addr = B_TRUE;
2845 } else {
2846 lower_addr = B_TRUE;
2847 }
2848 /*
2849 * if the buffer lives both above and below the segment
2850 * boundary, or the current page is the page immediately
2851 * after the segment, we will use a copy/bounce buffer for
2852 * all pages > seg.
2853 */
2854 if ((lower_addr && upper_addr) ||
2855 (raddr == (sglinfo->si_segmask + 1))) {
2856 return (B_TRUE);
2857 }
2858
2859 size -= psize;
2860 }
2861
2862 return (B_FALSE);
2863 }
2864
2865 /*
2866 * rootnex_get_sgl()
2867 * Called in bind fastpath to get the sgl. Most of this will be replaced
2868 * with a call to the vm layer when vm2.0 comes around...
2869 */
2870 static void
2871 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2872 rootnex_sglinfo_t *sglinfo)
2873 {
2874 ddi_dma_atyp_t buftype;
2875 rootnex_addr_t raddr;
2876 uint64_t last_page;
2877 uint64_t offset;
2878 uint64_t addrhi;
2879 uint64_t addrlo;
2880 uint64_t maxseg;
2881 page_t **pplist;
2882 uint64_t paddr;
2883 uint32_t psize;
2884 uint32_t size;
2885 caddr_t vaddr;
2886 uint_t pcnt;
2887 page_t *pp;
2888 uint_t cnt;
2889
2890
2891 /* shortcuts */
2892 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2893 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2894 maxseg = sglinfo->si_max_cookie_size;
2895 buftype = dmar_object->dmao_type;
2896 addrhi = sglinfo->si_max_addr;
2897 addrlo = sglinfo->si_min_addr;
2898 size = dmar_object->dmao_size;
2899
2900 pcnt = 0;
2901 cnt = 0;
2902
2903
2904 /*
2905 * check to see if we need to use the copy buffer for pages over
2906 * the segment attr.
2907 */
2908 sglinfo->si_bounce_on_seg = B_FALSE;
2909 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2910 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2911 dmar_object, sglinfo);
2912 }
2913
2914 /*
2915 * if we were passed down a linked list of pages, i.e. pointer to
2916 * page_t, use this to get our physical address and buf offset.
2917 */
2918 if (buftype == DMA_OTYP_PAGES) {
2919 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2920 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2921 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2922 MMU_PAGEOFFSET;
2923 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2924 psize = MIN(size, (MMU_PAGESIZE - offset));
2925 pp = pp->p_next;
2926 sglinfo->si_asp = NULL;
2927
2928 /*
2929 * We weren't passed down a linked list of pages, but if we were passed
2930 * down an array of pages, use this to get our physical address and buf
2931 * offset.
2932 */
2933 } else if (pplist != NULL) {
2934 ASSERT((buftype == DMA_OTYP_VADDR) ||
2935 (buftype == DMA_OTYP_BUFVADDR));
2936
2937 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2938 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2939 if (sglinfo->si_asp == NULL) {
2940 sglinfo->si_asp = &kas;
2941 }
2942
2943 ASSERT(!PP_ISFREE(pplist[pcnt]));
2944 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2945 paddr += offset;
2946 psize = MIN(size, (MMU_PAGESIZE - offset));
2947 pcnt++;
2948
2949 /*
2950 * All we have is a virtual address, we'll need to call into the VM
2951 * to get the physical address.
2952 */
2953 } else {
2954 ASSERT((buftype == DMA_OTYP_VADDR) ||
2955 (buftype == DMA_OTYP_BUFVADDR));
2956
2957 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2958 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2959 if (sglinfo->si_asp == NULL) {
2960 sglinfo->si_asp = &kas;
2961 }
2962
2963 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2964 paddr += offset;
2965 psize = MIN(size, (MMU_PAGESIZE - offset));
2966 vaddr += psize;
2967 }
2968
2969 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2970
2971 /*
2972 * Setup the first cookie with the physical address of the page and the
2973 * size of the page (which takes into account the initial offset into
2974 * the page.
2975 */
2976 sgl[cnt].dmac_laddress = raddr;
2977 sgl[cnt].dmac_size = psize;
2978 sgl[cnt].dmac_type = 0;
2979
2980 /*
2981 * Save away the buffer offset into the page. We'll need this later in
2982 * the copy buffer code to help figure out the page index within the
2983 * buffer and the offset into the current page.
2984 */
2985 sglinfo->si_buf_offset = offset;
2986
2987 /*
2988 * If we are using the copy buffer for anything over the segment
2989 * boundary, and this page is over the segment boundary.
2990 * OR
2991 * if the DMA engine can't reach the physical address.
2992 */
2993 if (((sglinfo->si_bounce_on_seg) &&
2994 ((raddr + psize) > sglinfo->si_segmask)) ||
2995 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2996 /*
2997 * Increase how much copy buffer we use. We always increase by
2998 * pagesize so we don't have to worry about converting offsets.
2999 * Set a flag in the cookies dmac_type to indicate that it uses
3000 * the copy buffer. If this isn't the last cookie, go to the
3001 * next cookie (since we separate each page which uses the copy
3002 * buffer in case the copy buffer is not physically contiguous.
3003 */
3004 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3005 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3006 if ((cnt + 1) < sglinfo->si_max_pages) {
3007 cnt++;
3008 sgl[cnt].dmac_laddress = 0;
3009 sgl[cnt].dmac_size = 0;
3010 sgl[cnt].dmac_type = 0;
3011 }
3012 }
3013
3014 /*
3015 * save this page's physical address so we can figure out if the next
3016 * page is physically contiguous. Keep decrementing size until we are
3017 * done with the buffer.
3018 */
3019 last_page = raddr & MMU_PAGEMASK;
3020 size -= psize;
3021
3022 while (size > 0) {
3023 /* Get the size for this page (i.e. partial or full page) */
3024 psize = MIN(size, MMU_PAGESIZE);
3025
3026 if (buftype == DMA_OTYP_PAGES) {
3027 /* get the paddr from the page_t */
3028 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3029 paddr = pfn_to_pa(pp->p_pagenum);
3030 pp = pp->p_next;
3031 } else if (pplist != NULL) {
3032 /* index into the array of page_t's to get the paddr */
3033 ASSERT(!PP_ISFREE(pplist[pcnt]));
3034 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3035 pcnt++;
3036 } else {
3037 /* call into the VM to get the paddr */
3038 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3039 vaddr));
3040 vaddr += psize;
3041 }
3042
3043 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3044
3045 /*
3046 * If we are using the copy buffer for anything over the
3047 * segment boundary, and this page is over the segment
3048 * boundary.
3049 * OR
3050 * if the DMA engine can't reach the physical address.
3051 */
3052 if (((sglinfo->si_bounce_on_seg) &&
3053 ((raddr + psize) > sglinfo->si_segmask)) ||
3054 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3055
3056 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3057
3058 /*
3059 * if there is something in the current cookie, go to
3060 * the next one. We only want one page in a cookie which
3061 * uses the copybuf since the copybuf doesn't have to
3062 * be physically contiguous.
3063 */
3064 if (sgl[cnt].dmac_size != 0) {
3065 cnt++;
3066 }
3067 sgl[cnt].dmac_laddress = raddr;
3068 sgl[cnt].dmac_size = psize;
3069 #if defined(__amd64)
3070 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3071 #else
3072 /*
3073 * save the buf offset for 32-bit kernel. used in the
3074 * obsoleted interfaces.
3075 */
3076 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3077 (dmar_object->dmao_size - size);
3078 #endif
3079 /* if this isn't the last cookie, go to the next one */
3080 if ((cnt + 1) < sglinfo->si_max_pages) {
3081 cnt++;
3082 sgl[cnt].dmac_laddress = 0;
3083 sgl[cnt].dmac_size = 0;
3084 sgl[cnt].dmac_type = 0;
3085 }
3086
3087 /*
3088 * this page didn't need the copy buffer, if it's not physically
3089 * contiguous, or it would put us over a segment boundary, or it
3090 * puts us over the max cookie size, or the current sgl doesn't
3091 * have anything in it.
3092 */
3093 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3094 !(raddr & sglinfo->si_segmask) ||
3095 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3096 (sgl[cnt].dmac_size == 0)) {
3097 /*
3098 * if we're not already in a new cookie, go to the next
3099 * cookie.
3100 */
3101 if (sgl[cnt].dmac_size != 0) {
3102 cnt++;
3103 }
3104
3105 /* save the cookie information */
3106 sgl[cnt].dmac_laddress = raddr;
3107 sgl[cnt].dmac_size = psize;
3108 #if defined(__amd64)
3109 sgl[cnt].dmac_type = 0;
3110 #else
3111 /*
3112 * save the buf offset for 32-bit kernel. used in the
3113 * obsoleted interfaces.
3114 */
3115 sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3116 #endif
3117
3118 /*
3119 * this page didn't need the copy buffer, it is physically
3120 * contiguous with the last page, and it's <= the max cookie
3121 * size.
3122 */
3123 } else {
3124 sgl[cnt].dmac_size += psize;
3125
3126 /*
3127 * if this exactly == the maximum cookie size, and
3128 * it isn't the last cookie, go to the next cookie.
3129 */
3130 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3131 ((cnt + 1) < sglinfo->si_max_pages)) {
3132 cnt++;
3133 sgl[cnt].dmac_laddress = 0;
3134 sgl[cnt].dmac_size = 0;
3135 sgl[cnt].dmac_type = 0;
3136 }
3137 }
3138
3139 /*
3140 * save this page's physical address so we can figure out if the
3141 * next page is physically contiguous. Keep decrementing size
3142 * until we are done with the buffer.
3143 */
3144 last_page = raddr;
3145 size -= psize;
3146 }
3147
3148 /* we're done, save away how many cookies the sgl has */
3149 if (sgl[cnt].dmac_size == 0) {
3150 ASSERT(cnt < sglinfo->si_max_pages);
3151 sglinfo->si_sgl_size = cnt;
3152 } else {
3153 sglinfo->si_sgl_size = cnt + 1;
3154 }
3155 }
3156
3157 static void
3158 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3159 rootnex_sglinfo_t *sglinfo)
3160 {
3161 uint64_t offset;
3162 uint64_t maxseg;
3163 uint64_t dvaddr;
3164 struct dvmaseg *dvs;
3165 uint64_t paddr;
3166 uint32_t psize, ssize;
3167 uint32_t size;
3168 uint_t cnt;
3169 int physcontig;
3170
3171 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3172
3173 /* shortcuts */
3174 maxseg = sglinfo->si_max_cookie_size;
3175 size = dmar_object->dmao_size;
3176
3177 cnt = 0;
3178 sglinfo->si_bounce_on_seg = B_FALSE;
3179
3180 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3181 offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3182 ssize = dvs->dvs_len;
3183 paddr = dvs->dvs_start;
3184 paddr += offset;
3185 psize = MIN(ssize, (maxseg - offset));
3186 dvaddr = paddr + psize;
3187 ssize -= psize;
3188
3189 sgl[cnt].dmac_laddress = paddr;
3190 sgl[cnt].dmac_size = psize;
3191 sgl[cnt].dmac_type = 0;
3192
3193 size -= psize;
3194 while (size > 0) {
3195 if (ssize == 0) {
3196 dvs++;
3197 ssize = dvs->dvs_len;
3198 dvaddr = dvs->dvs_start;
3199 physcontig = 0;
3200 } else
3201 physcontig = 1;
3202
3203 paddr = dvaddr;
3204 psize = MIN(ssize, maxseg);
3205 dvaddr += psize;
3206 ssize -= psize;
3207
3208 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3209 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3210 (sgl[cnt].dmac_size == 0)) {
3211 /*
3212 * if we're not already in a new cookie, go to the next
3213 * cookie.
3214 */
3215 if (sgl[cnt].dmac_size != 0) {
3216 cnt++;
3217 }
3218
3219 /* save the cookie information */
3220 sgl[cnt].dmac_laddress = paddr;
3221 sgl[cnt].dmac_size = psize;
3222 sgl[cnt].dmac_type = 0;
3223 } else {
3224 sgl[cnt].dmac_size += psize;
3225
3226 /*
3227 * if this exactly == the maximum cookie size, and
3228 * it isn't the last cookie, go to the next cookie.
3229 */
3230 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3231 ((cnt + 1) < sglinfo->si_max_pages)) {
3232 cnt++;
3233 sgl[cnt].dmac_laddress = 0;
3234 sgl[cnt].dmac_size = 0;
3235 sgl[cnt].dmac_type = 0;
3236 }
3237 }
3238 size -= psize;
3239 }
3240
3241 /* we're done, save away how many cookies the sgl has */
3242 if (sgl[cnt].dmac_size == 0) {
3243 sglinfo->si_sgl_size = cnt;
3244 } else {
3245 sglinfo->si_sgl_size = cnt + 1;
3246 }
3247 }
3248
3249 /*
3250 * rootnex_bind_slowpath()
3251 * Call in the bind path if the calling driver can't use the sgl without
3252 * modifying it. We either need to use the copy buffer and/or we will end up
3253 * with a partial bind.
3254 */
3255 static int
3256 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3257 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3258 {
3259 rootnex_sglinfo_t *sinfo;
3260 rootnex_window_t *window;
3261 ddi_dma_cookie_t *cookie;
3262 size_t copybuf_used;
3263 size_t dmac_size;
3264 boolean_t partial;
3265 off_t cur_offset;
3266 page_t *cur_pp;
3267 major_t mnum;
3268 int e;
3269 int i;
3270
3271
3272 sinfo = &dma->dp_sglinfo;
3273 copybuf_used = 0;
3274 partial = B_FALSE;
3275
3276 /*
3277 * If we're using the copybuf, set the copybuf state in dma struct.
3278 * Needs to be first since it sets the copy buffer size.
3279 */
3280 if (sinfo->si_copybuf_req != 0) {
3281 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3282 if (e != DDI_SUCCESS) {
3283 return (e);
3284 }
3285 } else {
3286 dma->dp_copybuf_size = 0;
3287 }
3288
3289 /*
3290 * Figure out if we need to do a partial mapping. If so, figure out
3291 * if we need to trim the buffers when we munge the sgl.
3292 */
3293 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3294 (dmao->dmao_size > dma->dp_maxxfer) ||
3295 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3296 dma->dp_partial_required = B_TRUE;
3297 if (attr->dma_attr_granular != 1) {
3298 dma->dp_trim_required = B_TRUE;
3299 }
3300 } else {
3301 dma->dp_partial_required = B_FALSE;
3302 dma->dp_trim_required = B_FALSE;
3303 }
3304
3305 /* If we need to do a partial bind, make sure the driver supports it */
3306 if (dma->dp_partial_required &&
3307 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3308
3309 mnum = ddi_driver_major(dma->dp_dip);
3310 /*
3311 * patchable which allows us to print one warning per major
3312 * number.
3313 */
3314 if ((rootnex_bind_warn) &&
3315 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3316 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3317 cmn_err(CE_WARN, "!%s: coding error detected, the "
3318 "driver is using ddi_dma_attr(9S) incorrectly. "
3319 "There is a small risk of data corruption in "
3320 "particular with large I/Os. The driver should be "
3321 "replaced with a corrected version for proper "
3322 "system operation. To disable this warning, add "
3323 "'set rootnex:rootnex_bind_warn=0' to "
3324 "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3325 }
3326 return (DDI_DMA_TOOBIG);
3327 }
3328
3329 /*
3330 * we might need multiple windows, setup state to handle them. In this
3331 * code path, we will have at least one window.
3332 */
3333 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3334 if (e != DDI_SUCCESS) {
3335 rootnex_teardown_copybuf(dma);
3336 return (e);
3337 }
3338
3339 window = &dma->dp_window[0];
3340 cookie = &dma->dp_cookies[0];
3341 cur_offset = 0;
3342 rootnex_init_win(hp, dma, window, cookie, cur_offset);
3343 if (dmao->dmao_type == DMA_OTYP_PAGES) {
3344 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3345 }
3346
3347 /* loop though all the cookies we got back from get_sgl() */
3348 for (i = 0; i < sinfo->si_sgl_size; i++) {
3349 /*
3350 * If we're using the copy buffer, check this cookie and setup
3351 * its associated copy buffer state. If this cookie uses the
3352 * copy buffer, make sure we sync this window during dma_sync.
3353 */
3354 if (dma->dp_copybuf_size > 0) {
3355 rootnex_setup_cookie(dmao, dma, cookie,
3356 cur_offset, ©buf_used, &cur_pp);
3357 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3358 window->wd_dosync = B_TRUE;
3359 }
3360 }
3361
3362 /*
3363 * save away the cookie size, since it could be modified in
3364 * the windowing code.
3365 */
3366 dmac_size = cookie->dmac_size;
3367
3368 /* if we went over max copybuf size */
3369 if (dma->dp_copybuf_size &&
3370 (copybuf_used > dma->dp_copybuf_size)) {
3371 partial = B_TRUE;
3372 e = rootnex_copybuf_window_boundary(hp, dma, &window,
3373 cookie, cur_offset, ©buf_used);
3374 if (e != DDI_SUCCESS) {
3375 rootnex_teardown_copybuf(dma);
3376 rootnex_teardown_windows(dma);
3377 return (e);
3378 }
3379
3380 /*
3381 * if the coookie uses the copy buffer, make sure the
3382 * new window we just moved to is set to sync.
3383 */
3384 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3385 window->wd_dosync = B_TRUE;
3386 }
3387 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3388 dma->dp_dip);
3389
3390 /* if the cookie cnt == max sgllen, move to the next window */
3391 } else if (window->wd_cookie_cnt >=
3392 (unsigned)attr->dma_attr_sgllen) {
3393 partial = B_TRUE;
3394 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3395 e = rootnex_sgllen_window_boundary(hp, dma, &window,
3396 cookie, attr, cur_offset);
3397 if (e != DDI_SUCCESS) {
3398 rootnex_teardown_copybuf(dma);
3399 rootnex_teardown_windows(dma);
3400 return (e);
3401 }
3402
3403 /*
3404 * if the coookie uses the copy buffer, make sure the
3405 * new window we just moved to is set to sync.
3406 */
3407 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3408 window->wd_dosync = B_TRUE;
3409 }
3410 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3411 dma->dp_dip);
3412
3413 /* else if we will be over maxxfer */
3414 } else if ((window->wd_size + dmac_size) >
3415 dma->dp_maxxfer) {
3416 partial = B_TRUE;
3417 e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3418 cookie);
3419 if (e != DDI_SUCCESS) {
3420 rootnex_teardown_copybuf(dma);
3421 rootnex_teardown_windows(dma);
3422 return (e);
3423 }
3424
3425 /*
3426 * if the coookie uses the copy buffer, make sure the
3427 * new window we just moved to is set to sync.
3428 */
3429 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3430 window->wd_dosync = B_TRUE;
3431 }
3432 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3433 dma->dp_dip);
3434
3435 /* else this cookie fits in the current window */
3436 } else {
3437 window->wd_cookie_cnt++;
3438 window->wd_size += dmac_size;
3439 }
3440
3441 /* track our offset into the buffer, go to the next cookie */
3442 ASSERT(dmac_size <= dmao->dmao_size);
3443 ASSERT(cookie->dmac_size <= dmac_size);
3444 cur_offset += dmac_size;
3445 cookie++;
3446 }
3447
3448 /* if we ended up with a zero sized window in the end, clean it up */
3449 if (window->wd_size == 0) {
3450 hp->dmai_nwin--;
3451 window--;
3452 }
3453
3454 ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3455
3456 if (!partial) {
3457 return (DDI_DMA_MAPPED);
3458 }
3459
3460 ASSERT(dma->dp_partial_required);
3461 return (DDI_DMA_PARTIAL_MAP);
3462 }
3463
3464 /*
3465 * rootnex_setup_copybuf()
3466 * Called in bind slowpath. Figures out if we're going to use the copy
3467 * buffer, and if we do, sets up the basic state to handle it.
3468 */
3469 static int
3470 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3471 rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3472 {
3473 rootnex_sglinfo_t *sinfo;
3474 ddi_dma_attr_t lattr;
3475 size_t max_copybuf;
3476 int cansleep;
3477 int e;
3478 #if !defined(__amd64)
3479 int vmflag;
3480 #endif
3481
3482 ASSERT(!dma->dp_dvma_used);
3483
3484 sinfo = &dma->dp_sglinfo;
3485
3486 /* read this first so it's consistent through the routine */
3487 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3488
3489 /* We need to call into the rootnex on ddi_dma_sync() */
3490 hp->dmai_rflags &= ~DMP_NOSYNC;
3491
3492 /* make sure the copybuf size <= the max size */
3493 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3494 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3495
3496 #if !defined(__amd64)
3497 /*
3498 * if we don't have kva space to copy to/from, allocate the KVA space
3499 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3500 * the 64-bit kernel.
3501 */
3502 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3503 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3504
3505 /* convert the sleep flags */
3506 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3507 vmflag = VM_SLEEP;
3508 } else {
3509 vmflag = VM_NOSLEEP;
3510 }
3511
3512 /* allocate Kernel VA space that we can bcopy to/from */
3513 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3514 vmflag);
3515 if (dma->dp_kva == NULL) {
3516 return (DDI_DMA_NORESOURCES);
3517 }
3518 }
3519 #endif
3520
3521 /* convert the sleep flags */
3522 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3523 cansleep = 1;
3524 } else {
3525 cansleep = 0;
3526 }
3527
3528 /*
3529 * Allocate the actual copy buffer. This needs to fit within the DMA
3530 * engine limits, so we can't use kmem_alloc... We don't need
3531 * contiguous memory (sgllen) since we will be forcing windows on
3532 * sgllen anyway.
3533 */
3534 lattr = *attr;
3535 lattr.dma_attr_align = MMU_PAGESIZE;
3536 lattr.dma_attr_sgllen = -1; /* no limit */
3537 /*
3538 * if we're using the copy buffer because of seg, use that for our
3539 * upper address limit.
3540 */
3541 if (sinfo->si_bounce_on_seg) {
3542 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3543 }
3544 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3545 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3546 if (e != DDI_SUCCESS) {
3547 #if !defined(__amd64)
3548 if (dma->dp_kva != NULL) {
3549 vmem_free(heap_arena, dma->dp_kva,
3550 dma->dp_copybuf_size);
3551 }
3552 #endif
3553 return (DDI_DMA_NORESOURCES);
3554 }
3555
3556 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3557 size_t, dma->dp_copybuf_size);
3558
3559 return (DDI_SUCCESS);
3560 }
3561
3562
3563 /*
3564 * rootnex_setup_windows()
3565 * Called in bind slowpath to setup the window state. We always have windows
3566 * in the slowpath. Even if the window count = 1.
3567 */
3568 static int
3569 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3570 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3571 {
3572 rootnex_window_t *windowp;
3573 rootnex_sglinfo_t *sinfo;
3574 size_t copy_state_size;
3575 size_t win_state_size;
3576 size_t state_available;
3577 size_t space_needed;
3578 uint_t copybuf_win;
3579 uint_t maxxfer_win;
3580 size_t space_used;
3581 uint_t sglwin;
3582
3583
3584 sinfo = &dma->dp_sglinfo;
3585
3586 dma->dp_current_win = 0;
3587 hp->dmai_nwin = 0;
3588
3589 /* If we don't need to do a partial, we only have one window */
3590 if (!dma->dp_partial_required) {
3591 dma->dp_max_win = 1;
3592
3593 /*
3594 * we need multiple windows, need to figure out the worse case number
3595 * of windows.
3596 */
3597 } else {
3598 /*
3599 * if we need windows because we need more copy buffer that
3600 * we allow, the worse case number of windows we could need
3601 * here would be (copybuf space required / copybuf space that
3602 * we have) plus one for remainder, and plus 2 to handle the
3603 * extra pages on the trim for the first and last pages of the
3604 * buffer (a page is the minimum window size so under the right
3605 * attr settings, you could have a window for each page).
3606 * The last page will only be hit here if the size is not a
3607 * multiple of the granularity (which theoretically shouldn't
3608 * be the case but never has been enforced, so we could have
3609 * broken things without it).
3610 */
3611 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3612 ASSERT(dma->dp_copybuf_size > 0);
3613 copybuf_win = (sinfo->si_copybuf_req /
3614 dma->dp_copybuf_size) + 1 + 2;
3615 } else {
3616 copybuf_win = 0;
3617 }
3618
3619 /*
3620 * if we need windows because we have more cookies than the H/W
3621 * can handle, the number of windows we would need here would
3622 * be (cookie count / cookies count H/W supports minus 1[for
3623 * trim]) plus one for remainder.
3624 */
3625 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3626 sglwin = (sinfo->si_sgl_size /
3627 (attr->dma_attr_sgllen - 1)) + 1;
3628 } else {
3629 sglwin = 0;
3630 }
3631
3632 /*
3633 * if we need windows because we're binding more memory than the
3634 * H/W can transfer at once, the number of windows we would need
3635 * here would be (xfer count / max xfer H/W supports) plus one
3636 * for remainder, and plus 2 to handle the extra pages on the
3637 * trim (see above comment about trim)
3638 */
3639 if (dmao->dmao_size > dma->dp_maxxfer) {
3640 maxxfer_win = (dmao->dmao_size /
3641 dma->dp_maxxfer) + 1 + 2;
3642 } else {
3643 maxxfer_win = 0;
3644 }
3645 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
3646 ASSERT(dma->dp_max_win > 0);
3647 }
3648 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3649
3650 /*
3651 * Get space for window and potential copy buffer state. Before we
3652 * go and allocate memory, see if we can get away with using what's
3653 * left in the pre-allocted state or the dynamically allocated sgl.
3654 */
3655 space_used = (uintptr_t)(sinfo->si_sgl_size *
3656 sizeof (ddi_dma_cookie_t));
3657
3658 /* if we dynamically allocated space for the cookies */
3659 if (dma->dp_need_to_free_cookie) {
3660 /* if we have more space in the pre-allocted buffer, use it */
3661 ASSERT(space_used <= dma->dp_cookie_size);
3662 if ((dma->dp_cookie_size - space_used) <=
3663 rootnex_state->r_prealloc_size) {
3664 state_available = rootnex_state->r_prealloc_size;
3665 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3666
3667 /*
3668 * else, we have more free space in the dynamically allocated
3669 * buffer, i.e. the buffer wasn't worse case fragmented so we
3670 * didn't need a lot of cookies.
3671 */
3672 } else {
3673 state_available = dma->dp_cookie_size - space_used;
3674 windowp = (rootnex_window_t *)
3675 &dma->dp_cookies[sinfo->si_sgl_size];
3676 }
3677
3678 /* we used the pre-alloced buffer */
3679 } else {
3680 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3681 state_available = rootnex_state->r_prealloc_size - space_used;
3682 windowp = (rootnex_window_t *)
3683 &dma->dp_cookies[sinfo->si_sgl_size];
3684 }
3685
3686 /*
3687 * figure out how much state we need to track the copy buffer. Add an
3688 * addition 8 bytes for pointer alignemnt later.
3689 */
3690 if (dma->dp_copybuf_size > 0) {
3691 copy_state_size = sinfo->si_max_pages *
3692 sizeof (rootnex_pgmap_t);
3693 } else {
3694 copy_state_size = 0;
3695 }
3696 /* add an additional 8 bytes for pointer alignment */
3697 space_needed = win_state_size + copy_state_size + 0x8;
3698
3699 /* if we have enough space already, use it */
3700 if (state_available >= space_needed) {
3701 dma->dp_window = windowp;
3702 dma->dp_need_to_free_window = B_FALSE;
3703
3704 /* not enough space, need to allocate more. */
3705 } else {
3706 dma->dp_window = kmem_alloc(space_needed, kmflag);
3707 if (dma->dp_window == NULL) {
3708 return (DDI_DMA_NORESOURCES);
3709 }
3710 dma->dp_need_to_free_window = B_TRUE;
3711 dma->dp_window_size = space_needed;
3712 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3713 dma->dp_dip, size_t, space_needed);
3714 }
3715
3716 /*
3717 * we allocate copy buffer state and window state at the same time.
3718 * setup our copy buffer state pointers. Make sure it's aligned.
3719 */
3720 if (dma->dp_copybuf_size > 0) {
3721 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3722 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3723
3724 #if !defined(__amd64)
3725 /*
3726 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3727 * false/NULL. Should be quicker to bzero vs loop and set.
3728 */
3729 bzero(dma->dp_pgmap, copy_state_size);
3730 #endif
3731 } else {
3732 dma->dp_pgmap = NULL;
3733 }
3734
3735 return (DDI_SUCCESS);
3736 }
3737
3738
3739 /*
3740 * rootnex_teardown_copybuf()
3741 * cleans up after rootnex_setup_copybuf()
3742 */
3743 static void
3744 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3745 {
3746 #if !defined(__amd64)
3747 int i;
3748
3749 /*
3750 * if we allocated kernel heap VMEM space, go through all the pages and
3751 * map out any of the ones that we're mapped into the kernel heap VMEM
3752 * arena. Then free the VMEM space.
3753 */
3754 if (dma->dp_kva != NULL) {
3755 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3756 if (dma->dp_pgmap[i].pm_mapped) {
3757 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3758 MMU_PAGESIZE, HAT_UNLOAD);
3759 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3760 }
3761 }
3762
3763 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3764 }
3765
3766 #endif
3767
3768 /* if we allocated a copy buffer, free it */
3769 if (dma->dp_cbaddr != NULL) {
3770 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3771 }
3772 }
3773
3774
3775 /*
3776 * rootnex_teardown_windows()
3777 * cleans up after rootnex_setup_windows()
3778 */
3779 static void
3780 rootnex_teardown_windows(rootnex_dma_t *dma)
3781 {
3782 /*
3783 * if we had to allocate window state on the last bind (because we
3784 * didn't have enough pre-allocated space in the handle), free it.
3785 */
3786 if (dma->dp_need_to_free_window) {
3787 kmem_free(dma->dp_window, dma->dp_window_size);
3788 }
3789 }
3790
3791
3792 /*
3793 * rootnex_init_win()
3794 * Called in bind slow path during creation of a new window. Initializes
3795 * window state to default values.
3796 */
3797 /*ARGSUSED*/
3798 static void
3799 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3800 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3801 {
3802 hp->dmai_nwin++;
3803 window->wd_dosync = B_FALSE;
3804 window->wd_offset = cur_offset;
3805 window->wd_size = 0;
3806 window->wd_first_cookie = cookie;
3807 window->wd_cookie_cnt = 0;
3808 window->wd_trim.tr_trim_first = B_FALSE;
3809 window->wd_trim.tr_trim_last = B_FALSE;
3810 window->wd_trim.tr_first_copybuf_win = B_FALSE;
3811 window->wd_trim.tr_last_copybuf_win = B_FALSE;
3812 #if !defined(__amd64)
3813 window->wd_remap_copybuf = dma->dp_cb_remaping;
3814 #endif
3815 }
3816
3817
3818 /*
3819 * rootnex_setup_cookie()
3820 * Called in the bind slow path when the sgl uses the copy buffer. If any of
3821 * the sgl uses the copy buffer, we need to go through each cookie, figure
3822 * out if it uses the copy buffer, and if it does, save away everything we'll
3823 * need during sync.
3824 */
3825 static void
3826 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3827 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3828 page_t **cur_pp)
3829 {
3830 boolean_t copybuf_sz_power_2;
3831 rootnex_sglinfo_t *sinfo;
3832 paddr_t paddr;
3833 uint_t pidx;
3834 uint_t pcnt;
3835 off_t poff;
3836 #if defined(__amd64)
3837 pfn_t pfn;
3838 #else
3839 page_t **pplist;
3840 #endif
3841
3842 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3843
3844 sinfo = &dma->dp_sglinfo;
3845
3846 /*
3847 * Calculate the page index relative to the start of the buffer. The
3848 * index to the current page for our buffer is the offset into the
3849 * first page of the buffer plus our current offset into the buffer
3850 * itself, shifted of course...
3851 */
3852 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3853 ASSERT(pidx < sinfo->si_max_pages);
3854
3855 /* if this cookie uses the copy buffer */
3856 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3857 /*
3858 * NOTE: we know that since this cookie uses the copy buffer, it
3859 * is <= MMU_PAGESIZE.
3860 */
3861
3862 /*
3863 * get the offset into the page. For the 64-bit kernel, get the
3864 * pfn which we'll use with seg kpm.
3865 */
3866 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3867 #if defined(__amd64)
3868 /* mfn_to_pfn() is a NOP on i86pc */
3869 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3870 #endif /* __amd64 */
3871
3872 /* figure out if the copybuf size is a power of 2 */
3873 if (!ISP2(dma->dp_copybuf_size)) {
3874 copybuf_sz_power_2 = B_FALSE;
3875 } else {
3876 copybuf_sz_power_2 = B_TRUE;
3877 }
3878
3879 /* This page uses the copy buffer */
3880 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3881
3882 /*
3883 * save the copy buffer KVA that we'll use with this page.
3884 * if we still fit within the copybuf, it's a simple add.
3885 * otherwise, we need to wrap over using & or % accordingly.
3886 */
3887 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3888 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3889 *copybuf_used;
3890 } else {
3891 if (copybuf_sz_power_2) {
3892 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3893 (uintptr_t)dma->dp_cbaddr +
3894 (*copybuf_used &
3895 (dma->dp_copybuf_size - 1)));
3896 } else {
3897 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3898 (uintptr_t)dma->dp_cbaddr +
3899 (*copybuf_used % dma->dp_copybuf_size));
3900 }
3901 }
3902
3903 /*
3904 * over write the cookie physical address with the address of
3905 * the physical address of the copy buffer page that we will
3906 * use.
3907 */
3908 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3909 dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3910
3911 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3912
3913 /* if we have a kernel VA, it's easy, just save that address */
3914 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3915 (sinfo->si_asp == &kas)) {
3916 /*
3917 * save away the page aligned virtual address of the
3918 * driver buffer. Offsets are handled in the sync code.
3919 */
3920 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3921 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3922 & MMU_PAGEMASK);
3923 #if !defined(__amd64)
3924 /*
3925 * we didn't need to, and will never need to map this
3926 * page.
3927 */
3928 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3929 #endif
3930
3931 /* we don't have a kernel VA. We need one for the bcopy. */
3932 } else {
3933 #if defined(__amd64)
3934 /*
3935 * for the 64-bit kernel, it's easy. We use seg kpm to
3936 * get a Kernel VA for the corresponding pfn.
3937 */
3938 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3939 #else
3940 /*
3941 * for the 32-bit kernel, this is a pain. First we'll
3942 * save away the page_t or user VA for this page. This
3943 * is needed in rootnex_dma_win() when we switch to a
3944 * new window which requires us to re-map the copy
3945 * buffer.
3946 */
3947 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3948 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3949 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3950 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3951 } else if (pplist != NULL) {
3952 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3953 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3954 } else {
3955 dma->dp_pgmap[pidx].pm_pp = NULL;
3956 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3957 (((uintptr_t)
3958 dmar_object->dmao_obj.virt_obj.v_addr +
3959 cur_offset) & MMU_PAGEMASK);
3960 }
3961
3962 /*
3963 * save away the page aligned virtual address which was
3964 * allocated from the kernel heap arena (taking into
3965 * account if we need more copy buffer than we alloced
3966 * and use multiple windows to handle this, i.e. &,%).
3967 * NOTE: there isn't and physical memory backing up this
3968 * virtual address space currently.
3969 */
3970 if ((*copybuf_used + MMU_PAGESIZE) <=
3971 dma->dp_copybuf_size) {
3972 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3973 (((uintptr_t)dma->dp_kva + *copybuf_used) &
3974 MMU_PAGEMASK);
3975 } else {
3976 if (copybuf_sz_power_2) {
3977 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3978 (((uintptr_t)dma->dp_kva +
3979 (*copybuf_used &
3980 (dma->dp_copybuf_size - 1))) &
3981 MMU_PAGEMASK);
3982 } else {
3983 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3984 (((uintptr_t)dma->dp_kva +
3985 (*copybuf_used %
3986 dma->dp_copybuf_size)) &
3987 MMU_PAGEMASK);
3988 }
3989 }
3990
3991 /*
3992 * if we haven't used up the available copy buffer yet,
3993 * map the kva to the physical page.
3994 */
3995 if (!dma->dp_cb_remaping && ((*copybuf_used +
3996 MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3997 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3998 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3999 i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
4000 dma->dp_pgmap[pidx].pm_kaddr);
4001 } else {
4002 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4003 sinfo->si_asp,
4004 dma->dp_pgmap[pidx].pm_kaddr);
4005 }
4006
4007 /*
4008 * we've used up the available copy buffer, this page
4009 * will have to be mapped during rootnex_dma_win() when
4010 * we switch to a new window which requires a re-map
4011 * the copy buffer. (32-bit kernel only)
4012 */
4013 } else {
4014 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4015 }
4016 #endif
4017 /* go to the next page_t */
4018 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4019 *cur_pp = (*cur_pp)->p_next;
4020 }
4021 }
4022
4023 /* add to the copy buffer count */
4024 *copybuf_used += MMU_PAGESIZE;
4025
4026 /*
4027 * This cookie doesn't use the copy buffer. Walk through the pages this
4028 * cookie occupies to reflect this.
4029 */
4030 } else {
4031 /*
4032 * figure out how many pages the cookie occupies. We need to
4033 * use the original page offset of the buffer and the cookies
4034 * offset in the buffer to do this.
4035 */
4036 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4037 pcnt = mmu_btopr(cookie->dmac_size + poff);
4038
4039 while (pcnt > 0) {
4040 #if !defined(__amd64)
4041 /*
4042 * the 32-bit kernel doesn't have seg kpm, so we need
4043 * to map in the driver buffer (if it didn't come down
4044 * with a kernel VA) on the fly. Since this page doesn't
4045 * use the copy buffer, it's not, or will it ever, have
4046 * to be mapped in.
4047 */
4048 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4049 #endif
4050 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4051
4052 /*
4053 * we need to update pidx and cur_pp or we'll loose
4054 * track of where we are.
4055 */
4056 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4057 *cur_pp = (*cur_pp)->p_next;
4058 }
4059 pidx++;
4060 pcnt--;
4061 }
4062 }
4063 }
4064
4065
4066 /*
4067 * rootnex_sgllen_window_boundary()
4068 * Called in the bind slow path when the next cookie causes us to exceed (in
4069 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4070 * length supported by the DMA H/W.
4071 */
4072 static int
4073 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4074 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4075 off_t cur_offset)
4076 {
4077 off_t new_offset;
4078 size_t trim_sz;
4079 off_t coffset;
4080
4081
4082 /*
4083 * if we know we'll never have to trim, it's pretty easy. Just move to
4084 * the next window and init it. We're done.
4085 */
4086 if (!dma->dp_trim_required) {
4087 (*windowp)++;
4088 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4089 (*windowp)->wd_cookie_cnt++;
4090 (*windowp)->wd_size = cookie->dmac_size;
4091 return (DDI_SUCCESS);
4092 }
4093
4094 /* figure out how much we need to trim from the window */
4095 ASSERT(attr->dma_attr_granular != 0);
4096 if (dma->dp_granularity_power_2) {
4097 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4098 } else {
4099 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4100 }
4101
4102 /* The window's a whole multiple of granularity. We're done */
4103 if (trim_sz == 0) {
4104 (*windowp)++;
4105 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4106 (*windowp)->wd_cookie_cnt++;
4107 (*windowp)->wd_size = cookie->dmac_size;
4108 return (DDI_SUCCESS);
4109 }
4110
4111 /*
4112 * The window's not a whole multiple of granularity, since we know this
4113 * is due to the sgllen, we need to go back to the last cookie and trim
4114 * that one, add the left over part of the old cookie into the new
4115 * window, and then add in the new cookie into the new window.
4116 */
4117
4118 /*
4119 * make sure the driver isn't making us do something bad... Trimming and
4120 * sgllen == 1 don't go together.
4121 */
4122 if (attr->dma_attr_sgllen == 1) {
4123 return (DDI_DMA_NOMAPPING);
4124 }
4125
4126 /*
4127 * first, setup the current window to account for the trim. Need to go
4128 * back to the last cookie for this.
4129 */
4130 cookie--;
4131 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4132 (*windowp)->wd_trim.tr_last_cookie = cookie;
4133 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4134 ASSERT(cookie->dmac_size > trim_sz);
4135 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4136 (*windowp)->wd_size -= trim_sz;
4137
4138 /* save the buffer offsets for the next window */
4139 coffset = cookie->dmac_size - trim_sz;
4140 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4141
4142 /*
4143 * set this now in case this is the first window. all other cases are
4144 * set in dma_win()
4145 */
4146 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4147
4148 /*
4149 * initialize the next window using what's left over in the previous
4150 * cookie.
4151 */
4152 (*windowp)++;
4153 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4154 (*windowp)->wd_cookie_cnt++;
4155 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4156 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4157 (*windowp)->wd_trim.tr_first_size = trim_sz;
4158 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4159 (*windowp)->wd_dosync = B_TRUE;
4160 }
4161
4162 /*
4163 * now go back to the current cookie and add it to the new window. set
4164 * the new window size to the what was left over from the previous
4165 * cookie and what's in the current cookie.
4166 */
4167 cookie++;
4168 (*windowp)->wd_cookie_cnt++;
4169 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4170
4171 /*
4172 * trim plus the next cookie could put us over maxxfer (a cookie can be
4173 * a max size of maxxfer). Handle that case.
4174 */
4175 if ((*windowp)->wd_size > dma->dp_maxxfer) {
4176 /*
4177 * maxxfer is already a whole multiple of granularity, and this
4178 * trim will be <= the previous trim (since a cookie can't be
4179 * larger than maxxfer). Make things simple here.
4180 */
4181 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4182 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4183 (*windowp)->wd_trim.tr_last_cookie = cookie;
4184 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4185 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4186 (*windowp)->wd_size -= trim_sz;
4187 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4188
4189 /* save the buffer offsets for the next window */
4190 coffset = cookie->dmac_size - trim_sz;
4191 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4192
4193 /* setup the next window */
4194 (*windowp)++;
4195 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4196 (*windowp)->wd_cookie_cnt++;
4197 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4198 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4199 coffset;
4200 (*windowp)->wd_trim.tr_first_size = trim_sz;
4201 }
4202
4203 return (DDI_SUCCESS);
4204 }
4205
4206
4207 /*
4208 * rootnex_copybuf_window_boundary()
4209 * Called in bind slowpath when we get to a window boundary because we used
4210 * up all the copy buffer that we have.
4211 */
4212 static int
4213 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4214 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4215 size_t *copybuf_used)
4216 {
4217 rootnex_sglinfo_t *sinfo;
4218 off_t new_offset;
4219 size_t trim_sz;
4220 paddr_t paddr;
4221 off_t coffset;
4222 uint_t pidx;
4223 off_t poff;
4224
4225
4226 sinfo = &dma->dp_sglinfo;
4227
4228 /*
4229 * the copy buffer should be a whole multiple of page size. We know that
4230 * this cookie is <= MMU_PAGESIZE.
4231 */
4232 ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4233
4234 /*
4235 * from now on, all new windows in this bind need to be re-mapped during
4236 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4237 * space...
4238 */
4239 #if !defined(__amd64)
4240 dma->dp_cb_remaping = B_TRUE;
4241 #endif
4242
4243 /* reset copybuf used */
4244 *copybuf_used = 0;
4245
4246 /*
4247 * if we don't have to trim (since granularity is set to 1), go to the
4248 * next window and add the current cookie to it. We know the current
4249 * cookie uses the copy buffer since we're in this code path.
4250 */
4251 if (!dma->dp_trim_required) {
4252 (*windowp)++;
4253 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4254
4255 /* Add this cookie to the new window */
4256 (*windowp)->wd_cookie_cnt++;
4257 (*windowp)->wd_size += cookie->dmac_size;
4258 *copybuf_used += MMU_PAGESIZE;
4259 return (DDI_SUCCESS);
4260 }
4261
4262 /*
4263 * *** may need to trim, figure it out.
4264 */
4265
4266 /* figure out how much we need to trim from the window */
4267 if (dma->dp_granularity_power_2) {
4268 trim_sz = (*windowp)->wd_size &
4269 (hp->dmai_attr.dma_attr_granular - 1);
4270 } else {
4271 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4272 }
4273
4274 /*
4275 * if the window's a whole multiple of granularity, go to the next
4276 * window, init it, then add in the current cookie. We know the current
4277 * cookie uses the copy buffer since we're in this code path.
4278 */
4279 if (trim_sz == 0) {
4280 (*windowp)++;
4281 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4282
4283 /* Add this cookie to the new window */
4284 (*windowp)->wd_cookie_cnt++;
4285 (*windowp)->wd_size += cookie->dmac_size;
4286 *copybuf_used += MMU_PAGESIZE;
4287 return (DDI_SUCCESS);
4288 }
4289
4290 /*
4291 * *** We figured it out, we definitly need to trim
4292 */
4293
4294 /*
4295 * make sure the driver isn't making us do something bad...
4296 * Trimming and sgllen == 1 don't go together.
4297 */
4298 if (hp->dmai_attr.dma_attr_sgllen == 1) {
4299 return (DDI_DMA_NOMAPPING);
4300 }
4301
4302 /*
4303 * first, setup the current window to account for the trim. Need to go
4304 * back to the last cookie for this. Some of the last cookie will be in
4305 * the current window, and some of the last cookie will be in the new
4306 * window. All of the current cookie will be in the new window.
4307 */
4308 cookie--;
4309 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4310 (*windowp)->wd_trim.tr_last_cookie = cookie;
4311 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4312 ASSERT(cookie->dmac_size > trim_sz);
4313 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4314 (*windowp)->wd_size -= trim_sz;
4315
4316 /*
4317 * we're trimming the last cookie (not the current cookie). So that
4318 * last cookie may have or may not have been using the copy buffer (
4319 * we know the cookie passed in uses the copy buffer since we're in
4320 * this code path).
4321 *
4322 * If the last cookie doesn't use the copy buffer, nothing special to
4323 * do. However, if it does uses the copy buffer, it will be both the
4324 * last page in the current window and the first page in the next
4325 * window. Since we are reusing the copy buffer (and KVA space on the
4326 * 32-bit kernel), this page will use the end of the copy buffer in the
4327 * current window, and the start of the copy buffer in the next window.
4328 * Track that info... The cookie physical address was already set to
4329 * the copy buffer physical address in setup_cookie..
4330 */
4331 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4332 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4333 (*windowp)->wd_size) >> MMU_PAGESHIFT;
4334 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4335 (*windowp)->wd_trim.tr_last_pidx = pidx;
4336 (*windowp)->wd_trim.tr_last_cbaddr =
4337 dma->dp_pgmap[pidx].pm_cbaddr;
4338 #if !defined(__amd64)
4339 (*windowp)->wd_trim.tr_last_kaddr =
4340 dma->dp_pgmap[pidx].pm_kaddr;
4341 #endif
4342 }
4343
4344 /* save the buffer offsets for the next window */
4345 coffset = cookie->dmac_size - trim_sz;
4346 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4347
4348 /*
4349 * set this now in case this is the first window. all other cases are
4350 * set in dma_win()
4351 */
4352 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4353
4354 /*
4355 * initialize the next window using what's left over in the previous
4356 * cookie.
4357 */
4358 (*windowp)++;
4359 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4360 (*windowp)->wd_cookie_cnt++;
4361 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4362 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4363 (*windowp)->wd_trim.tr_first_size = trim_sz;
4364
4365 /*
4366 * again, we're tracking if the last cookie uses the copy buffer.
4367 * read the comment above for more info on why we need to track
4368 * additional state.
4369 *
4370 * For the first cookie in the new window, we need reset the physical
4371 * address to DMA into to the start of the copy buffer plus any
4372 * initial page offset which may be present.
4373 */
4374 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4375 (*windowp)->wd_dosync = B_TRUE;
4376 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4377 (*windowp)->wd_trim.tr_first_pidx = pidx;
4378 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4379 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4380
4381 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4382 poff;
4383 (*windowp)->wd_trim.tr_first_paddr =
4384 ROOTNEX_PADDR_TO_RBASE(paddr);
4385
4386 #if !defined(__amd64)
4387 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4388 #endif
4389 /* account for the cookie copybuf usage in the new window */
4390 *copybuf_used += MMU_PAGESIZE;
4391
4392 /*
4393 * every piece of code has to have a hack, and here is this
4394 * ones :-)
4395 *
4396 * There is a complex interaction between setup_cookie and the
4397 * copybuf window boundary. The complexity had to be in either
4398 * the maxxfer window, or the copybuf window, and I chose the
4399 * copybuf code.
4400 *
4401 * So in this code path, we have taken the last cookie,
4402 * virtually broken it in half due to the trim, and it happens
4403 * to use the copybuf which further complicates life. At the
4404 * same time, we have already setup the current cookie, which
4405 * is now wrong. More background info: the current cookie uses
4406 * the copybuf, so it is only a page long max. So we need to
4407 * fix the current cookies copy buffer address, physical
4408 * address, and kva for the 32-bit kernel. We due this by
4409 * bumping them by page size (of course, we can't due this on
4410 * the physical address since the copy buffer may not be
4411 * physically contiguous).
4412 */
4413 cookie++;
4414 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4415 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4416
4417 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4418 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4419 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4420
4421 #if !defined(__amd64)
4422 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4423 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4424 #endif
4425 } else {
4426 /* go back to the current cookie */
4427 cookie++;
4428 }
4429
4430 /*
4431 * add the current cookie to the new window. set the new window size to
4432 * the what was left over from the previous cookie and what's in the
4433 * current cookie.
4434 */
4435 (*windowp)->wd_cookie_cnt++;
4436 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4437 ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4438
4439 /*
4440 * we know that the cookie passed in always uses the copy buffer. We
4441 * wouldn't be here if it didn't.
4442 */
4443 *copybuf_used += MMU_PAGESIZE;
4444
4445 return (DDI_SUCCESS);
4446 }
4447
4448
4449 /*
4450 * rootnex_maxxfer_window_boundary()
4451 * Called in bind slowpath when we get to a window boundary because we will
4452 * go over maxxfer.
4453 */
4454 static int
4455 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4456 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4457 {
4458 size_t dmac_size;
4459 off_t new_offset;
4460 size_t trim_sz;
4461 off_t coffset;
4462
4463
4464 /*
4465 * calculate how much we have to trim off of the current cookie to equal
4466 * maxxfer. We don't have to account for granularity here since our
4467 * maxxfer already takes that into account.
4468 */
4469 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4470 ASSERT(trim_sz <= cookie->dmac_size);
4471 ASSERT(trim_sz <= dma->dp_maxxfer);
4472
4473 /* save cookie size since we need it later and we might change it */
4474 dmac_size = cookie->dmac_size;
4475
4476 /*
4477 * if we're not trimming the entire cookie, setup the current window to
4478 * account for the trim.
4479 */
4480 if (trim_sz < cookie->dmac_size) {
4481 (*windowp)->wd_cookie_cnt++;
4482 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4483 (*windowp)->wd_trim.tr_last_cookie = cookie;
4484 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4485 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4486 (*windowp)->wd_size = dma->dp_maxxfer;
4487
4488 /*
4489 * set the adjusted cookie size now in case this is the first
4490 * window. All other windows are taken care of in get win
4491 */
4492 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4493 }
4494
4495 /*
4496 * coffset is the current offset within the cookie, new_offset is the
4497 * current offset with the entire buffer.
4498 */
4499 coffset = dmac_size - trim_sz;
4500 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4501
4502 /* initialize the next window */
4503 (*windowp)++;
4504 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4505 (*windowp)->wd_cookie_cnt++;
4506 (*windowp)->wd_size = trim_sz;
4507 if (trim_sz < dmac_size) {
4508 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4509 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4510 coffset;
4511 (*windowp)->wd_trim.tr_first_size = trim_sz;
4512 }
4513
4514 return (DDI_SUCCESS);
4515 }
4516
4517
4518 /*ARGSUSED*/
4519 static int
4520 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4521 off_t off, size_t len, uint_t cache_flags)
4522 {
4523 rootnex_sglinfo_t *sinfo;
4524 rootnex_pgmap_t *cbpage;
4525 rootnex_window_t *win;
4526 ddi_dma_impl_t *hp;
4527 rootnex_dma_t *dma;
4528 caddr_t fromaddr;
4529 caddr_t toaddr;
4530 uint_t psize;
4531 off_t offset;
4532 uint_t pidx;
4533 size_t size;
4534 off_t poff;
4535 int e;
4536
4537
4538 hp = (ddi_dma_impl_t *)handle;
4539 dma = (rootnex_dma_t *)hp->dmai_private;
4540 sinfo = &dma->dp_sglinfo;
4541
4542 /*
4543 * if we don't have any windows, we don't need to sync. A copybuf
4544 * will cause us to have at least one window.
4545 */
4546 if (dma->dp_window == NULL) {
4547 return (DDI_SUCCESS);
4548 }
4549
4550 /* This window may not need to be sync'd */
4551 win = &dma->dp_window[dma->dp_current_win];
4552 if (!win->wd_dosync) {
4553 return (DDI_SUCCESS);
4554 }
4555
4556 /* handle off and len special cases */
4557 if ((off == 0) || (rootnex_sync_ignore_params)) {
4558 offset = win->wd_offset;
4559 } else {
4560 offset = off;
4561 }
4562 if ((len == 0) || (rootnex_sync_ignore_params)) {
4563 size = win->wd_size;
4564 } else {
4565 size = len;
4566 }
4567
4568 /* check the sync args to make sure they make a little sense */
4569 if (rootnex_sync_check_parms) {
4570 e = rootnex_valid_sync_parms(hp, win, offset, size,
4571 cache_flags);
4572 if (e != DDI_SUCCESS) {
4573 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4574 return (DDI_FAILURE);
4575 }
4576 }
4577
4578 /*
4579 * special case the first page to handle the offset into the page. The
4580 * offset to the current page for our buffer is the offset into the
4581 * first page of the buffer plus our current offset into the buffer
4582 * itself, masked of course.
4583 */
4584 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4585 psize = MIN((MMU_PAGESIZE - poff), size);
4586
4587 /* go through all the pages that we want to sync */
4588 while (size > 0) {
4589 /*
4590 * Calculate the page index relative to the start of the buffer.
4591 * The index to the current page for our buffer is the offset
4592 * into the first page of the buffer plus our current offset
4593 * into the buffer itself, shifted of course...
4594 */
4595 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4596 ASSERT(pidx < sinfo->si_max_pages);
4597
4598 /*
4599 * if this page uses the copy buffer, we need to sync it,
4600 * otherwise, go on to the next page.
4601 */
4602 cbpage = &dma->dp_pgmap[pidx];
4603 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4604 (cbpage->pm_uses_copybuf == B_FALSE));
4605 if (cbpage->pm_uses_copybuf) {
4606 /* cbaddr and kaddr should be page aligned */
4607 ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4608 MMU_PAGEOFFSET) == 0);
4609 ASSERT(((uintptr_t)cbpage->pm_kaddr &
4610 MMU_PAGEOFFSET) == 0);
4611
4612 /*
4613 * if we're copying for the device, we are going to
4614 * copy from the drivers buffer and to the rootnex
4615 * allocated copy buffer.
4616 */
4617 if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4618 fromaddr = cbpage->pm_kaddr + poff;
4619 toaddr = cbpage->pm_cbaddr + poff;
4620 ROOTNEX_DPROBE2(rootnex__sync__dev,
4621 dev_info_t *, dma->dp_dip, size_t, psize);
4622
4623 /*
4624 * if we're copying for the cpu/kernel, we are going to
4625 * copy from the rootnex allocated copy buffer to the
4626 * drivers buffer.
4627 */
4628 } else {
4629 fromaddr = cbpage->pm_cbaddr + poff;
4630 toaddr = cbpage->pm_kaddr + poff;
4631 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4632 dev_info_t *, dma->dp_dip, size_t, psize);
4633 }
4634
4635 bcopy(fromaddr, toaddr, psize);
4636 }
4637
4638 /*
4639 * decrement size until we're done, update our offset into the
4640 * buffer, and get the next page size.
4641 */
4642 size -= psize;
4643 offset += psize;
4644 psize = MIN(MMU_PAGESIZE, size);
4645
4646 /* page offset is zero for the rest of this loop */
4647 poff = 0;
4648 }
4649
4650 return (DDI_SUCCESS);
4651 }
4652
4653 /*
4654 * rootnex_dma_sync()
4655 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4656 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4657 * is set, ddi_dma_sync() returns immediately passing back success.
4658 */
4659 /*ARGSUSED*/
4660 static int
4661 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4662 off_t off, size_t len, uint_t cache_flags)
4663 {
4664 #if defined(__amd64) && !defined(__xpv)
4665 if (IOMMU_USED(rdip)) {
4666 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4667 cache_flags));
4668 }
4669 #endif
4670 return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4671 cache_flags));
4672 }
4673
4674 /*
4675 * rootnex_valid_sync_parms()
4676 * checks the parameters passed to sync to verify they are correct.
4677 */
4678 static int
4679 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4680 off_t offset, size_t size, uint_t cache_flags)
4681 {
4682 off_t woffset;
4683
4684
4685 /*
4686 * the first part of the test to make sure the offset passed in is
4687 * within the window.
4688 */
4689 if (offset < win->wd_offset) {
4690 return (DDI_FAILURE);
4691 }
4692
4693 /*
4694 * second and last part of the test to make sure the offset and length
4695 * passed in is within the window.
4696 */
4697 woffset = offset - win->wd_offset;
4698 if ((woffset + size) > win->wd_size) {
4699 return (DDI_FAILURE);
4700 }
4701
4702 /*
4703 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4704 * be set too.
4705 */
4706 if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4707 (hp->dmai_rflags & DDI_DMA_WRITE)) {
4708 return (DDI_SUCCESS);
4709 }
4710
4711 /*
4712 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4713 * should be set. Also DDI_DMA_READ should be set in the flags.
4714 */
4715 if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4716 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4717 (hp->dmai_rflags & DDI_DMA_READ)) {
4718 return (DDI_SUCCESS);
4719 }
4720
4721 return (DDI_FAILURE);
4722 }
4723
4724
4725 /*ARGSUSED*/
4726 static int
4727 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4728 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4729 uint_t *ccountp)
4730 {
4731 rootnex_window_t *window;
4732 rootnex_trim_t *trim;
4733 ddi_dma_impl_t *hp;
4734 rootnex_dma_t *dma;
4735 ddi_dma_obj_t *dmao;
4736 #if !defined(__amd64)
4737 rootnex_sglinfo_t *sinfo;
4738 rootnex_pgmap_t *pmap;
4739 uint_t pidx;
4740 uint_t pcnt;
4741 off_t poff;
4742 int i;
4743 #endif
4744
4745
4746 hp = (ddi_dma_impl_t *)handle;
4747 dma = (rootnex_dma_t *)hp->dmai_private;
4748 #if !defined(__amd64)
4749 sinfo = &dma->dp_sglinfo;
4750 #endif
4751
4752 /* If we try and get a window which doesn't exist, return failure */
4753 if (win >= hp->dmai_nwin) {
4754 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4755 return (DDI_FAILURE);
4756 }
4757
4758 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4759
4760 /*
4761 * if we don't have any windows, and they're asking for the first
4762 * window, setup the cookie pointer to the first cookie in the bind.
4763 * setup our return values, then increment the cookie since we return
4764 * the first cookie on the stack.
4765 */
4766 if (dma->dp_window == NULL) {
4767 if (win != 0) {
4768 ROOTNEX_DPROF_INC(
4769 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4770 return (DDI_FAILURE);
4771 }
4772 hp->dmai_cookie = dma->dp_cookies;
4773 *offp = 0;
4774 *lenp = dmao->dmao_size;
4775 *ccountp = dma->dp_sglinfo.si_sgl_size;
4776 *cookiep = hp->dmai_cookie[0];
4777 hp->dmai_cookie++;
4778 return (DDI_SUCCESS);
4779 }
4780
4781 /* sync the old window before moving on to the new one */
4782 window = &dma->dp_window[dma->dp_current_win];
4783 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4784 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4785 DDI_DMA_SYNC_FORCPU);
4786 }
4787
4788 #if !defined(__amd64)
4789 /*
4790 * before we move to the next window, if we need to re-map, unmap all
4791 * the pages in this window.
4792 */
4793 if (dma->dp_cb_remaping) {
4794 /*
4795 * If we switch to this window again, we'll need to map in
4796 * on the fly next time.
4797 */
4798 window->wd_remap_copybuf = B_TRUE;
4799
4800 /*
4801 * calculate the page index into the buffer where this window
4802 * starts, and the number of pages this window takes up.
4803 */
4804 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4805 MMU_PAGESHIFT;
4806 poff = (sinfo->si_buf_offset + window->wd_offset) &
4807 MMU_PAGEOFFSET;
4808 pcnt = mmu_btopr(window->wd_size + poff);
4809 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4810
4811 /* unmap pages which are currently mapped in this window */
4812 for (i = 0; i < pcnt; i++) {
4813 if (dma->dp_pgmap[pidx].pm_mapped) {
4814 hat_unload(kas.a_hat,
4815 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4816 HAT_UNLOAD);
4817 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4818 }
4819 pidx++;
4820 }
4821 }
4822 #endif
4823
4824 /*
4825 * Move to the new window.
4826 * NOTE: current_win must be set for sync to work right
4827 */
4828 dma->dp_current_win = win;
4829 window = &dma->dp_window[win];
4830
4831 /* if needed, adjust the first and/or last cookies for trim */
4832 trim = &window->wd_trim;
4833 if (trim->tr_trim_first) {
4834 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4835 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4836 #if !defined(__amd64)
4837 window->wd_first_cookie->dmac_type =
4838 (window->wd_first_cookie->dmac_type &
4839 ROOTNEX_USES_COPYBUF) + window->wd_offset;
4840 #endif
4841 if (trim->tr_first_copybuf_win) {
4842 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4843 trim->tr_first_cbaddr;
4844 #if !defined(__amd64)
4845 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4846 trim->tr_first_kaddr;
4847 #endif
4848 }
4849 }
4850 if (trim->tr_trim_last) {
4851 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4852 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4853 if (trim->tr_last_copybuf_win) {
4854 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4855 trim->tr_last_cbaddr;
4856 #if !defined(__amd64)
4857 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4858 trim->tr_last_kaddr;
4859 #endif
4860 }
4861 }
4862
4863 /*
4864 * setup the cookie pointer to the first cookie in the window. setup
4865 * our return values, then increment the cookie since we return the
4866 * first cookie on the stack.
4867 */
4868 hp->dmai_cookie = window->wd_first_cookie;
4869 *offp = window->wd_offset;
4870 *lenp = window->wd_size;
4871 *ccountp = window->wd_cookie_cnt;
4872 *cookiep = hp->dmai_cookie[0];
4873 hp->dmai_cookie++;
4874
4875 #if !defined(__amd64)
4876 /* re-map copybuf if required for this window */
4877 if (dma->dp_cb_remaping) {
4878 /*
4879 * calculate the page index into the buffer where this
4880 * window starts.
4881 */
4882 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4883 MMU_PAGESHIFT;
4884 ASSERT(pidx < sinfo->si_max_pages);
4885
4886 /*
4887 * the first page can get unmapped if it's shared with the
4888 * previous window. Even if the rest of this window is already
4889 * mapped in, we need to still check this one.
4890 */
4891 pmap = &dma->dp_pgmap[pidx];
4892 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4893 if (pmap->pm_pp != NULL) {
4894 pmap->pm_mapped = B_TRUE;
4895 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4896 } else if (pmap->pm_vaddr != NULL) {
4897 pmap->pm_mapped = B_TRUE;
4898 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4899 pmap->pm_kaddr);
4900 }
4901 }
4902 pidx++;
4903
4904 /* map in the rest of the pages if required */
4905 if (window->wd_remap_copybuf) {
4906 window->wd_remap_copybuf = B_FALSE;
4907
4908 /* figure out many pages this window takes up */
4909 poff = (sinfo->si_buf_offset + window->wd_offset) &
4910 MMU_PAGEOFFSET;
4911 pcnt = mmu_btopr(window->wd_size + poff);
4912 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4913
4914 /* map pages which require it */
4915 for (i = 1; i < pcnt; i++) {
4916 pmap = &dma->dp_pgmap[pidx];
4917 if (pmap->pm_uses_copybuf) {
4918 ASSERT(pmap->pm_mapped == B_FALSE);
4919 if (pmap->pm_pp != NULL) {
4920 pmap->pm_mapped = B_TRUE;
4921 i86_pp_map(pmap->pm_pp,
4922 pmap->pm_kaddr);
4923 } else if (pmap->pm_vaddr != NULL) {
4924 pmap->pm_mapped = B_TRUE;
4925 i86_va_map(pmap->pm_vaddr,
4926 sinfo->si_asp,
4927 pmap->pm_kaddr);
4928 }
4929 }
4930 pidx++;
4931 }
4932 }
4933 }
4934 #endif
4935
4936 /* if the new window uses the copy buffer, sync it for the device */
4937 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4938 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4939 DDI_DMA_SYNC_FORDEV);
4940 }
4941
4942 return (DDI_SUCCESS);
4943 }
4944
4945 /*
4946 * rootnex_dma_win()
4947 * called from ddi_dma_getwin()
4948 */
4949 /*ARGSUSED*/
4950 static int
4951 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4952 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4953 uint_t *ccountp)
4954 {
4955 #if defined(__amd64) && !defined(__xpv)
4956 if (IOMMU_USED(rdip)) {
4957 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4958 cookiep, ccountp));
4959 }
4960 #endif
4961
4962 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4963 cookiep, ccountp));
4964 }
4965
4966 #if defined(__amd64) && !defined(__xpv)
4967 /*ARGSUSED*/
4968 static int
4969 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4970 ddi_dma_handle_t handle, void *v)
4971 {
4972 ddi_dma_impl_t *hp;
4973 rootnex_dma_t *dma;
4974
4975 hp = (ddi_dma_impl_t *)handle;
4976 dma = (rootnex_dma_t *)hp->dmai_private;
4977 dma->dp_iommu_private = v;
4978
4979 return (DDI_SUCCESS);
4980 }
4981
4982 /*ARGSUSED*/
4983 static void *
4984 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4985 ddi_dma_handle_t handle)
4986 {
4987 ddi_dma_impl_t *hp;
4988 rootnex_dma_t *dma;
4989
4990 hp = (ddi_dma_impl_t *)handle;
4991 dma = (rootnex_dma_t *)hp->dmai_private;
4992
4993 return (dma->dp_iommu_private);
4994 }
4995 #endif
4996
4997 /*
4998 * ************************
4999 * obsoleted dma routines
5000 * ************************
5001 */
5002
5003 /*
5004 * rootnex_dma_mctl()
5005 *
5006 * We don't support this legacy interface any more on x86.
5007 */
5008 /* ARGSUSED */
5009 static int
5010 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5011 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5012 uint_t cache_flags)
5013 {
5014 /*
5015 * The only thing dma_mctl is usef for anymore is legacy SPARC
5016 * dvma and sbus-specific routines.
5017 */
5018 return (DDI_FAILURE);
5019 }
5020
5021 /*
5022 * *********
5023 * FMA Code
5024 * *********
5025 */
5026
5027 /*
5028 * rootnex_fm_init()
5029 * FMA init busop
5030 */
5031 /* ARGSUSED */
5032 static int
5033 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5034 ddi_iblock_cookie_t *ibc)
5035 {
5036 *ibc = rootnex_state->r_err_ibc;
5037
5038 return (ddi_system_fmcap);
5039 }
5040
5041 /*
5042 * rootnex_dma_check()
5043 * Function called after a dma fault occurred to find out whether the
5044 * fault address is associated with a driver that is able to handle faults
5045 * and recover from faults.
5046 */
5047 /* ARGSUSED */
5048 static int
5049 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5050 const void *not_used)
5051 {
5052 rootnex_window_t *window;
5053 uint64_t start_addr;
5054 uint64_t fault_addr;
5055 ddi_dma_impl_t *hp;
5056 rootnex_dma_t *dma;
5057 uint64_t end_addr;
5058 size_t csize;
5059 int i;
5060 int j;
5061
5062
5063 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5064 hp = (ddi_dma_impl_t *)handle;
5065 ASSERT(hp);
5066
5067 dma = (rootnex_dma_t *)hp->dmai_private;
5068
5069 /* Get the address that we need to search for */
5070 fault_addr = *(uint64_t *)addr;
5071
5072 /*
5073 * if we don't have any windows, we can just walk through all the
5074 * cookies.
5075 */
5076 if (dma->dp_window == NULL) {
5077 /* for each cookie */
5078 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5079 /*
5080 * if the faulted address is within the physical address
5081 * range of the cookie, return DDI_FM_NONFATAL.
5082 */
5083 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5084 (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5085 dma->dp_cookies[i].dmac_size))) {
5086 return (DDI_FM_NONFATAL);
5087 }
5088 }
5089
5090 /* fault_addr not within this DMA handle */
5091 return (DDI_FM_UNKNOWN);
5092 }
5093
5094 /* we have mutiple windows, walk through each window */
5095 for (i = 0; i < hp->dmai_nwin; i++) {
5096 window = &dma->dp_window[i];
5097
5098 /* Go through all the cookies in the window */
5099 for (j = 0; j < window->wd_cookie_cnt; j++) {
5100
5101 start_addr = window->wd_first_cookie[j].dmac_laddress;
5102 csize = window->wd_first_cookie[j].dmac_size;
5103
5104 /*
5105 * if we are trimming the first cookie in the window,
5106 * and this is the first cookie, adjust the start
5107 * address and size of the cookie to account for the
5108 * trim.
5109 */
5110 if (window->wd_trim.tr_trim_first && (j == 0)) {
5111 start_addr = window->wd_trim.tr_first_paddr;
5112 csize = window->wd_trim.tr_first_size;
5113 }
5114
5115 /*
5116 * if we are trimming the last cookie in the window,
5117 * and this is the last cookie, adjust the start
5118 * address and size of the cookie to account for the
5119 * trim.
5120 */
5121 if (window->wd_trim.tr_trim_last &&
5122 (j == (window->wd_cookie_cnt - 1))) {
5123 start_addr = window->wd_trim.tr_last_paddr;
5124 csize = window->wd_trim.tr_last_size;
5125 }
5126
5127 end_addr = start_addr + csize;
5128
5129 /*
5130 * if the faulted address is within the physical
5131 * address of the cookie, return DDI_FM_NONFATAL.
5132 */
5133 if ((fault_addr >= start_addr) &&
5134 (fault_addr <= end_addr)) {
5135 return (DDI_FM_NONFATAL);
5136 }
5137 }
5138 }
5139
5140 /* fault_addr not within this DMA handle */
5141 return (DDI_FM_UNKNOWN);
5142 }
5143
5144 /*ARGSUSED*/
5145 static int
5146 rootnex_quiesce(dev_info_t *dip)
5147 {
5148 #if defined(__amd64) && !defined(__xpv)
5149 return (immu_quiesce());
5150 #else
5151 return (DDI_SUCCESS);
5152 #endif
5153 }
5154
5155 #if defined(__xpv)
5156 void
5157 immu_init(void)
5158 {
5159 ;
5160 }
5161
5162 void
5163 immu_startup(void)
5164 {
5165 ;
5166 }
5167 /*ARGSUSED*/
5168 void
5169 immu_physmem_update(uint64_t addr, uint64_t size)
5170 {
5171 ;
5172 }
5173 #endif