Print this page
8624 xhci and nvme can't bind DMA memory with IOMMU enabled
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/rootnex.c
+++ new/usr/src/uts/i86pc/io/rootnex.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24 /*
25 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
28 28 */
29 29
30 30 /*
31 31 * x86 root nexus driver
32 32 */
33 33
34 34 #include <sys/sysmacros.h>
35 35 #include <sys/conf.h>
36 36 #include <sys/autoconf.h>
37 37 #include <sys/sysmacros.h>
38 38 #include <sys/debug.h>
39 39 #include <sys/psw.h>
40 40 #include <sys/ddidmareq.h>
41 41 #include <sys/promif.h>
42 42 #include <sys/devops.h>
43 43 #include <sys/kmem.h>
44 44 #include <sys/cmn_err.h>
45 45 #include <vm/seg.h>
46 46 #include <vm/seg_kmem.h>
47 47 #include <vm/seg_dev.h>
48 48 #include <sys/vmem.h>
49 49 #include <sys/mman.h>
50 50 #include <vm/hat.h>
51 51 #include <vm/as.h>
52 52 #include <vm/page.h>
53 53 #include <sys/avintr.h>
54 54 #include <sys/errno.h>
55 55 #include <sys/modctl.h>
56 56 #include <sys/ddi_impldefs.h>
57 57 #include <sys/sunddi.h>
58 58 #include <sys/sunndi.h>
59 59 #include <sys/mach_intr.h>
60 60 #include <sys/psm.h>
61 61 #include <sys/ontrap.h>
62 62 #include <sys/atomic.h>
63 63 #include <sys/sdt.h>
64 64 #include <sys/rootnex.h>
65 65 #include <vm/hat_i86.h>
66 66 #include <sys/ddifm.h>
67 67 #include <sys/ddi_isa.h>
68 68 #include <sys/apic.h>
69 69
70 70 #ifdef __xpv
71 71 #include <sys/bootinfo.h>
72 72 #include <sys/hypervisor.h>
73 73 #include <sys/bootconf.h>
74 74 #include <vm/kboot_mmu.h>
75 75 #endif
76 76
77 77 #if defined(__amd64) && !defined(__xpv)
78 78 #include <sys/immu.h>
79 79 #endif
80 80
81 81
82 82 /*
83 83 * enable/disable extra checking of function parameters. Useful for debugging
84 84 * drivers.
85 85 */
86 86 #ifdef DEBUG
87 87 int rootnex_alloc_check_parms = 1;
88 88 int rootnex_bind_check_parms = 1;
89 89 int rootnex_bind_check_inuse = 1;
90 90 int rootnex_unbind_verify_buffer = 0;
91 91 int rootnex_sync_check_parms = 1;
92 92 #else
93 93 int rootnex_alloc_check_parms = 0;
94 94 int rootnex_bind_check_parms = 0;
95 95 int rootnex_bind_check_inuse = 0;
96 96 int rootnex_unbind_verify_buffer = 0;
97 97 int rootnex_sync_check_parms = 0;
98 98 #endif
99 99
100 100 boolean_t rootnex_dmar_not_setup;
101 101
102 102 /* Master Abort and Target Abort panic flag */
103 103 int rootnex_fm_ma_ta_panic_flag = 0;
104 104
105 105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
106 106 int rootnex_bind_fail = 1;
107 107 int rootnex_bind_warn = 1;
108 108 uint8_t *rootnex_warn_list;
109 109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
110 110 #define ROOTNEX_BIND_WARNING (0x1 << 0)
111 111
112 112 /*
113 113 * revert back to old broken behavior of always sync'ing entire copy buffer.
114 114 * This is useful if be have a buggy driver which doesn't correctly pass in
115 115 * the offset and size into ddi_dma_sync().
116 116 */
117 117 int rootnex_sync_ignore_params = 0;
118 118
119 119 /*
120 120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
121 121 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
122 122 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
123 123 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
124 124 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
125 125 * (< 8K). We will still need to allocate the copy buffer during bind though
126 126 * (if we need one). These can only be modified in /etc/system before rootnex
127 127 * attach.
128 128 */
129 129 #if defined(__amd64)
130 130 int rootnex_prealloc_cookies = 65;
131 131 int rootnex_prealloc_windows = 4;
132 132 int rootnex_prealloc_copybuf = 2;
133 133 #else
134 134 int rootnex_prealloc_cookies = 33;
135 135 int rootnex_prealloc_windows = 4;
136 136 int rootnex_prealloc_copybuf = 2;
137 137 #endif
138 138
139 139 /* driver global state */
140 140 static rootnex_state_t *rootnex_state;
141 141
142 142 #ifdef DEBUG
143 143 /* shortcut to rootnex counters */
144 144 static uint64_t *rootnex_cnt;
145 145 #endif
146 146
147 147 /*
148 148 * XXX - does x86 even need these or are they left over from the SPARC days?
149 149 */
150 150 /* statically defined integer/boolean properties for the root node */
151 151 static rootnex_intprop_t rootnex_intprp[] = {
152 152 { "PAGESIZE", PAGESIZE },
153 153 { "MMU_PAGESIZE", MMU_PAGESIZE },
154 154 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
155 155 { DDI_RELATIVE_ADDRESSING, 1 },
156 156 };
157 157 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
158 158
159 159 /*
160 160 * If we're dom0, we're using a real device so we need to load
161 161 * the cookies with MFNs instead of PFNs.
162 162 */
163 163 #ifdef __xpv
164 164 typedef maddr_t rootnex_addr_t;
165 165 #define ROOTNEX_PADDR_TO_RBASE(pa) \
166 166 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
167 167 #else
168 168 typedef paddr_t rootnex_addr_t;
169 169 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
170 170 #endif
171 171
172 172 static struct cb_ops rootnex_cb_ops = {
173 173 nodev, /* open */
174 174 nodev, /* close */
175 175 nodev, /* strategy */
176 176 nodev, /* print */
177 177 nodev, /* dump */
178 178 nodev, /* read */
179 179 nodev, /* write */
180 180 nodev, /* ioctl */
181 181 nodev, /* devmap */
182 182 nodev, /* mmap */
183 183 nodev, /* segmap */
184 184 nochpoll, /* chpoll */
185 185 ddi_prop_op, /* cb_prop_op */
186 186 NULL, /* struct streamtab */
187 187 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
188 188 CB_REV, /* Rev */
189 189 nodev, /* cb_aread */
190 190 nodev /* cb_awrite */
191 191 };
192 192
193 193 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
194 194 off_t offset, off_t len, caddr_t *vaddrp);
195 195 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
196 196 struct hat *hat, struct seg *seg, caddr_t addr,
197 197 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
198 198 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
199 199 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
200 200 ddi_dma_handle_t *handlep);
201 201 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
202 202 ddi_dma_handle_t handle);
203 203 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
204 204 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
205 205 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
206 206 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
207 207 ddi_dma_handle_t handle);
208 208 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
209 209 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
210 210 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
211 211 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
212 212 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
213 213 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
214 214 ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
215 215 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
216 216 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
217 217 ddi_ctl_enum_t ctlop, void *arg, void *result);
218 218 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
219 219 ddi_iblock_cookie_t *ibc);
220 220 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
221 221 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
222 222 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
223 223 void *);
224 224 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
225 225
226 226 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
227 227 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
228 228 ddi_dma_handle_t *handlep);
229 229 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
230 230 ddi_dma_handle_t handle);
231 231 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
232 232 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
233 233 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
234 234 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
235 235 ddi_dma_handle_t handle);
236 236 #if defined(__amd64) && !defined(__xpv)
237 237 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
238 238 ddi_dma_handle_t handle);
239 239 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
240 240 ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
241 241 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
242 242 ddi_dma_cookie_t *cookiep, uint_t ccount);
243 243 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
244 244 ddi_dma_handle_t handle);
245 245 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
246 246 #endif
247 247 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
248 248 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
249 249 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
250 250 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
251 251 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
252 252
253 253 #if defined(__amd64) && !defined(__xpv)
254 254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
255 255 ddi_dma_handle_t handle, void *v);
256 256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
257 257 ddi_dma_handle_t handle);
258 258 #endif
259 259
260 260
261 261 static struct bus_ops rootnex_bus_ops = {
262 262 BUSO_REV,
263 263 rootnex_map,
264 264 NULL,
265 265 NULL,
266 266 NULL,
267 267 rootnex_map_fault,
268 268 0,
269 269 rootnex_dma_allochdl,
270 270 rootnex_dma_freehdl,
271 271 rootnex_dma_bindhdl,
272 272 rootnex_dma_unbindhdl,
273 273 rootnex_dma_sync,
274 274 rootnex_dma_win,
275 275 rootnex_dma_mctl,
276 276 rootnex_ctlops,
277 277 ddi_bus_prop_op,
278 278 i_ddi_rootnex_get_eventcookie,
279 279 i_ddi_rootnex_add_eventcall,
280 280 i_ddi_rootnex_remove_eventcall,
281 281 i_ddi_rootnex_post_event,
282 282 0, /* bus_intr_ctl */
283 283 0, /* bus_config */
284 284 0, /* bus_unconfig */
285 285 rootnex_fm_init, /* bus_fm_init */
286 286 NULL, /* bus_fm_fini */
287 287 NULL, /* bus_fm_access_enter */
288 288 NULL, /* bus_fm_access_exit */
289 289 NULL, /* bus_powr */
290 290 rootnex_intr_ops /* bus_intr_op */
291 291 };
292 292
293 293 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
294 294 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
295 295 static int rootnex_quiesce(dev_info_t *dip);
296 296
297 297 static struct dev_ops rootnex_ops = {
298 298 DEVO_REV,
299 299 0,
300 300 ddi_no_info,
301 301 nulldev,
302 302 nulldev,
303 303 rootnex_attach,
304 304 rootnex_detach,
305 305 nulldev,
306 306 &rootnex_cb_ops,
307 307 &rootnex_bus_ops,
308 308 NULL,
309 309 rootnex_quiesce, /* quiesce */
310 310 };
311 311
312 312 static struct modldrv rootnex_modldrv = {
313 313 &mod_driverops,
314 314 "i86pc root nexus",
315 315 &rootnex_ops
316 316 };
317 317
318 318 static struct modlinkage rootnex_modlinkage = {
319 319 MODREV_1,
320 320 (void *)&rootnex_modldrv,
321 321 NULL
322 322 };
323 323
324 324 #if defined(__amd64) && !defined(__xpv)
325 325 static iommulib_nexops_t iommulib_nexops = {
326 326 IOMMU_NEXOPS_VERSION,
327 327 "Rootnex IOMMU ops Vers 1.1",
328 328 NULL,
329 329 rootnex_coredma_allochdl,
330 330 rootnex_coredma_freehdl,
331 331 rootnex_coredma_bindhdl,
332 332 rootnex_coredma_unbindhdl,
333 333 rootnex_coredma_reset_cookies,
334 334 rootnex_coredma_get_cookies,
335 335 rootnex_coredma_set_cookies,
336 336 rootnex_coredma_clear_cookies,
337 337 rootnex_coredma_get_sleep_flags,
338 338 rootnex_coredma_sync,
339 339 rootnex_coredma_win,
340 340 rootnex_coredma_hdl_setprivate,
341 341 rootnex_coredma_hdl_getprivate
342 342 };
343 343 #endif
344 344
345 345 /*
346 346 * extern hacks
347 347 */
348 348 extern struct seg_ops segdev_ops;
349 349 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
350 350 #ifdef DDI_MAP_DEBUG
351 351 extern int ddi_map_debug_flag;
352 352 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
353 353 #endif
354 354 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
355 355 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
356 356 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
357 357 psm_intr_op_t, int *);
358 358 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
359 359 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
360 360
361 361 /*
362 362 * Use device arena to use for device control register mappings.
363 363 * Various kernel memory walkers (debugger, dtrace) need to know
364 364 * to avoid this address range to prevent undesired device activity.
365 365 */
366 366 extern void *device_arena_alloc(size_t size, int vm_flag);
367 367 extern void device_arena_free(void * vaddr, size_t size);
368 368
369 369
370 370 /*
371 371 * Internal functions
372 372 */
373 373 static int rootnex_dma_init();
374 374 static void rootnex_add_props(dev_info_t *);
375 375 static int rootnex_ctl_reportdev(dev_info_t *dip);
376 376 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
377 377 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
378 378 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
379 379 static int rootnex_map_handle(ddi_map_req_t *mp);
380 380 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
381 381 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
382 382 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
383 383 ddi_dma_attr_t *attr);
384 384 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
385 385 rootnex_sglinfo_t *sglinfo);
386 386 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
387 387 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
388 388 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
389 389 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
390 390 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
391 391 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
392 392 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
393 393 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
394 394 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
395 395 static void rootnex_teardown_windows(rootnex_dma_t *dma);
396 396 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
397 397 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
398 398 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
399 399 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
400 400 size_t *copybuf_used, page_t **cur_pp);
401 401 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
402 402 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
403 403 ddi_dma_attr_t *attr, off_t cur_offset);
404 404 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
405 405 rootnex_dma_t *dma, rootnex_window_t **windowp,
406 406 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
407 407 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
408 408 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
409 409 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
410 410 off_t offset, size_t size, uint_t cache_flags);
411 411 static int rootnex_verify_buffer(rootnex_dma_t *dma);
412 412 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
413 413 const void *comp_addr, const void *not_used);
414 414 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
415 415 rootnex_sglinfo_t *sglinfo);
416 416 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
417 417
418 418 /*
419 419 * _init()
420 420 *
421 421 */
422 422 int
423 423 _init(void)
424 424 {
425 425
426 426 rootnex_state = NULL;
427 427 return (mod_install(&rootnex_modlinkage));
428 428 }
429 429
430 430
431 431 /*
432 432 * _info()
433 433 *
434 434 */
435 435 int
436 436 _info(struct modinfo *modinfop)
437 437 {
438 438 return (mod_info(&rootnex_modlinkage, modinfop));
439 439 }
440 440
441 441
442 442 /*
443 443 * _fini()
444 444 *
445 445 */
446 446 int
447 447 _fini(void)
448 448 {
449 449 return (EBUSY);
450 450 }
451 451
452 452
453 453 /*
454 454 * rootnex_attach()
455 455 *
456 456 */
457 457 static int
458 458 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
459 459 {
460 460 int fmcap;
461 461 int e;
462 462
463 463 switch (cmd) {
464 464 case DDI_ATTACH:
465 465 break;
466 466 case DDI_RESUME:
467 467 #if defined(__amd64) && !defined(__xpv)
468 468 return (immu_unquiesce());
469 469 #else
470 470 return (DDI_SUCCESS);
471 471 #endif
472 472 default:
473 473 return (DDI_FAILURE);
474 474 }
475 475
476 476 /*
477 477 * We should only have one instance of rootnex. Save it away since we
478 478 * don't have an easy way to get it back later.
479 479 */
480 480 ASSERT(rootnex_state == NULL);
481 481 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
482 482
483 483 rootnex_state->r_dip = dip;
484 484 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
485 485 rootnex_state->r_reserved_msg_printed = B_FALSE;
486 486 #ifdef DEBUG
487 487 rootnex_cnt = &rootnex_state->r_counters[0];
488 488 #endif
489 489
490 490 /*
491 491 * Set minimum fm capability level for i86pc platforms and then
492 492 * initialize error handling. Since we're the rootnex, we don't
493 493 * care what's returned in the fmcap field.
494 494 */
495 495 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
496 496 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
497 497 fmcap = ddi_system_fmcap;
498 498 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
499 499
500 500 /* initialize DMA related state */
501 501 e = rootnex_dma_init();
502 502 if (e != DDI_SUCCESS) {
503 503 kmem_free(rootnex_state, sizeof (rootnex_state_t));
504 504 return (DDI_FAILURE);
505 505 }
506 506
507 507 /* Add static root node properties */
508 508 rootnex_add_props(dip);
509 509
510 510 /* since we can't call ddi_report_dev() */
511 511 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
512 512
513 513 /* Initialize rootnex event handle */
514 514 i_ddi_rootnex_init_events(dip);
515 515
516 516 #if defined(__amd64) && !defined(__xpv)
517 517 e = iommulib_nexus_register(dip, &iommulib_nexops,
518 518 &rootnex_state->r_iommulib_handle);
519 519
520 520 ASSERT(e == DDI_SUCCESS);
521 521 #endif
522 522
523 523 return (DDI_SUCCESS);
524 524 }
525 525
526 526
527 527 /*
528 528 * rootnex_detach()
529 529 *
530 530 */
531 531 /*ARGSUSED*/
532 532 static int
533 533 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
534 534 {
535 535 switch (cmd) {
536 536 case DDI_SUSPEND:
537 537 #if defined(__amd64) && !defined(__xpv)
538 538 return (immu_quiesce());
539 539 #else
540 540 return (DDI_SUCCESS);
541 541 #endif
542 542 default:
543 543 return (DDI_FAILURE);
544 544 }
545 545 /*NOTREACHED*/
546 546
547 547 }
548 548
549 549
550 550 /*
551 551 * rootnex_dma_init()
552 552 *
553 553 */
554 554 /*ARGSUSED*/
555 555 static int
556 556 rootnex_dma_init()
557 557 {
558 558 size_t bufsize;
559 559
560 560
561 561 /*
562 562 * size of our cookie/window/copybuf state needed in dma bind that we
563 563 * pre-alloc in dma_alloc_handle
564 564 */
565 565 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
566 566 rootnex_state->r_prealloc_size =
567 567 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
568 568 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
569 569 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
570 570
571 571 /*
572 572 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
573 573 * allocate 16 extra bytes for struct pointer alignment
574 574 * (p->dmai_private & dma->dp_prealloc_buffer)
575 575 */
576 576 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
577 577 rootnex_state->r_prealloc_size + 0x10;
578 578 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
579 579 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
580 580 if (rootnex_state->r_dmahdl_cache == NULL) {
581 581 return (DDI_FAILURE);
582 582 }
583 583
584 584 /*
585 585 * allocate array to track which major numbers we have printed warnings
586 586 * for.
587 587 */
588 588 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
589 589 KM_SLEEP);
590 590
591 591 return (DDI_SUCCESS);
592 592 }
593 593
594 594
595 595 /*
596 596 * rootnex_add_props()
597 597 *
598 598 */
599 599 static void
600 600 rootnex_add_props(dev_info_t *dip)
601 601 {
602 602 rootnex_intprop_t *rpp;
603 603 int i;
604 604
605 605 /* Add static integer/boolean properties to the root node */
606 606 rpp = rootnex_intprp;
607 607 for (i = 0; i < NROOT_INTPROPS; i++) {
608 608 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
609 609 rpp[i].prop_name, rpp[i].prop_value);
610 610 }
611 611 }
612 612
613 613
614 614
615 615 /*
616 616 * *************************
617 617 * ctlops related routines
618 618 * *************************
619 619 */
620 620
621 621 /*
622 622 * rootnex_ctlops()
623 623 *
624 624 */
625 625 /*ARGSUSED*/
626 626 static int
627 627 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
628 628 void *arg, void *result)
629 629 {
630 630 int n, *ptr;
631 631 struct ddi_parent_private_data *pdp;
632 632
633 633 switch (ctlop) {
634 634 case DDI_CTLOPS_DMAPMAPC:
635 635 /*
636 636 * Return 'partial' to indicate that dma mapping
637 637 * has to be done in the main MMU.
638 638 */
639 639 return (DDI_DMA_PARTIAL);
640 640
641 641 case DDI_CTLOPS_BTOP:
642 642 /*
643 643 * Convert byte count input to physical page units.
644 644 * (byte counts that are not a page-size multiple
645 645 * are rounded down)
646 646 */
647 647 *(ulong_t *)result = btop(*(ulong_t *)arg);
648 648 return (DDI_SUCCESS);
649 649
650 650 case DDI_CTLOPS_PTOB:
651 651 /*
652 652 * Convert size in physical pages to bytes
653 653 */
654 654 *(ulong_t *)result = ptob(*(ulong_t *)arg);
655 655 return (DDI_SUCCESS);
656 656
657 657 case DDI_CTLOPS_BTOPR:
658 658 /*
659 659 * Convert byte count input to physical page units
660 660 * (byte counts that are not a page-size multiple
661 661 * are rounded up)
662 662 */
663 663 *(ulong_t *)result = btopr(*(ulong_t *)arg);
664 664 return (DDI_SUCCESS);
665 665
666 666 case DDI_CTLOPS_INITCHILD:
667 667 return (impl_ddi_sunbus_initchild(arg));
668 668
669 669 case DDI_CTLOPS_UNINITCHILD:
670 670 impl_ddi_sunbus_removechild(arg);
671 671 return (DDI_SUCCESS);
672 672
673 673 case DDI_CTLOPS_REPORTDEV:
674 674 return (rootnex_ctl_reportdev(rdip));
675 675
676 676 case DDI_CTLOPS_IOMIN:
677 677 /*
678 678 * Nothing to do here but reflect back..
679 679 */
680 680 return (DDI_SUCCESS);
681 681
682 682 case DDI_CTLOPS_REGSIZE:
683 683 case DDI_CTLOPS_NREGS:
684 684 break;
685 685
686 686 case DDI_CTLOPS_SIDDEV:
687 687 if (ndi_dev_is_prom_node(rdip))
688 688 return (DDI_SUCCESS);
689 689 if (ndi_dev_is_persistent_node(rdip))
690 690 return (DDI_SUCCESS);
691 691 return (DDI_FAILURE);
692 692
693 693 case DDI_CTLOPS_POWER:
694 694 return ((*pm_platform_power)((power_req_t *)arg));
695 695
696 696 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
697 697 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
698 698 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
699 699 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
700 700 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
701 701 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
702 702 if (!rootnex_state->r_reserved_msg_printed) {
703 703 rootnex_state->r_reserved_msg_printed = B_TRUE;
704 704 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
705 705 "1 or more reserved/obsolete operations.");
706 706 }
707 707 return (DDI_FAILURE);
708 708
709 709 default:
710 710 return (DDI_FAILURE);
711 711 }
712 712 /*
713 713 * The rest are for "hardware" properties
714 714 */
715 715 if ((pdp = ddi_get_parent_data(rdip)) == NULL)
716 716 return (DDI_FAILURE);
717 717
718 718 if (ctlop == DDI_CTLOPS_NREGS) {
719 719 ptr = (int *)result;
720 720 *ptr = pdp->par_nreg;
721 721 } else {
722 722 off_t *size = (off_t *)result;
723 723
724 724 ptr = (int *)arg;
725 725 n = *ptr;
726 726 if (n >= pdp->par_nreg) {
727 727 return (DDI_FAILURE);
728 728 }
729 729 *size = (off_t)pdp->par_reg[n].regspec_size;
730 730 }
731 731 return (DDI_SUCCESS);
732 732 }
733 733
734 734
735 735 /*
736 736 * rootnex_ctl_reportdev()
737 737 *
738 738 */
739 739 static int
740 740 rootnex_ctl_reportdev(dev_info_t *dev)
741 741 {
742 742 int i, n, len, f_len = 0;
743 743 char *buf;
744 744
745 745 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
746 746 f_len += snprintf(buf, REPORTDEV_BUFSIZE,
747 747 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
748 748 len = strlen(buf);
749 749
750 750 for (i = 0; i < sparc_pd_getnreg(dev); i++) {
751 751
752 752 struct regspec *rp = sparc_pd_getreg(dev, i);
753 753
754 754 if (i == 0)
755 755 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
756 756 ": ");
757 757 else
758 758 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
759 759 " and ");
760 760 len = strlen(buf);
761 761
762 762 switch (rp->regspec_bustype) {
763 763
764 764 case BTEISA:
765 765 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
766 766 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
767 767 break;
768 768
769 769 case BTISA:
770 770 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
771 771 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
772 772 break;
773 773
774 774 default:
775 775 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
776 776 "space %x offset %x",
777 777 rp->regspec_bustype, rp->regspec_addr);
778 778 break;
779 779 }
780 780 len = strlen(buf);
781 781 }
782 782 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
783 783 int pri;
784 784
785 785 if (i != 0) {
786 786 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
787 787 ",");
788 788 len = strlen(buf);
789 789 }
790 790 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
791 791 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
792 792 " sparc ipl %d", pri);
793 793 len = strlen(buf);
794 794 }
795 795 #ifdef DEBUG
796 796 if (f_len + 1 >= REPORTDEV_BUFSIZE) {
797 797 cmn_err(CE_NOTE, "next message is truncated: "
798 798 "printed length 1024, real length %d", f_len);
799 799 }
800 800 #endif /* DEBUG */
801 801 cmn_err(CE_CONT, "?%s\n", buf);
802 802 kmem_free(buf, REPORTDEV_BUFSIZE);
803 803 return (DDI_SUCCESS);
804 804 }
805 805
806 806
807 807 /*
808 808 * ******************
809 809 * map related code
810 810 * ******************
811 811 */
812 812
813 813 /*
814 814 * rootnex_map()
815 815 *
816 816 */
817 817 static int
818 818 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
819 819 off_t len, caddr_t *vaddrp)
820 820 {
821 821 struct regspec *rp, tmp_reg;
822 822 ddi_map_req_t mr = *mp; /* Get private copy of request */
823 823 int error;
824 824
825 825 mp = &mr;
826 826
827 827 switch (mp->map_op) {
828 828 case DDI_MO_MAP_LOCKED:
829 829 case DDI_MO_UNMAP:
830 830 case DDI_MO_MAP_HANDLE:
831 831 break;
832 832 default:
833 833 #ifdef DDI_MAP_DEBUG
834 834 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
835 835 mp->map_op);
836 836 #endif /* DDI_MAP_DEBUG */
837 837 return (DDI_ME_UNIMPLEMENTED);
838 838 }
839 839
840 840 if (mp->map_flags & DDI_MF_USER_MAPPING) {
841 841 #ifdef DDI_MAP_DEBUG
842 842 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
843 843 #endif /* DDI_MAP_DEBUG */
844 844 return (DDI_ME_UNIMPLEMENTED);
845 845 }
846 846
847 847 /*
848 848 * First, if given an rnumber, convert it to a regspec...
849 849 * (Presumably, this is on behalf of a child of the root node?)
850 850 */
851 851
852 852 if (mp->map_type == DDI_MT_RNUMBER) {
853 853
854 854 int rnumber = mp->map_obj.rnumber;
855 855 #ifdef DDI_MAP_DEBUG
856 856 static char *out_of_range =
857 857 "rootnex_map: Out of range rnumber <%d>, device <%s>";
858 858 #endif /* DDI_MAP_DEBUG */
859 859
860 860 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
861 861 if (rp == NULL) {
862 862 #ifdef DDI_MAP_DEBUG
863 863 cmn_err(CE_WARN, out_of_range, rnumber,
864 864 ddi_get_name(rdip));
865 865 #endif /* DDI_MAP_DEBUG */
866 866 return (DDI_ME_RNUMBER_RANGE);
867 867 }
868 868
869 869 /*
870 870 * Convert the given ddi_map_req_t from rnumber to regspec...
871 871 */
872 872
873 873 mp->map_type = DDI_MT_REGSPEC;
874 874 mp->map_obj.rp = rp;
875 875 }
876 876
877 877 /*
878 878 * Adjust offset and length correspnding to called values...
879 879 * XXX: A non-zero length means override the one in the regspec
880 880 * XXX: (regardless of what's in the parent's range?)
881 881 */
882 882
883 883 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */
884 884 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */
885 885
886 886 #ifdef DDI_MAP_DEBUG
887 887 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
888 888 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
889 889 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
890 890 len, mp->map_handlep);
891 891 #endif /* DDI_MAP_DEBUG */
892 892
893 893 /*
894 894 * I/O or memory mapping:
895 895 *
896 896 * <bustype=0, addr=x, len=x>: memory
897 897 * <bustype=1, addr=x, len=x>: i/o
898 898 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
899 899 */
900 900
901 901 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
902 902 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
903 903 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
904 904 ddi_get_name(rdip), rp->regspec_bustype,
905 905 rp->regspec_addr, rp->regspec_size);
906 906 return (DDI_ME_INVAL);
907 907 }
908 908
909 909 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
910 910 /*
911 911 * compatibility i/o mapping
912 912 */
913 913 rp->regspec_bustype += (uint_t)offset;
914 914 } else {
915 915 /*
916 916 * Normal memory or i/o mapping
917 917 */
918 918 rp->regspec_addr += (uint_t)offset;
919 919 }
920 920
921 921 if (len != 0)
922 922 rp->regspec_size = (uint_t)len;
923 923
924 924 #ifdef DDI_MAP_DEBUG
925 925 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
926 926 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
927 927 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
928 928 offset, len, mp->map_handlep);
929 929 #endif /* DDI_MAP_DEBUG */
930 930
931 931 /*
932 932 * Apply any parent ranges at this level, if applicable.
933 933 * (This is where nexus specific regspec translation takes place.
934 934 * Use of this function is implicit agreement that translation is
935 935 * provided via ddi_apply_range.)
936 936 */
937 937
938 938 #ifdef DDI_MAP_DEBUG
939 939 ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
940 940 ddi_get_name(dip), ddi_get_name(rdip));
941 941 #endif /* DDI_MAP_DEBUG */
942 942
943 943 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
944 944 return (error);
945 945
946 946 switch (mp->map_op) {
947 947 case DDI_MO_MAP_LOCKED:
948 948
949 949 /*
950 950 * Set up the locked down kernel mapping to the regspec...
951 951 */
952 952
953 953 return (rootnex_map_regspec(mp, vaddrp));
954 954
955 955 case DDI_MO_UNMAP:
956 956
957 957 /*
958 958 * Release mapping...
959 959 */
960 960
961 961 return (rootnex_unmap_regspec(mp, vaddrp));
962 962
963 963 case DDI_MO_MAP_HANDLE:
964 964
965 965 return (rootnex_map_handle(mp));
966 966
967 967 default:
968 968 return (DDI_ME_UNIMPLEMENTED);
969 969 }
970 970 }
971 971
972 972
973 973 /*
974 974 * rootnex_map_fault()
975 975 *
976 976 * fault in mappings for requestors
977 977 */
978 978 /*ARGSUSED*/
979 979 static int
980 980 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
981 981 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
982 982 uint_t lock)
983 983 {
984 984
985 985 #ifdef DDI_MAP_DEBUG
986 986 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
987 987 ddi_map_debug(" Seg <%s>\n",
988 988 seg->s_ops == &segdev_ops ? "segdev" :
989 989 seg == &kvseg ? "segkmem" : "NONE!");
990 990 #endif /* DDI_MAP_DEBUG */
991 991
992 992 /*
993 993 * This is all terribly broken, but it is a start
994 994 *
995 995 * XXX Note that this test means that segdev_ops
996 996 * must be exported from seg_dev.c.
997 997 * XXX What about devices with their own segment drivers?
998 998 */
999 999 if (seg->s_ops == &segdev_ops) {
1000 1000 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1001 1001
1002 1002 if (hat == NULL) {
1003 1003 /*
1004 1004 * This is one plausible interpretation of
1005 1005 * a null hat i.e. use the first hat on the
1006 1006 * address space hat list which by convention is
1007 1007 * the hat of the system MMU. At alternative
1008 1008 * would be to panic .. this might well be better ..
1009 1009 */
1010 1010 ASSERT(AS_READ_HELD(seg->s_as));
1011 1011 hat = seg->s_as->a_hat;
1012 1012 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1013 1013 }
1014 1014 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1015 1015 (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1016 1016 } else if (seg == &kvseg && dp == NULL) {
1017 1017 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1018 1018 HAT_LOAD_LOCK);
1019 1019 } else
1020 1020 return (DDI_FAILURE);
1021 1021 return (DDI_SUCCESS);
1022 1022 }
1023 1023
1024 1024
1025 1025 /*
1026 1026 * rootnex_map_regspec()
1027 1027 * we don't support mapping of I/O cards above 4Gb
1028 1028 */
1029 1029 static int
1030 1030 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1031 1031 {
1032 1032 rootnex_addr_t rbase;
1033 1033 void *cvaddr;
1034 1034 uint_t npages, pgoffset;
1035 1035 struct regspec *rp;
1036 1036 ddi_acc_hdl_t *hp;
1037 1037 ddi_acc_impl_t *ap;
1038 1038 uint_t hat_acc_flags;
1039 1039 paddr_t pbase;
1040 1040
1041 1041 rp = mp->map_obj.rp;
1042 1042 hp = mp->map_handlep;
1043 1043
1044 1044 #ifdef DDI_MAP_DEBUG
1045 1045 ddi_map_debug(
1046 1046 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1047 1047 rp->regspec_bustype, rp->regspec_addr,
1048 1048 rp->regspec_size, mp->map_handlep);
1049 1049 #endif /* DDI_MAP_DEBUG */
1050 1050
1051 1051 /*
1052 1052 * I/O or memory mapping
1053 1053 *
1054 1054 * <bustype=0, addr=x, len=x>: memory
1055 1055 * <bustype=1, addr=x, len=x>: i/o
1056 1056 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1057 1057 */
1058 1058
1059 1059 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1060 1060 cmn_err(CE_WARN, "rootnex: invalid register spec"
1061 1061 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1062 1062 rp->regspec_addr, rp->regspec_size);
1063 1063 return (DDI_FAILURE);
1064 1064 }
1065 1065
1066 1066 if (rp->regspec_bustype != 0) {
1067 1067 /*
1068 1068 * I/O space - needs a handle.
1069 1069 */
1070 1070 if (hp == NULL) {
1071 1071 return (DDI_FAILURE);
1072 1072 }
1073 1073 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1074 1074 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1075 1075 impl_acc_hdl_init(hp);
1076 1076
1077 1077 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1078 1078 #ifdef DDI_MAP_DEBUG
1079 1079 ddi_map_debug("rootnex_map_regspec: mmap() "
1080 1080 "to I/O space is not supported.\n");
1081 1081 #endif /* DDI_MAP_DEBUG */
1082 1082 return (DDI_ME_INVAL);
1083 1083 } else {
1084 1084 /*
1085 1085 * 1275-compliant vs. compatibility i/o mapping
1086 1086 */
1087 1087 *vaddrp =
1088 1088 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1089 1089 ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1090 1090 ((caddr_t)(uintptr_t)rp->regspec_addr);
1091 1091 #ifdef __xpv
1092 1092 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1093 1093 hp->ah_pfn = xen_assign_pfn(
1094 1094 mmu_btop((ulong_t)rp->regspec_addr &
1095 1095 MMU_PAGEMASK));
1096 1096 } else {
1097 1097 hp->ah_pfn = mmu_btop(
1098 1098 (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1099 1099 }
1100 1100 #else
1101 1101 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1102 1102 MMU_PAGEMASK);
1103 1103 #endif
1104 1104 hp->ah_pnum = mmu_btopr(rp->regspec_size +
1105 1105 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1106 1106 }
1107 1107
1108 1108 #ifdef DDI_MAP_DEBUG
1109 1109 ddi_map_debug(
1110 1110 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1111 1111 rp->regspec_size, *vaddrp);
1112 1112 #endif /* DDI_MAP_DEBUG */
1113 1113 return (DDI_SUCCESS);
1114 1114 }
1115 1115
1116 1116 /*
1117 1117 * Memory space
1118 1118 */
1119 1119
1120 1120 if (hp != NULL) {
1121 1121 /*
1122 1122 * hat layer ignores
1123 1123 * hp->ah_acc.devacc_attr_endian_flags.
1124 1124 */
1125 1125 switch (hp->ah_acc.devacc_attr_dataorder) {
1126 1126 case DDI_STRICTORDER_ACC:
1127 1127 hat_acc_flags = HAT_STRICTORDER;
1128 1128 break;
1129 1129 case DDI_UNORDERED_OK_ACC:
1130 1130 hat_acc_flags = HAT_UNORDERED_OK;
1131 1131 break;
1132 1132 case DDI_MERGING_OK_ACC:
1133 1133 hat_acc_flags = HAT_MERGING_OK;
1134 1134 break;
1135 1135 case DDI_LOADCACHING_OK_ACC:
1136 1136 hat_acc_flags = HAT_LOADCACHING_OK;
1137 1137 break;
1138 1138 case DDI_STORECACHING_OK_ACC:
1139 1139 hat_acc_flags = HAT_STORECACHING_OK;
1140 1140 break;
1141 1141 }
1142 1142 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1143 1143 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1144 1144 impl_acc_hdl_init(hp);
1145 1145 hp->ah_hat_flags = hat_acc_flags;
1146 1146 } else {
1147 1147 hat_acc_flags = HAT_STRICTORDER;
1148 1148 }
1149 1149
1150 1150 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1151 1151 #ifdef __xpv
1152 1152 /*
1153 1153 * If we're dom0, we're using a real device so we need to translate
1154 1154 * the MA to a PA.
1155 1155 */
1156 1156 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1157 1157 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1158 1158 } else {
1159 1159 pbase = rbase;
1160 1160 }
1161 1161 #else
1162 1162 pbase = rbase;
1163 1163 #endif
1164 1164 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1165 1165
1166 1166 if (rp->regspec_size == 0) {
1167 1167 #ifdef DDI_MAP_DEBUG
1168 1168 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1169 1169 #endif /* DDI_MAP_DEBUG */
1170 1170 return (DDI_ME_INVAL);
1171 1171 }
1172 1172
1173 1173 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1174 1174 /* extra cast to make gcc happy */
1175 1175 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1176 1176 } else {
1177 1177 npages = mmu_btopr(rp->regspec_size + pgoffset);
1178 1178
1179 1179 #ifdef DDI_MAP_DEBUG
1180 1180 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1181 1181 "physical %llx", npages, pbase);
1182 1182 #endif /* DDI_MAP_DEBUG */
1183 1183
1184 1184 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1185 1185 if (cvaddr == NULL)
1186 1186 return (DDI_ME_NORESOURCES);
1187 1187
1188 1188 /*
1189 1189 * Now map in the pages we've allocated...
1190 1190 */
1191 1191 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1192 1192 mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1193 1193 HAT_LOAD_LOCK);
1194 1194 *vaddrp = (caddr_t)cvaddr + pgoffset;
1195 1195
1196 1196 /* save away pfn and npages for FMA */
1197 1197 hp = mp->map_handlep;
1198 1198 if (hp) {
1199 1199 hp->ah_pfn = mmu_btop(pbase);
1200 1200 hp->ah_pnum = npages;
1201 1201 }
1202 1202 }
1203 1203
1204 1204 #ifdef DDI_MAP_DEBUG
1205 1205 ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1206 1206 #endif /* DDI_MAP_DEBUG */
1207 1207 return (DDI_SUCCESS);
1208 1208 }
1209 1209
1210 1210
1211 1211 /*
1212 1212 * rootnex_unmap_regspec()
1213 1213 *
1214 1214 */
1215 1215 static int
1216 1216 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1217 1217 {
1218 1218 caddr_t addr = (caddr_t)*vaddrp;
1219 1219 uint_t npages, pgoffset;
1220 1220 struct regspec *rp;
1221 1221
1222 1222 if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1223 1223 return (0);
1224 1224
1225 1225 rp = mp->map_obj.rp;
1226 1226
1227 1227 if (rp->regspec_size == 0) {
1228 1228 #ifdef DDI_MAP_DEBUG
1229 1229 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1230 1230 #endif /* DDI_MAP_DEBUG */
1231 1231 return (DDI_ME_INVAL);
1232 1232 }
1233 1233
1234 1234 /*
1235 1235 * I/O or memory mapping:
1236 1236 *
1237 1237 * <bustype=0, addr=x, len=x>: memory
1238 1238 * <bustype=1, addr=x, len=x>: i/o
1239 1239 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1240 1240 */
1241 1241 if (rp->regspec_bustype != 0) {
1242 1242 /*
1243 1243 * This is I/O space, which requires no particular
1244 1244 * processing on unmap since it isn't mapped in the
1245 1245 * first place.
1246 1246 */
1247 1247 return (DDI_SUCCESS);
1248 1248 }
1249 1249
1250 1250 /*
1251 1251 * Memory space
1252 1252 */
1253 1253 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1254 1254 npages = mmu_btopr(rp->regspec_size + pgoffset);
1255 1255 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1256 1256 device_arena_free(addr - pgoffset, ptob(npages));
1257 1257
1258 1258 /*
1259 1259 * Destroy the pointer - the mapping has logically gone
1260 1260 */
1261 1261 *vaddrp = NULL;
1262 1262
1263 1263 return (DDI_SUCCESS);
1264 1264 }
1265 1265
1266 1266
1267 1267 /*
1268 1268 * rootnex_map_handle()
1269 1269 *
1270 1270 */
1271 1271 static int
1272 1272 rootnex_map_handle(ddi_map_req_t *mp)
1273 1273 {
1274 1274 rootnex_addr_t rbase;
1275 1275 ddi_acc_hdl_t *hp;
1276 1276 uint_t pgoffset;
1277 1277 struct regspec *rp;
1278 1278 paddr_t pbase;
1279 1279
1280 1280 rp = mp->map_obj.rp;
1281 1281
1282 1282 #ifdef DDI_MAP_DEBUG
1283 1283 ddi_map_debug(
1284 1284 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1285 1285 rp->regspec_bustype, rp->regspec_addr,
1286 1286 rp->regspec_size, mp->map_handlep);
1287 1287 #endif /* DDI_MAP_DEBUG */
1288 1288
1289 1289 /*
1290 1290 * I/O or memory mapping:
1291 1291 *
1292 1292 * <bustype=0, addr=x, len=x>: memory
1293 1293 * <bustype=1, addr=x, len=x>: i/o
1294 1294 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1295 1295 */
1296 1296 if (rp->regspec_bustype != 0) {
1297 1297 /*
1298 1298 * This refers to I/O space, and we don't support "mapping"
1299 1299 * I/O space to a user.
1300 1300 */
1301 1301 return (DDI_FAILURE);
1302 1302 }
1303 1303
1304 1304 /*
1305 1305 * Set up the hat_flags for the mapping.
1306 1306 */
1307 1307 hp = mp->map_handlep;
1308 1308
1309 1309 switch (hp->ah_acc.devacc_attr_endian_flags) {
1310 1310 case DDI_NEVERSWAP_ACC:
1311 1311 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1312 1312 break;
1313 1313 case DDI_STRUCTURE_LE_ACC:
1314 1314 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1315 1315 break;
1316 1316 case DDI_STRUCTURE_BE_ACC:
1317 1317 return (DDI_FAILURE);
1318 1318 default:
1319 1319 return (DDI_REGS_ACC_CONFLICT);
1320 1320 }
1321 1321
1322 1322 switch (hp->ah_acc.devacc_attr_dataorder) {
1323 1323 case DDI_STRICTORDER_ACC:
1324 1324 break;
1325 1325 case DDI_UNORDERED_OK_ACC:
1326 1326 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1327 1327 break;
1328 1328 case DDI_MERGING_OK_ACC:
1329 1329 hp->ah_hat_flags |= HAT_MERGING_OK;
1330 1330 break;
1331 1331 case DDI_LOADCACHING_OK_ACC:
1332 1332 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1333 1333 break;
1334 1334 case DDI_STORECACHING_OK_ACC:
1335 1335 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1336 1336 break;
1337 1337 default:
1338 1338 return (DDI_FAILURE);
1339 1339 }
1340 1340
1341 1341 rbase = (rootnex_addr_t)rp->regspec_addr &
1342 1342 (~(rootnex_addr_t)MMU_PAGEOFFSET);
1343 1343 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1344 1344
1345 1345 if (rp->regspec_size == 0)
1346 1346 return (DDI_ME_INVAL);
1347 1347
1348 1348 #ifdef __xpv
1349 1349 /*
1350 1350 * If we're dom0, we're using a real device so we need to translate
1351 1351 * the MA to a PA.
1352 1352 */
1353 1353 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1354 1354 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1355 1355 (rbase & MMU_PAGEOFFSET);
1356 1356 } else {
1357 1357 pbase = rbase;
1358 1358 }
1359 1359 #else
1360 1360 pbase = rbase;
1361 1361 #endif
1362 1362
1363 1363 hp->ah_pfn = mmu_btop(pbase);
1364 1364 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1365 1365
1366 1366 return (DDI_SUCCESS);
1367 1367 }
1368 1368
1369 1369
1370 1370
1371 1371 /*
1372 1372 * ************************
1373 1373 * interrupt related code
1374 1374 * ************************
1375 1375 */
1376 1376
1377 1377 /*
1378 1378 * rootnex_intr_ops()
1379 1379 * bus_intr_op() function for interrupt support
1380 1380 */
1381 1381 /* ARGSUSED */
1382 1382 static int
1383 1383 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1384 1384 ddi_intr_handle_impl_t *hdlp, void *result)
1385 1385 {
1386 1386 struct intrspec *ispec;
1387 1387
1388 1388 DDI_INTR_NEXDBG((CE_CONT,
1389 1389 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1390 1390 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1391 1391
1392 1392 /* Process the interrupt operation */
1393 1393 switch (intr_op) {
1394 1394 case DDI_INTROP_GETCAP:
1395 1395 /* First check with pcplusmp */
1396 1396 if (psm_intr_ops == NULL)
1397 1397 return (DDI_FAILURE);
1398 1398
1399 1399 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1400 1400 *(int *)result = 0;
1401 1401 return (DDI_FAILURE);
1402 1402 }
1403 1403 break;
1404 1404 case DDI_INTROP_SETCAP:
1405 1405 if (psm_intr_ops == NULL)
1406 1406 return (DDI_FAILURE);
1407 1407
1408 1408 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1409 1409 return (DDI_FAILURE);
1410 1410 break;
1411 1411 case DDI_INTROP_ALLOC:
1412 1412 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1413 1413 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1414 1414 case DDI_INTROP_FREE:
1415 1415 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1416 1416 return (rootnex_free_intr_fixed(rdip, hdlp));
1417 1417 case DDI_INTROP_GETPRI:
1418 1418 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1419 1419 return (DDI_FAILURE);
1420 1420 *(int *)result = ispec->intrspec_pri;
1421 1421 break;
1422 1422 case DDI_INTROP_SETPRI:
1423 1423 /* Validate the interrupt priority passed to us */
1424 1424 if (*(int *)result > LOCK_LEVEL)
1425 1425 return (DDI_FAILURE);
1426 1426
1427 1427 /* Ensure that PSM is all initialized and ispec is ok */
1428 1428 if ((psm_intr_ops == NULL) ||
1429 1429 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1430 1430 return (DDI_FAILURE);
1431 1431
1432 1432 /* Change the priority */
1433 1433 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1434 1434 PSM_FAILURE)
1435 1435 return (DDI_FAILURE);
1436 1436
1437 1437 /* update the ispec with the new priority */
1438 1438 ispec->intrspec_pri = *(int *)result;
1439 1439 break;
1440 1440 case DDI_INTROP_ADDISR:
1441 1441 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1442 1442 return (DDI_FAILURE);
1443 1443 ispec->intrspec_func = hdlp->ih_cb_func;
1444 1444 break;
1445 1445 case DDI_INTROP_REMISR:
1446 1446 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1447 1447 return (DDI_FAILURE);
1448 1448 ispec->intrspec_func = (uint_t (*)()) 0;
1449 1449 break;
1450 1450 case DDI_INTROP_ENABLE:
1451 1451 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1452 1452 return (DDI_FAILURE);
1453 1453
1454 1454 /* Call psmi to translate irq with the dip */
1455 1455 if (psm_intr_ops == NULL)
1456 1456 return (DDI_FAILURE);
1457 1457
1458 1458 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1459 1459 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1460 1460 (int *)&hdlp->ih_vector) == PSM_FAILURE)
1461 1461 return (DDI_FAILURE);
1462 1462
1463 1463 /* Add the interrupt handler */
1464 1464 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1465 1465 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1466 1466 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1467 1467 return (DDI_FAILURE);
1468 1468 break;
1469 1469 case DDI_INTROP_DISABLE:
1470 1470 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1471 1471 return (DDI_FAILURE);
1472 1472
1473 1473 /* Call psm_ops() to translate irq with the dip */
1474 1474 if (psm_intr_ops == NULL)
1475 1475 return (DDI_FAILURE);
1476 1476
1477 1477 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1478 1478 (void) (*psm_intr_ops)(rdip, hdlp,
1479 1479 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1480 1480
1481 1481 /* Remove the interrupt handler */
1482 1482 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1483 1483 hdlp->ih_cb_func, hdlp->ih_vector);
1484 1484 break;
1485 1485 case DDI_INTROP_SETMASK:
1486 1486 if (psm_intr_ops == NULL)
1487 1487 return (DDI_FAILURE);
1488 1488
1489 1489 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1490 1490 return (DDI_FAILURE);
1491 1491 break;
1492 1492 case DDI_INTROP_CLRMASK:
1493 1493 if (psm_intr_ops == NULL)
1494 1494 return (DDI_FAILURE);
1495 1495
1496 1496 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1497 1497 return (DDI_FAILURE);
1498 1498 break;
1499 1499 case DDI_INTROP_GETPENDING:
1500 1500 if (psm_intr_ops == NULL)
1501 1501 return (DDI_FAILURE);
1502 1502
1503 1503 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1504 1504 result)) {
1505 1505 *(int *)result = 0;
1506 1506 return (DDI_FAILURE);
1507 1507 }
1508 1508 break;
1509 1509 case DDI_INTROP_NAVAIL:
1510 1510 case DDI_INTROP_NINTRS:
1511 1511 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1512 1512 if (*(int *)result == 0) {
1513 1513 /*
1514 1514 * Special case for 'pcic' driver' only. This driver
1515 1515 * driver is a child of 'isa' and 'rootnex' drivers.
1516 1516 *
1517 1517 * See detailed comments on this in the function
1518 1518 * rootnex_get_ispec().
1519 1519 *
1520 1520 * Children of 'pcic' send 'NINITR' request all the
1521 1521 * way to rootnex driver. But, the 'pdp->par_nintr'
1522 1522 * field may not initialized. So, we fake it here
1523 1523 * to return 1 (a la what PCMCIA nexus does).
1524 1524 */
1525 1525 if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1526 1526 *(int *)result = 1;
1527 1527 else
1528 1528 return (DDI_FAILURE);
1529 1529 }
1530 1530 break;
1531 1531 case DDI_INTROP_SUPPORTED_TYPES:
1532 1532 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
1533 1533 break;
1534 1534 default:
1535 1535 return (DDI_FAILURE);
1536 1536 }
1537 1537
1538 1538 return (DDI_SUCCESS);
1539 1539 }
1540 1540
1541 1541
1542 1542 /*
1543 1543 * rootnex_get_ispec()
1544 1544 * convert an interrupt number to an interrupt specification.
1545 1545 * The interrupt number determines which interrupt spec will be
1546 1546 * returned if more than one exists.
1547 1547 *
1548 1548 * Look into the parent private data area of the 'rdip' to find out
1549 1549 * the interrupt specification. First check to make sure there is
1550 1550 * one that matchs "inumber" and then return a pointer to it.
1551 1551 *
1552 1552 * Return NULL if one could not be found.
1553 1553 *
1554 1554 * NOTE: This is needed for rootnex_intr_ops()
1555 1555 */
1556 1556 static struct intrspec *
1557 1557 rootnex_get_ispec(dev_info_t *rdip, int inum)
1558 1558 {
1559 1559 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1560 1560
1561 1561 /*
1562 1562 * Special case handling for drivers that provide their own
1563 1563 * intrspec structures instead of relying on the DDI framework.
1564 1564 *
1565 1565 * A broken hardware driver in ON could potentially provide its
1566 1566 * own intrspec structure, instead of relying on the hardware.
1567 1567 * If these drivers are children of 'rootnex' then we need to
1568 1568 * continue to provide backward compatibility to them here.
1569 1569 *
1570 1570 * Following check is a special case for 'pcic' driver which
1571 1571 * was found to have broken hardwre andby provides its own intrspec.
1572 1572 *
1573 1573 * Verbatim comments from this driver are shown here:
1574 1574 * "Don't use the ddi_add_intr since we don't have a
1575 1575 * default intrspec in all cases."
1576 1576 *
1577 1577 * Since an 'ispec' may not be always created for it,
1578 1578 * check for that and create one if so.
1579 1579 *
1580 1580 * NOTE: Currently 'pcic' is the only driver found to do this.
1581 1581 */
1582 1582 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1583 1583 pdp->par_nintr = 1;
1584 1584 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1585 1585 pdp->par_nintr, KM_SLEEP);
1586 1586 }
1587 1587
1588 1588 /* Validate the interrupt number */
1589 1589 if (inum >= pdp->par_nintr)
1590 1590 return (NULL);
1591 1591
1592 1592 /* Get the interrupt structure pointer and return that */
1593 1593 return ((struct intrspec *)&pdp->par_intr[inum]);
1594 1594 }
1595 1595
1596 1596 /*
1597 1597 * Allocate interrupt vector for FIXED (legacy) type.
1598 1598 */
1599 1599 static int
1600 1600 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1601 1601 void *result)
1602 1602 {
1603 1603 struct intrspec *ispec;
1604 1604 ddi_intr_handle_impl_t info_hdl;
1605 1605 int ret;
1606 1606 int free_phdl = 0;
1607 1607 apic_get_type_t type_info;
1608 1608
1609 1609 if (psm_intr_ops == NULL)
1610 1610 return (DDI_FAILURE);
1611 1611
1612 1612 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1613 1613 return (DDI_FAILURE);
1614 1614
1615 1615 /*
1616 1616 * If the PSM module is "APIX" then pass the request for it
1617 1617 * to allocate the vector now.
1618 1618 */
1619 1619 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1620 1620 info_hdl.ih_private = &type_info;
1621 1621 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1622 1622 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1623 1623 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1624 1624 free_phdl = 1;
1625 1625 i_ddi_alloc_intr_phdl(hdlp);
1626 1626 }
1627 1627 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1628 1628 ret = (*psm_intr_ops)(rdip, hdlp,
1629 1629 PSM_INTR_OP_ALLOC_VECTORS, result);
1630 1630 if (free_phdl) { /* free up the phdl structure */
1631 1631 free_phdl = 0;
1632 1632 i_ddi_free_intr_phdl(hdlp);
1633 1633 hdlp->ih_private = NULL;
1634 1634 }
1635 1635 } else {
1636 1636 /*
1637 1637 * No APIX module; fall back to the old scheme where the
1638 1638 * interrupt vector is allocated during ddi_enable_intr() call.
1639 1639 */
1640 1640 hdlp->ih_pri = ispec->intrspec_pri;
1641 1641 *(int *)result = hdlp->ih_scratch1;
1642 1642 ret = DDI_SUCCESS;
1643 1643 }
1644 1644
1645 1645 return (ret);
1646 1646 }
1647 1647
1648 1648 /*
1649 1649 * Free up interrupt vector for FIXED (legacy) type.
1650 1650 */
1651 1651 static int
1652 1652 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1653 1653 {
1654 1654 struct intrspec *ispec;
1655 1655 struct ddi_parent_private_data *pdp;
1656 1656 ddi_intr_handle_impl_t info_hdl;
1657 1657 int ret;
1658 1658 apic_get_type_t type_info;
1659 1659
1660 1660 if (psm_intr_ops == NULL)
1661 1661 return (DDI_FAILURE);
1662 1662
1663 1663 /*
1664 1664 * If the PSM module is "APIX" then pass the request for it
1665 1665 * to free up the vector now.
1666 1666 */
1667 1667 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1668 1668 info_hdl.ih_private = &type_info;
1669 1669 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1670 1670 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1671 1671 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1672 1672 return (DDI_FAILURE);
1673 1673 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1674 1674 ret = (*psm_intr_ops)(rdip, hdlp,
1675 1675 PSM_INTR_OP_FREE_VECTORS, NULL);
1676 1676 } else {
1677 1677 /*
1678 1678 * No APIX module; fall back to the old scheme where
1679 1679 * the interrupt vector was already freed during
1680 1680 * ddi_disable_intr() call.
1681 1681 */
1682 1682 ret = DDI_SUCCESS;
1683 1683 }
1684 1684
1685 1685 pdp = ddi_get_parent_data(rdip);
1686 1686
1687 1687 /*
1688 1688 * Special case for 'pcic' driver' only.
1689 1689 * If an intrspec was created for it, clean it up here
1690 1690 * See detailed comments on this in the function
1691 1691 * rootnex_get_ispec().
1692 1692 */
1693 1693 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1694 1694 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1695 1695 pdp->par_nintr);
1696 1696 /*
1697 1697 * Set it to zero; so that
1698 1698 * DDI framework doesn't free it again
1699 1699 */
1700 1700 pdp->par_intr = NULL;
1701 1701 pdp->par_nintr = 0;
1702 1702 }
1703 1703
1704 1704 return (ret);
1705 1705 }
1706 1706
1707 1707
1708 1708 /*
1709 1709 * ******************
1710 1710 * dma related code
1711 1711 * ******************
1712 1712 */
1713 1713
1714 1714 /*ARGSUSED*/
1715 1715 static int
1716 1716 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1717 1717 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1718 1718 ddi_dma_handle_t *handlep)
1719 1719 {
1720 1720 uint64_t maxsegmentsize_ll;
1721 1721 uint_t maxsegmentsize;
1722 1722 ddi_dma_impl_t *hp;
1723 1723 rootnex_dma_t *dma;
1724 1724 uint64_t count_max;
1725 1725 uint64_t seg;
1726 1726 int kmflag;
1727 1727 int e;
1728 1728
1729 1729
1730 1730 /* convert our sleep flags */
1731 1731 if (waitfp == DDI_DMA_SLEEP) {
1732 1732 kmflag = KM_SLEEP;
1733 1733 } else {
1734 1734 kmflag = KM_NOSLEEP;
1735 1735 }
1736 1736
1737 1737 /*
1738 1738 * We try to do only one memory allocation here. We'll do a little
1739 1739 * pointer manipulation later. If the bind ends up taking more than
1740 1740 * our prealloc's space, we'll have to allocate more memory in the
1741 1741 * bind operation. Not great, but much better than before and the
1742 1742 * best we can do with the current bind interfaces.
1743 1743 */
1744 1744 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1745 1745 if (hp == NULL)
1746 1746 return (DDI_DMA_NORESOURCES);
1747 1747
1748 1748 /* Do our pointer manipulation now, align the structures */
1749 1749 hp->dmai_private = (void *)(((uintptr_t)hp +
1750 1750 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1751 1751 dma = (rootnex_dma_t *)hp->dmai_private;
1752 1752 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1753 1753 sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1754 1754
1755 1755 /* setup the handle */
1756 1756 rootnex_clean_dmahdl(hp);
1757 1757 hp->dmai_error.err_fep = NULL;
1758 1758 hp->dmai_error.err_cf = NULL;
1759 1759 dma->dp_dip = rdip;
1760 1760 dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1761 1761 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1762 1762
1763 1763 /*
1764 1764 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1765 1765 * is being used. Set the upper limit to the seg value.
1766 1766 * There will be enough DVMA space to always get addresses
1767 1767 * that will match the constraints.
1768 1768 */
1769 1769 if (IOMMU_USED(rdip) &&
1770 1770 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1771 1771 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1772 1772 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1773 1773 } else
1774 1774 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1775 1775
1776 1776 hp->dmai_minxfer = attr->dma_attr_minxfer;
1777 1777 hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1778 1778 hp->dmai_rdip = rdip;
1779 1779 hp->dmai_attr = *attr;
1780 1780
1781 1781 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1782 1782 dma->dp_sglinfo.si_cancross = B_FALSE;
1783 1783 else
1784 1784 dma->dp_sglinfo.si_cancross = B_TRUE;
1785 1785
1786 1786 /* we don't need to worry about the SPL since we do a tryenter */
1787 1787 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1788 1788
1789 1789 /*
1790 1790 * Figure out our maximum segment size. If the segment size is greater
1791 1791 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1792 1792 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1793 1793 * dma_attr_count_max are size-1 type values.
1794 1794 *
1795 1795 * Maximum segment size is the largest physically contiguous chunk of
1796 1796 * memory that we can return from a bind (i.e. the maximum size of a
1797 1797 * single cookie).
1798 1798 */
1799 1799
1800 1800 /* handle the rollover cases */
1801 1801 seg = attr->dma_attr_seg + 1;
1802 1802 if (seg < attr->dma_attr_seg) {
1803 1803 seg = attr->dma_attr_seg;
1804 1804 }
1805 1805 count_max = attr->dma_attr_count_max + 1;
1806 1806 if (count_max < attr->dma_attr_count_max) {
1807 1807 count_max = attr->dma_attr_count_max;
1808 1808 }
1809 1809
1810 1810 /*
1811 1811 * granularity may or may not be a power of two. If it isn't, we can't
1812 1812 * use a simple mask.
1813 1813 */
1814 1814 if (!ISP2(attr->dma_attr_granular)) {
1815 1815 dma->dp_granularity_power_2 = B_FALSE;
1816 1816 } else {
1817 1817 dma->dp_granularity_power_2 = B_TRUE;
1818 1818 }
1819 1819
1820 1820 /*
1821 1821 * maxxfer should be a whole multiple of granularity. If we're going to
1822 1822 * break up a window because we're greater than maxxfer, we might as
1823 1823 * well make sure it's maxxfer is a whole multiple so we don't have to
1824 1824 * worry about triming the window later on for this case.
1825 1825 */
1826 1826 if (attr->dma_attr_granular > 1) {
1827 1827 if (dma->dp_granularity_power_2) {
1828 1828 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1829 1829 (attr->dma_attr_maxxfer &
1830 1830 (attr->dma_attr_granular - 1));
1831 1831 } else {
1832 1832 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1833 1833 (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1834 1834 }
1835 1835 } else {
1836 1836 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1837 1837 }
1838 1838
1839 1839 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1840 1840 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1841 1841 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1842 1842 maxsegmentsize = 0xFFFFFFFF;
1843 1843 } else {
1844 1844 maxsegmentsize = maxsegmentsize_ll;
1845 1845 }
1846 1846 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1847 1847 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1848 1848
1849 1849 /* check the ddi_dma_attr arg to make sure it makes a little sense */
1850 1850 if (rootnex_alloc_check_parms) {
1851 1851 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1852 1852 if (e != DDI_SUCCESS) {
1853 1853 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1854 1854 (void) rootnex_dma_freehdl(dip, rdip,
1855 1855 (ddi_dma_handle_t)hp);
1856 1856 return (e);
1857 1857 }
1858 1858 }
1859 1859
1860 1860 *handlep = (ddi_dma_handle_t)hp;
1861 1861
1862 1862 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1863 1863 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1864 1864 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1865 1865
1866 1866 return (DDI_SUCCESS);
1867 1867 }
1868 1868
1869 1869
1870 1870 /*
1871 1871 * rootnex_dma_allochdl()
1872 1872 * called from ddi_dma_alloc_handle().
1873 1873 */
1874 1874 static int
1875 1875 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1876 1876 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1877 1877 {
1878 1878 int retval = DDI_SUCCESS;
1879 1879 #if defined(__amd64) && !defined(__xpv)
1880 1880
1881 1881 if (IOMMU_UNITIALIZED(rdip)) {
1882 1882 retval = iommulib_nex_open(dip, rdip);
1883 1883
1884 1884 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1885 1885 return (retval);
1886 1886 }
1887 1887
1888 1888 if (IOMMU_UNUSED(rdip)) {
1889 1889 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1890 1890 handlep);
1891 1891 } else {
1892 1892 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1893 1893 waitfp, arg, handlep);
1894 1894 }
1895 1895 #else
1896 1896 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1897 1897 handlep);
1898 1898 #endif
1899 1899 switch (retval) {
1900 1900 case DDI_DMA_NORESOURCES:
1901 1901 if (waitfp != DDI_DMA_DONTWAIT) {
1902 1902 ddi_set_callback(waitfp, arg,
1903 1903 &rootnex_state->r_dvma_call_list_id);
1904 1904 }
1905 1905 break;
1906 1906 case DDI_SUCCESS:
1907 1907 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1908 1908 break;
1909 1909 default:
1910 1910 break;
1911 1911 }
1912 1912 return (retval);
1913 1913 }
1914 1914
1915 1915 /*ARGSUSED*/
1916 1916 static int
1917 1917 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1918 1918 ddi_dma_handle_t handle)
1919 1919 {
1920 1920 ddi_dma_impl_t *hp;
1921 1921 rootnex_dma_t *dma;
1922 1922
1923 1923
1924 1924 hp = (ddi_dma_impl_t *)handle;
1925 1925 dma = (rootnex_dma_t *)hp->dmai_private;
1926 1926
1927 1927 /* unbind should have been called first */
1928 1928 ASSERT(!dma->dp_inuse);
1929 1929
1930 1930 mutex_destroy(&dma->dp_mutex);
1931 1931 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1932 1932
1933 1933 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1934 1934 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1935 1935 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1936 1936
1937 1937 return (DDI_SUCCESS);
1938 1938 }
1939 1939
1940 1940 /*
1941 1941 * rootnex_dma_freehdl()
1942 1942 * called from ddi_dma_free_handle().
1943 1943 */
1944 1944 static int
1945 1945 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1946 1946 {
1947 1947 int ret;
1948 1948
1949 1949 ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1950 1950 #if defined(__amd64) && !defined(__xpv)
1951 1951 if (IOMMU_USED(rdip))
1952 1952 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1953 1953 else
1954 1954 #endif
1955 1955 ret = rootnex_coredma_freehdl(dip, rdip, handle);
1956 1956
1957 1957 if (rootnex_state->r_dvma_call_list_id)
1958 1958 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1959 1959
1960 1960 return (ret);
1961 1961 }
1962 1962
1963 1963 /*ARGSUSED*/
1964 1964 static int
1965 1965 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1966 1966 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1967 1967 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1968 1968 {
1969 1969 rootnex_sglinfo_t *sinfo;
1970 1970 ddi_dma_obj_t *dmao;
1971 1971 #if defined(__amd64) && !defined(__xpv)
1972 1972 struct dvmaseg *dvs;
1973 1973 ddi_dma_cookie_t *cookie;
1974 1974 #endif
1975 1975 ddi_dma_attr_t *attr;
1976 1976 ddi_dma_impl_t *hp;
1977 1977 rootnex_dma_t *dma;
1978 1978 int kmflag;
1979 1979 int e;
1980 1980 uint_t ncookies;
1981 1981
1982 1982 hp = (ddi_dma_impl_t *)handle;
1983 1983 dma = (rootnex_dma_t *)hp->dmai_private;
1984 1984 dmao = &dma->dp_dma;
1985 1985 sinfo = &dma->dp_sglinfo;
1986 1986 attr = &hp->dmai_attr;
1987 1987
1988 1988 /* convert the sleep flags */
1989 1989 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1990 1990 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1991 1991 } else {
1992 1992 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1993 1993 }
1994 1994
1995 1995 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1996 1996
1997 1997 /*
1998 1998 * This is useful for debugging a driver. Not as useful in a production
1999 1999 * system. The only time this will fail is if you have a driver bug.
2000 2000 */
2001 2001 if (rootnex_bind_check_inuse) {
2002 2002 /*
2003 2003 * No one else should ever have this lock unless someone else
2004 2004 * is trying to use this handle. So contention on the lock
2005 2005 * is the same as inuse being set.
2006 2006 */
2007 2007 e = mutex_tryenter(&dma->dp_mutex);
2008 2008 if (e == 0) {
2009 2009 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2010 2010 return (DDI_DMA_INUSE);
2011 2011 }
2012 2012 if (dma->dp_inuse) {
2013 2013 mutex_exit(&dma->dp_mutex);
2014 2014 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2015 2015 return (DDI_DMA_INUSE);
2016 2016 }
2017 2017 dma->dp_inuse = B_TRUE;
2018 2018 mutex_exit(&dma->dp_mutex);
2019 2019 }
2020 2020
2021 2021 /* check the ddi_dma_attr arg to make sure it makes a little sense */
2022 2022 if (rootnex_bind_check_parms) {
2023 2023 e = rootnex_valid_bind_parms(dmareq, attr);
2024 2024 if (e != DDI_SUCCESS) {
2025 2025 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2026 2026 rootnex_clean_dmahdl(hp);
2027 2027 return (e);
2028 2028 }
2029 2029 }
2030 2030
2031 2031 /* save away the original bind info */
2032 2032 dma->dp_dma = dmareq->dmar_object;
2033 2033
2034 2034 #if defined(__amd64) && !defined(__xpv)
2035 2035 if (IOMMU_USED(rdip)) {
2036 2036 dmao = &dma->dp_dvma;
2037 2037 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2038 2038 switch (e) {
2039 2039 case DDI_SUCCESS:
2040 2040 if (sinfo->si_cancross ||
2041 2041 dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2042 2042 dmao->dmao_size > sinfo->si_max_cookie_size) {
2043 2043 dma->dp_dvma_used = B_TRUE;
2044 2044 break;
2045 2045 }
2046 2046 sinfo->si_sgl_size = 1;
2047 2047 hp->dmai_rflags |= DMP_NOSYNC;
2048 2048
2049 2049 dma->dp_dvma_used = B_TRUE;
2050 2050 dma->dp_need_to_free_cookie = B_FALSE;
2051 2051
2052 2052 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2053 2053 cookie = hp->dmai_cookie = dma->dp_cookies =
2054 2054 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2055 2055 cookie->dmac_laddress = dvs->dvs_start +
2056 2056 dmao->dmao_obj.dvma_obj.dv_off;
2057 2057 cookie->dmac_size = dvs->dvs_len;
2058 2058 cookie->dmac_type = 0;
2059 2059
2060 2060 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2061 2061 rdip);
2062 2062 goto fast;
2063 2063 case DDI_ENOTSUP:
2064 2064 break;
2065 2065 default:
2066 2066 rootnex_clean_dmahdl(hp);
2067 2067 return (e);
2068 2068 }
2069 2069 }
2070 2070 #endif
2071 2071
2072 2072 /*
2073 2073 * Figure out a rough estimate of what maximum number of pages
2074 2074 * this buffer could use (a high estimate of course).
2075 2075 */
2076 2076 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2077 2077
2078 2078 if (dma->dp_dvma_used) {
2079 2079 /*
2080 2080 * The number of physical pages is the worst case.
2081 2081 *
2082 2082 * For DVMA, the worst case is the length divided
2083 2083 * by the maximum cookie length, plus 1. Add to that
2084 2084 * the number of segment boundaries potentially crossed, and
2085 2085 * the additional number of DVMA segments that was returned.
2086 2086 *
2087 2087 * In the normal case, for modern devices, si_cancross will
2088 2088 * be false, and dv_nseg will be 1, and the fast path will
2089 2089 * have been taken above.
2090 2090 */
2091 2091 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2092 2092 + 1;
2093 2093 if (sinfo->si_cancross)
2094 2094 ncookies +=
2095 2095 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2096 2096 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2097 2097
2098 2098 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2099 2099 }
2100 2100
2101 2101 /*
2102 2102 * We'll use the pre-allocated cookies for any bind that will *always*
2103 2103 * fit (more important to be consistent, we don't want to create
2104 2104 * additional degenerate cases).
2105 2105 */
2106 2106 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2107 2107 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2108 2108 dma->dp_need_to_free_cookie = B_FALSE;
2109 2109 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2110 2110 uint_t, sinfo->si_max_pages);
2111 2111
2112 2112 /*
2113 2113 * For anything larger than that, we'll go ahead and allocate the
2114 2114 * maximum number of pages we expect to see. Hopefuly, we won't be
2115 2115 * seeing this path in the fast path for high performance devices very
2116 2116 * frequently.
2117 2117 *
2118 2118 * a ddi bind interface that allowed the driver to provide storage to
2119 2119 * the bind interface would speed this case up.
2120 2120 */
2121 2121 } else {
2122 2122 /*
2123 2123 * Save away how much memory we allocated. If we're doing a
2124 2124 * nosleep, the alloc could fail...
2125 2125 */
2126 2126 dma->dp_cookie_size = sinfo->si_max_pages *
2127 2127 sizeof (ddi_dma_cookie_t);
2128 2128 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2129 2129 if (dma->dp_cookies == NULL) {
2130 2130 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2131 2131 rootnex_clean_dmahdl(hp);
2132 2132 return (DDI_DMA_NORESOURCES);
2133 2133 }
2134 2134 dma->dp_need_to_free_cookie = B_TRUE;
2135 2135 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2136 2136 uint_t, sinfo->si_max_pages);
2137 2137 }
2138 2138 hp->dmai_cookie = dma->dp_cookies;
2139 2139
2140 2140 /*
2141 2141 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2142 2142 * looking at the constraints in the dma structure. It will then put
2143 2143 * some additional state about the sgl in the dma struct (i.e. is
2144 2144 * the sgl clean, or do we need to do some munging; how many pages
2145 2145 * need to be copied, etc.)
2146 2146 */
2147 2147 if (dma->dp_dvma_used)
2148 2148 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2149 2149 else
2150 2150 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2151 2151
2152 2152 out:
2153 2153 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2154 2154 /* if we don't need a copy buffer, we don't need to sync */
2155 2155 if (sinfo->si_copybuf_req == 0) {
2156 2156 hp->dmai_rflags |= DMP_NOSYNC;
2157 2157 }
2158 2158
2159 2159 /*
2160 2160 * if we don't need the copybuf and we don't need to do a partial, we
2161 2161 * hit the fast path. All the high performance devices should be trying
2162 2162 * to hit this path. To hit this path, a device should be able to reach
2163 2163 * all of memory, shouldn't try to bind more than it can transfer, and
|
↓ open down ↓ |
2163 lines elided |
↑ open up ↑ |
2164 2164 * the buffer shouldn't require more cookies than the driver/device can
2165 2165 * handle [sgllen]).
2166 2166 *
2167 2167 * Note that negative values of dma_attr_sgllen are supposed
2168 2168 * to mean unlimited, but we just cast them to mean a
2169 2169 * "ridiculous large limit". This saves some extra checks on
2170 2170 * hot paths.
2171 2171 */
2172 2172 if ((sinfo->si_copybuf_req == 0) &&
2173 2173 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2174 - (dmao->dmao_size < dma->dp_maxxfer)) {
2174 + (dmao->dmao_size <= dma->dp_maxxfer)) {
2175 2175 fast:
2176 2176 /*
2177 2177 * If the driver supports FMA, insert the handle in the FMA DMA
2178 2178 * handle cache.
2179 2179 */
2180 2180 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2181 2181 hp->dmai_error.err_cf = rootnex_dma_check;
2182 2182
2183 2183 /*
2184 2184 * copy out the first cookie and ccountp, set the cookie
2185 2185 * pointer to the second cookie. The first cookie is passed
2186 2186 * back on the stack. Additional cookies are accessed via
2187 2187 * ddi_dma_nextcookie()
2188 2188 */
2189 2189 *cookiep = dma->dp_cookies[0];
2190 2190 *ccountp = sinfo->si_sgl_size;
2191 2191 hp->dmai_cookie++;
2192 2192 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2193 2193 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2194 2194 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2195 2195 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2196 2196 uint_t, dmao->dmao_size, uint_t, *ccountp);
2197 2197
2198 2198
2199 2199 return (DDI_DMA_MAPPED);
2200 2200 }
2201 2201
2202 2202 /*
2203 2203 * go to the slow path, we may need to alloc more memory, create
2204 2204 * multiple windows, and munge up a sgl to make the device happy.
2205 2205 */
2206 2206
2207 2207 /*
2208 2208 * With the IOMMU mapobject method used, we should never hit
2209 2209 * the slow path. If we do, something is seriously wrong.
2210 2210 * Clean up and return an error.
2211 2211 */
2212 2212
2213 2213 #if defined(__amd64) && !defined(__xpv)
2214 2214
2215 2215 if (dma->dp_dvma_used) {
2216 2216 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2217 2217 &dma->dp_dvma);
2218 2218 e = DDI_DMA_NOMAPPING;
2219 2219 } else {
2220 2220 #endif
2221 2221 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2222 2222 kmflag);
2223 2223 #if defined(__amd64) && !defined(__xpv)
2224 2224 }
2225 2225 #endif
2226 2226 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2227 2227 if (dma->dp_need_to_free_cookie) {
2228 2228 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2229 2229 }
2230 2230 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2231 2231 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2232 2232 return (e);
2233 2233 }
2234 2234
2235 2235 /*
2236 2236 * If the driver supports FMA, insert the handle in the FMA DMA handle
2237 2237 * cache.
2238 2238 */
2239 2239 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2240 2240 hp->dmai_error.err_cf = rootnex_dma_check;
2241 2241
2242 2242 /* if the first window uses the copy buffer, sync it for the device */
2243 2243 if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2244 2244 (hp->dmai_rflags & DDI_DMA_WRITE)) {
2245 2245 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2246 2246 DDI_DMA_SYNC_FORDEV);
2247 2247 }
2248 2248
2249 2249 /*
2250 2250 * copy out the first cookie and ccountp, set the cookie pointer to the
2251 2251 * second cookie. Make sure the partial flag is set/cleared correctly.
2252 2252 * If we have a partial map (i.e. multiple windows), the number of
2253 2253 * cookies we return is the number of cookies in the first window.
2254 2254 */
2255 2255 if (e == DDI_DMA_MAPPED) {
2256 2256 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2257 2257 *ccountp = sinfo->si_sgl_size;
2258 2258 hp->dmai_nwin = 1;
2259 2259 } else {
2260 2260 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2261 2261 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2262 2262 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2263 2263 }
2264 2264 *cookiep = dma->dp_cookies[0];
2265 2265 hp->dmai_cookie++;
2266 2266
2267 2267 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2268 2268 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2269 2269 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2270 2270 dmao->dmao_size, uint_t, *ccountp);
2271 2271 return (e);
2272 2272 }
2273 2273
2274 2274 /*
2275 2275 * rootnex_dma_bindhdl()
2276 2276 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2277 2277 */
2278 2278 static int
2279 2279 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2280 2280 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2281 2281 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2282 2282 {
2283 2283 int ret;
2284 2284 #if defined(__amd64) && !defined(__xpv)
2285 2285 if (IOMMU_USED(rdip))
2286 2286 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2287 2287 cookiep, ccountp);
2288 2288 else
2289 2289 #endif
2290 2290 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2291 2291 cookiep, ccountp);
2292 2292
2293 2293 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2294 2294 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2295 2295 &rootnex_state->r_dvma_call_list_id);
2296 2296 }
2297 2297
2298 2298 return (ret);
2299 2299 }
2300 2300
2301 2301
2302 2302
2303 2303 /*ARGSUSED*/
2304 2304 static int
2305 2305 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2306 2306 ddi_dma_handle_t handle)
2307 2307 {
2308 2308 ddi_dma_impl_t *hp;
2309 2309 rootnex_dma_t *dma;
2310 2310 int e;
2311 2311
2312 2312 hp = (ddi_dma_impl_t *)handle;
2313 2313 dma = (rootnex_dma_t *)hp->dmai_private;
2314 2314
2315 2315 /* make sure the buffer wasn't free'd before calling unbind */
2316 2316 if (rootnex_unbind_verify_buffer) {
2317 2317 e = rootnex_verify_buffer(dma);
2318 2318 if (e != DDI_SUCCESS) {
2319 2319 ASSERT(0);
2320 2320 return (DDI_FAILURE);
2321 2321 }
2322 2322 }
2323 2323
2324 2324 /* sync the current window before unbinding the buffer */
2325 2325 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2326 2326 (hp->dmai_rflags & DDI_DMA_READ)) {
2327 2327 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2328 2328 DDI_DMA_SYNC_FORCPU);
2329 2329 }
2330 2330
2331 2331 /*
2332 2332 * cleanup and copy buffer or window state. if we didn't use the copy
2333 2333 * buffer or windows, there won't be much to do :-)
2334 2334 */
2335 2335 rootnex_teardown_copybuf(dma);
2336 2336 rootnex_teardown_windows(dma);
2337 2337
2338 2338 #if defined(__amd64) && !defined(__xpv)
2339 2339 if (IOMMU_USED(rdip))
2340 2340 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2341 2341 &dma->dp_dvma);
2342 2342 #endif
2343 2343
2344 2344 /*
2345 2345 * If we had to allocate space to for the worse case sgl (it didn't
2346 2346 * fit into our pre-allocate buffer), free that up now
2347 2347 */
2348 2348 if (dma->dp_need_to_free_cookie) {
2349 2349 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2350 2350 }
2351 2351
2352 2352 /*
2353 2353 * clean up the handle so it's ready for the next bind (i.e. if the
2354 2354 * handle is reused).
2355 2355 */
2356 2356 rootnex_clean_dmahdl(hp);
2357 2357 hp->dmai_error.err_cf = NULL;
2358 2358
2359 2359 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2360 2360 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2361 2361 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2362 2362
2363 2363 return (DDI_SUCCESS);
2364 2364 }
2365 2365
2366 2366 /*
2367 2367 * rootnex_dma_unbindhdl()
2368 2368 * called from ddi_dma_unbind_handle()
2369 2369 */
2370 2370 /*ARGSUSED*/
2371 2371 static int
2372 2372 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2373 2373 ddi_dma_handle_t handle)
2374 2374 {
2375 2375 int ret;
2376 2376
2377 2377 #if defined(__amd64) && !defined(__xpv)
2378 2378 if (IOMMU_USED(rdip))
2379 2379 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2380 2380 else
2381 2381 #endif
2382 2382 ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2383 2383
2384 2384 if (rootnex_state->r_dvma_call_list_id)
2385 2385 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2386 2386
2387 2387 return (ret);
2388 2388 }
2389 2389
2390 2390 #if defined(__amd64) && !defined(__xpv)
2391 2391
2392 2392 static int
2393 2393 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2394 2394 {
2395 2395 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2396 2396 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2397 2397
2398 2398 if (dma->dp_sleep_flags != KM_SLEEP &&
2399 2399 dma->dp_sleep_flags != KM_NOSLEEP)
2400 2400 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2401 2401 return (dma->dp_sleep_flags);
2402 2402 }
2403 2403 /*ARGSUSED*/
2404 2404 static void
2405 2405 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2406 2406 {
2407 2407 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2408 2408 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2409 2409 rootnex_window_t *window;
2410 2410
2411 2411 if (dma->dp_window) {
2412 2412 window = &dma->dp_window[dma->dp_current_win];
2413 2413 hp->dmai_cookie = window->wd_first_cookie;
2414 2414 } else {
2415 2415 hp->dmai_cookie = dma->dp_cookies;
2416 2416 }
2417 2417 hp->dmai_cookie++;
2418 2418 }
2419 2419
2420 2420 /*ARGSUSED*/
2421 2421 static int
2422 2422 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2423 2423 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2424 2424 {
2425 2425 int i;
2426 2426 int km_flags;
2427 2427 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2428 2428 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2429 2429 rootnex_window_t *window;
2430 2430 ddi_dma_cookie_t *cp;
2431 2431 ddi_dma_cookie_t *cookie;
2432 2432
2433 2433 ASSERT(*cookiepp == NULL);
2434 2434 ASSERT(*ccountp == 0);
2435 2435
2436 2436 if (dma->dp_window) {
2437 2437 window = &dma->dp_window[dma->dp_current_win];
2438 2438 cp = window->wd_first_cookie;
2439 2439 *ccountp = window->wd_cookie_cnt;
2440 2440 } else {
2441 2441 cp = dma->dp_cookies;
2442 2442 *ccountp = dma->dp_sglinfo.si_sgl_size;
2443 2443 }
2444 2444
2445 2445 km_flags = rootnex_coredma_get_sleep_flags(handle);
2446 2446 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2447 2447 if (cookie == NULL) {
2448 2448 return (DDI_DMA_NORESOURCES);
2449 2449 }
2450 2450
2451 2451 for (i = 0; i < *ccountp; i++) {
2452 2452 cookie[i].dmac_notused = cp[i].dmac_notused;
2453 2453 cookie[i].dmac_type = cp[i].dmac_type;
2454 2454 cookie[i].dmac_address = cp[i].dmac_address;
2455 2455 cookie[i].dmac_size = cp[i].dmac_size;
2456 2456 }
2457 2457
2458 2458 *cookiepp = cookie;
2459 2459
2460 2460 return (DDI_SUCCESS);
2461 2461 }
2462 2462
2463 2463 /*ARGSUSED*/
2464 2464 static int
2465 2465 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2466 2466 ddi_dma_cookie_t *cookiep, uint_t ccount)
2467 2467 {
2468 2468 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2469 2469 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2470 2470 rootnex_window_t *window;
2471 2471 ddi_dma_cookie_t *cur_cookiep;
2472 2472
2473 2473 ASSERT(cookiep);
2474 2474 ASSERT(ccount != 0);
2475 2475 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2476 2476
2477 2477 if (dma->dp_window) {
2478 2478 window = &dma->dp_window[dma->dp_current_win];
2479 2479 dma->dp_saved_cookies = window->wd_first_cookie;
2480 2480 window->wd_first_cookie = cookiep;
2481 2481 ASSERT(ccount == window->wd_cookie_cnt);
2482 2482 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2483 2483 + window->wd_first_cookie;
2484 2484 } else {
2485 2485 dma->dp_saved_cookies = dma->dp_cookies;
2486 2486 dma->dp_cookies = cookiep;
2487 2487 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2488 2488 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2489 2489 + dma->dp_cookies;
2490 2490 }
2491 2491
2492 2492 dma->dp_need_to_switch_cookies = B_TRUE;
2493 2493 hp->dmai_cookie = cur_cookiep;
2494 2494
2495 2495 return (DDI_SUCCESS);
2496 2496 }
2497 2497
2498 2498 /*ARGSUSED*/
2499 2499 static int
2500 2500 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2501 2501 {
2502 2502 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2503 2503 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2504 2504 rootnex_window_t *window;
2505 2505 ddi_dma_cookie_t *cur_cookiep;
2506 2506 ddi_dma_cookie_t *cookie_array;
2507 2507 uint_t ccount;
2508 2508
2509 2509 /* check if cookies have not been switched */
2510 2510 if (dma->dp_need_to_switch_cookies == B_FALSE)
2511 2511 return (DDI_SUCCESS);
2512 2512
2513 2513 ASSERT(dma->dp_saved_cookies);
2514 2514
2515 2515 if (dma->dp_window) {
2516 2516 window = &dma->dp_window[dma->dp_current_win];
2517 2517 cookie_array = window->wd_first_cookie;
2518 2518 window->wd_first_cookie = dma->dp_saved_cookies;
2519 2519 dma->dp_saved_cookies = NULL;
2520 2520 ccount = window->wd_cookie_cnt;
2521 2521 cur_cookiep = (hp->dmai_cookie - cookie_array)
2522 2522 + window->wd_first_cookie;
2523 2523 } else {
2524 2524 cookie_array = dma->dp_cookies;
2525 2525 dma->dp_cookies = dma->dp_saved_cookies;
2526 2526 dma->dp_saved_cookies = NULL;
2527 2527 ccount = dma->dp_sglinfo.si_sgl_size;
2528 2528 cur_cookiep = (hp->dmai_cookie - cookie_array)
2529 2529 + dma->dp_cookies;
2530 2530 }
2531 2531
2532 2532 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2533 2533
2534 2534 hp->dmai_cookie = cur_cookiep;
2535 2535
2536 2536 dma->dp_need_to_switch_cookies = B_FALSE;
2537 2537
2538 2538 return (DDI_SUCCESS);
2539 2539 }
2540 2540
2541 2541 #endif
2542 2542
2543 2543 static struct as *
2544 2544 rootnex_get_as(ddi_dma_obj_t *dmao)
2545 2545 {
2546 2546 struct as *asp;
2547 2547
2548 2548 switch (dmao->dmao_type) {
2549 2549 case DMA_OTYP_VADDR:
2550 2550 case DMA_OTYP_BUFVADDR:
2551 2551 asp = dmao->dmao_obj.virt_obj.v_as;
2552 2552 if (asp == NULL)
2553 2553 asp = &kas;
2554 2554 break;
2555 2555 default:
2556 2556 asp = NULL;
2557 2557 break;
2558 2558 }
2559 2559 return (asp);
2560 2560 }
2561 2561
2562 2562 /*
2563 2563 * rootnex_verify_buffer()
2564 2564 * verify buffer wasn't free'd
2565 2565 */
2566 2566 static int
2567 2567 rootnex_verify_buffer(rootnex_dma_t *dma)
2568 2568 {
2569 2569 page_t **pplist;
2570 2570 caddr_t vaddr;
2571 2571 uint_t pcnt;
2572 2572 uint_t poff;
2573 2573 page_t *pp;
2574 2574 char b;
2575 2575 int i;
2576 2576
2577 2577 /* Figure out how many pages this buffer occupies */
2578 2578 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2579 2579 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2580 2580 } else {
2581 2581 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2582 2582 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2583 2583 }
2584 2584 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2585 2585
2586 2586 switch (dma->dp_dma.dmao_type) {
2587 2587 case DMA_OTYP_PAGES:
2588 2588 /*
2589 2589 * for a linked list of pp's walk through them to make sure
2590 2590 * they're locked and not free.
2591 2591 */
2592 2592 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2593 2593 for (i = 0; i < pcnt; i++) {
2594 2594 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2595 2595 return (DDI_FAILURE);
2596 2596 }
2597 2597 pp = pp->p_next;
2598 2598 }
2599 2599 break;
2600 2600
2601 2601 case DMA_OTYP_VADDR:
2602 2602 case DMA_OTYP_BUFVADDR:
2603 2603 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2604 2604 /*
2605 2605 * for an array of pp's walk through them to make sure they're
2606 2606 * not free. It's possible that they may not be locked.
2607 2607 */
2608 2608 if (pplist) {
2609 2609 for (i = 0; i < pcnt; i++) {
2610 2610 if (PP_ISFREE(pplist[i])) {
2611 2611 return (DDI_FAILURE);
2612 2612 }
2613 2613 }
2614 2614
2615 2615 /* For a virtual address, try to peek at each page */
2616 2616 } else {
2617 2617 if (rootnex_get_as(&dma->dp_dma) == &kas) {
2618 2618 for (i = 0; i < pcnt; i++) {
2619 2619 if (ddi_peek8(NULL, vaddr, &b) ==
2620 2620 DDI_FAILURE)
2621 2621 return (DDI_FAILURE);
2622 2622 vaddr += MMU_PAGESIZE;
2623 2623 }
2624 2624 }
2625 2625 }
2626 2626 break;
2627 2627
2628 2628 default:
2629 2629 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2630 2630 break;
2631 2631 }
2632 2632
2633 2633 return (DDI_SUCCESS);
2634 2634 }
2635 2635
2636 2636
2637 2637 /*
2638 2638 * rootnex_clean_dmahdl()
2639 2639 * Clean the dma handle. This should be called on a handle alloc and an
2640 2640 * unbind handle. Set the handle state to the default settings.
2641 2641 */
2642 2642 static void
2643 2643 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2644 2644 {
2645 2645 rootnex_dma_t *dma;
2646 2646
2647 2647
2648 2648 dma = (rootnex_dma_t *)hp->dmai_private;
2649 2649
2650 2650 hp->dmai_nwin = 0;
2651 2651 dma->dp_current_cookie = 0;
2652 2652 dma->dp_copybuf_size = 0;
2653 2653 dma->dp_window = NULL;
2654 2654 dma->dp_cbaddr = NULL;
2655 2655 dma->dp_inuse = B_FALSE;
2656 2656 dma->dp_dvma_used = B_FALSE;
2657 2657 dma->dp_need_to_free_cookie = B_FALSE;
2658 2658 dma->dp_need_to_switch_cookies = B_FALSE;
2659 2659 dma->dp_saved_cookies = NULL;
2660 2660 dma->dp_sleep_flags = KM_PANIC;
2661 2661 dma->dp_need_to_free_window = B_FALSE;
2662 2662 dma->dp_partial_required = B_FALSE;
2663 2663 dma->dp_trim_required = B_FALSE;
2664 2664 dma->dp_sglinfo.si_copybuf_req = 0;
2665 2665 #if !defined(__amd64)
2666 2666 dma->dp_cb_remaping = B_FALSE;
2667 2667 dma->dp_kva = NULL;
2668 2668 #endif
2669 2669
2670 2670 /* FMA related initialization */
2671 2671 hp->dmai_fault = 0;
2672 2672 hp->dmai_fault_check = NULL;
2673 2673 hp->dmai_fault_notify = NULL;
2674 2674 hp->dmai_error.err_ena = 0;
2675 2675 hp->dmai_error.err_status = DDI_FM_OK;
2676 2676 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2677 2677 hp->dmai_error.err_ontrap = NULL;
2678 2678 }
2679 2679
2680 2680
2681 2681 /*
2682 2682 * rootnex_valid_alloc_parms()
2683 2683 * Called in ddi_dma_alloc_handle path to validate its parameters.
2684 2684 */
2685 2685 static int
2686 2686 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2687 2687 {
2688 2688 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2689 2689 (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2690 2690 (attr->dma_attr_granular > MMU_PAGESIZE) ||
2691 2691 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2692 2692 return (DDI_DMA_BADATTR);
2693 2693 }
2694 2694
2695 2695 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2696 2696 return (DDI_DMA_BADATTR);
2697 2697 }
2698 2698
2699 2699 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2700 2700 MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2701 2701 attr->dma_attr_sgllen == 0) {
2702 2702 return (DDI_DMA_BADATTR);
2703 2703 }
2704 2704
2705 2705 /* We should be able to DMA into every byte offset in a page */
2706 2706 if (maxsegmentsize < MMU_PAGESIZE) {
2707 2707 return (DDI_DMA_BADATTR);
2708 2708 }
2709 2709
2710 2710 /* if we're bouncing on seg, seg must be <= addr_hi */
2711 2711 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2712 2712 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2713 2713 return (DDI_DMA_BADATTR);
2714 2714 }
2715 2715 return (DDI_SUCCESS);
2716 2716 }
2717 2717
2718 2718 /*
2719 2719 * rootnex_valid_bind_parms()
2720 2720 * Called in ddi_dma_*_bind_handle path to validate its parameters.
2721 2721 */
2722 2722 /* ARGSUSED */
2723 2723 static int
2724 2724 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2725 2725 {
2726 2726 #if !defined(__amd64)
2727 2727 /*
2728 2728 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2729 2729 * we can track the offset for the obsoleted interfaces.
2730 2730 */
2731 2731 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2732 2732 return (DDI_DMA_TOOBIG);
2733 2733 }
2734 2734 #endif
2735 2735
2736 2736 return (DDI_SUCCESS);
2737 2737 }
2738 2738
2739 2739
2740 2740 /*
2741 2741 * rootnex_need_bounce_seg()
2742 2742 * check to see if the buffer lives on both side of the seg.
2743 2743 */
2744 2744 static boolean_t
2745 2745 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2746 2746 {
2747 2747 ddi_dma_atyp_t buftype;
2748 2748 rootnex_addr_t raddr;
2749 2749 boolean_t lower_addr;
2750 2750 boolean_t upper_addr;
2751 2751 uint64_t offset;
2752 2752 page_t **pplist;
2753 2753 uint64_t paddr;
2754 2754 uint32_t psize;
2755 2755 uint32_t size;
2756 2756 caddr_t vaddr;
2757 2757 uint_t pcnt;
2758 2758 page_t *pp;
2759 2759
2760 2760
2761 2761 /* shortcuts */
2762 2762 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2763 2763 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2764 2764 buftype = dmar_object->dmao_type;
2765 2765 size = dmar_object->dmao_size;
2766 2766
2767 2767 lower_addr = B_FALSE;
2768 2768 upper_addr = B_FALSE;
2769 2769 pcnt = 0;
2770 2770
2771 2771 /*
2772 2772 * Process the first page to handle the initial offset of the buffer.
2773 2773 * We'll use the base address we get later when we loop through all
2774 2774 * the pages.
2775 2775 */
2776 2776 if (buftype == DMA_OTYP_PAGES) {
2777 2777 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2778 2778 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2779 2779 MMU_PAGEOFFSET;
2780 2780 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2781 2781 psize = MIN(size, (MMU_PAGESIZE - offset));
2782 2782 pp = pp->p_next;
2783 2783 sglinfo->si_asp = NULL;
2784 2784 } else if (pplist != NULL) {
2785 2785 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2786 2786 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2787 2787 if (sglinfo->si_asp == NULL) {
2788 2788 sglinfo->si_asp = &kas;
2789 2789 }
2790 2790 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2791 2791 paddr += offset;
2792 2792 psize = MIN(size, (MMU_PAGESIZE - offset));
2793 2793 pcnt++;
2794 2794 } else {
2795 2795 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2796 2796 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2797 2797 if (sglinfo->si_asp == NULL) {
2798 2798 sglinfo->si_asp = &kas;
2799 2799 }
2800 2800 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2801 2801 paddr += offset;
2802 2802 psize = MIN(size, (MMU_PAGESIZE - offset));
2803 2803 vaddr += psize;
2804 2804 }
2805 2805
2806 2806 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2807 2807
2808 2808 if ((raddr + psize) > sglinfo->si_segmask) {
2809 2809 upper_addr = B_TRUE;
2810 2810 } else {
2811 2811 lower_addr = B_TRUE;
2812 2812 }
2813 2813 size -= psize;
2814 2814
2815 2815 /*
2816 2816 * Walk through the rest of the pages in the buffer. Track to see
2817 2817 * if we have pages on both sides of the segment boundary.
2818 2818 */
2819 2819 while (size > 0) {
2820 2820 /* partial or full page */
2821 2821 psize = MIN(size, MMU_PAGESIZE);
2822 2822
2823 2823 if (buftype == DMA_OTYP_PAGES) {
2824 2824 /* get the paddr from the page_t */
2825 2825 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2826 2826 paddr = pfn_to_pa(pp->p_pagenum);
2827 2827 pp = pp->p_next;
2828 2828 } else if (pplist != NULL) {
2829 2829 /* index into the array of page_t's to get the paddr */
2830 2830 ASSERT(!PP_ISFREE(pplist[pcnt]));
2831 2831 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2832 2832 pcnt++;
2833 2833 } else {
2834 2834 /* call into the VM to get the paddr */
2835 2835 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2836 2836 vaddr));
2837 2837 vaddr += psize;
2838 2838 }
2839 2839
2840 2840 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2841 2841
2842 2842 if ((raddr + psize) > sglinfo->si_segmask) {
2843 2843 upper_addr = B_TRUE;
2844 2844 } else {
2845 2845 lower_addr = B_TRUE;
2846 2846 }
2847 2847 /*
2848 2848 * if the buffer lives both above and below the segment
2849 2849 * boundary, or the current page is the page immediately
2850 2850 * after the segment, we will use a copy/bounce buffer for
2851 2851 * all pages > seg.
2852 2852 */
2853 2853 if ((lower_addr && upper_addr) ||
2854 2854 (raddr == (sglinfo->si_segmask + 1))) {
2855 2855 return (B_TRUE);
2856 2856 }
2857 2857
2858 2858 size -= psize;
2859 2859 }
2860 2860
2861 2861 return (B_FALSE);
2862 2862 }
2863 2863
2864 2864 /*
2865 2865 * rootnex_get_sgl()
2866 2866 * Called in bind fastpath to get the sgl. Most of this will be replaced
2867 2867 * with a call to the vm layer when vm2.0 comes around...
2868 2868 */
2869 2869 static void
2870 2870 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2871 2871 rootnex_sglinfo_t *sglinfo)
2872 2872 {
2873 2873 ddi_dma_atyp_t buftype;
2874 2874 rootnex_addr_t raddr;
2875 2875 uint64_t last_page;
2876 2876 uint64_t offset;
2877 2877 uint64_t addrhi;
2878 2878 uint64_t addrlo;
2879 2879 uint64_t maxseg;
2880 2880 page_t **pplist;
2881 2881 uint64_t paddr;
2882 2882 uint32_t psize;
2883 2883 uint32_t size;
2884 2884 caddr_t vaddr;
2885 2885 uint_t pcnt;
2886 2886 page_t *pp;
2887 2887 uint_t cnt;
2888 2888
2889 2889
2890 2890 /* shortcuts */
2891 2891 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2892 2892 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2893 2893 maxseg = sglinfo->si_max_cookie_size;
2894 2894 buftype = dmar_object->dmao_type;
2895 2895 addrhi = sglinfo->si_max_addr;
2896 2896 addrlo = sglinfo->si_min_addr;
2897 2897 size = dmar_object->dmao_size;
2898 2898
2899 2899 pcnt = 0;
2900 2900 cnt = 0;
2901 2901
2902 2902
2903 2903 /*
2904 2904 * check to see if we need to use the copy buffer for pages over
2905 2905 * the segment attr.
2906 2906 */
2907 2907 sglinfo->si_bounce_on_seg = B_FALSE;
2908 2908 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2909 2909 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2910 2910 dmar_object, sglinfo);
2911 2911 }
2912 2912
2913 2913 /*
2914 2914 * if we were passed down a linked list of pages, i.e. pointer to
2915 2915 * page_t, use this to get our physical address and buf offset.
2916 2916 */
2917 2917 if (buftype == DMA_OTYP_PAGES) {
2918 2918 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2919 2919 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2920 2920 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2921 2921 MMU_PAGEOFFSET;
2922 2922 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2923 2923 psize = MIN(size, (MMU_PAGESIZE - offset));
2924 2924 pp = pp->p_next;
2925 2925 sglinfo->si_asp = NULL;
2926 2926
2927 2927 /*
2928 2928 * We weren't passed down a linked list of pages, but if we were passed
2929 2929 * down an array of pages, use this to get our physical address and buf
2930 2930 * offset.
2931 2931 */
2932 2932 } else if (pplist != NULL) {
2933 2933 ASSERT((buftype == DMA_OTYP_VADDR) ||
2934 2934 (buftype == DMA_OTYP_BUFVADDR));
2935 2935
2936 2936 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2937 2937 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2938 2938 if (sglinfo->si_asp == NULL) {
2939 2939 sglinfo->si_asp = &kas;
2940 2940 }
2941 2941
2942 2942 ASSERT(!PP_ISFREE(pplist[pcnt]));
2943 2943 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2944 2944 paddr += offset;
2945 2945 psize = MIN(size, (MMU_PAGESIZE - offset));
2946 2946 pcnt++;
2947 2947
2948 2948 /*
2949 2949 * All we have is a virtual address, we'll need to call into the VM
2950 2950 * to get the physical address.
2951 2951 */
2952 2952 } else {
2953 2953 ASSERT((buftype == DMA_OTYP_VADDR) ||
2954 2954 (buftype == DMA_OTYP_BUFVADDR));
2955 2955
2956 2956 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2957 2957 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2958 2958 if (sglinfo->si_asp == NULL) {
2959 2959 sglinfo->si_asp = &kas;
2960 2960 }
2961 2961
2962 2962 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2963 2963 paddr += offset;
2964 2964 psize = MIN(size, (MMU_PAGESIZE - offset));
2965 2965 vaddr += psize;
2966 2966 }
2967 2967
2968 2968 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2969 2969
2970 2970 /*
2971 2971 * Setup the first cookie with the physical address of the page and the
2972 2972 * size of the page (which takes into account the initial offset into
2973 2973 * the page.
2974 2974 */
2975 2975 sgl[cnt].dmac_laddress = raddr;
2976 2976 sgl[cnt].dmac_size = psize;
2977 2977 sgl[cnt].dmac_type = 0;
2978 2978
2979 2979 /*
2980 2980 * Save away the buffer offset into the page. We'll need this later in
2981 2981 * the copy buffer code to help figure out the page index within the
2982 2982 * buffer and the offset into the current page.
2983 2983 */
2984 2984 sglinfo->si_buf_offset = offset;
2985 2985
2986 2986 /*
2987 2987 * If we are using the copy buffer for anything over the segment
2988 2988 * boundary, and this page is over the segment boundary.
2989 2989 * OR
2990 2990 * if the DMA engine can't reach the physical address.
2991 2991 */
2992 2992 if (((sglinfo->si_bounce_on_seg) &&
2993 2993 ((raddr + psize) > sglinfo->si_segmask)) ||
2994 2994 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2995 2995 /*
2996 2996 * Increase how much copy buffer we use. We always increase by
2997 2997 * pagesize so we don't have to worry about converting offsets.
2998 2998 * Set a flag in the cookies dmac_type to indicate that it uses
2999 2999 * the copy buffer. If this isn't the last cookie, go to the
3000 3000 * next cookie (since we separate each page which uses the copy
3001 3001 * buffer in case the copy buffer is not physically contiguous.
3002 3002 */
3003 3003 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3004 3004 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3005 3005 if ((cnt + 1) < sglinfo->si_max_pages) {
3006 3006 cnt++;
3007 3007 sgl[cnt].dmac_laddress = 0;
3008 3008 sgl[cnt].dmac_size = 0;
3009 3009 sgl[cnt].dmac_type = 0;
3010 3010 }
3011 3011 }
3012 3012
3013 3013 /*
3014 3014 * save this page's physical address so we can figure out if the next
3015 3015 * page is physically contiguous. Keep decrementing size until we are
3016 3016 * done with the buffer.
3017 3017 */
3018 3018 last_page = raddr & MMU_PAGEMASK;
3019 3019 size -= psize;
3020 3020
3021 3021 while (size > 0) {
3022 3022 /* Get the size for this page (i.e. partial or full page) */
3023 3023 psize = MIN(size, MMU_PAGESIZE);
3024 3024
3025 3025 if (buftype == DMA_OTYP_PAGES) {
3026 3026 /* get the paddr from the page_t */
3027 3027 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3028 3028 paddr = pfn_to_pa(pp->p_pagenum);
3029 3029 pp = pp->p_next;
3030 3030 } else if (pplist != NULL) {
3031 3031 /* index into the array of page_t's to get the paddr */
3032 3032 ASSERT(!PP_ISFREE(pplist[pcnt]));
3033 3033 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3034 3034 pcnt++;
3035 3035 } else {
3036 3036 /* call into the VM to get the paddr */
3037 3037 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3038 3038 vaddr));
3039 3039 vaddr += psize;
3040 3040 }
3041 3041
3042 3042 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3043 3043
3044 3044 /*
3045 3045 * If we are using the copy buffer for anything over the
3046 3046 * segment boundary, and this page is over the segment
3047 3047 * boundary.
3048 3048 * OR
3049 3049 * if the DMA engine can't reach the physical address.
3050 3050 */
3051 3051 if (((sglinfo->si_bounce_on_seg) &&
3052 3052 ((raddr + psize) > sglinfo->si_segmask)) ||
3053 3053 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3054 3054
3055 3055 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3056 3056
3057 3057 /*
3058 3058 * if there is something in the current cookie, go to
3059 3059 * the next one. We only want one page in a cookie which
3060 3060 * uses the copybuf since the copybuf doesn't have to
3061 3061 * be physically contiguous.
3062 3062 */
3063 3063 if (sgl[cnt].dmac_size != 0) {
3064 3064 cnt++;
3065 3065 }
3066 3066 sgl[cnt].dmac_laddress = raddr;
3067 3067 sgl[cnt].dmac_size = psize;
3068 3068 #if defined(__amd64)
3069 3069 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3070 3070 #else
3071 3071 /*
3072 3072 * save the buf offset for 32-bit kernel. used in the
3073 3073 * obsoleted interfaces.
3074 3074 */
3075 3075 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3076 3076 (dmar_object->dmao_size - size);
3077 3077 #endif
3078 3078 /* if this isn't the last cookie, go to the next one */
3079 3079 if ((cnt + 1) < sglinfo->si_max_pages) {
3080 3080 cnt++;
3081 3081 sgl[cnt].dmac_laddress = 0;
3082 3082 sgl[cnt].dmac_size = 0;
3083 3083 sgl[cnt].dmac_type = 0;
3084 3084 }
3085 3085
3086 3086 /*
3087 3087 * this page didn't need the copy buffer, if it's not physically
3088 3088 * contiguous, or it would put us over a segment boundary, or it
3089 3089 * puts us over the max cookie size, or the current sgl doesn't
3090 3090 * have anything in it.
3091 3091 */
3092 3092 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3093 3093 !(raddr & sglinfo->si_segmask) ||
3094 3094 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3095 3095 (sgl[cnt].dmac_size == 0)) {
3096 3096 /*
3097 3097 * if we're not already in a new cookie, go to the next
3098 3098 * cookie.
3099 3099 */
3100 3100 if (sgl[cnt].dmac_size != 0) {
3101 3101 cnt++;
3102 3102 }
3103 3103
3104 3104 /* save the cookie information */
3105 3105 sgl[cnt].dmac_laddress = raddr;
3106 3106 sgl[cnt].dmac_size = psize;
3107 3107 #if defined(__amd64)
3108 3108 sgl[cnt].dmac_type = 0;
3109 3109 #else
3110 3110 /*
3111 3111 * save the buf offset for 32-bit kernel. used in the
3112 3112 * obsoleted interfaces.
3113 3113 */
3114 3114 sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3115 3115 #endif
3116 3116
3117 3117 /*
3118 3118 * this page didn't need the copy buffer, it is physically
3119 3119 * contiguous with the last page, and it's <= the max cookie
3120 3120 * size.
3121 3121 */
3122 3122 } else {
3123 3123 sgl[cnt].dmac_size += psize;
3124 3124
3125 3125 /*
3126 3126 * if this exactly == the maximum cookie size, and
3127 3127 * it isn't the last cookie, go to the next cookie.
3128 3128 */
3129 3129 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3130 3130 ((cnt + 1) < sglinfo->si_max_pages)) {
3131 3131 cnt++;
3132 3132 sgl[cnt].dmac_laddress = 0;
3133 3133 sgl[cnt].dmac_size = 0;
3134 3134 sgl[cnt].dmac_type = 0;
3135 3135 }
3136 3136 }
3137 3137
3138 3138 /*
3139 3139 * save this page's physical address so we can figure out if the
3140 3140 * next page is physically contiguous. Keep decrementing size
3141 3141 * until we are done with the buffer.
3142 3142 */
3143 3143 last_page = raddr;
3144 3144 size -= psize;
3145 3145 }
3146 3146
3147 3147 /* we're done, save away how many cookies the sgl has */
3148 3148 if (sgl[cnt].dmac_size == 0) {
3149 3149 ASSERT(cnt < sglinfo->si_max_pages);
3150 3150 sglinfo->si_sgl_size = cnt;
3151 3151 } else {
3152 3152 sglinfo->si_sgl_size = cnt + 1;
3153 3153 }
3154 3154 }
3155 3155
3156 3156 static void
3157 3157 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3158 3158 rootnex_sglinfo_t *sglinfo)
3159 3159 {
3160 3160 uint64_t offset;
3161 3161 uint64_t maxseg;
3162 3162 uint64_t dvaddr;
3163 3163 struct dvmaseg *dvs;
3164 3164 uint64_t paddr;
3165 3165 uint32_t psize, ssize;
3166 3166 uint32_t size;
3167 3167 uint_t cnt;
3168 3168 int physcontig;
3169 3169
3170 3170 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3171 3171
3172 3172 /* shortcuts */
3173 3173 maxseg = sglinfo->si_max_cookie_size;
3174 3174 size = dmar_object->dmao_size;
3175 3175
3176 3176 cnt = 0;
3177 3177 sglinfo->si_bounce_on_seg = B_FALSE;
3178 3178
3179 3179 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3180 3180 offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3181 3181 ssize = dvs->dvs_len;
3182 3182 paddr = dvs->dvs_start;
3183 3183 paddr += offset;
3184 3184 psize = MIN(ssize, (maxseg - offset));
3185 3185 dvaddr = paddr + psize;
3186 3186 ssize -= psize;
3187 3187
3188 3188 sgl[cnt].dmac_laddress = paddr;
3189 3189 sgl[cnt].dmac_size = psize;
3190 3190 sgl[cnt].dmac_type = 0;
3191 3191
3192 3192 size -= psize;
3193 3193 while (size > 0) {
3194 3194 if (ssize == 0) {
3195 3195 dvs++;
3196 3196 ssize = dvs->dvs_len;
3197 3197 dvaddr = dvs->dvs_start;
3198 3198 physcontig = 0;
3199 3199 } else
3200 3200 physcontig = 1;
3201 3201
3202 3202 paddr = dvaddr;
3203 3203 psize = MIN(ssize, maxseg);
3204 3204 dvaddr += psize;
3205 3205 ssize -= psize;
3206 3206
3207 3207 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3208 3208 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3209 3209 (sgl[cnt].dmac_size == 0)) {
3210 3210 /*
3211 3211 * if we're not already in a new cookie, go to the next
3212 3212 * cookie.
3213 3213 */
3214 3214 if (sgl[cnt].dmac_size != 0) {
3215 3215 cnt++;
3216 3216 }
3217 3217
3218 3218 /* save the cookie information */
3219 3219 sgl[cnt].dmac_laddress = paddr;
3220 3220 sgl[cnt].dmac_size = psize;
3221 3221 sgl[cnt].dmac_type = 0;
3222 3222 } else {
3223 3223 sgl[cnt].dmac_size += psize;
3224 3224
3225 3225 /*
3226 3226 * if this exactly == the maximum cookie size, and
3227 3227 * it isn't the last cookie, go to the next cookie.
3228 3228 */
3229 3229 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3230 3230 ((cnt + 1) < sglinfo->si_max_pages)) {
3231 3231 cnt++;
3232 3232 sgl[cnt].dmac_laddress = 0;
3233 3233 sgl[cnt].dmac_size = 0;
3234 3234 sgl[cnt].dmac_type = 0;
3235 3235 }
3236 3236 }
3237 3237 size -= psize;
3238 3238 }
3239 3239
3240 3240 /* we're done, save away how many cookies the sgl has */
3241 3241 if (sgl[cnt].dmac_size == 0) {
3242 3242 sglinfo->si_sgl_size = cnt;
3243 3243 } else {
3244 3244 sglinfo->si_sgl_size = cnt + 1;
3245 3245 }
3246 3246 }
3247 3247
3248 3248 /*
3249 3249 * rootnex_bind_slowpath()
3250 3250 * Call in the bind path if the calling driver can't use the sgl without
3251 3251 * modifying it. We either need to use the copy buffer and/or we will end up
3252 3252 * with a partial bind.
3253 3253 */
3254 3254 static int
3255 3255 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3256 3256 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3257 3257 {
3258 3258 rootnex_sglinfo_t *sinfo;
3259 3259 rootnex_window_t *window;
3260 3260 ddi_dma_cookie_t *cookie;
3261 3261 size_t copybuf_used;
3262 3262 size_t dmac_size;
3263 3263 boolean_t partial;
3264 3264 off_t cur_offset;
3265 3265 page_t *cur_pp;
3266 3266 major_t mnum;
3267 3267 int e;
3268 3268 int i;
3269 3269
3270 3270
3271 3271 sinfo = &dma->dp_sglinfo;
3272 3272 copybuf_used = 0;
3273 3273 partial = B_FALSE;
3274 3274
3275 3275 /*
3276 3276 * If we're using the copybuf, set the copybuf state in dma struct.
3277 3277 * Needs to be first since it sets the copy buffer size.
3278 3278 */
3279 3279 if (sinfo->si_copybuf_req != 0) {
3280 3280 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3281 3281 if (e != DDI_SUCCESS) {
3282 3282 return (e);
3283 3283 }
3284 3284 } else {
3285 3285 dma->dp_copybuf_size = 0;
3286 3286 }
3287 3287
3288 3288 /*
3289 3289 * Figure out if we need to do a partial mapping. If so, figure out
3290 3290 * if we need to trim the buffers when we munge the sgl.
3291 3291 */
3292 3292 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3293 3293 (dmao->dmao_size > dma->dp_maxxfer) ||
3294 3294 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3295 3295 dma->dp_partial_required = B_TRUE;
3296 3296 if (attr->dma_attr_granular != 1) {
3297 3297 dma->dp_trim_required = B_TRUE;
3298 3298 }
3299 3299 } else {
3300 3300 dma->dp_partial_required = B_FALSE;
3301 3301 dma->dp_trim_required = B_FALSE;
3302 3302 }
3303 3303
3304 3304 /* If we need to do a partial bind, make sure the driver supports it */
3305 3305 if (dma->dp_partial_required &&
3306 3306 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3307 3307
3308 3308 mnum = ddi_driver_major(dma->dp_dip);
3309 3309 /*
3310 3310 * patchable which allows us to print one warning per major
3311 3311 * number.
3312 3312 */
3313 3313 if ((rootnex_bind_warn) &&
3314 3314 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3315 3315 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3316 3316 cmn_err(CE_WARN, "!%s: coding error detected, the "
3317 3317 "driver is using ddi_dma_attr(9S) incorrectly. "
3318 3318 "There is a small risk of data corruption in "
3319 3319 "particular with large I/Os. The driver should be "
3320 3320 "replaced with a corrected version for proper "
3321 3321 "system operation. To disable this warning, add "
3322 3322 "'set rootnex:rootnex_bind_warn=0' to "
3323 3323 "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3324 3324 }
3325 3325 return (DDI_DMA_TOOBIG);
3326 3326 }
3327 3327
3328 3328 /*
3329 3329 * we might need multiple windows, setup state to handle them. In this
3330 3330 * code path, we will have at least one window.
3331 3331 */
3332 3332 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3333 3333 if (e != DDI_SUCCESS) {
3334 3334 rootnex_teardown_copybuf(dma);
3335 3335 return (e);
3336 3336 }
3337 3337
3338 3338 window = &dma->dp_window[0];
3339 3339 cookie = &dma->dp_cookies[0];
3340 3340 cur_offset = 0;
3341 3341 rootnex_init_win(hp, dma, window, cookie, cur_offset);
3342 3342 if (dmao->dmao_type == DMA_OTYP_PAGES) {
3343 3343 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3344 3344 }
3345 3345
3346 3346 /* loop though all the cookies we got back from get_sgl() */
3347 3347 for (i = 0; i < sinfo->si_sgl_size; i++) {
3348 3348 /*
3349 3349 * If we're using the copy buffer, check this cookie and setup
3350 3350 * its associated copy buffer state. If this cookie uses the
3351 3351 * copy buffer, make sure we sync this window during dma_sync.
3352 3352 */
3353 3353 if (dma->dp_copybuf_size > 0) {
3354 3354 rootnex_setup_cookie(dmao, dma, cookie,
3355 3355 cur_offset, ©buf_used, &cur_pp);
3356 3356 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3357 3357 window->wd_dosync = B_TRUE;
3358 3358 }
3359 3359 }
3360 3360
3361 3361 /*
3362 3362 * save away the cookie size, since it could be modified in
3363 3363 * the windowing code.
3364 3364 */
3365 3365 dmac_size = cookie->dmac_size;
3366 3366
3367 3367 /* if we went over max copybuf size */
3368 3368 if (dma->dp_copybuf_size &&
3369 3369 (copybuf_used > dma->dp_copybuf_size)) {
3370 3370 partial = B_TRUE;
3371 3371 e = rootnex_copybuf_window_boundary(hp, dma, &window,
3372 3372 cookie, cur_offset, ©buf_used);
3373 3373 if (e != DDI_SUCCESS) {
3374 3374 rootnex_teardown_copybuf(dma);
3375 3375 rootnex_teardown_windows(dma);
3376 3376 return (e);
3377 3377 }
3378 3378
3379 3379 /*
3380 3380 * if the coookie uses the copy buffer, make sure the
3381 3381 * new window we just moved to is set to sync.
3382 3382 */
3383 3383 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3384 3384 window->wd_dosync = B_TRUE;
3385 3385 }
3386 3386 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3387 3387 dma->dp_dip);
3388 3388
3389 3389 /* if the cookie cnt == max sgllen, move to the next window */
3390 3390 } else if (window->wd_cookie_cnt >=
3391 3391 (unsigned)attr->dma_attr_sgllen) {
3392 3392 partial = B_TRUE;
3393 3393 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3394 3394 e = rootnex_sgllen_window_boundary(hp, dma, &window,
3395 3395 cookie, attr, cur_offset);
3396 3396 if (e != DDI_SUCCESS) {
3397 3397 rootnex_teardown_copybuf(dma);
3398 3398 rootnex_teardown_windows(dma);
3399 3399 return (e);
3400 3400 }
3401 3401
3402 3402 /*
3403 3403 * if the coookie uses the copy buffer, make sure the
3404 3404 * new window we just moved to is set to sync.
3405 3405 */
3406 3406 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3407 3407 window->wd_dosync = B_TRUE;
3408 3408 }
3409 3409 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3410 3410 dma->dp_dip);
3411 3411
3412 3412 /* else if we will be over maxxfer */
3413 3413 } else if ((window->wd_size + dmac_size) >
3414 3414 dma->dp_maxxfer) {
3415 3415 partial = B_TRUE;
3416 3416 e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3417 3417 cookie);
3418 3418 if (e != DDI_SUCCESS) {
3419 3419 rootnex_teardown_copybuf(dma);
3420 3420 rootnex_teardown_windows(dma);
3421 3421 return (e);
3422 3422 }
3423 3423
3424 3424 /*
3425 3425 * if the coookie uses the copy buffer, make sure the
3426 3426 * new window we just moved to is set to sync.
3427 3427 */
3428 3428 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3429 3429 window->wd_dosync = B_TRUE;
3430 3430 }
3431 3431 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3432 3432 dma->dp_dip);
3433 3433
3434 3434 /* else this cookie fits in the current window */
3435 3435 } else {
3436 3436 window->wd_cookie_cnt++;
3437 3437 window->wd_size += dmac_size;
3438 3438 }
3439 3439
3440 3440 /* track our offset into the buffer, go to the next cookie */
3441 3441 ASSERT(dmac_size <= dmao->dmao_size);
3442 3442 ASSERT(cookie->dmac_size <= dmac_size);
3443 3443 cur_offset += dmac_size;
3444 3444 cookie++;
3445 3445 }
3446 3446
3447 3447 /* if we ended up with a zero sized window in the end, clean it up */
3448 3448 if (window->wd_size == 0) {
3449 3449 hp->dmai_nwin--;
3450 3450 window--;
3451 3451 }
3452 3452
3453 3453 ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3454 3454
3455 3455 if (!partial) {
3456 3456 return (DDI_DMA_MAPPED);
3457 3457 }
3458 3458
3459 3459 ASSERT(dma->dp_partial_required);
3460 3460 return (DDI_DMA_PARTIAL_MAP);
3461 3461 }
3462 3462
3463 3463 /*
3464 3464 * rootnex_setup_copybuf()
3465 3465 * Called in bind slowpath. Figures out if we're going to use the copy
3466 3466 * buffer, and if we do, sets up the basic state to handle it.
3467 3467 */
3468 3468 static int
3469 3469 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3470 3470 rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3471 3471 {
3472 3472 rootnex_sglinfo_t *sinfo;
3473 3473 ddi_dma_attr_t lattr;
3474 3474 size_t max_copybuf;
3475 3475 int cansleep;
3476 3476 int e;
3477 3477 #if !defined(__amd64)
3478 3478 int vmflag;
3479 3479 #endif
3480 3480
3481 3481 ASSERT(!dma->dp_dvma_used);
3482 3482
3483 3483 sinfo = &dma->dp_sglinfo;
3484 3484
3485 3485 /* read this first so it's consistent through the routine */
3486 3486 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3487 3487
3488 3488 /* We need to call into the rootnex on ddi_dma_sync() */
3489 3489 hp->dmai_rflags &= ~DMP_NOSYNC;
3490 3490
3491 3491 /* make sure the copybuf size <= the max size */
3492 3492 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3493 3493 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3494 3494
3495 3495 #if !defined(__amd64)
3496 3496 /*
3497 3497 * if we don't have kva space to copy to/from, allocate the KVA space
3498 3498 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3499 3499 * the 64-bit kernel.
3500 3500 */
3501 3501 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3502 3502 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3503 3503
3504 3504 /* convert the sleep flags */
3505 3505 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3506 3506 vmflag = VM_SLEEP;
3507 3507 } else {
3508 3508 vmflag = VM_NOSLEEP;
3509 3509 }
3510 3510
3511 3511 /* allocate Kernel VA space that we can bcopy to/from */
3512 3512 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3513 3513 vmflag);
3514 3514 if (dma->dp_kva == NULL) {
3515 3515 return (DDI_DMA_NORESOURCES);
3516 3516 }
3517 3517 }
3518 3518 #endif
3519 3519
3520 3520 /* convert the sleep flags */
3521 3521 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3522 3522 cansleep = 1;
3523 3523 } else {
3524 3524 cansleep = 0;
3525 3525 }
3526 3526
3527 3527 /*
3528 3528 * Allocate the actual copy buffer. This needs to fit within the DMA
3529 3529 * engine limits, so we can't use kmem_alloc... We don't need
3530 3530 * contiguous memory (sgllen) since we will be forcing windows on
3531 3531 * sgllen anyway.
3532 3532 */
3533 3533 lattr = *attr;
3534 3534 lattr.dma_attr_align = MMU_PAGESIZE;
3535 3535 lattr.dma_attr_sgllen = -1; /* no limit */
3536 3536 /*
3537 3537 * if we're using the copy buffer because of seg, use that for our
3538 3538 * upper address limit.
3539 3539 */
3540 3540 if (sinfo->si_bounce_on_seg) {
3541 3541 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3542 3542 }
3543 3543 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3544 3544 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3545 3545 if (e != DDI_SUCCESS) {
3546 3546 #if !defined(__amd64)
3547 3547 if (dma->dp_kva != NULL) {
3548 3548 vmem_free(heap_arena, dma->dp_kva,
3549 3549 dma->dp_copybuf_size);
3550 3550 }
3551 3551 #endif
3552 3552 return (DDI_DMA_NORESOURCES);
3553 3553 }
3554 3554
3555 3555 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3556 3556 size_t, dma->dp_copybuf_size);
3557 3557
3558 3558 return (DDI_SUCCESS);
3559 3559 }
3560 3560
3561 3561
3562 3562 /*
3563 3563 * rootnex_setup_windows()
3564 3564 * Called in bind slowpath to setup the window state. We always have windows
3565 3565 * in the slowpath. Even if the window count = 1.
3566 3566 */
3567 3567 static int
3568 3568 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3569 3569 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3570 3570 {
3571 3571 rootnex_window_t *windowp;
3572 3572 rootnex_sglinfo_t *sinfo;
3573 3573 size_t copy_state_size;
3574 3574 size_t win_state_size;
3575 3575 size_t state_available;
3576 3576 size_t space_needed;
3577 3577 uint_t copybuf_win;
3578 3578 uint_t maxxfer_win;
3579 3579 size_t space_used;
3580 3580 uint_t sglwin;
3581 3581
3582 3582
3583 3583 sinfo = &dma->dp_sglinfo;
3584 3584
3585 3585 dma->dp_current_win = 0;
3586 3586 hp->dmai_nwin = 0;
3587 3587
3588 3588 /* If we don't need to do a partial, we only have one window */
3589 3589 if (!dma->dp_partial_required) {
3590 3590 dma->dp_max_win = 1;
3591 3591
3592 3592 /*
3593 3593 * we need multiple windows, need to figure out the worse case number
3594 3594 * of windows.
3595 3595 */
3596 3596 } else {
3597 3597 /*
3598 3598 * if we need windows because we need more copy buffer that
3599 3599 * we allow, the worse case number of windows we could need
3600 3600 * here would be (copybuf space required / copybuf space that
3601 3601 * we have) plus one for remainder, and plus 2 to handle the
3602 3602 * extra pages on the trim for the first and last pages of the
3603 3603 * buffer (a page is the minimum window size so under the right
3604 3604 * attr settings, you could have a window for each page).
3605 3605 * The last page will only be hit here if the size is not a
3606 3606 * multiple of the granularity (which theoretically shouldn't
3607 3607 * be the case but never has been enforced, so we could have
3608 3608 * broken things without it).
3609 3609 */
3610 3610 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3611 3611 ASSERT(dma->dp_copybuf_size > 0);
3612 3612 copybuf_win = (sinfo->si_copybuf_req /
3613 3613 dma->dp_copybuf_size) + 1 + 2;
3614 3614 } else {
3615 3615 copybuf_win = 0;
3616 3616 }
3617 3617
3618 3618 /*
3619 3619 * if we need windows because we have more cookies than the H/W
3620 3620 * can handle, the number of windows we would need here would
3621 3621 * be (cookie count / cookies count H/W supports minus 1[for
3622 3622 * trim]) plus one for remainder.
3623 3623 */
3624 3624 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3625 3625 sglwin = (sinfo->si_sgl_size /
3626 3626 (attr->dma_attr_sgllen - 1)) + 1;
3627 3627 } else {
3628 3628 sglwin = 0;
3629 3629 }
3630 3630
3631 3631 /*
3632 3632 * if we need windows because we're binding more memory than the
3633 3633 * H/W can transfer at once, the number of windows we would need
3634 3634 * here would be (xfer count / max xfer H/W supports) plus one
3635 3635 * for remainder, and plus 2 to handle the extra pages on the
3636 3636 * trim (see above comment about trim)
3637 3637 */
3638 3638 if (dmao->dmao_size > dma->dp_maxxfer) {
3639 3639 maxxfer_win = (dmao->dmao_size /
3640 3640 dma->dp_maxxfer) + 1 + 2;
3641 3641 } else {
3642 3642 maxxfer_win = 0;
3643 3643 }
3644 3644 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
3645 3645 ASSERT(dma->dp_max_win > 0);
3646 3646 }
3647 3647 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3648 3648
3649 3649 /*
3650 3650 * Get space for window and potential copy buffer state. Before we
3651 3651 * go and allocate memory, see if we can get away with using what's
3652 3652 * left in the pre-allocted state or the dynamically allocated sgl.
3653 3653 */
3654 3654 space_used = (uintptr_t)(sinfo->si_sgl_size *
3655 3655 sizeof (ddi_dma_cookie_t));
3656 3656
3657 3657 /* if we dynamically allocated space for the cookies */
3658 3658 if (dma->dp_need_to_free_cookie) {
3659 3659 /* if we have more space in the pre-allocted buffer, use it */
3660 3660 ASSERT(space_used <= dma->dp_cookie_size);
3661 3661 if ((dma->dp_cookie_size - space_used) <=
3662 3662 rootnex_state->r_prealloc_size) {
3663 3663 state_available = rootnex_state->r_prealloc_size;
3664 3664 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3665 3665
3666 3666 /*
3667 3667 * else, we have more free space in the dynamically allocated
3668 3668 * buffer, i.e. the buffer wasn't worse case fragmented so we
3669 3669 * didn't need a lot of cookies.
3670 3670 */
3671 3671 } else {
3672 3672 state_available = dma->dp_cookie_size - space_used;
3673 3673 windowp = (rootnex_window_t *)
3674 3674 &dma->dp_cookies[sinfo->si_sgl_size];
3675 3675 }
3676 3676
3677 3677 /* we used the pre-alloced buffer */
3678 3678 } else {
3679 3679 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3680 3680 state_available = rootnex_state->r_prealloc_size - space_used;
3681 3681 windowp = (rootnex_window_t *)
3682 3682 &dma->dp_cookies[sinfo->si_sgl_size];
3683 3683 }
3684 3684
3685 3685 /*
3686 3686 * figure out how much state we need to track the copy buffer. Add an
3687 3687 * addition 8 bytes for pointer alignemnt later.
3688 3688 */
3689 3689 if (dma->dp_copybuf_size > 0) {
3690 3690 copy_state_size = sinfo->si_max_pages *
3691 3691 sizeof (rootnex_pgmap_t);
3692 3692 } else {
3693 3693 copy_state_size = 0;
3694 3694 }
3695 3695 /* add an additional 8 bytes for pointer alignment */
3696 3696 space_needed = win_state_size + copy_state_size + 0x8;
3697 3697
3698 3698 /* if we have enough space already, use it */
3699 3699 if (state_available >= space_needed) {
3700 3700 dma->dp_window = windowp;
3701 3701 dma->dp_need_to_free_window = B_FALSE;
3702 3702
3703 3703 /* not enough space, need to allocate more. */
3704 3704 } else {
3705 3705 dma->dp_window = kmem_alloc(space_needed, kmflag);
3706 3706 if (dma->dp_window == NULL) {
3707 3707 return (DDI_DMA_NORESOURCES);
3708 3708 }
3709 3709 dma->dp_need_to_free_window = B_TRUE;
3710 3710 dma->dp_window_size = space_needed;
3711 3711 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3712 3712 dma->dp_dip, size_t, space_needed);
3713 3713 }
3714 3714
3715 3715 /*
3716 3716 * we allocate copy buffer state and window state at the same time.
3717 3717 * setup our copy buffer state pointers. Make sure it's aligned.
3718 3718 */
3719 3719 if (dma->dp_copybuf_size > 0) {
3720 3720 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3721 3721 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3722 3722
3723 3723 #if !defined(__amd64)
3724 3724 /*
3725 3725 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3726 3726 * false/NULL. Should be quicker to bzero vs loop and set.
3727 3727 */
3728 3728 bzero(dma->dp_pgmap, copy_state_size);
3729 3729 #endif
3730 3730 } else {
3731 3731 dma->dp_pgmap = NULL;
3732 3732 }
3733 3733
3734 3734 return (DDI_SUCCESS);
3735 3735 }
3736 3736
3737 3737
3738 3738 /*
3739 3739 * rootnex_teardown_copybuf()
3740 3740 * cleans up after rootnex_setup_copybuf()
3741 3741 */
3742 3742 static void
3743 3743 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3744 3744 {
3745 3745 #if !defined(__amd64)
3746 3746 int i;
3747 3747
3748 3748 /*
3749 3749 * if we allocated kernel heap VMEM space, go through all the pages and
3750 3750 * map out any of the ones that we're mapped into the kernel heap VMEM
3751 3751 * arena. Then free the VMEM space.
3752 3752 */
3753 3753 if (dma->dp_kva != NULL) {
3754 3754 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3755 3755 if (dma->dp_pgmap[i].pm_mapped) {
3756 3756 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3757 3757 MMU_PAGESIZE, HAT_UNLOAD);
3758 3758 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3759 3759 }
3760 3760 }
3761 3761
3762 3762 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3763 3763 }
3764 3764
3765 3765 #endif
3766 3766
3767 3767 /* if we allocated a copy buffer, free it */
3768 3768 if (dma->dp_cbaddr != NULL) {
3769 3769 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3770 3770 }
3771 3771 }
3772 3772
3773 3773
3774 3774 /*
3775 3775 * rootnex_teardown_windows()
3776 3776 * cleans up after rootnex_setup_windows()
3777 3777 */
3778 3778 static void
3779 3779 rootnex_teardown_windows(rootnex_dma_t *dma)
3780 3780 {
3781 3781 /*
3782 3782 * if we had to allocate window state on the last bind (because we
3783 3783 * didn't have enough pre-allocated space in the handle), free it.
3784 3784 */
3785 3785 if (dma->dp_need_to_free_window) {
3786 3786 kmem_free(dma->dp_window, dma->dp_window_size);
3787 3787 }
3788 3788 }
3789 3789
3790 3790
3791 3791 /*
3792 3792 * rootnex_init_win()
3793 3793 * Called in bind slow path during creation of a new window. Initializes
3794 3794 * window state to default values.
3795 3795 */
3796 3796 /*ARGSUSED*/
3797 3797 static void
3798 3798 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3799 3799 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3800 3800 {
3801 3801 hp->dmai_nwin++;
3802 3802 window->wd_dosync = B_FALSE;
3803 3803 window->wd_offset = cur_offset;
3804 3804 window->wd_size = 0;
3805 3805 window->wd_first_cookie = cookie;
3806 3806 window->wd_cookie_cnt = 0;
3807 3807 window->wd_trim.tr_trim_first = B_FALSE;
3808 3808 window->wd_trim.tr_trim_last = B_FALSE;
3809 3809 window->wd_trim.tr_first_copybuf_win = B_FALSE;
3810 3810 window->wd_trim.tr_last_copybuf_win = B_FALSE;
3811 3811 #if !defined(__amd64)
3812 3812 window->wd_remap_copybuf = dma->dp_cb_remaping;
3813 3813 #endif
3814 3814 }
3815 3815
3816 3816
3817 3817 /*
3818 3818 * rootnex_setup_cookie()
3819 3819 * Called in the bind slow path when the sgl uses the copy buffer. If any of
3820 3820 * the sgl uses the copy buffer, we need to go through each cookie, figure
3821 3821 * out if it uses the copy buffer, and if it does, save away everything we'll
3822 3822 * need during sync.
3823 3823 */
3824 3824 static void
3825 3825 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3826 3826 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3827 3827 page_t **cur_pp)
3828 3828 {
3829 3829 boolean_t copybuf_sz_power_2;
3830 3830 rootnex_sglinfo_t *sinfo;
3831 3831 paddr_t paddr;
3832 3832 uint_t pidx;
3833 3833 uint_t pcnt;
3834 3834 off_t poff;
3835 3835 #if defined(__amd64)
3836 3836 pfn_t pfn;
3837 3837 #else
3838 3838 page_t **pplist;
3839 3839 #endif
3840 3840
3841 3841 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3842 3842
3843 3843 sinfo = &dma->dp_sglinfo;
3844 3844
3845 3845 /*
3846 3846 * Calculate the page index relative to the start of the buffer. The
3847 3847 * index to the current page for our buffer is the offset into the
3848 3848 * first page of the buffer plus our current offset into the buffer
3849 3849 * itself, shifted of course...
3850 3850 */
3851 3851 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3852 3852 ASSERT(pidx < sinfo->si_max_pages);
3853 3853
3854 3854 /* if this cookie uses the copy buffer */
3855 3855 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3856 3856 /*
3857 3857 * NOTE: we know that since this cookie uses the copy buffer, it
3858 3858 * is <= MMU_PAGESIZE.
3859 3859 */
3860 3860
3861 3861 /*
3862 3862 * get the offset into the page. For the 64-bit kernel, get the
3863 3863 * pfn which we'll use with seg kpm.
3864 3864 */
3865 3865 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3866 3866 #if defined(__amd64)
3867 3867 /* mfn_to_pfn() is a NOP on i86pc */
3868 3868 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3869 3869 #endif /* __amd64 */
3870 3870
3871 3871 /* figure out if the copybuf size is a power of 2 */
3872 3872 if (!ISP2(dma->dp_copybuf_size)) {
3873 3873 copybuf_sz_power_2 = B_FALSE;
3874 3874 } else {
3875 3875 copybuf_sz_power_2 = B_TRUE;
3876 3876 }
3877 3877
3878 3878 /* This page uses the copy buffer */
3879 3879 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3880 3880
3881 3881 /*
3882 3882 * save the copy buffer KVA that we'll use with this page.
3883 3883 * if we still fit within the copybuf, it's a simple add.
3884 3884 * otherwise, we need to wrap over using & or % accordingly.
3885 3885 */
3886 3886 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3887 3887 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3888 3888 *copybuf_used;
3889 3889 } else {
3890 3890 if (copybuf_sz_power_2) {
3891 3891 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3892 3892 (uintptr_t)dma->dp_cbaddr +
3893 3893 (*copybuf_used &
3894 3894 (dma->dp_copybuf_size - 1)));
3895 3895 } else {
3896 3896 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3897 3897 (uintptr_t)dma->dp_cbaddr +
3898 3898 (*copybuf_used % dma->dp_copybuf_size));
3899 3899 }
3900 3900 }
3901 3901
3902 3902 /*
3903 3903 * over write the cookie physical address with the address of
3904 3904 * the physical address of the copy buffer page that we will
3905 3905 * use.
3906 3906 */
3907 3907 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3908 3908 dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3909 3909
3910 3910 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3911 3911
3912 3912 /* if we have a kernel VA, it's easy, just save that address */
3913 3913 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3914 3914 (sinfo->si_asp == &kas)) {
3915 3915 /*
3916 3916 * save away the page aligned virtual address of the
3917 3917 * driver buffer. Offsets are handled in the sync code.
3918 3918 */
3919 3919 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3920 3920 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3921 3921 & MMU_PAGEMASK);
3922 3922 #if !defined(__amd64)
3923 3923 /*
3924 3924 * we didn't need to, and will never need to map this
3925 3925 * page.
3926 3926 */
3927 3927 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3928 3928 #endif
3929 3929
3930 3930 /* we don't have a kernel VA. We need one for the bcopy. */
3931 3931 } else {
3932 3932 #if defined(__amd64)
3933 3933 /*
3934 3934 * for the 64-bit kernel, it's easy. We use seg kpm to
3935 3935 * get a Kernel VA for the corresponding pfn.
3936 3936 */
3937 3937 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3938 3938 #else
3939 3939 /*
3940 3940 * for the 32-bit kernel, this is a pain. First we'll
3941 3941 * save away the page_t or user VA for this page. This
3942 3942 * is needed in rootnex_dma_win() when we switch to a
3943 3943 * new window which requires us to re-map the copy
3944 3944 * buffer.
3945 3945 */
3946 3946 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3947 3947 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3948 3948 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3949 3949 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3950 3950 } else if (pplist != NULL) {
3951 3951 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3952 3952 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3953 3953 } else {
3954 3954 dma->dp_pgmap[pidx].pm_pp = NULL;
3955 3955 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3956 3956 (((uintptr_t)
3957 3957 dmar_object->dmao_obj.virt_obj.v_addr +
3958 3958 cur_offset) & MMU_PAGEMASK);
3959 3959 }
3960 3960
3961 3961 /*
3962 3962 * save away the page aligned virtual address which was
3963 3963 * allocated from the kernel heap arena (taking into
3964 3964 * account if we need more copy buffer than we alloced
3965 3965 * and use multiple windows to handle this, i.e. &,%).
3966 3966 * NOTE: there isn't and physical memory backing up this
3967 3967 * virtual address space currently.
3968 3968 */
3969 3969 if ((*copybuf_used + MMU_PAGESIZE) <=
3970 3970 dma->dp_copybuf_size) {
3971 3971 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3972 3972 (((uintptr_t)dma->dp_kva + *copybuf_used) &
3973 3973 MMU_PAGEMASK);
3974 3974 } else {
3975 3975 if (copybuf_sz_power_2) {
3976 3976 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3977 3977 (((uintptr_t)dma->dp_kva +
3978 3978 (*copybuf_used &
3979 3979 (dma->dp_copybuf_size - 1))) &
3980 3980 MMU_PAGEMASK);
3981 3981 } else {
3982 3982 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3983 3983 (((uintptr_t)dma->dp_kva +
3984 3984 (*copybuf_used %
3985 3985 dma->dp_copybuf_size)) &
3986 3986 MMU_PAGEMASK);
3987 3987 }
3988 3988 }
3989 3989
3990 3990 /*
3991 3991 * if we haven't used up the available copy buffer yet,
3992 3992 * map the kva to the physical page.
3993 3993 */
3994 3994 if (!dma->dp_cb_remaping && ((*copybuf_used +
3995 3995 MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3996 3996 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3997 3997 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3998 3998 i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3999 3999 dma->dp_pgmap[pidx].pm_kaddr);
4000 4000 } else {
4001 4001 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4002 4002 sinfo->si_asp,
4003 4003 dma->dp_pgmap[pidx].pm_kaddr);
4004 4004 }
4005 4005
4006 4006 /*
4007 4007 * we've used up the available copy buffer, this page
4008 4008 * will have to be mapped during rootnex_dma_win() when
4009 4009 * we switch to a new window which requires a re-map
4010 4010 * the copy buffer. (32-bit kernel only)
4011 4011 */
4012 4012 } else {
4013 4013 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4014 4014 }
4015 4015 #endif
4016 4016 /* go to the next page_t */
4017 4017 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4018 4018 *cur_pp = (*cur_pp)->p_next;
4019 4019 }
4020 4020 }
4021 4021
4022 4022 /* add to the copy buffer count */
4023 4023 *copybuf_used += MMU_PAGESIZE;
4024 4024
4025 4025 /*
4026 4026 * This cookie doesn't use the copy buffer. Walk through the pages this
4027 4027 * cookie occupies to reflect this.
4028 4028 */
4029 4029 } else {
4030 4030 /*
4031 4031 * figure out how many pages the cookie occupies. We need to
4032 4032 * use the original page offset of the buffer and the cookies
4033 4033 * offset in the buffer to do this.
4034 4034 */
4035 4035 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4036 4036 pcnt = mmu_btopr(cookie->dmac_size + poff);
4037 4037
4038 4038 while (pcnt > 0) {
4039 4039 #if !defined(__amd64)
4040 4040 /*
4041 4041 * the 32-bit kernel doesn't have seg kpm, so we need
4042 4042 * to map in the driver buffer (if it didn't come down
4043 4043 * with a kernel VA) on the fly. Since this page doesn't
4044 4044 * use the copy buffer, it's not, or will it ever, have
4045 4045 * to be mapped in.
4046 4046 */
4047 4047 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4048 4048 #endif
4049 4049 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4050 4050
4051 4051 /*
4052 4052 * we need to update pidx and cur_pp or we'll loose
4053 4053 * track of where we are.
4054 4054 */
4055 4055 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4056 4056 *cur_pp = (*cur_pp)->p_next;
4057 4057 }
4058 4058 pidx++;
4059 4059 pcnt--;
4060 4060 }
4061 4061 }
4062 4062 }
4063 4063
4064 4064
4065 4065 /*
4066 4066 * rootnex_sgllen_window_boundary()
4067 4067 * Called in the bind slow path when the next cookie causes us to exceed (in
4068 4068 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4069 4069 * length supported by the DMA H/W.
4070 4070 */
4071 4071 static int
4072 4072 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4073 4073 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4074 4074 off_t cur_offset)
4075 4075 {
4076 4076 off_t new_offset;
4077 4077 size_t trim_sz;
4078 4078 off_t coffset;
4079 4079
4080 4080
4081 4081 /*
4082 4082 * if we know we'll never have to trim, it's pretty easy. Just move to
4083 4083 * the next window and init it. We're done.
4084 4084 */
4085 4085 if (!dma->dp_trim_required) {
4086 4086 (*windowp)++;
4087 4087 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4088 4088 (*windowp)->wd_cookie_cnt++;
4089 4089 (*windowp)->wd_size = cookie->dmac_size;
4090 4090 return (DDI_SUCCESS);
4091 4091 }
4092 4092
4093 4093 /* figure out how much we need to trim from the window */
4094 4094 ASSERT(attr->dma_attr_granular != 0);
4095 4095 if (dma->dp_granularity_power_2) {
4096 4096 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4097 4097 } else {
4098 4098 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4099 4099 }
4100 4100
4101 4101 /* The window's a whole multiple of granularity. We're done */
4102 4102 if (trim_sz == 0) {
4103 4103 (*windowp)++;
4104 4104 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4105 4105 (*windowp)->wd_cookie_cnt++;
4106 4106 (*windowp)->wd_size = cookie->dmac_size;
4107 4107 return (DDI_SUCCESS);
4108 4108 }
4109 4109
4110 4110 /*
4111 4111 * The window's not a whole multiple of granularity, since we know this
4112 4112 * is due to the sgllen, we need to go back to the last cookie and trim
4113 4113 * that one, add the left over part of the old cookie into the new
4114 4114 * window, and then add in the new cookie into the new window.
4115 4115 */
4116 4116
4117 4117 /*
4118 4118 * make sure the driver isn't making us do something bad... Trimming and
4119 4119 * sgllen == 1 don't go together.
4120 4120 */
4121 4121 if (attr->dma_attr_sgllen == 1) {
4122 4122 return (DDI_DMA_NOMAPPING);
4123 4123 }
4124 4124
4125 4125 /*
4126 4126 * first, setup the current window to account for the trim. Need to go
4127 4127 * back to the last cookie for this.
4128 4128 */
4129 4129 cookie--;
4130 4130 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4131 4131 (*windowp)->wd_trim.tr_last_cookie = cookie;
4132 4132 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4133 4133 ASSERT(cookie->dmac_size > trim_sz);
4134 4134 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4135 4135 (*windowp)->wd_size -= trim_sz;
4136 4136
4137 4137 /* save the buffer offsets for the next window */
4138 4138 coffset = cookie->dmac_size - trim_sz;
4139 4139 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4140 4140
4141 4141 /*
4142 4142 * set this now in case this is the first window. all other cases are
4143 4143 * set in dma_win()
4144 4144 */
4145 4145 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4146 4146
4147 4147 /*
4148 4148 * initialize the next window using what's left over in the previous
4149 4149 * cookie.
4150 4150 */
4151 4151 (*windowp)++;
4152 4152 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4153 4153 (*windowp)->wd_cookie_cnt++;
4154 4154 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4155 4155 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4156 4156 (*windowp)->wd_trim.tr_first_size = trim_sz;
4157 4157 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4158 4158 (*windowp)->wd_dosync = B_TRUE;
4159 4159 }
4160 4160
4161 4161 /*
4162 4162 * now go back to the current cookie and add it to the new window. set
4163 4163 * the new window size to the what was left over from the previous
4164 4164 * cookie and what's in the current cookie.
4165 4165 */
4166 4166 cookie++;
4167 4167 (*windowp)->wd_cookie_cnt++;
4168 4168 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4169 4169
4170 4170 /*
4171 4171 * trim plus the next cookie could put us over maxxfer (a cookie can be
4172 4172 * a max size of maxxfer). Handle that case.
4173 4173 */
4174 4174 if ((*windowp)->wd_size > dma->dp_maxxfer) {
4175 4175 /*
4176 4176 * maxxfer is already a whole multiple of granularity, and this
4177 4177 * trim will be <= the previous trim (since a cookie can't be
4178 4178 * larger than maxxfer). Make things simple here.
4179 4179 */
4180 4180 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4181 4181 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4182 4182 (*windowp)->wd_trim.tr_last_cookie = cookie;
4183 4183 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4184 4184 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4185 4185 (*windowp)->wd_size -= trim_sz;
4186 4186 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4187 4187
4188 4188 /* save the buffer offsets for the next window */
4189 4189 coffset = cookie->dmac_size - trim_sz;
4190 4190 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4191 4191
4192 4192 /* setup the next window */
4193 4193 (*windowp)++;
4194 4194 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4195 4195 (*windowp)->wd_cookie_cnt++;
4196 4196 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4197 4197 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4198 4198 coffset;
4199 4199 (*windowp)->wd_trim.tr_first_size = trim_sz;
4200 4200 }
4201 4201
4202 4202 return (DDI_SUCCESS);
4203 4203 }
4204 4204
4205 4205
4206 4206 /*
4207 4207 * rootnex_copybuf_window_boundary()
4208 4208 * Called in bind slowpath when we get to a window boundary because we used
4209 4209 * up all the copy buffer that we have.
4210 4210 */
4211 4211 static int
4212 4212 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4213 4213 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4214 4214 size_t *copybuf_used)
4215 4215 {
4216 4216 rootnex_sglinfo_t *sinfo;
4217 4217 off_t new_offset;
4218 4218 size_t trim_sz;
4219 4219 paddr_t paddr;
4220 4220 off_t coffset;
4221 4221 uint_t pidx;
4222 4222 off_t poff;
4223 4223
4224 4224
4225 4225 sinfo = &dma->dp_sglinfo;
4226 4226
4227 4227 /*
4228 4228 * the copy buffer should be a whole multiple of page size. We know that
4229 4229 * this cookie is <= MMU_PAGESIZE.
4230 4230 */
4231 4231 ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4232 4232
4233 4233 /*
4234 4234 * from now on, all new windows in this bind need to be re-mapped during
4235 4235 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4236 4236 * space...
4237 4237 */
4238 4238 #if !defined(__amd64)
4239 4239 dma->dp_cb_remaping = B_TRUE;
4240 4240 #endif
4241 4241
4242 4242 /* reset copybuf used */
4243 4243 *copybuf_used = 0;
4244 4244
4245 4245 /*
4246 4246 * if we don't have to trim (since granularity is set to 1), go to the
4247 4247 * next window and add the current cookie to it. We know the current
4248 4248 * cookie uses the copy buffer since we're in this code path.
4249 4249 */
4250 4250 if (!dma->dp_trim_required) {
4251 4251 (*windowp)++;
4252 4252 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4253 4253
4254 4254 /* Add this cookie to the new window */
4255 4255 (*windowp)->wd_cookie_cnt++;
4256 4256 (*windowp)->wd_size += cookie->dmac_size;
4257 4257 *copybuf_used += MMU_PAGESIZE;
4258 4258 return (DDI_SUCCESS);
4259 4259 }
4260 4260
4261 4261 /*
4262 4262 * *** may need to trim, figure it out.
4263 4263 */
4264 4264
4265 4265 /* figure out how much we need to trim from the window */
4266 4266 if (dma->dp_granularity_power_2) {
4267 4267 trim_sz = (*windowp)->wd_size &
4268 4268 (hp->dmai_attr.dma_attr_granular - 1);
4269 4269 } else {
4270 4270 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4271 4271 }
4272 4272
4273 4273 /*
4274 4274 * if the window's a whole multiple of granularity, go to the next
4275 4275 * window, init it, then add in the current cookie. We know the current
4276 4276 * cookie uses the copy buffer since we're in this code path.
4277 4277 */
4278 4278 if (trim_sz == 0) {
4279 4279 (*windowp)++;
4280 4280 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4281 4281
4282 4282 /* Add this cookie to the new window */
4283 4283 (*windowp)->wd_cookie_cnt++;
4284 4284 (*windowp)->wd_size += cookie->dmac_size;
4285 4285 *copybuf_used += MMU_PAGESIZE;
4286 4286 return (DDI_SUCCESS);
4287 4287 }
4288 4288
4289 4289 /*
4290 4290 * *** We figured it out, we definitly need to trim
4291 4291 */
4292 4292
4293 4293 /*
4294 4294 * make sure the driver isn't making us do something bad...
4295 4295 * Trimming and sgllen == 1 don't go together.
4296 4296 */
4297 4297 if (hp->dmai_attr.dma_attr_sgllen == 1) {
4298 4298 return (DDI_DMA_NOMAPPING);
4299 4299 }
4300 4300
4301 4301 /*
4302 4302 * first, setup the current window to account for the trim. Need to go
4303 4303 * back to the last cookie for this. Some of the last cookie will be in
4304 4304 * the current window, and some of the last cookie will be in the new
4305 4305 * window. All of the current cookie will be in the new window.
4306 4306 */
4307 4307 cookie--;
4308 4308 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4309 4309 (*windowp)->wd_trim.tr_last_cookie = cookie;
4310 4310 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4311 4311 ASSERT(cookie->dmac_size > trim_sz);
4312 4312 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4313 4313 (*windowp)->wd_size -= trim_sz;
4314 4314
4315 4315 /*
4316 4316 * we're trimming the last cookie (not the current cookie). So that
4317 4317 * last cookie may have or may not have been using the copy buffer (
4318 4318 * we know the cookie passed in uses the copy buffer since we're in
4319 4319 * this code path).
4320 4320 *
4321 4321 * If the last cookie doesn't use the copy buffer, nothing special to
4322 4322 * do. However, if it does uses the copy buffer, it will be both the
4323 4323 * last page in the current window and the first page in the next
4324 4324 * window. Since we are reusing the copy buffer (and KVA space on the
4325 4325 * 32-bit kernel), this page will use the end of the copy buffer in the
4326 4326 * current window, and the start of the copy buffer in the next window.
4327 4327 * Track that info... The cookie physical address was already set to
4328 4328 * the copy buffer physical address in setup_cookie..
4329 4329 */
4330 4330 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4331 4331 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4332 4332 (*windowp)->wd_size) >> MMU_PAGESHIFT;
4333 4333 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4334 4334 (*windowp)->wd_trim.tr_last_pidx = pidx;
4335 4335 (*windowp)->wd_trim.tr_last_cbaddr =
4336 4336 dma->dp_pgmap[pidx].pm_cbaddr;
4337 4337 #if !defined(__amd64)
4338 4338 (*windowp)->wd_trim.tr_last_kaddr =
4339 4339 dma->dp_pgmap[pidx].pm_kaddr;
4340 4340 #endif
4341 4341 }
4342 4342
4343 4343 /* save the buffer offsets for the next window */
4344 4344 coffset = cookie->dmac_size - trim_sz;
4345 4345 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4346 4346
4347 4347 /*
4348 4348 * set this now in case this is the first window. all other cases are
4349 4349 * set in dma_win()
4350 4350 */
4351 4351 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4352 4352
4353 4353 /*
4354 4354 * initialize the next window using what's left over in the previous
4355 4355 * cookie.
4356 4356 */
4357 4357 (*windowp)++;
4358 4358 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4359 4359 (*windowp)->wd_cookie_cnt++;
4360 4360 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4361 4361 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4362 4362 (*windowp)->wd_trim.tr_first_size = trim_sz;
4363 4363
4364 4364 /*
4365 4365 * again, we're tracking if the last cookie uses the copy buffer.
4366 4366 * read the comment above for more info on why we need to track
4367 4367 * additional state.
4368 4368 *
4369 4369 * For the first cookie in the new window, we need reset the physical
4370 4370 * address to DMA into to the start of the copy buffer plus any
4371 4371 * initial page offset which may be present.
4372 4372 */
4373 4373 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4374 4374 (*windowp)->wd_dosync = B_TRUE;
4375 4375 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4376 4376 (*windowp)->wd_trim.tr_first_pidx = pidx;
4377 4377 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4378 4378 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4379 4379
4380 4380 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4381 4381 poff;
4382 4382 (*windowp)->wd_trim.tr_first_paddr =
4383 4383 ROOTNEX_PADDR_TO_RBASE(paddr);
4384 4384
4385 4385 #if !defined(__amd64)
4386 4386 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4387 4387 #endif
4388 4388 /* account for the cookie copybuf usage in the new window */
4389 4389 *copybuf_used += MMU_PAGESIZE;
4390 4390
4391 4391 /*
4392 4392 * every piece of code has to have a hack, and here is this
4393 4393 * ones :-)
4394 4394 *
4395 4395 * There is a complex interaction between setup_cookie and the
4396 4396 * copybuf window boundary. The complexity had to be in either
4397 4397 * the maxxfer window, or the copybuf window, and I chose the
4398 4398 * copybuf code.
4399 4399 *
4400 4400 * So in this code path, we have taken the last cookie,
4401 4401 * virtually broken it in half due to the trim, and it happens
4402 4402 * to use the copybuf which further complicates life. At the
4403 4403 * same time, we have already setup the current cookie, which
4404 4404 * is now wrong. More background info: the current cookie uses
4405 4405 * the copybuf, so it is only a page long max. So we need to
4406 4406 * fix the current cookies copy buffer address, physical
4407 4407 * address, and kva for the 32-bit kernel. We due this by
4408 4408 * bumping them by page size (of course, we can't due this on
4409 4409 * the physical address since the copy buffer may not be
4410 4410 * physically contiguous).
4411 4411 */
4412 4412 cookie++;
4413 4413 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4414 4414 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4415 4415
4416 4416 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4417 4417 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4418 4418 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4419 4419
4420 4420 #if !defined(__amd64)
4421 4421 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4422 4422 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4423 4423 #endif
4424 4424 } else {
4425 4425 /* go back to the current cookie */
4426 4426 cookie++;
4427 4427 }
4428 4428
4429 4429 /*
4430 4430 * add the current cookie to the new window. set the new window size to
4431 4431 * the what was left over from the previous cookie and what's in the
4432 4432 * current cookie.
4433 4433 */
4434 4434 (*windowp)->wd_cookie_cnt++;
4435 4435 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4436 4436 ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4437 4437
4438 4438 /*
4439 4439 * we know that the cookie passed in always uses the copy buffer. We
4440 4440 * wouldn't be here if it didn't.
4441 4441 */
4442 4442 *copybuf_used += MMU_PAGESIZE;
4443 4443
4444 4444 return (DDI_SUCCESS);
4445 4445 }
4446 4446
4447 4447
4448 4448 /*
4449 4449 * rootnex_maxxfer_window_boundary()
4450 4450 * Called in bind slowpath when we get to a window boundary because we will
4451 4451 * go over maxxfer.
4452 4452 */
4453 4453 static int
4454 4454 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4455 4455 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4456 4456 {
4457 4457 size_t dmac_size;
4458 4458 off_t new_offset;
4459 4459 size_t trim_sz;
4460 4460 off_t coffset;
4461 4461
4462 4462
4463 4463 /*
4464 4464 * calculate how much we have to trim off of the current cookie to equal
4465 4465 * maxxfer. We don't have to account for granularity here since our
4466 4466 * maxxfer already takes that into account.
4467 4467 */
4468 4468 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4469 4469 ASSERT(trim_sz <= cookie->dmac_size);
4470 4470 ASSERT(trim_sz <= dma->dp_maxxfer);
4471 4471
4472 4472 /* save cookie size since we need it later and we might change it */
4473 4473 dmac_size = cookie->dmac_size;
4474 4474
4475 4475 /*
4476 4476 * if we're not trimming the entire cookie, setup the current window to
4477 4477 * account for the trim.
4478 4478 */
4479 4479 if (trim_sz < cookie->dmac_size) {
4480 4480 (*windowp)->wd_cookie_cnt++;
4481 4481 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4482 4482 (*windowp)->wd_trim.tr_last_cookie = cookie;
4483 4483 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4484 4484 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4485 4485 (*windowp)->wd_size = dma->dp_maxxfer;
4486 4486
4487 4487 /*
4488 4488 * set the adjusted cookie size now in case this is the first
4489 4489 * window. All other windows are taken care of in get win
4490 4490 */
4491 4491 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4492 4492 }
4493 4493
4494 4494 /*
4495 4495 * coffset is the current offset within the cookie, new_offset is the
4496 4496 * current offset with the entire buffer.
4497 4497 */
4498 4498 coffset = dmac_size - trim_sz;
4499 4499 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4500 4500
4501 4501 /* initialize the next window */
4502 4502 (*windowp)++;
4503 4503 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4504 4504 (*windowp)->wd_cookie_cnt++;
4505 4505 (*windowp)->wd_size = trim_sz;
4506 4506 if (trim_sz < dmac_size) {
4507 4507 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4508 4508 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4509 4509 coffset;
4510 4510 (*windowp)->wd_trim.tr_first_size = trim_sz;
4511 4511 }
4512 4512
4513 4513 return (DDI_SUCCESS);
4514 4514 }
4515 4515
4516 4516
4517 4517 /*ARGSUSED*/
4518 4518 static int
4519 4519 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4520 4520 off_t off, size_t len, uint_t cache_flags)
4521 4521 {
4522 4522 rootnex_sglinfo_t *sinfo;
4523 4523 rootnex_pgmap_t *cbpage;
4524 4524 rootnex_window_t *win;
4525 4525 ddi_dma_impl_t *hp;
4526 4526 rootnex_dma_t *dma;
4527 4527 caddr_t fromaddr;
4528 4528 caddr_t toaddr;
4529 4529 uint_t psize;
4530 4530 off_t offset;
4531 4531 uint_t pidx;
4532 4532 size_t size;
4533 4533 off_t poff;
4534 4534 int e;
4535 4535
4536 4536
4537 4537 hp = (ddi_dma_impl_t *)handle;
4538 4538 dma = (rootnex_dma_t *)hp->dmai_private;
4539 4539 sinfo = &dma->dp_sglinfo;
4540 4540
4541 4541 /*
4542 4542 * if we don't have any windows, we don't need to sync. A copybuf
4543 4543 * will cause us to have at least one window.
4544 4544 */
4545 4545 if (dma->dp_window == NULL) {
4546 4546 return (DDI_SUCCESS);
4547 4547 }
4548 4548
4549 4549 /* This window may not need to be sync'd */
4550 4550 win = &dma->dp_window[dma->dp_current_win];
4551 4551 if (!win->wd_dosync) {
4552 4552 return (DDI_SUCCESS);
4553 4553 }
4554 4554
4555 4555 /* handle off and len special cases */
4556 4556 if ((off == 0) || (rootnex_sync_ignore_params)) {
4557 4557 offset = win->wd_offset;
4558 4558 } else {
4559 4559 offset = off;
4560 4560 }
4561 4561 if ((len == 0) || (rootnex_sync_ignore_params)) {
4562 4562 size = win->wd_size;
4563 4563 } else {
4564 4564 size = len;
4565 4565 }
4566 4566
4567 4567 /* check the sync args to make sure they make a little sense */
4568 4568 if (rootnex_sync_check_parms) {
4569 4569 e = rootnex_valid_sync_parms(hp, win, offset, size,
4570 4570 cache_flags);
4571 4571 if (e != DDI_SUCCESS) {
4572 4572 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4573 4573 return (DDI_FAILURE);
4574 4574 }
4575 4575 }
4576 4576
4577 4577 /*
4578 4578 * special case the first page to handle the offset into the page. The
4579 4579 * offset to the current page for our buffer is the offset into the
4580 4580 * first page of the buffer plus our current offset into the buffer
4581 4581 * itself, masked of course.
4582 4582 */
4583 4583 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4584 4584 psize = MIN((MMU_PAGESIZE - poff), size);
4585 4585
4586 4586 /* go through all the pages that we want to sync */
4587 4587 while (size > 0) {
4588 4588 /*
4589 4589 * Calculate the page index relative to the start of the buffer.
4590 4590 * The index to the current page for our buffer is the offset
4591 4591 * into the first page of the buffer plus our current offset
4592 4592 * into the buffer itself, shifted of course...
4593 4593 */
4594 4594 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4595 4595 ASSERT(pidx < sinfo->si_max_pages);
4596 4596
4597 4597 /*
4598 4598 * if this page uses the copy buffer, we need to sync it,
4599 4599 * otherwise, go on to the next page.
4600 4600 */
4601 4601 cbpage = &dma->dp_pgmap[pidx];
4602 4602 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4603 4603 (cbpage->pm_uses_copybuf == B_FALSE));
4604 4604 if (cbpage->pm_uses_copybuf) {
4605 4605 /* cbaddr and kaddr should be page aligned */
4606 4606 ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4607 4607 MMU_PAGEOFFSET) == 0);
4608 4608 ASSERT(((uintptr_t)cbpage->pm_kaddr &
4609 4609 MMU_PAGEOFFSET) == 0);
4610 4610
4611 4611 /*
4612 4612 * if we're copying for the device, we are going to
4613 4613 * copy from the drivers buffer and to the rootnex
4614 4614 * allocated copy buffer.
4615 4615 */
4616 4616 if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4617 4617 fromaddr = cbpage->pm_kaddr + poff;
4618 4618 toaddr = cbpage->pm_cbaddr + poff;
4619 4619 ROOTNEX_DPROBE2(rootnex__sync__dev,
4620 4620 dev_info_t *, dma->dp_dip, size_t, psize);
4621 4621
4622 4622 /*
4623 4623 * if we're copying for the cpu/kernel, we are going to
4624 4624 * copy from the rootnex allocated copy buffer to the
4625 4625 * drivers buffer.
4626 4626 */
4627 4627 } else {
4628 4628 fromaddr = cbpage->pm_cbaddr + poff;
4629 4629 toaddr = cbpage->pm_kaddr + poff;
4630 4630 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4631 4631 dev_info_t *, dma->dp_dip, size_t, psize);
4632 4632 }
4633 4633
4634 4634 bcopy(fromaddr, toaddr, psize);
4635 4635 }
4636 4636
4637 4637 /*
4638 4638 * decrement size until we're done, update our offset into the
4639 4639 * buffer, and get the next page size.
4640 4640 */
4641 4641 size -= psize;
4642 4642 offset += psize;
4643 4643 psize = MIN(MMU_PAGESIZE, size);
4644 4644
4645 4645 /* page offset is zero for the rest of this loop */
4646 4646 poff = 0;
4647 4647 }
4648 4648
4649 4649 return (DDI_SUCCESS);
4650 4650 }
4651 4651
4652 4652 /*
4653 4653 * rootnex_dma_sync()
4654 4654 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4655 4655 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4656 4656 * is set, ddi_dma_sync() returns immediately passing back success.
4657 4657 */
4658 4658 /*ARGSUSED*/
4659 4659 static int
4660 4660 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4661 4661 off_t off, size_t len, uint_t cache_flags)
4662 4662 {
4663 4663 #if defined(__amd64) && !defined(__xpv)
4664 4664 if (IOMMU_USED(rdip)) {
4665 4665 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4666 4666 cache_flags));
4667 4667 }
4668 4668 #endif
4669 4669 return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4670 4670 cache_flags));
4671 4671 }
4672 4672
4673 4673 /*
4674 4674 * rootnex_valid_sync_parms()
4675 4675 * checks the parameters passed to sync to verify they are correct.
4676 4676 */
4677 4677 static int
4678 4678 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4679 4679 off_t offset, size_t size, uint_t cache_flags)
4680 4680 {
4681 4681 off_t woffset;
4682 4682
4683 4683
4684 4684 /*
4685 4685 * the first part of the test to make sure the offset passed in is
4686 4686 * within the window.
4687 4687 */
4688 4688 if (offset < win->wd_offset) {
4689 4689 return (DDI_FAILURE);
4690 4690 }
4691 4691
4692 4692 /*
4693 4693 * second and last part of the test to make sure the offset and length
4694 4694 * passed in is within the window.
4695 4695 */
4696 4696 woffset = offset - win->wd_offset;
4697 4697 if ((woffset + size) > win->wd_size) {
4698 4698 return (DDI_FAILURE);
4699 4699 }
4700 4700
4701 4701 /*
4702 4702 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4703 4703 * be set too.
4704 4704 */
4705 4705 if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4706 4706 (hp->dmai_rflags & DDI_DMA_WRITE)) {
4707 4707 return (DDI_SUCCESS);
4708 4708 }
4709 4709
4710 4710 /*
4711 4711 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4712 4712 * should be set. Also DDI_DMA_READ should be set in the flags.
4713 4713 */
4714 4714 if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4715 4715 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4716 4716 (hp->dmai_rflags & DDI_DMA_READ)) {
4717 4717 return (DDI_SUCCESS);
4718 4718 }
4719 4719
4720 4720 return (DDI_FAILURE);
4721 4721 }
4722 4722
4723 4723
4724 4724 /*ARGSUSED*/
4725 4725 static int
4726 4726 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4727 4727 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4728 4728 uint_t *ccountp)
4729 4729 {
4730 4730 rootnex_window_t *window;
4731 4731 rootnex_trim_t *trim;
4732 4732 ddi_dma_impl_t *hp;
4733 4733 rootnex_dma_t *dma;
4734 4734 ddi_dma_obj_t *dmao;
4735 4735 #if !defined(__amd64)
4736 4736 rootnex_sglinfo_t *sinfo;
4737 4737 rootnex_pgmap_t *pmap;
4738 4738 uint_t pidx;
4739 4739 uint_t pcnt;
4740 4740 off_t poff;
4741 4741 int i;
4742 4742 #endif
4743 4743
4744 4744
4745 4745 hp = (ddi_dma_impl_t *)handle;
4746 4746 dma = (rootnex_dma_t *)hp->dmai_private;
4747 4747 #if !defined(__amd64)
4748 4748 sinfo = &dma->dp_sglinfo;
4749 4749 #endif
4750 4750
4751 4751 /* If we try and get a window which doesn't exist, return failure */
4752 4752 if (win >= hp->dmai_nwin) {
4753 4753 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4754 4754 return (DDI_FAILURE);
4755 4755 }
4756 4756
4757 4757 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4758 4758
4759 4759 /*
4760 4760 * if we don't have any windows, and they're asking for the first
4761 4761 * window, setup the cookie pointer to the first cookie in the bind.
4762 4762 * setup our return values, then increment the cookie since we return
4763 4763 * the first cookie on the stack.
4764 4764 */
4765 4765 if (dma->dp_window == NULL) {
4766 4766 if (win != 0) {
4767 4767 ROOTNEX_DPROF_INC(
4768 4768 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4769 4769 return (DDI_FAILURE);
4770 4770 }
4771 4771 hp->dmai_cookie = dma->dp_cookies;
4772 4772 *offp = 0;
4773 4773 *lenp = dmao->dmao_size;
4774 4774 *ccountp = dma->dp_sglinfo.si_sgl_size;
4775 4775 *cookiep = hp->dmai_cookie[0];
4776 4776 hp->dmai_cookie++;
4777 4777 return (DDI_SUCCESS);
4778 4778 }
4779 4779
4780 4780 /* sync the old window before moving on to the new one */
4781 4781 window = &dma->dp_window[dma->dp_current_win];
4782 4782 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4783 4783 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4784 4784 DDI_DMA_SYNC_FORCPU);
4785 4785 }
4786 4786
4787 4787 #if !defined(__amd64)
4788 4788 /*
4789 4789 * before we move to the next window, if we need to re-map, unmap all
4790 4790 * the pages in this window.
4791 4791 */
4792 4792 if (dma->dp_cb_remaping) {
4793 4793 /*
4794 4794 * If we switch to this window again, we'll need to map in
4795 4795 * on the fly next time.
4796 4796 */
4797 4797 window->wd_remap_copybuf = B_TRUE;
4798 4798
4799 4799 /*
4800 4800 * calculate the page index into the buffer where this window
4801 4801 * starts, and the number of pages this window takes up.
4802 4802 */
4803 4803 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4804 4804 MMU_PAGESHIFT;
4805 4805 poff = (sinfo->si_buf_offset + window->wd_offset) &
4806 4806 MMU_PAGEOFFSET;
4807 4807 pcnt = mmu_btopr(window->wd_size + poff);
4808 4808 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4809 4809
4810 4810 /* unmap pages which are currently mapped in this window */
4811 4811 for (i = 0; i < pcnt; i++) {
4812 4812 if (dma->dp_pgmap[pidx].pm_mapped) {
4813 4813 hat_unload(kas.a_hat,
4814 4814 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4815 4815 HAT_UNLOAD);
4816 4816 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4817 4817 }
4818 4818 pidx++;
4819 4819 }
4820 4820 }
4821 4821 #endif
4822 4822
4823 4823 /*
4824 4824 * Move to the new window.
4825 4825 * NOTE: current_win must be set for sync to work right
4826 4826 */
4827 4827 dma->dp_current_win = win;
4828 4828 window = &dma->dp_window[win];
4829 4829
4830 4830 /* if needed, adjust the first and/or last cookies for trim */
4831 4831 trim = &window->wd_trim;
4832 4832 if (trim->tr_trim_first) {
4833 4833 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4834 4834 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4835 4835 #if !defined(__amd64)
4836 4836 window->wd_first_cookie->dmac_type =
4837 4837 (window->wd_first_cookie->dmac_type &
4838 4838 ROOTNEX_USES_COPYBUF) + window->wd_offset;
4839 4839 #endif
4840 4840 if (trim->tr_first_copybuf_win) {
4841 4841 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4842 4842 trim->tr_first_cbaddr;
4843 4843 #if !defined(__amd64)
4844 4844 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4845 4845 trim->tr_first_kaddr;
4846 4846 #endif
4847 4847 }
4848 4848 }
4849 4849 if (trim->tr_trim_last) {
4850 4850 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4851 4851 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4852 4852 if (trim->tr_last_copybuf_win) {
4853 4853 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4854 4854 trim->tr_last_cbaddr;
4855 4855 #if !defined(__amd64)
4856 4856 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4857 4857 trim->tr_last_kaddr;
4858 4858 #endif
4859 4859 }
4860 4860 }
4861 4861
4862 4862 /*
4863 4863 * setup the cookie pointer to the first cookie in the window. setup
4864 4864 * our return values, then increment the cookie since we return the
4865 4865 * first cookie on the stack.
4866 4866 */
4867 4867 hp->dmai_cookie = window->wd_first_cookie;
4868 4868 *offp = window->wd_offset;
4869 4869 *lenp = window->wd_size;
4870 4870 *ccountp = window->wd_cookie_cnt;
4871 4871 *cookiep = hp->dmai_cookie[0];
4872 4872 hp->dmai_cookie++;
4873 4873
4874 4874 #if !defined(__amd64)
4875 4875 /* re-map copybuf if required for this window */
4876 4876 if (dma->dp_cb_remaping) {
4877 4877 /*
4878 4878 * calculate the page index into the buffer where this
4879 4879 * window starts.
4880 4880 */
4881 4881 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4882 4882 MMU_PAGESHIFT;
4883 4883 ASSERT(pidx < sinfo->si_max_pages);
4884 4884
4885 4885 /*
4886 4886 * the first page can get unmapped if it's shared with the
4887 4887 * previous window. Even if the rest of this window is already
4888 4888 * mapped in, we need to still check this one.
4889 4889 */
4890 4890 pmap = &dma->dp_pgmap[pidx];
4891 4891 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4892 4892 if (pmap->pm_pp != NULL) {
4893 4893 pmap->pm_mapped = B_TRUE;
4894 4894 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4895 4895 } else if (pmap->pm_vaddr != NULL) {
4896 4896 pmap->pm_mapped = B_TRUE;
4897 4897 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4898 4898 pmap->pm_kaddr);
4899 4899 }
4900 4900 }
4901 4901 pidx++;
4902 4902
4903 4903 /* map in the rest of the pages if required */
4904 4904 if (window->wd_remap_copybuf) {
4905 4905 window->wd_remap_copybuf = B_FALSE;
4906 4906
4907 4907 /* figure out many pages this window takes up */
4908 4908 poff = (sinfo->si_buf_offset + window->wd_offset) &
4909 4909 MMU_PAGEOFFSET;
4910 4910 pcnt = mmu_btopr(window->wd_size + poff);
4911 4911 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4912 4912
4913 4913 /* map pages which require it */
4914 4914 for (i = 1; i < pcnt; i++) {
4915 4915 pmap = &dma->dp_pgmap[pidx];
4916 4916 if (pmap->pm_uses_copybuf) {
4917 4917 ASSERT(pmap->pm_mapped == B_FALSE);
4918 4918 if (pmap->pm_pp != NULL) {
4919 4919 pmap->pm_mapped = B_TRUE;
4920 4920 i86_pp_map(pmap->pm_pp,
4921 4921 pmap->pm_kaddr);
4922 4922 } else if (pmap->pm_vaddr != NULL) {
4923 4923 pmap->pm_mapped = B_TRUE;
4924 4924 i86_va_map(pmap->pm_vaddr,
4925 4925 sinfo->si_asp,
4926 4926 pmap->pm_kaddr);
4927 4927 }
4928 4928 }
4929 4929 pidx++;
4930 4930 }
4931 4931 }
4932 4932 }
4933 4933 #endif
4934 4934
4935 4935 /* if the new window uses the copy buffer, sync it for the device */
4936 4936 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4937 4937 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4938 4938 DDI_DMA_SYNC_FORDEV);
4939 4939 }
4940 4940
4941 4941 return (DDI_SUCCESS);
4942 4942 }
4943 4943
4944 4944 /*
4945 4945 * rootnex_dma_win()
4946 4946 * called from ddi_dma_getwin()
4947 4947 */
4948 4948 /*ARGSUSED*/
4949 4949 static int
4950 4950 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4951 4951 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4952 4952 uint_t *ccountp)
4953 4953 {
4954 4954 #if defined(__amd64) && !defined(__xpv)
4955 4955 if (IOMMU_USED(rdip)) {
4956 4956 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4957 4957 cookiep, ccountp));
4958 4958 }
4959 4959 #endif
4960 4960
4961 4961 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4962 4962 cookiep, ccountp));
4963 4963 }
4964 4964
4965 4965 #if defined(__amd64) && !defined(__xpv)
4966 4966 /*ARGSUSED*/
4967 4967 static int
4968 4968 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4969 4969 ddi_dma_handle_t handle, void *v)
4970 4970 {
4971 4971 ddi_dma_impl_t *hp;
4972 4972 rootnex_dma_t *dma;
4973 4973
4974 4974 hp = (ddi_dma_impl_t *)handle;
4975 4975 dma = (rootnex_dma_t *)hp->dmai_private;
4976 4976 dma->dp_iommu_private = v;
4977 4977
4978 4978 return (DDI_SUCCESS);
4979 4979 }
4980 4980
4981 4981 /*ARGSUSED*/
4982 4982 static void *
4983 4983 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4984 4984 ddi_dma_handle_t handle)
4985 4985 {
4986 4986 ddi_dma_impl_t *hp;
4987 4987 rootnex_dma_t *dma;
4988 4988
4989 4989 hp = (ddi_dma_impl_t *)handle;
4990 4990 dma = (rootnex_dma_t *)hp->dmai_private;
4991 4991
4992 4992 return (dma->dp_iommu_private);
4993 4993 }
4994 4994 #endif
4995 4995
4996 4996 /*
4997 4997 * ************************
4998 4998 * obsoleted dma routines
4999 4999 * ************************
5000 5000 */
5001 5001
5002 5002 /*
5003 5003 * rootnex_dma_mctl()
5004 5004 *
5005 5005 * We don't support this legacy interface any more on x86.
5006 5006 */
5007 5007 /* ARGSUSED */
5008 5008 static int
5009 5009 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5010 5010 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5011 5011 uint_t cache_flags)
5012 5012 {
5013 5013 /*
5014 5014 * The only thing dma_mctl is usef for anymore is legacy SPARC
5015 5015 * dvma and sbus-specific routines.
5016 5016 */
5017 5017 return (DDI_FAILURE);
5018 5018 }
5019 5019
5020 5020 /*
5021 5021 * *********
5022 5022 * FMA Code
5023 5023 * *********
5024 5024 */
5025 5025
5026 5026 /*
5027 5027 * rootnex_fm_init()
5028 5028 * FMA init busop
5029 5029 */
5030 5030 /* ARGSUSED */
5031 5031 static int
5032 5032 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5033 5033 ddi_iblock_cookie_t *ibc)
5034 5034 {
5035 5035 *ibc = rootnex_state->r_err_ibc;
5036 5036
5037 5037 return (ddi_system_fmcap);
5038 5038 }
5039 5039
5040 5040 /*
5041 5041 * rootnex_dma_check()
5042 5042 * Function called after a dma fault occurred to find out whether the
5043 5043 * fault address is associated with a driver that is able to handle faults
5044 5044 * and recover from faults.
5045 5045 */
5046 5046 /* ARGSUSED */
5047 5047 static int
5048 5048 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5049 5049 const void *not_used)
5050 5050 {
5051 5051 rootnex_window_t *window;
5052 5052 uint64_t start_addr;
5053 5053 uint64_t fault_addr;
5054 5054 ddi_dma_impl_t *hp;
5055 5055 rootnex_dma_t *dma;
5056 5056 uint64_t end_addr;
5057 5057 size_t csize;
5058 5058 int i;
5059 5059 int j;
5060 5060
5061 5061
5062 5062 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5063 5063 hp = (ddi_dma_impl_t *)handle;
5064 5064 ASSERT(hp);
5065 5065
5066 5066 dma = (rootnex_dma_t *)hp->dmai_private;
5067 5067
5068 5068 /* Get the address that we need to search for */
5069 5069 fault_addr = *(uint64_t *)addr;
5070 5070
5071 5071 /*
5072 5072 * if we don't have any windows, we can just walk through all the
5073 5073 * cookies.
5074 5074 */
5075 5075 if (dma->dp_window == NULL) {
5076 5076 /* for each cookie */
5077 5077 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5078 5078 /*
5079 5079 * if the faulted address is within the physical address
5080 5080 * range of the cookie, return DDI_FM_NONFATAL.
5081 5081 */
5082 5082 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5083 5083 (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5084 5084 dma->dp_cookies[i].dmac_size))) {
5085 5085 return (DDI_FM_NONFATAL);
5086 5086 }
5087 5087 }
5088 5088
5089 5089 /* fault_addr not within this DMA handle */
5090 5090 return (DDI_FM_UNKNOWN);
5091 5091 }
5092 5092
5093 5093 /* we have mutiple windows, walk through each window */
5094 5094 for (i = 0; i < hp->dmai_nwin; i++) {
5095 5095 window = &dma->dp_window[i];
5096 5096
5097 5097 /* Go through all the cookies in the window */
5098 5098 for (j = 0; j < window->wd_cookie_cnt; j++) {
5099 5099
5100 5100 start_addr = window->wd_first_cookie[j].dmac_laddress;
5101 5101 csize = window->wd_first_cookie[j].dmac_size;
5102 5102
5103 5103 /*
5104 5104 * if we are trimming the first cookie in the window,
5105 5105 * and this is the first cookie, adjust the start
5106 5106 * address and size of the cookie to account for the
5107 5107 * trim.
5108 5108 */
5109 5109 if (window->wd_trim.tr_trim_first && (j == 0)) {
5110 5110 start_addr = window->wd_trim.tr_first_paddr;
5111 5111 csize = window->wd_trim.tr_first_size;
5112 5112 }
5113 5113
5114 5114 /*
5115 5115 * if we are trimming the last cookie in the window,
5116 5116 * and this is the last cookie, adjust the start
5117 5117 * address and size of the cookie to account for the
5118 5118 * trim.
5119 5119 */
5120 5120 if (window->wd_trim.tr_trim_last &&
5121 5121 (j == (window->wd_cookie_cnt - 1))) {
5122 5122 start_addr = window->wd_trim.tr_last_paddr;
5123 5123 csize = window->wd_trim.tr_last_size;
5124 5124 }
5125 5125
5126 5126 end_addr = start_addr + csize;
5127 5127
5128 5128 /*
5129 5129 * if the faulted address is within the physical
5130 5130 * address of the cookie, return DDI_FM_NONFATAL.
5131 5131 */
5132 5132 if ((fault_addr >= start_addr) &&
5133 5133 (fault_addr <= end_addr)) {
5134 5134 return (DDI_FM_NONFATAL);
5135 5135 }
5136 5136 }
5137 5137 }
5138 5138
5139 5139 /* fault_addr not within this DMA handle */
5140 5140 return (DDI_FM_UNKNOWN);
5141 5141 }
5142 5142
5143 5143 /*ARGSUSED*/
5144 5144 static int
5145 5145 rootnex_quiesce(dev_info_t *dip)
5146 5146 {
5147 5147 #if defined(__amd64) && !defined(__xpv)
5148 5148 return (immu_quiesce());
5149 5149 #else
5150 5150 return (DDI_SUCCESS);
5151 5151 #endif
5152 5152 }
5153 5153
5154 5154 #if defined(__xpv)
5155 5155 void
5156 5156 immu_init(void)
5157 5157 {
5158 5158 ;
5159 5159 }
5160 5160
5161 5161 void
5162 5162 immu_startup(void)
5163 5163 {
5164 5164 ;
5165 5165 }
5166 5166 /*ARGSUSED*/
5167 5167 void
5168 5168 immu_physmem_update(uint64_t addr, uint64_t size)
5169 5169 {
5170 5170 ;
5171 5171 }
5172 5172 #endif
|
↓ open down ↓ |
2988 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX