1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  26  * Copyright (c) 2011 Bayard G. Bell.  All rights reserved.
  27  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
  28  */
  29 
  30 /*
  31  * x86 root nexus driver
  32  */
  33 
  34 #include <sys/sysmacros.h>
  35 #include <sys/conf.h>
  36 #include <sys/autoconf.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/debug.h>
  39 #include <sys/psw.h>
  40 #include <sys/ddidmareq.h>
  41 #include <sys/promif.h>
  42 #include <sys/devops.h>
  43 #include <sys/kmem.h>
  44 #include <sys/cmn_err.h>
  45 #include <vm/seg.h>
  46 #include <vm/seg_kmem.h>
  47 #include <vm/seg_dev.h>
  48 #include <sys/vmem.h>
  49 #include <sys/mman.h>
  50 #include <vm/hat.h>
  51 #include <vm/as.h>
  52 #include <vm/page.h>
  53 #include <sys/avintr.h>
  54 #include <sys/errno.h>
  55 #include <sys/modctl.h>
  56 #include <sys/ddi_impldefs.h>
  57 #include <sys/sunddi.h>
  58 #include <sys/sunndi.h>
  59 #include <sys/mach_intr.h>
  60 #include <sys/psm.h>
  61 #include <sys/ontrap.h>
  62 #include <sys/atomic.h>
  63 #include <sys/sdt.h>
  64 #include <sys/rootnex.h>
  65 #include <vm/hat_i86.h>
  66 #include <sys/ddifm.h>
  67 #include <sys/ddi_isa.h>
  68 #include <sys/apic.h>
  69 
  70 #ifdef __xpv
  71 #include <sys/bootinfo.h>
  72 #include <sys/hypervisor.h>
  73 #include <sys/bootconf.h>
  74 #include <vm/kboot_mmu.h>
  75 #endif
  76 
  77 #if defined(__amd64) && !defined(__xpv)
  78 #include <sys/immu.h>
  79 #endif
  80 
  81 
  82 /*
  83  * enable/disable extra checking of function parameters. Useful for debugging
  84  * drivers.
  85  */
  86 #ifdef  DEBUG
  87 int rootnex_alloc_check_parms = 1;
  88 int rootnex_bind_check_parms = 1;
  89 int rootnex_bind_check_inuse = 1;
  90 int rootnex_unbind_verify_buffer = 0;
  91 int rootnex_sync_check_parms = 1;
  92 #else
  93 int rootnex_alloc_check_parms = 0;
  94 int rootnex_bind_check_parms = 0;
  95 int rootnex_bind_check_inuse = 0;
  96 int rootnex_unbind_verify_buffer = 0;
  97 int rootnex_sync_check_parms = 0;
  98 #endif
  99 
 100 boolean_t rootnex_dmar_not_setup;
 101 
 102 /* Master Abort and Target Abort panic flag */
 103 int rootnex_fm_ma_ta_panic_flag = 0;
 104 
 105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
 106 int rootnex_bind_fail = 1;
 107 int rootnex_bind_warn = 1;
 108 uint8_t *rootnex_warn_list;
 109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
 110 #define ROOTNEX_BIND_WARNING    (0x1 << 0)
 111 
 112 /*
 113  * revert back to old broken behavior of always sync'ing entire copy buffer.
 114  * This is useful if be have a buggy driver which doesn't correctly pass in
 115  * the offset and size into ddi_dma_sync().
 116  */
 117 int rootnex_sync_ignore_params = 0;
 118 
 119 /*
 120  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
 121  * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
 122  * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
 123  * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
 124  * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
 125  * (< 8K). We will still need to allocate the copy buffer during bind though
 126  * (if we need one). These can only be modified in /etc/system before rootnex
 127  * attach.
 128  */
 129 #if defined(__amd64)
 130 int rootnex_prealloc_cookies = 65;
 131 int rootnex_prealloc_windows = 4;
 132 int rootnex_prealloc_copybuf = 2;
 133 #else
 134 int rootnex_prealloc_cookies = 33;
 135 int rootnex_prealloc_windows = 4;
 136 int rootnex_prealloc_copybuf = 2;
 137 #endif
 138 
 139 /* driver global state */
 140 static rootnex_state_t *rootnex_state;
 141 
 142 #ifdef DEBUG
 143 /* shortcut to rootnex counters */
 144 static uint64_t *rootnex_cnt;
 145 #endif
 146 
 147 /*
 148  * XXX - does x86 even need these or are they left over from the SPARC days?
 149  */
 150 /* statically defined integer/boolean properties for the root node */
 151 static rootnex_intprop_t rootnex_intprp[] = {
 152         { "PAGESIZE",                   PAGESIZE },
 153         { "MMU_PAGESIZE",               MMU_PAGESIZE },
 154         { "MMU_PAGEOFFSET",             MMU_PAGEOFFSET },
 155         { DDI_RELATIVE_ADDRESSING,      1 },
 156 };
 157 #define NROOT_INTPROPS  (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
 158 
 159 /*
 160  * If we're dom0, we're using a real device so we need to load
 161  * the cookies with MFNs instead of PFNs.
 162  */
 163 #ifdef __xpv
 164 typedef maddr_t rootnex_addr_t;
 165 #define ROOTNEX_PADDR_TO_RBASE(pa)      \
 166         (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
 167 #else
 168 typedef paddr_t rootnex_addr_t;
 169 #define ROOTNEX_PADDR_TO_RBASE(pa)      (pa)
 170 #endif
 171 
 172 static struct cb_ops rootnex_cb_ops = {
 173         nodev,          /* open */
 174         nodev,          /* close */
 175         nodev,          /* strategy */
 176         nodev,          /* print */
 177         nodev,          /* dump */
 178         nodev,          /* read */
 179         nodev,          /* write */
 180         nodev,          /* ioctl */
 181         nodev,          /* devmap */
 182         nodev,          /* mmap */
 183         nodev,          /* segmap */
 184         nochpoll,       /* chpoll */
 185         ddi_prop_op,    /* cb_prop_op */
 186         NULL,           /* struct streamtab */
 187         D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
 188         CB_REV,         /* Rev */
 189         nodev,          /* cb_aread */
 190         nodev           /* cb_awrite */
 191 };
 192 
 193 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
 194     off_t offset, off_t len, caddr_t *vaddrp);
 195 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
 196     struct hat *hat, struct seg *seg, caddr_t addr,
 197     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
 198 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
 199     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
 200     ddi_dma_handle_t *handlep);
 201 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
 202     ddi_dma_handle_t handle);
 203 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
 204     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
 205     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 206 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
 207     ddi_dma_handle_t handle);
 208 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
 209     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
 210 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
 211     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
 212     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 213 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
 214     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
 215     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
 216 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
 217     ddi_ctl_enum_t ctlop, void *arg, void *result);
 218 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
 219     ddi_iblock_cookie_t *ibc);
 220 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
 221     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
 222 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
 223     void *);
 224 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
 225 
 226 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
 227     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
 228     ddi_dma_handle_t *handlep);
 229 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
 230     ddi_dma_handle_t handle);
 231 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
 232     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
 233     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 234 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
 235     ddi_dma_handle_t handle);
 236 #if defined(__amd64) && !defined(__xpv)
 237 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
 238     ddi_dma_handle_t handle);
 239 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
 240     ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
 241 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
 242     ddi_dma_cookie_t *cookiep, uint_t ccount);
 243 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
 244     ddi_dma_handle_t handle);
 245 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
 246 #endif
 247 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
 248     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
 249 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
 250     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
 251     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 252 
 253 #if defined(__amd64) && !defined(__xpv)
 254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
 255     ddi_dma_handle_t handle, void *v);
 256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
 257     ddi_dma_handle_t handle);
 258 #endif
 259 
 260 
 261 static struct bus_ops rootnex_bus_ops = {
 262         BUSO_REV,
 263         rootnex_map,
 264         NULL,
 265         NULL,
 266         NULL,
 267         rootnex_map_fault,
 268         0,
 269         rootnex_dma_allochdl,
 270         rootnex_dma_freehdl,
 271         rootnex_dma_bindhdl,
 272         rootnex_dma_unbindhdl,
 273         rootnex_dma_sync,
 274         rootnex_dma_win,
 275         rootnex_dma_mctl,
 276         rootnex_ctlops,
 277         ddi_bus_prop_op,
 278         i_ddi_rootnex_get_eventcookie,
 279         i_ddi_rootnex_add_eventcall,
 280         i_ddi_rootnex_remove_eventcall,
 281         i_ddi_rootnex_post_event,
 282         0,                      /* bus_intr_ctl */
 283         0,                      /* bus_config */
 284         0,                      /* bus_unconfig */
 285         rootnex_fm_init,        /* bus_fm_init */
 286         NULL,                   /* bus_fm_fini */
 287         NULL,                   /* bus_fm_access_enter */
 288         NULL,                   /* bus_fm_access_exit */
 289         NULL,                   /* bus_powr */
 290         rootnex_intr_ops        /* bus_intr_op */
 291 };
 292 
 293 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
 294 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
 295 static int rootnex_quiesce(dev_info_t *dip);
 296 
 297 static struct dev_ops rootnex_ops = {
 298         DEVO_REV,
 299         0,
 300         ddi_no_info,
 301         nulldev,
 302         nulldev,
 303         rootnex_attach,
 304         rootnex_detach,
 305         nulldev,
 306         &rootnex_cb_ops,
 307         &rootnex_bus_ops,
 308         NULL,
 309         rootnex_quiesce,                /* quiesce */
 310 };
 311 
 312 static struct modldrv rootnex_modldrv = {
 313         &mod_driverops,
 314         "i86pc root nexus",
 315         &rootnex_ops
 316 };
 317 
 318 static struct modlinkage rootnex_modlinkage = {
 319         MODREV_1,
 320         (void *)&rootnex_modldrv,
 321         NULL
 322 };
 323 
 324 #if defined(__amd64) && !defined(__xpv)
 325 static iommulib_nexops_t iommulib_nexops = {
 326         IOMMU_NEXOPS_VERSION,
 327         "Rootnex IOMMU ops Vers 1.1",
 328         NULL,
 329         rootnex_coredma_allochdl,
 330         rootnex_coredma_freehdl,
 331         rootnex_coredma_bindhdl,
 332         rootnex_coredma_unbindhdl,
 333         rootnex_coredma_reset_cookies,
 334         rootnex_coredma_get_cookies,
 335         rootnex_coredma_set_cookies,
 336         rootnex_coredma_clear_cookies,
 337         rootnex_coredma_get_sleep_flags,
 338         rootnex_coredma_sync,
 339         rootnex_coredma_win,
 340         rootnex_coredma_hdl_setprivate,
 341         rootnex_coredma_hdl_getprivate
 342 };
 343 #endif
 344 
 345 /*
 346  *  extern hacks
 347  */
 348 extern struct seg_ops segdev_ops;
 349 extern int ignore_hardware_nodes;       /* force flag from ddi_impl.c */
 350 #ifdef  DDI_MAP_DEBUG
 351 extern int ddi_map_debug_flag;
 352 #define ddi_map_debug   if (ddi_map_debug_flag) prom_printf
 353 #endif
 354 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
 355 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
 356 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
 357     psm_intr_op_t, int *);
 358 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
 359 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
 360 
 361 /*
 362  * Use device arena to use for device control register mappings.
 363  * Various kernel memory walkers (debugger, dtrace) need to know
 364  * to avoid this address range to prevent undesired device activity.
 365  */
 366 extern void *device_arena_alloc(size_t size, int vm_flag);
 367 extern void device_arena_free(void * vaddr, size_t size);
 368 
 369 
 370 /*
 371  *  Internal functions
 372  */
 373 static int rootnex_dma_init();
 374 static void rootnex_add_props(dev_info_t *);
 375 static int rootnex_ctl_reportdev(dev_info_t *dip);
 376 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
 377 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
 378 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
 379 static int rootnex_map_handle(ddi_map_req_t *mp);
 380 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
 381 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
 382 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
 383     ddi_dma_attr_t *attr);
 384 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
 385     rootnex_sglinfo_t *sglinfo);
 386 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
 387     ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
 388 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
 389     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
 390 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
 391     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
 392 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
 393 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
 394     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
 395 static void rootnex_teardown_windows(rootnex_dma_t *dma);
 396 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
 397     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
 398 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
 399     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
 400     size_t *copybuf_used, page_t **cur_pp);
 401 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
 402     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
 403     ddi_dma_attr_t *attr, off_t cur_offset);
 404 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
 405     rootnex_dma_t *dma, rootnex_window_t **windowp,
 406     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
 407 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
 408     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
 409 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
 410     off_t offset, size_t size, uint_t cache_flags);
 411 static int rootnex_verify_buffer(rootnex_dma_t *dma);
 412 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
 413     const void *comp_addr, const void *not_used);
 414 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
 415     rootnex_sglinfo_t *sglinfo);
 416 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
 417 
 418 /*
 419  * _init()
 420  *
 421  */
 422 int
 423 _init(void)
 424 {
 425 
 426         rootnex_state = NULL;
 427         return (mod_install(&rootnex_modlinkage));
 428 }
 429 
 430 
 431 /*
 432  * _info()
 433  *
 434  */
 435 int
 436 _info(struct modinfo *modinfop)
 437 {
 438         return (mod_info(&rootnex_modlinkage, modinfop));
 439 }
 440 
 441 
 442 /*
 443  * _fini()
 444  *
 445  */
 446 int
 447 _fini(void)
 448 {
 449         return (EBUSY);
 450 }
 451 
 452 
 453 /*
 454  * rootnex_attach()
 455  *
 456  */
 457 static int
 458 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 459 {
 460         int fmcap;
 461         int e;
 462 
 463         switch (cmd) {
 464         case DDI_ATTACH:
 465                 break;
 466         case DDI_RESUME:
 467 #if defined(__amd64) && !defined(__xpv)
 468                 return (immu_unquiesce());
 469 #else
 470                 return (DDI_SUCCESS);
 471 #endif
 472         default:
 473                 return (DDI_FAILURE);
 474         }
 475 
 476         /*
 477          * We should only have one instance of rootnex. Save it away since we
 478          * don't have an easy way to get it back later.
 479          */
 480         ASSERT(rootnex_state == NULL);
 481         rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
 482 
 483         rootnex_state->r_dip = dip;
 484         rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
 485         rootnex_state->r_reserved_msg_printed = B_FALSE;
 486 #ifdef DEBUG
 487         rootnex_cnt = &rootnex_state->r_counters[0];
 488 #endif
 489 
 490         /*
 491          * Set minimum fm capability level for i86pc platforms and then
 492          * initialize error handling. Since we're the rootnex, we don't
 493          * care what's returned in the fmcap field.
 494          */
 495         ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
 496             DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
 497         fmcap = ddi_system_fmcap;
 498         ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
 499 
 500         /* initialize DMA related state */
 501         e = rootnex_dma_init();
 502         if (e != DDI_SUCCESS) {
 503                 kmem_free(rootnex_state, sizeof (rootnex_state_t));
 504                 return (DDI_FAILURE);
 505         }
 506 
 507         /* Add static root node properties */
 508         rootnex_add_props(dip);
 509 
 510         /* since we can't call ddi_report_dev() */
 511         cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
 512 
 513         /* Initialize rootnex event handle */
 514         i_ddi_rootnex_init_events(dip);
 515 
 516 #if defined(__amd64) && !defined(__xpv)
 517         e = iommulib_nexus_register(dip, &iommulib_nexops,
 518             &rootnex_state->r_iommulib_handle);
 519 
 520         ASSERT(e == DDI_SUCCESS);
 521 #endif
 522 
 523         return (DDI_SUCCESS);
 524 }
 525 
 526 
 527 /*
 528  * rootnex_detach()
 529  *
 530  */
 531 /*ARGSUSED*/
 532 static int
 533 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 534 {
 535         switch (cmd) {
 536         case DDI_SUSPEND:
 537 #if defined(__amd64) && !defined(__xpv)
 538                 return (immu_quiesce());
 539 #else
 540                 return (DDI_SUCCESS);
 541 #endif
 542         default:
 543                 return (DDI_FAILURE);
 544         }
 545         /*NOTREACHED*/
 546 
 547 }
 548 
 549 
 550 /*
 551  * rootnex_dma_init()
 552  *
 553  */
 554 /*ARGSUSED*/
 555 static int
 556 rootnex_dma_init()
 557 {
 558         size_t bufsize;
 559 
 560 
 561         /*
 562          * size of our cookie/window/copybuf state needed in dma bind that we
 563          * pre-alloc in dma_alloc_handle
 564          */
 565         rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
 566         rootnex_state->r_prealloc_size =
 567             (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
 568             (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
 569             (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
 570 
 571         /*
 572          * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
 573          * allocate 16 extra bytes for struct pointer alignment
 574          * (p->dmai_private & dma->dp_prealloc_buffer)
 575          */
 576         bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
 577             rootnex_state->r_prealloc_size + 0x10;
 578         rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
 579             bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
 580         if (rootnex_state->r_dmahdl_cache == NULL) {
 581                 return (DDI_FAILURE);
 582         }
 583 
 584         /*
 585          * allocate array to track which major numbers we have printed warnings
 586          * for.
 587          */
 588         rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
 589             KM_SLEEP);
 590 
 591         return (DDI_SUCCESS);
 592 }
 593 
 594 
 595 /*
 596  * rootnex_add_props()
 597  *
 598  */
 599 static void
 600 rootnex_add_props(dev_info_t *dip)
 601 {
 602         rootnex_intprop_t *rpp;
 603         int i;
 604 
 605         /* Add static integer/boolean properties to the root node */
 606         rpp = rootnex_intprp;
 607         for (i = 0; i < NROOT_INTPROPS; i++) {
 608                 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
 609                     rpp[i].prop_name, rpp[i].prop_value);
 610         }
 611 }
 612 
 613 
 614 
 615 /*
 616  * *************************
 617  *  ctlops related routines
 618  * *************************
 619  */
 620 
 621 /*
 622  * rootnex_ctlops()
 623  *
 624  */
 625 /*ARGSUSED*/
 626 static int
 627 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
 628     void *arg, void *result)
 629 {
 630         int n, *ptr;
 631         struct ddi_parent_private_data *pdp;
 632 
 633         switch (ctlop) {
 634         case DDI_CTLOPS_DMAPMAPC:
 635                 /*
 636                  * Return 'partial' to indicate that dma mapping
 637                  * has to be done in the main MMU.
 638                  */
 639                 return (DDI_DMA_PARTIAL);
 640 
 641         case DDI_CTLOPS_BTOP:
 642                 /*
 643                  * Convert byte count input to physical page units.
 644                  * (byte counts that are not a page-size multiple
 645                  * are rounded down)
 646                  */
 647                 *(ulong_t *)result = btop(*(ulong_t *)arg);
 648                 return (DDI_SUCCESS);
 649 
 650         case DDI_CTLOPS_PTOB:
 651                 /*
 652                  * Convert size in physical pages to bytes
 653                  */
 654                 *(ulong_t *)result = ptob(*(ulong_t *)arg);
 655                 return (DDI_SUCCESS);
 656 
 657         case DDI_CTLOPS_BTOPR:
 658                 /*
 659                  * Convert byte count input to physical page units
 660                  * (byte counts that are not a page-size multiple
 661                  * are rounded up)
 662                  */
 663                 *(ulong_t *)result = btopr(*(ulong_t *)arg);
 664                 return (DDI_SUCCESS);
 665 
 666         case DDI_CTLOPS_INITCHILD:
 667                 return (impl_ddi_sunbus_initchild(arg));
 668 
 669         case DDI_CTLOPS_UNINITCHILD:
 670                 impl_ddi_sunbus_removechild(arg);
 671                 return (DDI_SUCCESS);
 672 
 673         case DDI_CTLOPS_REPORTDEV:
 674                 return (rootnex_ctl_reportdev(rdip));
 675 
 676         case DDI_CTLOPS_IOMIN:
 677                 /*
 678                  * Nothing to do here but reflect back..
 679                  */
 680                 return (DDI_SUCCESS);
 681 
 682         case DDI_CTLOPS_REGSIZE:
 683         case DDI_CTLOPS_NREGS:
 684                 break;
 685 
 686         case DDI_CTLOPS_SIDDEV:
 687                 if (ndi_dev_is_prom_node(rdip))
 688                         return (DDI_SUCCESS);
 689                 if (ndi_dev_is_persistent_node(rdip))
 690                         return (DDI_SUCCESS);
 691                 return (DDI_FAILURE);
 692 
 693         case DDI_CTLOPS_POWER:
 694                 return ((*pm_platform_power)((power_req_t *)arg));
 695 
 696         case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
 697         case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
 698         case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
 699         case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
 700         case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
 701         case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
 702                 if (!rootnex_state->r_reserved_msg_printed) {
 703                         rootnex_state->r_reserved_msg_printed = B_TRUE;
 704                         cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
 705                             "1 or more reserved/obsolete operations.");
 706                 }
 707                 return (DDI_FAILURE);
 708 
 709         default:
 710                 return (DDI_FAILURE);
 711         }
 712         /*
 713          * The rest are for "hardware" properties
 714          */
 715         if ((pdp = ddi_get_parent_data(rdip)) == NULL)
 716                 return (DDI_FAILURE);
 717 
 718         if (ctlop == DDI_CTLOPS_NREGS) {
 719                 ptr = (int *)result;
 720                 *ptr = pdp->par_nreg;
 721         } else {
 722                 off_t *size = (off_t *)result;
 723 
 724                 ptr = (int *)arg;
 725                 n = *ptr;
 726                 if (n >= pdp->par_nreg) {
 727                         return (DDI_FAILURE);
 728                 }
 729                 *size = (off_t)pdp->par_reg[n].regspec_size;
 730         }
 731         return (DDI_SUCCESS);
 732 }
 733 
 734 
 735 /*
 736  * rootnex_ctl_reportdev()
 737  *
 738  */
 739 static int
 740 rootnex_ctl_reportdev(dev_info_t *dev)
 741 {
 742         int i, n, len, f_len = 0;
 743         char *buf;
 744 
 745         buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
 746         f_len += snprintf(buf, REPORTDEV_BUFSIZE,
 747             "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
 748         len = strlen(buf);
 749 
 750         for (i = 0; i < sparc_pd_getnreg(dev); i++) {
 751 
 752                 struct regspec *rp = sparc_pd_getreg(dev, i);
 753 
 754                 if (i == 0)
 755                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 756                             ": ");
 757                 else
 758                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 759                             " and ");
 760                 len = strlen(buf);
 761 
 762                 switch (rp->regspec_bustype) {
 763 
 764                 case BTEISA:
 765                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 766                             "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
 767                         break;
 768 
 769                 case BTISA:
 770                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 771                             "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
 772                         break;
 773 
 774                 default:
 775                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 776                             "space %x offset %x",
 777                             rp->regspec_bustype, rp->regspec_addr);
 778                         break;
 779                 }
 780                 len = strlen(buf);
 781         }
 782         for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
 783                 int pri;
 784 
 785                 if (i != 0) {
 786                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 787                             ",");
 788                         len = strlen(buf);
 789                 }
 790                 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
 791                 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 792                     " sparc ipl %d", pri);
 793                 len = strlen(buf);
 794         }
 795 #ifdef DEBUG
 796         if (f_len + 1 >= REPORTDEV_BUFSIZE) {
 797                 cmn_err(CE_NOTE, "next message is truncated: "
 798                     "printed length 1024, real length %d", f_len);
 799         }
 800 #endif /* DEBUG */
 801         cmn_err(CE_CONT, "?%s\n", buf);
 802         kmem_free(buf, REPORTDEV_BUFSIZE);
 803         return (DDI_SUCCESS);
 804 }
 805 
 806 
 807 /*
 808  * ******************
 809  *  map related code
 810  * ******************
 811  */
 812 
 813 /*
 814  * rootnex_map()
 815  *
 816  */
 817 static int
 818 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
 819     off_t len, caddr_t *vaddrp)
 820 {
 821         struct regspec *rp, tmp_reg;
 822         ddi_map_req_t mr = *mp;         /* Get private copy of request */
 823         int error;
 824 
 825         mp = &mr;
 826 
 827         switch (mp->map_op)  {
 828         case DDI_MO_MAP_LOCKED:
 829         case DDI_MO_UNMAP:
 830         case DDI_MO_MAP_HANDLE:
 831                 break;
 832         default:
 833 #ifdef  DDI_MAP_DEBUG
 834                 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
 835                     mp->map_op);
 836 #endif  /* DDI_MAP_DEBUG */
 837                 return (DDI_ME_UNIMPLEMENTED);
 838         }
 839 
 840         if (mp->map_flags & DDI_MF_USER_MAPPING)  {
 841 #ifdef  DDI_MAP_DEBUG
 842                 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
 843 #endif  /* DDI_MAP_DEBUG */
 844                 return (DDI_ME_UNIMPLEMENTED);
 845         }
 846 
 847         /*
 848          * First, if given an rnumber, convert it to a regspec...
 849          * (Presumably, this is on behalf of a child of the root node?)
 850          */
 851 
 852         if (mp->map_type == DDI_MT_RNUMBER)  {
 853 
 854                 int rnumber = mp->map_obj.rnumber;
 855 #ifdef  DDI_MAP_DEBUG
 856                 static char *out_of_range =
 857                     "rootnex_map: Out of range rnumber <%d>, device <%s>";
 858 #endif  /* DDI_MAP_DEBUG */
 859 
 860                 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
 861                 if (rp == NULL)  {
 862 #ifdef  DDI_MAP_DEBUG
 863                         cmn_err(CE_WARN, out_of_range, rnumber,
 864                             ddi_get_name(rdip));
 865 #endif  /* DDI_MAP_DEBUG */
 866                         return (DDI_ME_RNUMBER_RANGE);
 867                 }
 868 
 869                 /*
 870                  * Convert the given ddi_map_req_t from rnumber to regspec...
 871                  */
 872 
 873                 mp->map_type = DDI_MT_REGSPEC;
 874                 mp->map_obj.rp = rp;
 875         }
 876 
 877         /*
 878          * Adjust offset and length correspnding to called values...
 879          * XXX: A non-zero length means override the one in the regspec
 880          * XXX: (regardless of what's in the parent's range?)
 881          */
 882 
 883         tmp_reg = *(mp->map_obj.rp);         /* Preserve underlying data */
 884         rp = mp->map_obj.rp = &tmp_reg;          /* Use tmp_reg in request */
 885 
 886 #ifdef  DDI_MAP_DEBUG
 887         cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
 888             "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
 889             rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
 890             len, mp->map_handlep);
 891 #endif  /* DDI_MAP_DEBUG */
 892 
 893         /*
 894          * I/O or memory mapping:
 895          *
 896          *      <bustype=0, addr=x, len=x>: memory
 897          *      <bustype=1, addr=x, len=x>: i/o
 898          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
 899          */
 900 
 901         if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
 902                 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
 903                     " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
 904                     ddi_get_name(rdip), rp->regspec_bustype,
 905                     rp->regspec_addr, rp->regspec_size);
 906                 return (DDI_ME_INVAL);
 907         }
 908 
 909         if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
 910                 /*
 911                  * compatibility i/o mapping
 912                  */
 913                 rp->regspec_bustype += (uint_t)offset;
 914         } else {
 915                 /*
 916                  * Normal memory or i/o mapping
 917                  */
 918                 rp->regspec_addr += (uint_t)offset;
 919         }
 920 
 921         if (len != 0)
 922                 rp->regspec_size = (uint_t)len;
 923 
 924 #ifdef  DDI_MAP_DEBUG
 925         cmn_err(CE_CONT, "             <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
 926             "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
 927             rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
 928             offset, len, mp->map_handlep);
 929 #endif  /* DDI_MAP_DEBUG */
 930 
 931         /*
 932          * Apply any parent ranges at this level, if applicable.
 933          * (This is where nexus specific regspec translation takes place.
 934          * Use of this function is implicit agreement that translation is
 935          * provided via ddi_apply_range.)
 936          */
 937 
 938 #ifdef  DDI_MAP_DEBUG
 939         ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
 940             ddi_get_name(dip), ddi_get_name(rdip));
 941 #endif  /* DDI_MAP_DEBUG */
 942 
 943         if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
 944                 return (error);
 945 
 946         switch (mp->map_op)  {
 947         case DDI_MO_MAP_LOCKED:
 948 
 949                 /*
 950                  * Set up the locked down kernel mapping to the regspec...
 951                  */
 952 
 953                 return (rootnex_map_regspec(mp, vaddrp));
 954 
 955         case DDI_MO_UNMAP:
 956 
 957                 /*
 958                  * Release mapping...
 959                  */
 960 
 961                 return (rootnex_unmap_regspec(mp, vaddrp));
 962 
 963         case DDI_MO_MAP_HANDLE:
 964 
 965                 return (rootnex_map_handle(mp));
 966 
 967         default:
 968                 return (DDI_ME_UNIMPLEMENTED);
 969         }
 970 }
 971 
 972 
 973 /*
 974  * rootnex_map_fault()
 975  *
 976  *      fault in mappings for requestors
 977  */
 978 /*ARGSUSED*/
 979 static int
 980 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
 981     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
 982     uint_t lock)
 983 {
 984 
 985 #ifdef  DDI_MAP_DEBUG
 986         ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
 987         ddi_map_debug(" Seg <%s>\n",
 988             seg->s_ops == &segdev_ops ? "segdev" :
 989             seg == &kvseg ? "segkmem" : "NONE!");
 990 #endif  /* DDI_MAP_DEBUG */
 991 
 992         /*
 993          * This is all terribly broken, but it is a start
 994          *
 995          * XXX  Note that this test means that segdev_ops
 996          *      must be exported from seg_dev.c.
 997          * XXX  What about devices with their own segment drivers?
 998          */
 999         if (seg->s_ops == &segdev_ops) {
1000                 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1001 
1002                 if (hat == NULL) {
1003                         /*
1004                          * This is one plausible interpretation of
1005                          * a null hat i.e. use the first hat on the
1006                          * address space hat list which by convention is
1007                          * the hat of the system MMU.  At alternative
1008                          * would be to panic .. this might well be better ..
1009                          */
1010                         ASSERT(AS_READ_HELD(seg->s_as));
1011                         hat = seg->s_as->a_hat;
1012                         cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1013                 }
1014                 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1015                     (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1016         } else if (seg == &kvseg && dp == NULL) {
1017                 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1018                     HAT_LOAD_LOCK);
1019         } else
1020                 return (DDI_FAILURE);
1021         return (DDI_SUCCESS);
1022 }
1023 
1024 
1025 /*
1026  * rootnex_map_regspec()
1027  *     we don't support mapping of I/O cards above 4Gb
1028  */
1029 static int
1030 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1031 {
1032         rootnex_addr_t rbase;
1033         void *cvaddr;
1034         uint_t npages, pgoffset;
1035         struct regspec *rp;
1036         ddi_acc_hdl_t *hp;
1037         ddi_acc_impl_t *ap;
1038         uint_t  hat_acc_flags;
1039         paddr_t pbase;
1040 
1041         rp = mp->map_obj.rp;
1042         hp = mp->map_handlep;
1043 
1044 #ifdef  DDI_MAP_DEBUG
1045         ddi_map_debug(
1046             "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1047             rp->regspec_bustype, rp->regspec_addr,
1048             rp->regspec_size, mp->map_handlep);
1049 #endif  /* DDI_MAP_DEBUG */
1050 
1051         /*
1052          * I/O or memory mapping
1053          *
1054          *      <bustype=0, addr=x, len=x>: memory
1055          *      <bustype=1, addr=x, len=x>: i/o
1056          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
1057          */
1058 
1059         if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1060                 cmn_err(CE_WARN, "rootnex: invalid register spec"
1061                     " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1062                     rp->regspec_addr, rp->regspec_size);
1063                 return (DDI_FAILURE);
1064         }
1065 
1066         if (rp->regspec_bustype != 0) {
1067                 /*
1068                  * I/O space - needs a handle.
1069                  */
1070                 if (hp == NULL) {
1071                         return (DDI_FAILURE);
1072                 }
1073                 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1074                 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1075                 impl_acc_hdl_init(hp);
1076 
1077                 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1078 #ifdef  DDI_MAP_DEBUG
1079                         ddi_map_debug("rootnex_map_regspec: mmap() "
1080                             "to I/O space is not supported.\n");
1081 #endif  /* DDI_MAP_DEBUG */
1082                         return (DDI_ME_INVAL);
1083                 } else {
1084                         /*
1085                          * 1275-compliant vs. compatibility i/o mapping
1086                          */
1087                         *vaddrp =
1088                             (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1089                             ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1090                             ((caddr_t)(uintptr_t)rp->regspec_addr);
1091 #ifdef __xpv
1092                         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1093                                 hp->ah_pfn = xen_assign_pfn(
1094                                     mmu_btop((ulong_t)rp->regspec_addr &
1095                                     MMU_PAGEMASK));
1096                         } else {
1097                                 hp->ah_pfn = mmu_btop(
1098                                     (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1099                         }
1100 #else
1101                         hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1102                             MMU_PAGEMASK);
1103 #endif
1104                         hp->ah_pnum = mmu_btopr(rp->regspec_size +
1105                             (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1106                 }
1107 
1108 #ifdef  DDI_MAP_DEBUG
1109                 ddi_map_debug(
1110             "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1111                     rp->regspec_size, *vaddrp);
1112 #endif  /* DDI_MAP_DEBUG */
1113                 return (DDI_SUCCESS);
1114         }
1115 
1116         /*
1117          * Memory space
1118          */
1119 
1120         if (hp != NULL) {
1121                 /*
1122                  * hat layer ignores
1123                  * hp->ah_acc.devacc_attr_endian_flags.
1124                  */
1125                 switch (hp->ah_acc.devacc_attr_dataorder) {
1126                 case DDI_STRICTORDER_ACC:
1127                         hat_acc_flags = HAT_STRICTORDER;
1128                         break;
1129                 case DDI_UNORDERED_OK_ACC:
1130                         hat_acc_flags = HAT_UNORDERED_OK;
1131                         break;
1132                 case DDI_MERGING_OK_ACC:
1133                         hat_acc_flags = HAT_MERGING_OK;
1134                         break;
1135                 case DDI_LOADCACHING_OK_ACC:
1136                         hat_acc_flags = HAT_LOADCACHING_OK;
1137                         break;
1138                 case DDI_STORECACHING_OK_ACC:
1139                         hat_acc_flags = HAT_STORECACHING_OK;
1140                         break;
1141                 }
1142                 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1143                 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1144                 impl_acc_hdl_init(hp);
1145                 hp->ah_hat_flags = hat_acc_flags;
1146         } else {
1147                 hat_acc_flags = HAT_STRICTORDER;
1148         }
1149 
1150         rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1151 #ifdef __xpv
1152         /*
1153          * If we're dom0, we're using a real device so we need to translate
1154          * the MA to a PA.
1155          */
1156         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1157                 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1158         } else {
1159                 pbase = rbase;
1160         }
1161 #else
1162         pbase = rbase;
1163 #endif
1164         pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1165 
1166         if (rp->regspec_size == 0) {
1167 #ifdef  DDI_MAP_DEBUG
1168                 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1169 #endif  /* DDI_MAP_DEBUG */
1170                 return (DDI_ME_INVAL);
1171         }
1172 
1173         if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1174                 /* extra cast to make gcc happy */
1175                 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1176         } else {
1177                 npages = mmu_btopr(rp->regspec_size + pgoffset);
1178 
1179 #ifdef  DDI_MAP_DEBUG
1180                 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1181                     "physical %llx", npages, pbase);
1182 #endif  /* DDI_MAP_DEBUG */
1183 
1184                 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1185                 if (cvaddr == NULL)
1186                         return (DDI_ME_NORESOURCES);
1187 
1188                 /*
1189                  * Now map in the pages we've allocated...
1190                  */
1191                 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1192                     mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1193                     HAT_LOAD_LOCK);
1194                 *vaddrp = (caddr_t)cvaddr + pgoffset;
1195 
1196                 /* save away pfn and npages for FMA */
1197                 hp = mp->map_handlep;
1198                 if (hp) {
1199                         hp->ah_pfn = mmu_btop(pbase);
1200                         hp->ah_pnum = npages;
1201                 }
1202         }
1203 
1204 #ifdef  DDI_MAP_DEBUG
1205         ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1206 #endif  /* DDI_MAP_DEBUG */
1207         return (DDI_SUCCESS);
1208 }
1209 
1210 
1211 /*
1212  * rootnex_unmap_regspec()
1213  *
1214  */
1215 static int
1216 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1217 {
1218         caddr_t addr = (caddr_t)*vaddrp;
1219         uint_t npages, pgoffset;
1220         struct regspec *rp;
1221 
1222         if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1223                 return (0);
1224 
1225         rp = mp->map_obj.rp;
1226 
1227         if (rp->regspec_size == 0) {
1228 #ifdef  DDI_MAP_DEBUG
1229                 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1230 #endif  /* DDI_MAP_DEBUG */
1231                 return (DDI_ME_INVAL);
1232         }
1233 
1234         /*
1235          * I/O or memory mapping:
1236          *
1237          *      <bustype=0, addr=x, len=x>: memory
1238          *      <bustype=1, addr=x, len=x>: i/o
1239          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
1240          */
1241         if (rp->regspec_bustype != 0) {
1242                 /*
1243                  * This is I/O space, which requires no particular
1244                  * processing on unmap since it isn't mapped in the
1245                  * first place.
1246                  */
1247                 return (DDI_SUCCESS);
1248         }
1249 
1250         /*
1251          * Memory space
1252          */
1253         pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1254         npages = mmu_btopr(rp->regspec_size + pgoffset);
1255         hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1256         device_arena_free(addr - pgoffset, ptob(npages));
1257 
1258         /*
1259          * Destroy the pointer - the mapping has logically gone
1260          */
1261         *vaddrp = NULL;
1262 
1263         return (DDI_SUCCESS);
1264 }
1265 
1266 
1267 /*
1268  * rootnex_map_handle()
1269  *
1270  */
1271 static int
1272 rootnex_map_handle(ddi_map_req_t *mp)
1273 {
1274         rootnex_addr_t rbase;
1275         ddi_acc_hdl_t *hp;
1276         uint_t pgoffset;
1277         struct regspec *rp;
1278         paddr_t pbase;
1279 
1280         rp = mp->map_obj.rp;
1281 
1282 #ifdef  DDI_MAP_DEBUG
1283         ddi_map_debug(
1284             "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1285             rp->regspec_bustype, rp->regspec_addr,
1286             rp->regspec_size, mp->map_handlep);
1287 #endif  /* DDI_MAP_DEBUG */
1288 
1289         /*
1290          * I/O or memory mapping:
1291          *
1292          *      <bustype=0, addr=x, len=x>: memory
1293          *      <bustype=1, addr=x, len=x>: i/o
1294          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
1295          */
1296         if (rp->regspec_bustype != 0) {
1297                 /*
1298                  * This refers to I/O space, and we don't support "mapping"
1299                  * I/O space to a user.
1300                  */
1301                 return (DDI_FAILURE);
1302         }
1303 
1304         /*
1305          * Set up the hat_flags for the mapping.
1306          */
1307         hp = mp->map_handlep;
1308 
1309         switch (hp->ah_acc.devacc_attr_endian_flags) {
1310         case DDI_NEVERSWAP_ACC:
1311                 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1312                 break;
1313         case DDI_STRUCTURE_LE_ACC:
1314                 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1315                 break;
1316         case DDI_STRUCTURE_BE_ACC:
1317                 return (DDI_FAILURE);
1318         default:
1319                 return (DDI_REGS_ACC_CONFLICT);
1320         }
1321 
1322         switch (hp->ah_acc.devacc_attr_dataorder) {
1323         case DDI_STRICTORDER_ACC:
1324                 break;
1325         case DDI_UNORDERED_OK_ACC:
1326                 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1327                 break;
1328         case DDI_MERGING_OK_ACC:
1329                 hp->ah_hat_flags |= HAT_MERGING_OK;
1330                 break;
1331         case DDI_LOADCACHING_OK_ACC:
1332                 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1333                 break;
1334         case DDI_STORECACHING_OK_ACC:
1335                 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1336                 break;
1337         default:
1338                 return (DDI_FAILURE);
1339         }
1340 
1341         rbase = (rootnex_addr_t)rp->regspec_addr &
1342             (~(rootnex_addr_t)MMU_PAGEOFFSET);
1343         pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1344 
1345         if (rp->regspec_size == 0)
1346                 return (DDI_ME_INVAL);
1347 
1348 #ifdef __xpv
1349         /*
1350          * If we're dom0, we're using a real device so we need to translate
1351          * the MA to a PA.
1352          */
1353         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1354                 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1355                     (rbase & MMU_PAGEOFFSET);
1356         } else {
1357                 pbase = rbase;
1358         }
1359 #else
1360         pbase = rbase;
1361 #endif
1362 
1363         hp->ah_pfn = mmu_btop(pbase);
1364         hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1365 
1366         return (DDI_SUCCESS);
1367 }
1368 
1369 
1370 
1371 /*
1372  * ************************
1373  *  interrupt related code
1374  * ************************
1375  */
1376 
1377 /*
1378  * rootnex_intr_ops()
1379  *      bus_intr_op() function for interrupt support
1380  */
1381 /* ARGSUSED */
1382 static int
1383 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1384     ddi_intr_handle_impl_t *hdlp, void *result)
1385 {
1386         struct intrspec                 *ispec;
1387 
1388         DDI_INTR_NEXDBG((CE_CONT,
1389             "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1390             (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1391 
1392         /* Process the interrupt operation */
1393         switch (intr_op) {
1394         case DDI_INTROP_GETCAP:
1395                 /* First check with pcplusmp */
1396                 if (psm_intr_ops == NULL)
1397                         return (DDI_FAILURE);
1398 
1399                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1400                         *(int *)result = 0;
1401                         return (DDI_FAILURE);
1402                 }
1403                 break;
1404         case DDI_INTROP_SETCAP:
1405                 if (psm_intr_ops == NULL)
1406                         return (DDI_FAILURE);
1407 
1408                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1409                         return (DDI_FAILURE);
1410                 break;
1411         case DDI_INTROP_ALLOC:
1412                 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1413                 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1414         case DDI_INTROP_FREE:
1415                 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1416                 return (rootnex_free_intr_fixed(rdip, hdlp));
1417         case DDI_INTROP_GETPRI:
1418                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1419                         return (DDI_FAILURE);
1420                 *(int *)result = ispec->intrspec_pri;
1421                 break;
1422         case DDI_INTROP_SETPRI:
1423                 /* Validate the interrupt priority passed to us */
1424                 if (*(int *)result > LOCK_LEVEL)
1425                         return (DDI_FAILURE);
1426 
1427                 /* Ensure that PSM is all initialized and ispec is ok */
1428                 if ((psm_intr_ops == NULL) ||
1429                     ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1430                         return (DDI_FAILURE);
1431 
1432                 /* Change the priority */
1433                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1434                     PSM_FAILURE)
1435                         return (DDI_FAILURE);
1436 
1437                 /* update the ispec with the new priority */
1438                 ispec->intrspec_pri =  *(int *)result;
1439                 break;
1440         case DDI_INTROP_ADDISR:
1441                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1442                         return (DDI_FAILURE);
1443                 ispec->intrspec_func = hdlp->ih_cb_func;
1444                 break;
1445         case DDI_INTROP_REMISR:
1446                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1447                         return (DDI_FAILURE);
1448                 ispec->intrspec_func = (uint_t (*)()) 0;
1449                 break;
1450         case DDI_INTROP_ENABLE:
1451                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1452                         return (DDI_FAILURE);
1453 
1454                 /* Call psmi to translate irq with the dip */
1455                 if (psm_intr_ops == NULL)
1456                         return (DDI_FAILURE);
1457 
1458                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1459                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1460                     (int *)&hdlp->ih_vector) == PSM_FAILURE)
1461                         return (DDI_FAILURE);
1462 
1463                 /* Add the interrupt handler */
1464                 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1465                     hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1466                     hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1467                         return (DDI_FAILURE);
1468                 break;
1469         case DDI_INTROP_DISABLE:
1470                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1471                         return (DDI_FAILURE);
1472 
1473                 /* Call psm_ops() to translate irq with the dip */
1474                 if (psm_intr_ops == NULL)
1475                         return (DDI_FAILURE);
1476 
1477                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1478                 (void) (*psm_intr_ops)(rdip, hdlp,
1479                     PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1480 
1481                 /* Remove the interrupt handler */
1482                 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1483                     hdlp->ih_cb_func, hdlp->ih_vector);
1484                 break;
1485         case DDI_INTROP_SETMASK:
1486                 if (psm_intr_ops == NULL)
1487                         return (DDI_FAILURE);
1488 
1489                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1490                         return (DDI_FAILURE);
1491                 break;
1492         case DDI_INTROP_CLRMASK:
1493                 if (psm_intr_ops == NULL)
1494                         return (DDI_FAILURE);
1495 
1496                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1497                         return (DDI_FAILURE);
1498                 break;
1499         case DDI_INTROP_GETPENDING:
1500                 if (psm_intr_ops == NULL)
1501                         return (DDI_FAILURE);
1502 
1503                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1504                     result)) {
1505                         *(int *)result = 0;
1506                         return (DDI_FAILURE);
1507                 }
1508                 break;
1509         case DDI_INTROP_NAVAIL:
1510         case DDI_INTROP_NINTRS:
1511                 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1512                 if (*(int *)result == 0) {
1513                         /*
1514                          * Special case for 'pcic' driver' only. This driver
1515                          * driver is a child of 'isa' and 'rootnex' drivers.
1516                          *
1517                          * See detailed comments on this in the function
1518                          * rootnex_get_ispec().
1519                          *
1520                          * Children of 'pcic' send 'NINITR' request all the
1521                          * way to rootnex driver. But, the 'pdp->par_nintr'
1522                          * field may not initialized. So, we fake it here
1523                          * to return 1 (a la what PCMCIA nexus does).
1524                          */
1525                         if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1526                                 *(int *)result = 1;
1527                         else
1528                                 return (DDI_FAILURE);
1529                 }
1530                 break;
1531         case DDI_INTROP_SUPPORTED_TYPES:
1532                 *(int *)result = DDI_INTR_TYPE_FIXED;   /* Always ... */
1533                 break;
1534         default:
1535                 return (DDI_FAILURE);
1536         }
1537 
1538         return (DDI_SUCCESS);
1539 }
1540 
1541 
1542 /*
1543  * rootnex_get_ispec()
1544  *      convert an interrupt number to an interrupt specification.
1545  *      The interrupt number determines which interrupt spec will be
1546  *      returned if more than one exists.
1547  *
1548  *      Look into the parent private data area of the 'rdip' to find out
1549  *      the interrupt specification.  First check to make sure there is
1550  *      one that matchs "inumber" and then return a pointer to it.
1551  *
1552  *      Return NULL if one could not be found.
1553  *
1554  *      NOTE: This is needed for rootnex_intr_ops()
1555  */
1556 static struct intrspec *
1557 rootnex_get_ispec(dev_info_t *rdip, int inum)
1558 {
1559         struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1560 
1561         /*
1562          * Special case handling for drivers that provide their own
1563          * intrspec structures instead of relying on the DDI framework.
1564          *
1565          * A broken hardware driver in ON could potentially provide its
1566          * own intrspec structure, instead of relying on the hardware.
1567          * If these drivers are children of 'rootnex' then we need to
1568          * continue to provide backward compatibility to them here.
1569          *
1570          * Following check is a special case for 'pcic' driver which
1571          * was found to have broken hardwre andby provides its own intrspec.
1572          *
1573          * Verbatim comments from this driver are shown here:
1574          * "Don't use the ddi_add_intr since we don't have a
1575          * default intrspec in all cases."
1576          *
1577          * Since an 'ispec' may not be always created for it,
1578          * check for that and create one if so.
1579          *
1580          * NOTE: Currently 'pcic' is the only driver found to do this.
1581          */
1582         if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1583                 pdp->par_nintr = 1;
1584                 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1585                     pdp->par_nintr, KM_SLEEP);
1586         }
1587 
1588         /* Validate the interrupt number */
1589         if (inum >= pdp->par_nintr)
1590                 return (NULL);
1591 
1592         /* Get the interrupt structure pointer and return that */
1593         return ((struct intrspec *)&pdp->par_intr[inum]);
1594 }
1595 
1596 /*
1597  * Allocate interrupt vector for FIXED (legacy) type.
1598  */
1599 static int
1600 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1601     void *result)
1602 {
1603         struct intrspec         *ispec;
1604         ddi_intr_handle_impl_t  info_hdl;
1605         int                     ret;
1606         int                     free_phdl = 0;
1607         apic_get_type_t         type_info;
1608 
1609         if (psm_intr_ops == NULL)
1610                 return (DDI_FAILURE);
1611 
1612         if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1613                 return (DDI_FAILURE);
1614 
1615         /*
1616          * If the PSM module is "APIX" then pass the request for it
1617          * to allocate the vector now.
1618          */
1619         bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1620         info_hdl.ih_private = &type_info;
1621         if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1622             PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1623                 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1624                         free_phdl = 1;
1625                         i_ddi_alloc_intr_phdl(hdlp);
1626                 }
1627                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1628                 ret = (*psm_intr_ops)(rdip, hdlp,
1629                     PSM_INTR_OP_ALLOC_VECTORS, result);
1630                 if (free_phdl) { /* free up the phdl structure */
1631                         free_phdl = 0;
1632                         i_ddi_free_intr_phdl(hdlp);
1633                         hdlp->ih_private = NULL;
1634                 }
1635         } else {
1636                 /*
1637                  * No APIX module; fall back to the old scheme where the
1638                  * interrupt vector is allocated during ddi_enable_intr() call.
1639                  */
1640                 hdlp->ih_pri = ispec->intrspec_pri;
1641                 *(int *)result = hdlp->ih_scratch1;
1642                 ret = DDI_SUCCESS;
1643         }
1644 
1645         return (ret);
1646 }
1647 
1648 /*
1649  * Free up interrupt vector for FIXED (legacy) type.
1650  */
1651 static int
1652 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1653 {
1654         struct intrspec                 *ispec;
1655         struct ddi_parent_private_data  *pdp;
1656         ddi_intr_handle_impl_t          info_hdl;
1657         int                             ret;
1658         apic_get_type_t                 type_info;
1659 
1660         if (psm_intr_ops == NULL)
1661                 return (DDI_FAILURE);
1662 
1663         /*
1664          * If the PSM module is "APIX" then pass the request for it
1665          * to free up the vector now.
1666          */
1667         bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1668         info_hdl.ih_private = &type_info;
1669         if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1670             PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1671                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1672                         return (DDI_FAILURE);
1673                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1674                 ret = (*psm_intr_ops)(rdip, hdlp,
1675                     PSM_INTR_OP_FREE_VECTORS, NULL);
1676         } else {
1677                 /*
1678                  * No APIX module; fall back to the old scheme where
1679                  * the interrupt vector was already freed during
1680                  * ddi_disable_intr() call.
1681                  */
1682                 ret = DDI_SUCCESS;
1683         }
1684 
1685         pdp = ddi_get_parent_data(rdip);
1686 
1687         /*
1688          * Special case for 'pcic' driver' only.
1689          * If an intrspec was created for it, clean it up here
1690          * See detailed comments on this in the function
1691          * rootnex_get_ispec().
1692          */
1693         if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1694                 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1695                     pdp->par_nintr);
1696                 /*
1697                  * Set it to zero; so that
1698                  * DDI framework doesn't free it again
1699                  */
1700                 pdp->par_intr = NULL;
1701                 pdp->par_nintr = 0;
1702         }
1703 
1704         return (ret);
1705 }
1706 
1707 
1708 /*
1709  * ******************
1710  *  dma related code
1711  * ******************
1712  */
1713 
1714 /*ARGSUSED*/
1715 static int
1716 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1717     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1718     ddi_dma_handle_t *handlep)
1719 {
1720         uint64_t maxsegmentsize_ll;
1721         uint_t maxsegmentsize;
1722         ddi_dma_impl_t *hp;
1723         rootnex_dma_t *dma;
1724         uint64_t count_max;
1725         uint64_t seg;
1726         int kmflag;
1727         int e;
1728 
1729 
1730         /* convert our sleep flags */
1731         if (waitfp == DDI_DMA_SLEEP) {
1732                 kmflag = KM_SLEEP;
1733         } else {
1734                 kmflag = KM_NOSLEEP;
1735         }
1736 
1737         /*
1738          * We try to do only one memory allocation here. We'll do a little
1739          * pointer manipulation later. If the bind ends up taking more than
1740          * our prealloc's space, we'll have to allocate more memory in the
1741          * bind operation. Not great, but much better than before and the
1742          * best we can do with the current bind interfaces.
1743          */
1744         hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1745         if (hp == NULL)
1746                 return (DDI_DMA_NORESOURCES);
1747 
1748         /* Do our pointer manipulation now, align the structures */
1749         hp->dmai_private = (void *)(((uintptr_t)hp +
1750             (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1751         dma = (rootnex_dma_t *)hp->dmai_private;
1752         dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1753             sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1754 
1755         /* setup the handle */
1756         rootnex_clean_dmahdl(hp);
1757         hp->dmai_error.err_fep = NULL;
1758         hp->dmai_error.err_cf = NULL;
1759         dma->dp_dip = rdip;
1760         dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1761         dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1762 
1763         /*
1764          * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1765          * is being used. Set the upper limit to the seg value.
1766          * There will be enough DVMA space to always get addresses
1767          * that will match the constraints.
1768          */
1769         if (IOMMU_USED(rdip) &&
1770             (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1771                 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1772                 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1773         } else
1774                 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1775 
1776         hp->dmai_minxfer = attr->dma_attr_minxfer;
1777         hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1778         hp->dmai_rdip = rdip;
1779         hp->dmai_attr = *attr;
1780 
1781         if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1782                 dma->dp_sglinfo.si_cancross = B_FALSE;
1783         else
1784                 dma->dp_sglinfo.si_cancross = B_TRUE;
1785 
1786         /* we don't need to worry about the SPL since we do a tryenter */
1787         mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1788 
1789         /*
1790          * Figure out our maximum segment size. If the segment size is greater
1791          * than 4G, we will limit it to (4G - 1) since the max size of a dma
1792          * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1793          * dma_attr_count_max are size-1 type values.
1794          *
1795          * Maximum segment size is the largest physically contiguous chunk of
1796          * memory that we can return from a bind (i.e. the maximum size of a
1797          * single cookie).
1798          */
1799 
1800         /* handle the rollover cases */
1801         seg = attr->dma_attr_seg + 1;
1802         if (seg < attr->dma_attr_seg) {
1803                 seg = attr->dma_attr_seg;
1804         }
1805         count_max = attr->dma_attr_count_max + 1;
1806         if (count_max < attr->dma_attr_count_max) {
1807                 count_max = attr->dma_attr_count_max;
1808         }
1809 
1810         /*
1811          * granularity may or may not be a power of two. If it isn't, we can't
1812          * use a simple mask.
1813          */
1814         if (!ISP2(attr->dma_attr_granular)) {
1815                 dma->dp_granularity_power_2 = B_FALSE;
1816         } else {
1817                 dma->dp_granularity_power_2 = B_TRUE;
1818         }
1819 
1820         /*
1821          * maxxfer should be a whole multiple of granularity. If we're going to
1822          * break up a window because we're greater than maxxfer, we might as
1823          * well make sure it's maxxfer is a whole multiple so we don't have to
1824          * worry about triming the window later on for this case.
1825          */
1826         if (attr->dma_attr_granular > 1) {
1827                 if (dma->dp_granularity_power_2) {
1828                         dma->dp_maxxfer = attr->dma_attr_maxxfer -
1829                             (attr->dma_attr_maxxfer &
1830                             (attr->dma_attr_granular - 1));
1831                 } else {
1832                         dma->dp_maxxfer = attr->dma_attr_maxxfer -
1833                             (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1834                 }
1835         } else {
1836                 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1837         }
1838 
1839         maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1840         maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1841         if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1842                 maxsegmentsize = 0xFFFFFFFF;
1843         } else {
1844                 maxsegmentsize = maxsegmentsize_ll;
1845         }
1846         dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1847         dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1848 
1849         /* check the ddi_dma_attr arg to make sure it makes a little sense */
1850         if (rootnex_alloc_check_parms) {
1851                 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1852                 if (e != DDI_SUCCESS) {
1853                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1854                         (void) rootnex_dma_freehdl(dip, rdip,
1855                             (ddi_dma_handle_t)hp);
1856                         return (e);
1857                 }
1858         }
1859 
1860         *handlep = (ddi_dma_handle_t)hp;
1861 
1862         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1863         ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1864             rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1865 
1866         return (DDI_SUCCESS);
1867 }
1868 
1869 
1870 /*
1871  * rootnex_dma_allochdl()
1872  *    called from ddi_dma_alloc_handle().
1873  */
1874 static int
1875 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1876     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1877 {
1878         int retval = DDI_SUCCESS;
1879 #if defined(__amd64) && !defined(__xpv)
1880 
1881         if (IOMMU_UNITIALIZED(rdip)) {
1882                 retval = iommulib_nex_open(dip, rdip);
1883 
1884                 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1885                         return (retval);
1886         }
1887 
1888         if (IOMMU_UNUSED(rdip)) {
1889                 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1890                     handlep);
1891         } else {
1892                 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1893                     waitfp, arg, handlep);
1894         }
1895 #else
1896         retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1897             handlep);
1898 #endif
1899         switch (retval) {
1900         case DDI_DMA_NORESOURCES:
1901                 if (waitfp != DDI_DMA_DONTWAIT) {
1902                         ddi_set_callback(waitfp, arg,
1903                             &rootnex_state->r_dvma_call_list_id);
1904                 }
1905                 break;
1906         case DDI_SUCCESS:
1907                 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1908                 break;
1909         default:
1910                 break;
1911         }
1912         return (retval);
1913 }
1914 
1915 /*ARGSUSED*/
1916 static int
1917 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1918     ddi_dma_handle_t handle)
1919 {
1920         ddi_dma_impl_t *hp;
1921         rootnex_dma_t *dma;
1922 
1923 
1924         hp = (ddi_dma_impl_t *)handle;
1925         dma = (rootnex_dma_t *)hp->dmai_private;
1926 
1927         /* unbind should have been called first */
1928         ASSERT(!dma->dp_inuse);
1929 
1930         mutex_destroy(&dma->dp_mutex);
1931         kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1932 
1933         ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1934         ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1935             rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1936 
1937         return (DDI_SUCCESS);
1938 }
1939 
1940 /*
1941  * rootnex_dma_freehdl()
1942  *    called from ddi_dma_free_handle().
1943  */
1944 static int
1945 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1946 {
1947         int ret;
1948 
1949         ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1950 #if defined(__amd64) && !defined(__xpv)
1951         if (IOMMU_USED(rdip))
1952                 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1953         else
1954 #endif
1955         ret = rootnex_coredma_freehdl(dip, rdip, handle);
1956 
1957         if (rootnex_state->r_dvma_call_list_id)
1958                 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1959 
1960         return (ret);
1961 }
1962 
1963 /*ARGSUSED*/
1964 static int
1965 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1966     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1967     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1968 {
1969         rootnex_sglinfo_t *sinfo;
1970         ddi_dma_obj_t *dmao;
1971 #if defined(__amd64) && !defined(__xpv)
1972         struct dvmaseg *dvs;
1973         ddi_dma_cookie_t *cookie;
1974 #endif
1975         ddi_dma_attr_t *attr;
1976         ddi_dma_impl_t *hp;
1977         rootnex_dma_t *dma;
1978         int kmflag;
1979         int e;
1980         uint_t ncookies;
1981 
1982         hp = (ddi_dma_impl_t *)handle;
1983         dma = (rootnex_dma_t *)hp->dmai_private;
1984         dmao = &dma->dp_dma;
1985         sinfo = &dma->dp_sglinfo;
1986         attr = &hp->dmai_attr;
1987 
1988         /* convert the sleep flags */
1989         if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1990                 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1991         } else {
1992                 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1993         }
1994 
1995         hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1996 
1997         /*
1998          * This is useful for debugging a driver. Not as useful in a production
1999          * system. The only time this will fail is if you have a driver bug.
2000          */
2001         if (rootnex_bind_check_inuse) {
2002                 /*
2003                  * No one else should ever have this lock unless someone else
2004                  * is trying to use this handle. So contention on the lock
2005                  * is the same as inuse being set.
2006                  */
2007                 e = mutex_tryenter(&dma->dp_mutex);
2008                 if (e == 0) {
2009                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2010                         return (DDI_DMA_INUSE);
2011                 }
2012                 if (dma->dp_inuse) {
2013                         mutex_exit(&dma->dp_mutex);
2014                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2015                         return (DDI_DMA_INUSE);
2016                 }
2017                 dma->dp_inuse = B_TRUE;
2018                 mutex_exit(&dma->dp_mutex);
2019         }
2020 
2021         /* check the ddi_dma_attr arg to make sure it makes a little sense */
2022         if (rootnex_bind_check_parms) {
2023                 e = rootnex_valid_bind_parms(dmareq, attr);
2024                 if (e != DDI_SUCCESS) {
2025                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2026                         rootnex_clean_dmahdl(hp);
2027                         return (e);
2028                 }
2029         }
2030 
2031         /* save away the original bind info */
2032         dma->dp_dma = dmareq->dmar_object;
2033 
2034 #if defined(__amd64) && !defined(__xpv)
2035         if (IOMMU_USED(rdip)) {
2036                 dmao = &dma->dp_dvma;
2037                 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2038                 switch (e) {
2039                 case DDI_SUCCESS:
2040                         if (sinfo->si_cancross ||
2041                             dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2042                             dmao->dmao_size > sinfo->si_max_cookie_size) {
2043                                 dma->dp_dvma_used = B_TRUE;
2044                                 break;
2045                         }
2046                         sinfo->si_sgl_size = 1;
2047                         hp->dmai_rflags |= DMP_NOSYNC;
2048 
2049                         dma->dp_dvma_used = B_TRUE;
2050                         dma->dp_need_to_free_cookie = B_FALSE;
2051 
2052                         dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2053                         cookie = hp->dmai_cookie = dma->dp_cookies =
2054                             (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2055                         cookie->dmac_laddress = dvs->dvs_start +
2056                             dmao->dmao_obj.dvma_obj.dv_off;
2057                         cookie->dmac_size = dvs->dvs_len;
2058                         cookie->dmac_type = 0;
2059 
2060                         ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2061                             rdip);
2062                         goto fast;
2063                 case DDI_ENOTSUP:
2064                         break;
2065                 default:
2066                         rootnex_clean_dmahdl(hp);
2067                         return (e);
2068                 }
2069         }
2070 #endif
2071 
2072         /*
2073          * Figure out a rough estimate of what maximum number of pages
2074          * this buffer could use (a high estimate of course).
2075          */
2076         sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2077 
2078         if (dma->dp_dvma_used) {
2079                 /*
2080                  * The number of physical pages is the worst case.
2081                  *
2082                  * For DVMA, the worst case is the length divided
2083                  * by the maximum cookie length, plus 1. Add to that
2084                  * the number of segment boundaries potentially crossed, and
2085                  * the additional number of DVMA segments that was returned.
2086                  *
2087                  * In the normal case, for modern devices, si_cancross will
2088                  * be false, and dv_nseg will be 1, and the fast path will
2089                  * have been taken above.
2090                  */
2091                 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2092                     + 1;
2093                 if (sinfo->si_cancross)
2094                         ncookies +=
2095                             (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2096                 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2097 
2098                 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2099         }
2100 
2101         /*
2102          * We'll use the pre-allocated cookies for any bind that will *always*
2103          * fit (more important to be consistent, we don't want to create
2104          * additional degenerate cases).
2105          */
2106         if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2107                 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2108                 dma->dp_need_to_free_cookie = B_FALSE;
2109                 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2110                     uint_t, sinfo->si_max_pages);
2111 
2112         /*
2113          * For anything larger than that, we'll go ahead and allocate the
2114          * maximum number of pages we expect to see. Hopefuly, we won't be
2115          * seeing this path in the fast path for high performance devices very
2116          * frequently.
2117          *
2118          * a ddi bind interface that allowed the driver to provide storage to
2119          * the bind interface would speed this case up.
2120          */
2121         } else {
2122                 /*
2123                  * Save away how much memory we allocated. If we're doing a
2124                  * nosleep, the alloc could fail...
2125                  */
2126                 dma->dp_cookie_size = sinfo->si_max_pages *
2127                     sizeof (ddi_dma_cookie_t);
2128                 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2129                 if (dma->dp_cookies == NULL) {
2130                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2131                         rootnex_clean_dmahdl(hp);
2132                         return (DDI_DMA_NORESOURCES);
2133                 }
2134                 dma->dp_need_to_free_cookie = B_TRUE;
2135                 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2136                     uint_t, sinfo->si_max_pages);
2137         }
2138         hp->dmai_cookie = dma->dp_cookies;
2139 
2140         /*
2141          * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2142          * looking at the constraints in the dma structure. It will then put
2143          * some additional state about the sgl in the dma struct (i.e. is
2144          * the sgl clean, or do we need to do some munging; how many pages
2145          * need to be copied, etc.)
2146          */
2147         if (dma->dp_dvma_used)
2148                 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2149         else
2150                 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2151 
2152 out:
2153         ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2154         /* if we don't need a copy buffer, we don't need to sync */
2155         if (sinfo->si_copybuf_req == 0) {
2156                 hp->dmai_rflags |= DMP_NOSYNC;
2157         }
2158 
2159         /*
2160          * if we don't need the copybuf and we don't need to do a partial,  we
2161          * hit the fast path. All the high performance devices should be trying
2162          * to hit this path. To hit this path, a device should be able to reach
2163          * all of memory, shouldn't try to bind more than it can transfer, and
2164          * the buffer shouldn't require more cookies than the driver/device can
2165          * handle [sgllen]).
2166          *
2167          * Note that negative values of dma_attr_sgllen are supposed
2168          * to mean unlimited, but we just cast them to mean a
2169          * "ridiculous large limit".  This saves some extra checks on
2170          * hot paths.
2171          */
2172         if ((sinfo->si_copybuf_req == 0) &&
2173             (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2174             (dmao->dmao_size < dma->dp_maxxfer)) {
2175 fast:
2176                 /*
2177                  * If the driver supports FMA, insert the handle in the FMA DMA
2178                  * handle cache.
2179                  */
2180                 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2181                         hp->dmai_error.err_cf = rootnex_dma_check;
2182 
2183                 /*
2184                  * copy out the first cookie and ccountp, set the cookie
2185                  * pointer to the second cookie. The first cookie is passed
2186                  * back on the stack. Additional cookies are accessed via
2187                  * ddi_dma_nextcookie()
2188                  */
2189                 *cookiep = dma->dp_cookies[0];
2190                 *ccountp = sinfo->si_sgl_size;
2191                 hp->dmai_cookie++;
2192                 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2193                 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2194                 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2195                     uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2196                     uint_t, dmao->dmao_size, uint_t, *ccountp);
2197 
2198 
2199                 return (DDI_DMA_MAPPED);
2200         }
2201 
2202         /*
2203          * go to the slow path, we may need to alloc more memory, create
2204          * multiple windows, and munge up a sgl to make the device happy.
2205          */
2206 
2207         /*
2208          * With the IOMMU mapobject method used, we should never hit
2209          * the slow path. If we do, something is seriously wrong.
2210          * Clean up and return an error.
2211          */
2212 
2213 #if defined(__amd64) && !defined(__xpv)
2214 
2215         if (dma->dp_dvma_used) {
2216                 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2217                     &dma->dp_dvma);
2218                 e = DDI_DMA_NOMAPPING;
2219         } else {
2220 #endif
2221                 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2222                     kmflag);
2223 #if defined(__amd64) && !defined(__xpv)
2224         }
2225 #endif
2226         if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2227                 if (dma->dp_need_to_free_cookie) {
2228                         kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2229                 }
2230                 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2231                 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2232                 return (e);
2233         }
2234 
2235         /*
2236          * If the driver supports FMA, insert the handle in the FMA DMA handle
2237          * cache.
2238          */
2239         if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2240                 hp->dmai_error.err_cf = rootnex_dma_check;
2241 
2242         /* if the first window uses the copy buffer, sync it for the device */
2243         if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2244             (hp->dmai_rflags & DDI_DMA_WRITE)) {
2245                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2246                     DDI_DMA_SYNC_FORDEV);
2247         }
2248 
2249         /*
2250          * copy out the first cookie and ccountp, set the cookie pointer to the
2251          * second cookie. Make sure the partial flag is set/cleared correctly.
2252          * If we have a partial map (i.e. multiple windows), the number of
2253          * cookies we return is the number of cookies in the first window.
2254          */
2255         if (e == DDI_DMA_MAPPED) {
2256                 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2257                 *ccountp = sinfo->si_sgl_size;
2258                 hp->dmai_nwin = 1;
2259         } else {
2260                 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2261                 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2262                 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2263         }
2264         *cookiep = dma->dp_cookies[0];
2265         hp->dmai_cookie++;
2266 
2267         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2268         ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2269             rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2270             dmao->dmao_size, uint_t, *ccountp);
2271         return (e);
2272 }
2273 
2274 /*
2275  * rootnex_dma_bindhdl()
2276  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2277  */
2278 static int
2279 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2280     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2281     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2282 {
2283         int ret;
2284 #if defined(__amd64) && !defined(__xpv)
2285         if (IOMMU_USED(rdip))
2286                 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2287                     cookiep, ccountp);
2288         else
2289 #endif
2290         ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2291             cookiep, ccountp);
2292 
2293         if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2294                 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2295                     &rootnex_state->r_dvma_call_list_id);
2296         }
2297 
2298         return (ret);
2299 }
2300 
2301 
2302 
2303 /*ARGSUSED*/
2304 static int
2305 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2306     ddi_dma_handle_t handle)
2307 {
2308         ddi_dma_impl_t *hp;
2309         rootnex_dma_t *dma;
2310         int e;
2311 
2312         hp = (ddi_dma_impl_t *)handle;
2313         dma = (rootnex_dma_t *)hp->dmai_private;
2314 
2315         /* make sure the buffer wasn't free'd before calling unbind */
2316         if (rootnex_unbind_verify_buffer) {
2317                 e = rootnex_verify_buffer(dma);
2318                 if (e != DDI_SUCCESS) {
2319                         ASSERT(0);
2320                         return (DDI_FAILURE);
2321                 }
2322         }
2323 
2324         /* sync the current window before unbinding the buffer */
2325         if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2326             (hp->dmai_rflags & DDI_DMA_READ)) {
2327                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2328                     DDI_DMA_SYNC_FORCPU);
2329         }
2330 
2331         /*
2332          * cleanup and copy buffer or window state. if we didn't use the copy
2333          * buffer or windows, there won't be much to do :-)
2334          */
2335         rootnex_teardown_copybuf(dma);
2336         rootnex_teardown_windows(dma);
2337 
2338 #if defined(__amd64) && !defined(__xpv)
2339         if (IOMMU_USED(rdip))
2340                 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2341                     &dma->dp_dvma);
2342 #endif
2343 
2344         /*
2345          * If we had to allocate space to for the worse case sgl (it didn't
2346          * fit into our pre-allocate buffer), free that up now
2347          */
2348         if (dma->dp_need_to_free_cookie) {
2349                 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2350         }
2351 
2352         /*
2353          * clean up the handle so it's ready for the next bind (i.e. if the
2354          * handle is reused).
2355          */
2356         rootnex_clean_dmahdl(hp);
2357         hp->dmai_error.err_cf = NULL;
2358 
2359         ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2360         ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2361             rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2362 
2363         return (DDI_SUCCESS);
2364 }
2365 
2366 /*
2367  * rootnex_dma_unbindhdl()
2368  *    called from ddi_dma_unbind_handle()
2369  */
2370 /*ARGSUSED*/
2371 static int
2372 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2373     ddi_dma_handle_t handle)
2374 {
2375         int ret;
2376 
2377 #if defined(__amd64) && !defined(__xpv)
2378         if (IOMMU_USED(rdip))
2379                 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2380         else
2381 #endif
2382         ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2383 
2384         if (rootnex_state->r_dvma_call_list_id)
2385                 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2386 
2387         return (ret);
2388 }
2389 
2390 #if defined(__amd64) && !defined(__xpv)
2391 
2392 static int
2393 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2394 {
2395         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2396         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2397 
2398         if (dma->dp_sleep_flags != KM_SLEEP &&
2399             dma->dp_sleep_flags != KM_NOSLEEP)
2400                 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2401         return (dma->dp_sleep_flags);
2402 }
2403 /*ARGSUSED*/
2404 static void
2405 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2406 {
2407         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2408         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2409         rootnex_window_t *window;
2410 
2411         if (dma->dp_window) {
2412                 window = &dma->dp_window[dma->dp_current_win];
2413                 hp->dmai_cookie = window->wd_first_cookie;
2414         } else {
2415                 hp->dmai_cookie = dma->dp_cookies;
2416         }
2417         hp->dmai_cookie++;
2418 }
2419 
2420 /*ARGSUSED*/
2421 static int
2422 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2423     ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2424 {
2425         int i;
2426         int km_flags;
2427         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2428         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2429         rootnex_window_t *window;
2430         ddi_dma_cookie_t *cp;
2431         ddi_dma_cookie_t *cookie;
2432 
2433         ASSERT(*cookiepp == NULL);
2434         ASSERT(*ccountp == 0);
2435 
2436         if (dma->dp_window) {
2437                 window = &dma->dp_window[dma->dp_current_win];
2438                 cp = window->wd_first_cookie;
2439                 *ccountp = window->wd_cookie_cnt;
2440         } else {
2441                 cp = dma->dp_cookies;
2442                 *ccountp = dma->dp_sglinfo.si_sgl_size;
2443         }
2444 
2445         km_flags = rootnex_coredma_get_sleep_flags(handle);
2446         cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2447         if (cookie == NULL) {
2448                 return (DDI_DMA_NORESOURCES);
2449         }
2450 
2451         for (i = 0; i < *ccountp; i++) {
2452                 cookie[i].dmac_notused = cp[i].dmac_notused;
2453                 cookie[i].dmac_type = cp[i].dmac_type;
2454                 cookie[i].dmac_address = cp[i].dmac_address;
2455                 cookie[i].dmac_size = cp[i].dmac_size;
2456         }
2457 
2458         *cookiepp = cookie;
2459 
2460         return (DDI_SUCCESS);
2461 }
2462 
2463 /*ARGSUSED*/
2464 static int
2465 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2466     ddi_dma_cookie_t *cookiep, uint_t ccount)
2467 {
2468         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2469         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2470         rootnex_window_t *window;
2471         ddi_dma_cookie_t *cur_cookiep;
2472 
2473         ASSERT(cookiep);
2474         ASSERT(ccount != 0);
2475         ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2476 
2477         if (dma->dp_window) {
2478                 window = &dma->dp_window[dma->dp_current_win];
2479                 dma->dp_saved_cookies = window->wd_first_cookie;
2480                 window->wd_first_cookie = cookiep;
2481                 ASSERT(ccount == window->wd_cookie_cnt);
2482                 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2483                     + window->wd_first_cookie;
2484         } else {
2485                 dma->dp_saved_cookies = dma->dp_cookies;
2486                 dma->dp_cookies = cookiep;
2487                 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2488                 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2489                     + dma->dp_cookies;
2490         }
2491 
2492         dma->dp_need_to_switch_cookies = B_TRUE;
2493         hp->dmai_cookie = cur_cookiep;
2494 
2495         return (DDI_SUCCESS);
2496 }
2497 
2498 /*ARGSUSED*/
2499 static int
2500 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2501 {
2502         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2503         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2504         rootnex_window_t *window;
2505         ddi_dma_cookie_t *cur_cookiep;
2506         ddi_dma_cookie_t *cookie_array;
2507         uint_t ccount;
2508 
2509         /* check if cookies have not been switched */
2510         if (dma->dp_need_to_switch_cookies == B_FALSE)
2511                 return (DDI_SUCCESS);
2512 
2513         ASSERT(dma->dp_saved_cookies);
2514 
2515         if (dma->dp_window) {
2516                 window = &dma->dp_window[dma->dp_current_win];
2517                 cookie_array = window->wd_first_cookie;
2518                 window->wd_first_cookie = dma->dp_saved_cookies;
2519                 dma->dp_saved_cookies = NULL;
2520                 ccount = window->wd_cookie_cnt;
2521                 cur_cookiep = (hp->dmai_cookie - cookie_array)
2522                     + window->wd_first_cookie;
2523         } else {
2524                 cookie_array = dma->dp_cookies;
2525                 dma->dp_cookies = dma->dp_saved_cookies;
2526                 dma->dp_saved_cookies = NULL;
2527                 ccount = dma->dp_sglinfo.si_sgl_size;
2528                 cur_cookiep = (hp->dmai_cookie - cookie_array)
2529                     + dma->dp_cookies;
2530         }
2531 
2532         kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2533 
2534         hp->dmai_cookie = cur_cookiep;
2535 
2536         dma->dp_need_to_switch_cookies = B_FALSE;
2537 
2538         return (DDI_SUCCESS);
2539 }
2540 
2541 #endif
2542 
2543 static struct as *
2544 rootnex_get_as(ddi_dma_obj_t *dmao)
2545 {
2546         struct as *asp;
2547 
2548         switch (dmao->dmao_type) {
2549         case DMA_OTYP_VADDR:
2550         case DMA_OTYP_BUFVADDR:
2551                 asp = dmao->dmao_obj.virt_obj.v_as;
2552                 if (asp == NULL)
2553                         asp = &kas;
2554                 break;
2555         default:
2556                 asp = NULL;
2557                 break;
2558         }
2559         return (asp);
2560 }
2561 
2562 /*
2563  * rootnex_verify_buffer()
2564  *   verify buffer wasn't free'd
2565  */
2566 static int
2567 rootnex_verify_buffer(rootnex_dma_t *dma)
2568 {
2569         page_t **pplist;
2570         caddr_t vaddr;
2571         uint_t pcnt;
2572         uint_t poff;
2573         page_t *pp;
2574         char b;
2575         int i;
2576 
2577         /* Figure out how many pages this buffer occupies */
2578         if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2579                 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2580         } else {
2581                 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2582                 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2583         }
2584         pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2585 
2586         switch (dma->dp_dma.dmao_type) {
2587         case DMA_OTYP_PAGES:
2588                 /*
2589                  * for a linked list of pp's walk through them to make sure
2590                  * they're locked and not free.
2591                  */
2592                 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2593                 for (i = 0; i < pcnt; i++) {
2594                         if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2595                                 return (DDI_FAILURE);
2596                         }
2597                         pp = pp->p_next;
2598                 }
2599                 break;
2600 
2601         case DMA_OTYP_VADDR:
2602         case DMA_OTYP_BUFVADDR:
2603                 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2604                 /*
2605                  * for an array of pp's walk through them to make sure they're
2606                  * not free. It's possible that they may not be locked.
2607                  */
2608                 if (pplist) {
2609                         for (i = 0; i < pcnt; i++) {
2610                                 if (PP_ISFREE(pplist[i])) {
2611                                         return (DDI_FAILURE);
2612                                 }
2613                         }
2614 
2615                 /* For a virtual address, try to peek at each page */
2616                 } else {
2617                         if (rootnex_get_as(&dma->dp_dma) == &kas) {
2618                                 for (i = 0; i < pcnt; i++) {
2619                                         if (ddi_peek8(NULL, vaddr, &b) ==
2620                                             DDI_FAILURE)
2621                                                 return (DDI_FAILURE);
2622                                         vaddr += MMU_PAGESIZE;
2623                                 }
2624                         }
2625                 }
2626                 break;
2627 
2628         default:
2629                 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2630                 break;
2631         }
2632 
2633         return (DDI_SUCCESS);
2634 }
2635 
2636 
2637 /*
2638  * rootnex_clean_dmahdl()
2639  *    Clean the dma handle. This should be called on a handle alloc and an
2640  *    unbind handle. Set the handle state to the default settings.
2641  */
2642 static void
2643 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2644 {
2645         rootnex_dma_t *dma;
2646 
2647 
2648         dma = (rootnex_dma_t *)hp->dmai_private;
2649 
2650         hp->dmai_nwin = 0;
2651         dma->dp_current_cookie = 0;
2652         dma->dp_copybuf_size = 0;
2653         dma->dp_window = NULL;
2654         dma->dp_cbaddr = NULL;
2655         dma->dp_inuse = B_FALSE;
2656         dma->dp_dvma_used = B_FALSE;
2657         dma->dp_need_to_free_cookie = B_FALSE;
2658         dma->dp_need_to_switch_cookies = B_FALSE;
2659         dma->dp_saved_cookies = NULL;
2660         dma->dp_sleep_flags = KM_PANIC;
2661         dma->dp_need_to_free_window = B_FALSE;
2662         dma->dp_partial_required = B_FALSE;
2663         dma->dp_trim_required = B_FALSE;
2664         dma->dp_sglinfo.si_copybuf_req = 0;
2665 #if !defined(__amd64)
2666         dma->dp_cb_remaping = B_FALSE;
2667         dma->dp_kva = NULL;
2668 #endif
2669 
2670         /* FMA related initialization */
2671         hp->dmai_fault = 0;
2672         hp->dmai_fault_check = NULL;
2673         hp->dmai_fault_notify = NULL;
2674         hp->dmai_error.err_ena = 0;
2675         hp->dmai_error.err_status = DDI_FM_OK;
2676         hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2677         hp->dmai_error.err_ontrap = NULL;
2678 }
2679 
2680 
2681 /*
2682  * rootnex_valid_alloc_parms()
2683  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2684  */
2685 static int
2686 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2687 {
2688         if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2689             (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2690             (attr->dma_attr_granular > MMU_PAGESIZE) ||
2691             (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2692                 return (DDI_DMA_BADATTR);
2693         }
2694 
2695         if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2696                 return (DDI_DMA_BADATTR);
2697         }
2698 
2699         if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2700             MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2701             attr->dma_attr_sgllen == 0) {
2702                 return (DDI_DMA_BADATTR);
2703         }
2704 
2705         /* We should be able to DMA into every byte offset in a page */
2706         if (maxsegmentsize < MMU_PAGESIZE) {
2707                 return (DDI_DMA_BADATTR);
2708         }
2709 
2710         /* if we're bouncing on seg, seg must be <= addr_hi */
2711         if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2712             (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2713                 return (DDI_DMA_BADATTR);
2714         }
2715         return (DDI_SUCCESS);
2716 }
2717 
2718 /*
2719  * rootnex_valid_bind_parms()
2720  *    Called in ddi_dma_*_bind_handle path to validate its parameters.
2721  */
2722 /* ARGSUSED */
2723 static int
2724 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2725 {
2726 #if !defined(__amd64)
2727         /*
2728          * we only support up to a 2G-1 transfer size on 32-bit kernels so
2729          * we can track the offset for the obsoleted interfaces.
2730          */
2731         if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2732                 return (DDI_DMA_TOOBIG);
2733         }
2734 #endif
2735 
2736         return (DDI_SUCCESS);
2737 }
2738 
2739 
2740 /*
2741  * rootnex_need_bounce_seg()
2742  *    check to see if the buffer lives on both side of the seg.
2743  */
2744 static boolean_t
2745 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2746 {
2747         ddi_dma_atyp_t buftype;
2748         rootnex_addr_t raddr;
2749         boolean_t lower_addr;
2750         boolean_t upper_addr;
2751         uint64_t offset;
2752         page_t **pplist;
2753         uint64_t paddr;
2754         uint32_t psize;
2755         uint32_t size;
2756         caddr_t vaddr;
2757         uint_t pcnt;
2758         page_t *pp;
2759 
2760 
2761         /* shortcuts */
2762         pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2763         vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2764         buftype = dmar_object->dmao_type;
2765         size = dmar_object->dmao_size;
2766 
2767         lower_addr = B_FALSE;
2768         upper_addr = B_FALSE;
2769         pcnt = 0;
2770 
2771         /*
2772          * Process the first page to handle the initial offset of the buffer.
2773          * We'll use the base address we get later when we loop through all
2774          * the pages.
2775          */
2776         if (buftype == DMA_OTYP_PAGES) {
2777                 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2778                 offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2779                     MMU_PAGEOFFSET;
2780                 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2781                 psize = MIN(size, (MMU_PAGESIZE - offset));
2782                 pp = pp->p_next;
2783                 sglinfo->si_asp = NULL;
2784         } else if (pplist != NULL) {
2785                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2786                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2787                 if (sglinfo->si_asp == NULL) {
2788                         sglinfo->si_asp = &kas;
2789                 }
2790                 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2791                 paddr += offset;
2792                 psize = MIN(size, (MMU_PAGESIZE - offset));
2793                 pcnt++;
2794         } else {
2795                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2796                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2797                 if (sglinfo->si_asp == NULL) {
2798                         sglinfo->si_asp = &kas;
2799                 }
2800                 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2801                 paddr += offset;
2802                 psize = MIN(size, (MMU_PAGESIZE - offset));
2803                 vaddr += psize;
2804         }
2805 
2806         raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2807 
2808         if ((raddr + psize) > sglinfo->si_segmask) {
2809                 upper_addr = B_TRUE;
2810         } else {
2811                 lower_addr = B_TRUE;
2812         }
2813         size -= psize;
2814 
2815         /*
2816          * Walk through the rest of the pages in the buffer. Track to see
2817          * if we have pages on both sides of the segment boundary.
2818          */
2819         while (size > 0) {
2820                 /* partial or full page */
2821                 psize = MIN(size, MMU_PAGESIZE);
2822 
2823                 if (buftype == DMA_OTYP_PAGES) {
2824                         /* get the paddr from the page_t */
2825                         ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2826                         paddr = pfn_to_pa(pp->p_pagenum);
2827                         pp = pp->p_next;
2828                 } else if (pplist != NULL) {
2829                         /* index into the array of page_t's to get the paddr */
2830                         ASSERT(!PP_ISFREE(pplist[pcnt]));
2831                         paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2832                         pcnt++;
2833                 } else {
2834                         /* call into the VM to get the paddr */
2835                         paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2836                             vaddr));
2837                         vaddr += psize;
2838                 }
2839 
2840                 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2841 
2842                 if ((raddr + psize) > sglinfo->si_segmask) {
2843                         upper_addr = B_TRUE;
2844                 } else {
2845                         lower_addr = B_TRUE;
2846                 }
2847                 /*
2848                  * if the buffer lives both above and below the segment
2849                  * boundary, or the current page is the page immediately
2850                  * after the segment, we will use a copy/bounce buffer for
2851                  * all pages > seg.
2852                  */
2853                 if ((lower_addr && upper_addr) ||
2854                     (raddr == (sglinfo->si_segmask + 1))) {
2855                         return (B_TRUE);
2856                 }
2857 
2858                 size -= psize;
2859         }
2860 
2861         return (B_FALSE);
2862 }
2863 
2864 /*
2865  * rootnex_get_sgl()
2866  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2867  *    with a call to the vm layer when vm2.0 comes around...
2868  */
2869 static void
2870 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2871     rootnex_sglinfo_t *sglinfo)
2872 {
2873         ddi_dma_atyp_t buftype;
2874         rootnex_addr_t raddr;
2875         uint64_t last_page;
2876         uint64_t offset;
2877         uint64_t addrhi;
2878         uint64_t addrlo;
2879         uint64_t maxseg;
2880         page_t **pplist;
2881         uint64_t paddr;
2882         uint32_t psize;
2883         uint32_t size;
2884         caddr_t vaddr;
2885         uint_t pcnt;
2886         page_t *pp;
2887         uint_t cnt;
2888 
2889 
2890         /* shortcuts */
2891         pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2892         vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2893         maxseg = sglinfo->si_max_cookie_size;
2894         buftype = dmar_object->dmao_type;
2895         addrhi = sglinfo->si_max_addr;
2896         addrlo = sglinfo->si_min_addr;
2897         size = dmar_object->dmao_size;
2898 
2899         pcnt = 0;
2900         cnt = 0;
2901 
2902 
2903         /*
2904          * check to see if we need to use the copy buffer for pages over
2905          * the segment attr.
2906          */
2907         sglinfo->si_bounce_on_seg = B_FALSE;
2908         if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2909                 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2910                     dmar_object, sglinfo);
2911         }
2912 
2913         /*
2914          * if we were passed down a linked list of pages, i.e. pointer to
2915          * page_t, use this to get our physical address and buf offset.
2916          */
2917         if (buftype == DMA_OTYP_PAGES) {
2918                 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2919                 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2920                 offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2921                     MMU_PAGEOFFSET;
2922                 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2923                 psize = MIN(size, (MMU_PAGESIZE - offset));
2924                 pp = pp->p_next;
2925                 sglinfo->si_asp = NULL;
2926 
2927         /*
2928          * We weren't passed down a linked list of pages, but if we were passed
2929          * down an array of pages, use this to get our physical address and buf
2930          * offset.
2931          */
2932         } else if (pplist != NULL) {
2933                 ASSERT((buftype == DMA_OTYP_VADDR) ||
2934                     (buftype == DMA_OTYP_BUFVADDR));
2935 
2936                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2937                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2938                 if (sglinfo->si_asp == NULL) {
2939                         sglinfo->si_asp = &kas;
2940                 }
2941 
2942                 ASSERT(!PP_ISFREE(pplist[pcnt]));
2943                 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2944                 paddr += offset;
2945                 psize = MIN(size, (MMU_PAGESIZE - offset));
2946                 pcnt++;
2947 
2948         /*
2949          * All we have is a virtual address, we'll need to call into the VM
2950          * to get the physical address.
2951          */
2952         } else {
2953                 ASSERT((buftype == DMA_OTYP_VADDR) ||
2954                     (buftype == DMA_OTYP_BUFVADDR));
2955 
2956                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2957                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2958                 if (sglinfo->si_asp == NULL) {
2959                         sglinfo->si_asp = &kas;
2960                 }
2961 
2962                 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2963                 paddr += offset;
2964                 psize = MIN(size, (MMU_PAGESIZE - offset));
2965                 vaddr += psize;
2966         }
2967 
2968         raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2969 
2970         /*
2971          * Setup the first cookie with the physical address of the page and the
2972          * size of the page (which takes into account the initial offset into
2973          * the page.
2974          */
2975         sgl[cnt].dmac_laddress = raddr;
2976         sgl[cnt].dmac_size = psize;
2977         sgl[cnt].dmac_type = 0;
2978 
2979         /*
2980          * Save away the buffer offset into the page. We'll need this later in
2981          * the copy buffer code to help figure out the page index within the
2982          * buffer and the offset into the current page.
2983          */
2984         sglinfo->si_buf_offset = offset;
2985 
2986         /*
2987          * If we are using the copy buffer for anything over the segment
2988          * boundary, and this page is over the segment boundary.
2989          *   OR
2990          * if the DMA engine can't reach the physical address.
2991          */
2992         if (((sglinfo->si_bounce_on_seg) &&
2993             ((raddr + psize) > sglinfo->si_segmask)) ||
2994             ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2995                 /*
2996                  * Increase how much copy buffer we use. We always increase by
2997                  * pagesize so we don't have to worry about converting offsets.
2998                  * Set a flag in the cookies dmac_type to indicate that it uses
2999                  * the copy buffer. If this isn't the last cookie, go to the
3000                  * next cookie (since we separate each page which uses the copy
3001                  * buffer in case the copy buffer is not physically contiguous.
3002                  */
3003                 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3004                 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3005                 if ((cnt + 1) < sglinfo->si_max_pages) {
3006                         cnt++;
3007                         sgl[cnt].dmac_laddress = 0;
3008                         sgl[cnt].dmac_size = 0;
3009                         sgl[cnt].dmac_type = 0;
3010                 }
3011         }
3012 
3013         /*
3014          * save this page's physical address so we can figure out if the next
3015          * page is physically contiguous. Keep decrementing size until we are
3016          * done with the buffer.
3017          */
3018         last_page = raddr & MMU_PAGEMASK;
3019         size -= psize;
3020 
3021         while (size > 0) {
3022                 /* Get the size for this page (i.e. partial or full page) */
3023                 psize = MIN(size, MMU_PAGESIZE);
3024 
3025                 if (buftype == DMA_OTYP_PAGES) {
3026                         /* get the paddr from the page_t */
3027                         ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3028                         paddr = pfn_to_pa(pp->p_pagenum);
3029                         pp = pp->p_next;
3030                 } else if (pplist != NULL) {
3031                         /* index into the array of page_t's to get the paddr */
3032                         ASSERT(!PP_ISFREE(pplist[pcnt]));
3033                         paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3034                         pcnt++;
3035                 } else {
3036                         /* call into the VM to get the paddr */
3037                         paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3038                             vaddr));
3039                         vaddr += psize;
3040                 }
3041 
3042                 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3043 
3044                 /*
3045                  * If we are using the copy buffer for anything over the
3046                  * segment boundary, and this page is over the segment
3047                  * boundary.
3048                  *   OR
3049                  * if the DMA engine can't reach the physical address.
3050                  */
3051                 if (((sglinfo->si_bounce_on_seg) &&
3052                     ((raddr + psize) > sglinfo->si_segmask)) ||
3053                     ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3054 
3055                         sglinfo->si_copybuf_req += MMU_PAGESIZE;
3056 
3057                         /*
3058                          * if there is something in the current cookie, go to
3059                          * the next one. We only want one page in a cookie which
3060                          * uses the copybuf since the copybuf doesn't have to
3061                          * be physically contiguous.
3062                          */
3063                         if (sgl[cnt].dmac_size != 0) {
3064                                 cnt++;
3065                         }
3066                         sgl[cnt].dmac_laddress = raddr;
3067                         sgl[cnt].dmac_size = psize;
3068 #if defined(__amd64)
3069                         sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3070 #else
3071                         /*
3072                          * save the buf offset for 32-bit kernel. used in the
3073                          * obsoleted interfaces.
3074                          */
3075                         sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3076                             (dmar_object->dmao_size - size);
3077 #endif
3078                         /* if this isn't the last cookie, go to the next one */
3079                         if ((cnt + 1) < sglinfo->si_max_pages) {
3080                                 cnt++;
3081                                 sgl[cnt].dmac_laddress = 0;
3082                                 sgl[cnt].dmac_size = 0;
3083                                 sgl[cnt].dmac_type = 0;
3084                         }
3085 
3086                 /*
3087                  * this page didn't need the copy buffer, if it's not physically
3088                  * contiguous, or it would put us over a segment boundary, or it
3089                  * puts us over the max cookie size, or the current sgl doesn't
3090                  * have anything in it.
3091                  */
3092                 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3093                     !(raddr & sglinfo->si_segmask) ||
3094                     ((sgl[cnt].dmac_size + psize) > maxseg) ||
3095                     (sgl[cnt].dmac_size == 0)) {
3096                         /*
3097                          * if we're not already in a new cookie, go to the next
3098                          * cookie.
3099                          */
3100                         if (sgl[cnt].dmac_size != 0) {
3101                                 cnt++;
3102                         }
3103 
3104                         /* save the cookie information */
3105                         sgl[cnt].dmac_laddress = raddr;
3106                         sgl[cnt].dmac_size = psize;
3107 #if defined(__amd64)
3108                         sgl[cnt].dmac_type = 0;
3109 #else
3110                         /*
3111                          * save the buf offset for 32-bit kernel. used in the
3112                          * obsoleted interfaces.
3113                          */
3114                         sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3115 #endif
3116 
3117                 /*
3118                  * this page didn't need the copy buffer, it is physically
3119                  * contiguous with the last page, and it's <= the max cookie
3120                  * size.
3121                  */
3122                 } else {
3123                         sgl[cnt].dmac_size += psize;
3124 
3125                         /*
3126                          * if this exactly ==  the maximum cookie size, and
3127                          * it isn't the last cookie, go to the next cookie.
3128                          */
3129                         if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3130                             ((cnt + 1) < sglinfo->si_max_pages)) {
3131                                 cnt++;
3132                                 sgl[cnt].dmac_laddress = 0;
3133                                 sgl[cnt].dmac_size = 0;
3134                                 sgl[cnt].dmac_type = 0;
3135                         }
3136                 }
3137 
3138                 /*
3139                  * save this page's physical address so we can figure out if the
3140                  * next page is physically contiguous. Keep decrementing size
3141                  * until we are done with the buffer.
3142                  */
3143                 last_page = raddr;
3144                 size -= psize;
3145         }
3146 
3147         /* we're done, save away how many cookies the sgl has */
3148         if (sgl[cnt].dmac_size == 0) {
3149                 ASSERT(cnt < sglinfo->si_max_pages);
3150                 sglinfo->si_sgl_size = cnt;
3151         } else {
3152                 sglinfo->si_sgl_size = cnt + 1;
3153         }
3154 }
3155 
3156 static void
3157 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3158     rootnex_sglinfo_t *sglinfo)
3159 {
3160         uint64_t offset;
3161         uint64_t maxseg;
3162         uint64_t dvaddr;
3163         struct dvmaseg *dvs;
3164         uint64_t paddr;
3165         uint32_t psize, ssize;
3166         uint32_t size;
3167         uint_t cnt;
3168         int physcontig;
3169 
3170         ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3171 
3172         /* shortcuts */
3173         maxseg = sglinfo->si_max_cookie_size;
3174         size = dmar_object->dmao_size;
3175 
3176         cnt = 0;
3177         sglinfo->si_bounce_on_seg = B_FALSE;
3178 
3179         dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3180         offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3181         ssize = dvs->dvs_len;
3182         paddr = dvs->dvs_start;
3183         paddr += offset;
3184         psize = MIN(ssize, (maxseg - offset));
3185         dvaddr = paddr + psize;
3186         ssize -= psize;
3187 
3188         sgl[cnt].dmac_laddress = paddr;
3189         sgl[cnt].dmac_size = psize;
3190         sgl[cnt].dmac_type = 0;
3191 
3192         size -= psize;
3193         while (size > 0) {
3194                 if (ssize == 0) {
3195                         dvs++;
3196                         ssize = dvs->dvs_len;
3197                         dvaddr = dvs->dvs_start;
3198                         physcontig = 0;
3199                 } else
3200                         physcontig = 1;
3201 
3202                 paddr = dvaddr;
3203                 psize = MIN(ssize, maxseg);
3204                 dvaddr += psize;
3205                 ssize -= psize;
3206 
3207                 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3208                     ((sgl[cnt].dmac_size + psize) > maxseg) ||
3209                     (sgl[cnt].dmac_size == 0)) {
3210                         /*
3211                          * if we're not already in a new cookie, go to the next
3212                          * cookie.
3213                          */
3214                         if (sgl[cnt].dmac_size != 0) {
3215                                 cnt++;
3216                         }
3217 
3218                         /* save the cookie information */
3219                         sgl[cnt].dmac_laddress = paddr;
3220                         sgl[cnt].dmac_size = psize;
3221                         sgl[cnt].dmac_type = 0;
3222                 } else {
3223                         sgl[cnt].dmac_size += psize;
3224 
3225                         /*
3226                          * if this exactly ==  the maximum cookie size, and
3227                          * it isn't the last cookie, go to the next cookie.
3228                          */
3229                         if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3230                             ((cnt + 1) < sglinfo->si_max_pages)) {
3231                                 cnt++;
3232                                 sgl[cnt].dmac_laddress = 0;
3233                                 sgl[cnt].dmac_size = 0;
3234                                 sgl[cnt].dmac_type = 0;
3235                         }
3236                 }
3237                 size -= psize;
3238         }
3239 
3240         /* we're done, save away how many cookies the sgl has */
3241         if (sgl[cnt].dmac_size == 0) {
3242                 sglinfo->si_sgl_size = cnt;
3243         } else {
3244                 sglinfo->si_sgl_size = cnt + 1;
3245         }
3246 }
3247 
3248 /*
3249  * rootnex_bind_slowpath()
3250  *    Call in the bind path if the calling driver can't use the sgl without
3251  *    modifying it. We either need to use the copy buffer and/or we will end up
3252  *    with a partial bind.
3253  */
3254 static int
3255 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3256     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3257 {
3258         rootnex_sglinfo_t *sinfo;
3259         rootnex_window_t *window;
3260         ddi_dma_cookie_t *cookie;
3261         size_t copybuf_used;
3262         size_t dmac_size;
3263         boolean_t partial;
3264         off_t cur_offset;
3265         page_t *cur_pp;
3266         major_t mnum;
3267         int e;
3268         int i;
3269 
3270 
3271         sinfo = &dma->dp_sglinfo;
3272         copybuf_used = 0;
3273         partial = B_FALSE;
3274 
3275         /*
3276          * If we're using the copybuf, set the copybuf state in dma struct.
3277          * Needs to be first since it sets the copy buffer size.
3278          */
3279         if (sinfo->si_copybuf_req != 0) {
3280                 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3281                 if (e != DDI_SUCCESS) {
3282                         return (e);
3283                 }
3284         } else {
3285                 dma->dp_copybuf_size = 0;
3286         }
3287 
3288         /*
3289          * Figure out if we need to do a partial mapping. If so, figure out
3290          * if we need to trim the buffers when we munge the sgl.
3291          */
3292         if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3293             (dmao->dmao_size > dma->dp_maxxfer) ||
3294             ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3295                 dma->dp_partial_required = B_TRUE;
3296                 if (attr->dma_attr_granular != 1) {
3297                         dma->dp_trim_required = B_TRUE;
3298                 }
3299         } else {
3300                 dma->dp_partial_required = B_FALSE;
3301                 dma->dp_trim_required = B_FALSE;
3302         }
3303 
3304         /* If we need to do a partial bind, make sure the driver supports it */
3305         if (dma->dp_partial_required &&
3306             !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3307 
3308                 mnum = ddi_driver_major(dma->dp_dip);
3309                 /*
3310                  * patchable which allows us to print one warning per major
3311                  * number.
3312                  */
3313                 if ((rootnex_bind_warn) &&
3314                     ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3315                         rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3316                         cmn_err(CE_WARN, "!%s: coding error detected, the "
3317                             "driver is using ddi_dma_attr(9S) incorrectly. "
3318                             "There is a small risk of data corruption in "
3319                             "particular with large I/Os. The driver should be "
3320                             "replaced with a corrected version for proper "
3321                             "system operation. To disable this warning, add "
3322                             "'set rootnex:rootnex_bind_warn=0' to "
3323                             "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3324                 }
3325                 return (DDI_DMA_TOOBIG);
3326         }
3327 
3328         /*
3329          * we might need multiple windows, setup state to handle them. In this
3330          * code path, we will have at least one window.
3331          */
3332         e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3333         if (e != DDI_SUCCESS) {
3334                 rootnex_teardown_copybuf(dma);
3335                 return (e);
3336         }
3337 
3338         window = &dma->dp_window[0];
3339         cookie = &dma->dp_cookies[0];
3340         cur_offset = 0;
3341         rootnex_init_win(hp, dma, window, cookie, cur_offset);
3342         if (dmao->dmao_type == DMA_OTYP_PAGES) {
3343                 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3344         }
3345 
3346         /* loop though all the cookies we got back from get_sgl() */
3347         for (i = 0; i < sinfo->si_sgl_size; i++) {
3348                 /*
3349                  * If we're using the copy buffer, check this cookie and setup
3350                  * its associated copy buffer state. If this cookie uses the
3351                  * copy buffer, make sure we sync this window during dma_sync.
3352                  */
3353                 if (dma->dp_copybuf_size > 0) {
3354                         rootnex_setup_cookie(dmao, dma, cookie,
3355                             cur_offset, &copybuf_used, &cur_pp);
3356                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3357                                 window->wd_dosync = B_TRUE;
3358                         }
3359                 }
3360 
3361                 /*
3362                  * save away the cookie size, since it could be modified in
3363                  * the windowing code.
3364                  */
3365                 dmac_size = cookie->dmac_size;
3366 
3367                 /* if we went over max copybuf size */
3368                 if (dma->dp_copybuf_size &&
3369                     (copybuf_used > dma->dp_copybuf_size)) {
3370                         partial = B_TRUE;
3371                         e = rootnex_copybuf_window_boundary(hp, dma, &window,
3372                             cookie, cur_offset, &copybuf_used);
3373                         if (e != DDI_SUCCESS) {
3374                                 rootnex_teardown_copybuf(dma);
3375                                 rootnex_teardown_windows(dma);
3376                                 return (e);
3377                         }
3378 
3379                         /*
3380                          * if the coookie uses the copy buffer, make sure the
3381                          * new window we just moved to is set to sync.
3382                          */
3383                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3384                                 window->wd_dosync = B_TRUE;
3385                         }
3386                         ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3387                             dma->dp_dip);
3388 
3389                 /* if the cookie cnt == max sgllen, move to the next window */
3390                 } else if (window->wd_cookie_cnt >=
3391                     (unsigned)attr->dma_attr_sgllen) {
3392                         partial = B_TRUE;
3393                         ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3394                         e = rootnex_sgllen_window_boundary(hp, dma, &window,
3395                             cookie, attr, cur_offset);
3396                         if (e != DDI_SUCCESS) {
3397                                 rootnex_teardown_copybuf(dma);
3398                                 rootnex_teardown_windows(dma);
3399                                 return (e);
3400                         }
3401 
3402                         /*
3403                          * if the coookie uses the copy buffer, make sure the
3404                          * new window we just moved to is set to sync.
3405                          */
3406                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3407                                 window->wd_dosync = B_TRUE;
3408                         }
3409                         ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3410                             dma->dp_dip);
3411 
3412                 /* else if we will be over maxxfer */
3413                 } else if ((window->wd_size + dmac_size) >
3414                     dma->dp_maxxfer) {
3415                         partial = B_TRUE;
3416                         e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3417                             cookie);
3418                         if (e != DDI_SUCCESS) {
3419                                 rootnex_teardown_copybuf(dma);
3420                                 rootnex_teardown_windows(dma);
3421                                 return (e);
3422                         }
3423 
3424                         /*
3425                          * if the coookie uses the copy buffer, make sure the
3426                          * new window we just moved to is set to sync.
3427                          */
3428                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3429                                 window->wd_dosync = B_TRUE;
3430                         }
3431                         ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3432                             dma->dp_dip);
3433 
3434                 /* else this cookie fits in the current window */
3435                 } else {
3436                         window->wd_cookie_cnt++;
3437                         window->wd_size += dmac_size;
3438                 }
3439 
3440                 /* track our offset into the buffer, go to the next cookie */
3441                 ASSERT(dmac_size <= dmao->dmao_size);
3442                 ASSERT(cookie->dmac_size <= dmac_size);
3443                 cur_offset += dmac_size;
3444                 cookie++;
3445         }
3446 
3447         /* if we ended up with a zero sized window in the end, clean it up */
3448         if (window->wd_size == 0) {
3449                 hp->dmai_nwin--;
3450                 window--;
3451         }
3452 
3453         ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3454 
3455         if (!partial) {
3456                 return (DDI_DMA_MAPPED);
3457         }
3458 
3459         ASSERT(dma->dp_partial_required);
3460         return (DDI_DMA_PARTIAL_MAP);
3461 }
3462 
3463 /*
3464  * rootnex_setup_copybuf()
3465  *    Called in bind slowpath. Figures out if we're going to use the copy
3466  *    buffer, and if we do, sets up the basic state to handle it.
3467  */
3468 static int
3469 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3470     rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3471 {
3472         rootnex_sglinfo_t *sinfo;
3473         ddi_dma_attr_t lattr;
3474         size_t max_copybuf;
3475         int cansleep;
3476         int e;
3477 #if !defined(__amd64)
3478         int vmflag;
3479 #endif
3480 
3481         ASSERT(!dma->dp_dvma_used);
3482 
3483         sinfo = &dma->dp_sglinfo;
3484 
3485         /* read this first so it's consistent through the routine  */
3486         max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3487 
3488         /* We need to call into the rootnex on ddi_dma_sync() */
3489         hp->dmai_rflags &= ~DMP_NOSYNC;
3490 
3491         /* make sure the copybuf size <= the max size */
3492         dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3493         ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3494 
3495 #if !defined(__amd64)
3496         /*
3497          * if we don't have kva space to copy to/from, allocate the KVA space
3498          * now. We only do this for the 32-bit kernel. We use seg kpm space for
3499          * the 64-bit kernel.
3500          */
3501         if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3502             (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3503 
3504                 /* convert the sleep flags */
3505                 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3506                         vmflag = VM_SLEEP;
3507                 } else {
3508                         vmflag = VM_NOSLEEP;
3509                 }
3510 
3511                 /* allocate Kernel VA space that we can bcopy to/from */
3512                 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3513                     vmflag);
3514                 if (dma->dp_kva == NULL) {
3515                         return (DDI_DMA_NORESOURCES);
3516                 }
3517         }
3518 #endif
3519 
3520         /* convert the sleep flags */
3521         if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3522                 cansleep = 1;
3523         } else {
3524                 cansleep = 0;
3525         }
3526 
3527         /*
3528          * Allocate the actual copy buffer. This needs to fit within the DMA
3529          * engine limits, so we can't use kmem_alloc... We don't need
3530          * contiguous memory (sgllen) since we will be forcing windows on
3531          * sgllen anyway.
3532          */
3533         lattr = *attr;
3534         lattr.dma_attr_align = MMU_PAGESIZE;
3535         lattr.dma_attr_sgllen = -1;     /* no limit */
3536         /*
3537          * if we're using the copy buffer because of seg, use that for our
3538          * upper address limit.
3539          */
3540         if (sinfo->si_bounce_on_seg) {
3541                 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3542         }
3543         e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3544             0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3545         if (e != DDI_SUCCESS) {
3546 #if !defined(__amd64)
3547                 if (dma->dp_kva != NULL) {
3548                         vmem_free(heap_arena, dma->dp_kva,
3549                             dma->dp_copybuf_size);
3550                 }
3551 #endif
3552                 return (DDI_DMA_NORESOURCES);
3553         }
3554 
3555         ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3556             size_t, dma->dp_copybuf_size);
3557 
3558         return (DDI_SUCCESS);
3559 }
3560 
3561 
3562 /*
3563  * rootnex_setup_windows()
3564  *    Called in bind slowpath to setup the window state. We always have windows
3565  *    in the slowpath. Even if the window count = 1.
3566  */
3567 static int
3568 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3569     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3570 {
3571         rootnex_window_t *windowp;
3572         rootnex_sglinfo_t *sinfo;
3573         size_t copy_state_size;
3574         size_t win_state_size;
3575         size_t state_available;
3576         size_t space_needed;
3577         uint_t copybuf_win;
3578         uint_t maxxfer_win;
3579         size_t space_used;
3580         uint_t sglwin;
3581 
3582 
3583         sinfo = &dma->dp_sglinfo;
3584 
3585         dma->dp_current_win = 0;
3586         hp->dmai_nwin = 0;
3587 
3588         /* If we don't need to do a partial, we only have one window */
3589         if (!dma->dp_partial_required) {
3590                 dma->dp_max_win = 1;
3591 
3592         /*
3593          * we need multiple windows, need to figure out the worse case number
3594          * of windows.
3595          */
3596         } else {
3597                 /*
3598                  * if we need windows because we need more copy buffer that
3599                  * we allow, the worse case number of windows we could need
3600                  * here would be (copybuf space required / copybuf space that
3601                  * we have) plus one for remainder, and plus 2 to handle the
3602                  * extra pages on the trim for the first and last pages of the
3603                  * buffer (a page is the minimum window size so under the right
3604                  * attr settings, you could have a window for each page).
3605                  * The last page will only be hit here if the size is not a
3606                  * multiple of the granularity (which theoretically shouldn't
3607                  * be the case but never has been enforced, so we could have
3608                  * broken things without it).
3609                  */
3610                 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3611                         ASSERT(dma->dp_copybuf_size > 0);
3612                         copybuf_win = (sinfo->si_copybuf_req /
3613                             dma->dp_copybuf_size) + 1 + 2;
3614                 } else {
3615                         copybuf_win = 0;
3616                 }
3617 
3618                 /*
3619                  * if we need windows because we have more cookies than the H/W
3620                  * can handle, the number of windows we would need here would
3621                  * be (cookie count / cookies count H/W supports minus 1[for
3622                  * trim]) plus one for remainder.
3623                  */
3624                 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3625                         sglwin = (sinfo->si_sgl_size /
3626                             (attr->dma_attr_sgllen - 1)) + 1;
3627                 } else {
3628                         sglwin = 0;
3629                 }
3630 
3631                 /*
3632                  * if we need windows because we're binding more memory than the
3633                  * H/W can transfer at once, the number of windows we would need
3634                  * here would be (xfer count / max xfer H/W supports) plus one
3635                  * for remainder, and plus 2 to handle the extra pages on the
3636                  * trim (see above comment about trim)
3637                  */
3638                 if (dmao->dmao_size > dma->dp_maxxfer) {
3639                         maxxfer_win = (dmao->dmao_size /
3640                             dma->dp_maxxfer) + 1 + 2;
3641                 } else {
3642                         maxxfer_win = 0;
3643                 }
3644                 dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
3645                 ASSERT(dma->dp_max_win > 0);
3646         }
3647         win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3648 
3649         /*
3650          * Get space for window and potential copy buffer state. Before we
3651          * go and allocate memory, see if we can get away with using what's
3652          * left in the pre-allocted state or the dynamically allocated sgl.
3653          */
3654         space_used = (uintptr_t)(sinfo->si_sgl_size *
3655             sizeof (ddi_dma_cookie_t));
3656 
3657         /* if we dynamically allocated space for the cookies */
3658         if (dma->dp_need_to_free_cookie) {
3659                 /* if we have more space in the pre-allocted buffer, use it */
3660                 ASSERT(space_used <= dma->dp_cookie_size);
3661                 if ((dma->dp_cookie_size - space_used) <=
3662                     rootnex_state->r_prealloc_size) {
3663                         state_available = rootnex_state->r_prealloc_size;
3664                         windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3665 
3666                 /*
3667                  * else, we have more free space in the dynamically allocated
3668                  * buffer, i.e. the buffer wasn't worse case fragmented so we
3669                  * didn't need a lot of cookies.
3670                  */
3671                 } else {
3672                         state_available = dma->dp_cookie_size - space_used;
3673                         windowp = (rootnex_window_t *)
3674                             &dma->dp_cookies[sinfo->si_sgl_size];
3675                 }
3676 
3677         /* we used the pre-alloced buffer */
3678         } else {
3679                 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3680                 state_available = rootnex_state->r_prealloc_size - space_used;
3681                 windowp = (rootnex_window_t *)
3682                     &dma->dp_cookies[sinfo->si_sgl_size];
3683         }
3684 
3685         /*
3686          * figure out how much state we need to track the copy buffer. Add an
3687          * addition 8 bytes for pointer alignemnt later.
3688          */
3689         if (dma->dp_copybuf_size > 0) {
3690                 copy_state_size = sinfo->si_max_pages *
3691                     sizeof (rootnex_pgmap_t);
3692         } else {
3693                 copy_state_size = 0;
3694         }
3695         /* add an additional 8 bytes for pointer alignment */
3696         space_needed = win_state_size + copy_state_size + 0x8;
3697 
3698         /* if we have enough space already, use it */
3699         if (state_available >= space_needed) {
3700                 dma->dp_window = windowp;
3701                 dma->dp_need_to_free_window = B_FALSE;
3702 
3703         /* not enough space, need to allocate more. */
3704         } else {
3705                 dma->dp_window = kmem_alloc(space_needed, kmflag);
3706                 if (dma->dp_window == NULL) {
3707                         return (DDI_DMA_NORESOURCES);
3708                 }
3709                 dma->dp_need_to_free_window = B_TRUE;
3710                 dma->dp_window_size = space_needed;
3711                 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3712                     dma->dp_dip, size_t, space_needed);
3713         }
3714 
3715         /*
3716          * we allocate copy buffer state and window state at the same time.
3717          * setup our copy buffer state pointers. Make sure it's aligned.
3718          */
3719         if (dma->dp_copybuf_size > 0) {
3720                 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3721                     &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3722 
3723 #if !defined(__amd64)
3724                 /*
3725                  * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3726                  * false/NULL. Should be quicker to bzero vs loop and set.
3727                  */
3728                 bzero(dma->dp_pgmap, copy_state_size);
3729 #endif
3730         } else {
3731                 dma->dp_pgmap = NULL;
3732         }
3733 
3734         return (DDI_SUCCESS);
3735 }
3736 
3737 
3738 /*
3739  * rootnex_teardown_copybuf()
3740  *    cleans up after rootnex_setup_copybuf()
3741  */
3742 static void
3743 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3744 {
3745 #if !defined(__amd64)
3746         int i;
3747 
3748         /*
3749          * if we allocated kernel heap VMEM space, go through all the pages and
3750          * map out any of the ones that we're mapped into the kernel heap VMEM
3751          * arena. Then free the VMEM space.
3752          */
3753         if (dma->dp_kva != NULL) {
3754                 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3755                         if (dma->dp_pgmap[i].pm_mapped) {
3756                                 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3757                                     MMU_PAGESIZE, HAT_UNLOAD);
3758                                 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3759                         }
3760                 }
3761 
3762                 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3763         }
3764 
3765 #endif
3766 
3767         /* if we allocated a copy buffer, free it */
3768         if (dma->dp_cbaddr != NULL) {
3769                 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3770         }
3771 }
3772 
3773 
3774 /*
3775  * rootnex_teardown_windows()
3776  *    cleans up after rootnex_setup_windows()
3777  */
3778 static void
3779 rootnex_teardown_windows(rootnex_dma_t *dma)
3780 {
3781         /*
3782          * if we had to allocate window state on the last bind (because we
3783          * didn't have enough pre-allocated space in the handle), free it.
3784          */
3785         if (dma->dp_need_to_free_window) {
3786                 kmem_free(dma->dp_window, dma->dp_window_size);
3787         }
3788 }
3789 
3790 
3791 /*
3792  * rootnex_init_win()
3793  *    Called in bind slow path during creation of a new window. Initializes
3794  *    window state to default values.
3795  */
3796 /*ARGSUSED*/
3797 static void
3798 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3799     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3800 {
3801         hp->dmai_nwin++;
3802         window->wd_dosync = B_FALSE;
3803         window->wd_offset = cur_offset;
3804         window->wd_size = 0;
3805         window->wd_first_cookie = cookie;
3806         window->wd_cookie_cnt = 0;
3807         window->wd_trim.tr_trim_first = B_FALSE;
3808         window->wd_trim.tr_trim_last = B_FALSE;
3809         window->wd_trim.tr_first_copybuf_win = B_FALSE;
3810         window->wd_trim.tr_last_copybuf_win = B_FALSE;
3811 #if !defined(__amd64)
3812         window->wd_remap_copybuf = dma->dp_cb_remaping;
3813 #endif
3814 }
3815 
3816 
3817 /*
3818  * rootnex_setup_cookie()
3819  *    Called in the bind slow path when the sgl uses the copy buffer. If any of
3820  *    the sgl uses the copy buffer, we need to go through each cookie, figure
3821  *    out if it uses the copy buffer, and if it does, save away everything we'll
3822  *    need during sync.
3823  */
3824 static void
3825 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3826     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3827     page_t **cur_pp)
3828 {
3829         boolean_t copybuf_sz_power_2;
3830         rootnex_sglinfo_t *sinfo;
3831         paddr_t paddr;
3832         uint_t pidx;
3833         uint_t pcnt;
3834         off_t poff;
3835 #if defined(__amd64)
3836         pfn_t pfn;
3837 #else
3838         page_t **pplist;
3839 #endif
3840 
3841         ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3842 
3843         sinfo = &dma->dp_sglinfo;
3844 
3845         /*
3846          * Calculate the page index relative to the start of the buffer. The
3847          * index to the current page for our buffer is the offset into the
3848          * first page of the buffer plus our current offset into the buffer
3849          * itself, shifted of course...
3850          */
3851         pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3852         ASSERT(pidx < sinfo->si_max_pages);
3853 
3854         /* if this cookie uses the copy buffer */
3855         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3856                 /*
3857                  * NOTE: we know that since this cookie uses the copy buffer, it
3858                  * is <= MMU_PAGESIZE.
3859                  */
3860 
3861                 /*
3862                  * get the offset into the page. For the 64-bit kernel, get the
3863                  * pfn which we'll use with seg kpm.
3864                  */
3865                 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3866 #if defined(__amd64)
3867                 /* mfn_to_pfn() is a NOP on i86pc */
3868                 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3869 #endif /* __amd64 */
3870 
3871                 /* figure out if the copybuf size is a power of 2 */
3872                 if (!ISP2(dma->dp_copybuf_size)) {
3873                         copybuf_sz_power_2 = B_FALSE;
3874                 } else {
3875                         copybuf_sz_power_2 = B_TRUE;
3876                 }
3877 
3878                 /* This page uses the copy buffer */
3879                 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3880 
3881                 /*
3882                  * save the copy buffer KVA that we'll use with this page.
3883                  * if we still fit within the copybuf, it's a simple add.
3884                  * otherwise, we need to wrap over using & or % accordingly.
3885                  */
3886                 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3887                         dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3888                             *copybuf_used;
3889                 } else {
3890                         if (copybuf_sz_power_2) {
3891                                 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3892                                     (uintptr_t)dma->dp_cbaddr +
3893                                     (*copybuf_used &
3894                                     (dma->dp_copybuf_size - 1)));
3895                         } else {
3896                                 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3897                                     (uintptr_t)dma->dp_cbaddr +
3898                                     (*copybuf_used % dma->dp_copybuf_size));
3899                         }
3900                 }
3901 
3902                 /*
3903                  * over write the cookie physical address with the address of
3904                  * the physical address of the copy buffer page that we will
3905                  * use.
3906                  */
3907                 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3908                     dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3909 
3910                 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3911 
3912                 /* if we have a kernel VA, it's easy, just save that address */
3913                 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3914                     (sinfo->si_asp == &kas)) {
3915                         /*
3916                          * save away the page aligned virtual address of the
3917                          * driver buffer. Offsets are handled in the sync code.
3918                          */
3919                         dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3920                             dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3921                             & MMU_PAGEMASK);
3922 #if !defined(__amd64)
3923                         /*
3924                          * we didn't need to, and will never need to map this
3925                          * page.
3926                          */
3927                         dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3928 #endif
3929 
3930                 /* we don't have a kernel VA. We need one for the bcopy. */
3931                 } else {
3932 #if defined(__amd64)
3933                         /*
3934                          * for the 64-bit kernel, it's easy. We use seg kpm to
3935                          * get a Kernel VA for the corresponding pfn.
3936                          */
3937                         dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3938 #else
3939                         /*
3940                          * for the 32-bit kernel, this is a pain. First we'll
3941                          * save away the page_t or user VA for this page. This
3942                          * is needed in rootnex_dma_win() when we switch to a
3943                          * new window which requires us to re-map the copy
3944                          * buffer.
3945                          */
3946                         pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3947                         if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3948                                 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3949                                 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3950                         } else if (pplist != NULL) {
3951                                 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3952                                 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3953                         } else {
3954                                 dma->dp_pgmap[pidx].pm_pp = NULL;
3955                                 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3956                                     (((uintptr_t)
3957                                     dmar_object->dmao_obj.virt_obj.v_addr +
3958                                     cur_offset) & MMU_PAGEMASK);
3959                         }
3960 
3961                         /*
3962                          * save away the page aligned virtual address which was
3963                          * allocated from the kernel heap arena (taking into
3964                          * account if we need more copy buffer than we alloced
3965                          * and use multiple windows to handle this, i.e. &,%).
3966                          * NOTE: there isn't and physical memory backing up this
3967                          * virtual address space currently.
3968                          */
3969                         if ((*copybuf_used + MMU_PAGESIZE) <=
3970                             dma->dp_copybuf_size) {
3971                                 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3972                                     (((uintptr_t)dma->dp_kva + *copybuf_used) &
3973                                     MMU_PAGEMASK);
3974                         } else {
3975                                 if (copybuf_sz_power_2) {
3976                                         dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3977                                             (((uintptr_t)dma->dp_kva +
3978                                             (*copybuf_used &
3979                                             (dma->dp_copybuf_size - 1))) &
3980                                             MMU_PAGEMASK);
3981                                 } else {
3982                                         dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3983                                             (((uintptr_t)dma->dp_kva +
3984                                             (*copybuf_used %
3985                                             dma->dp_copybuf_size)) &
3986                                             MMU_PAGEMASK);
3987                                 }
3988                         }
3989 
3990                         /*
3991                          * if we haven't used up the available copy buffer yet,
3992                          * map the kva to the physical page.
3993                          */
3994                         if (!dma->dp_cb_remaping && ((*copybuf_used +
3995                             MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3996                                 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3997                                 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3998                                         i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3999                                             dma->dp_pgmap[pidx].pm_kaddr);
4000                                 } else {
4001                                         i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4002                                             sinfo->si_asp,
4003                                             dma->dp_pgmap[pidx].pm_kaddr);
4004                                 }
4005 
4006                         /*
4007                          * we've used up the available copy buffer, this page
4008                          * will have to be mapped during rootnex_dma_win() when
4009                          * we switch to a new window which requires a re-map
4010                          * the copy buffer. (32-bit kernel only)
4011                          */
4012                         } else {
4013                                 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4014                         }
4015 #endif
4016                         /* go to the next page_t */
4017                         if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4018                                 *cur_pp = (*cur_pp)->p_next;
4019                         }
4020                 }
4021 
4022                 /* add to the copy buffer count */
4023                 *copybuf_used += MMU_PAGESIZE;
4024 
4025         /*
4026          * This cookie doesn't use the copy buffer. Walk through the pages this
4027          * cookie occupies to reflect this.
4028          */
4029         } else {
4030                 /*
4031                  * figure out how many pages the cookie occupies. We need to
4032                  * use the original page offset of the buffer and the cookies
4033                  * offset in the buffer to do this.
4034                  */
4035                 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4036                 pcnt = mmu_btopr(cookie->dmac_size + poff);
4037 
4038                 while (pcnt > 0) {
4039 #if !defined(__amd64)
4040                         /*
4041                          * the 32-bit kernel doesn't have seg kpm, so we need
4042                          * to map in the driver buffer (if it didn't come down
4043                          * with a kernel VA) on the fly. Since this page doesn't
4044                          * use the copy buffer, it's not, or will it ever, have
4045                          * to be mapped in.
4046                          */
4047                         dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4048 #endif
4049                         dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4050 
4051                         /*
4052                          * we need to update pidx and cur_pp or we'll loose
4053                          * track of where we are.
4054                          */
4055                         if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4056                                 *cur_pp = (*cur_pp)->p_next;
4057                         }
4058                         pidx++;
4059                         pcnt--;
4060                 }
4061         }
4062 }
4063 
4064 
4065 /*
4066  * rootnex_sgllen_window_boundary()
4067  *    Called in the bind slow path when the next cookie causes us to exceed (in
4068  *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4069  *    length supported by the DMA H/W.
4070  */
4071 static int
4072 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4073     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4074     off_t cur_offset)
4075 {
4076         off_t new_offset;
4077         size_t trim_sz;
4078         off_t coffset;
4079 
4080 
4081         /*
4082          * if we know we'll never have to trim, it's pretty easy. Just move to
4083          * the next window and init it. We're done.
4084          */
4085         if (!dma->dp_trim_required) {
4086                 (*windowp)++;
4087                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4088                 (*windowp)->wd_cookie_cnt++;
4089                 (*windowp)->wd_size = cookie->dmac_size;
4090                 return (DDI_SUCCESS);
4091         }
4092 
4093         /* figure out how much we need to trim from the window */
4094         ASSERT(attr->dma_attr_granular != 0);
4095         if (dma->dp_granularity_power_2) {
4096                 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4097         } else {
4098                 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4099         }
4100 
4101         /* The window's a whole multiple of granularity. We're done */
4102         if (trim_sz == 0) {
4103                 (*windowp)++;
4104                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4105                 (*windowp)->wd_cookie_cnt++;
4106                 (*windowp)->wd_size = cookie->dmac_size;
4107                 return (DDI_SUCCESS);
4108         }
4109 
4110         /*
4111          * The window's not a whole multiple of granularity, since we know this
4112          * is due to the sgllen, we need to go back to the last cookie and trim
4113          * that one, add the left over part of the old cookie into the new
4114          * window, and then add in the new cookie into the new window.
4115          */
4116 
4117         /*
4118          * make sure the driver isn't making us do something bad... Trimming and
4119          * sgllen == 1 don't go together.
4120          */
4121         if (attr->dma_attr_sgllen == 1) {
4122                 return (DDI_DMA_NOMAPPING);
4123         }
4124 
4125         /*
4126          * first, setup the current window to account for the trim. Need to go
4127          * back to the last cookie for this.
4128          */
4129         cookie--;
4130         (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4131         (*windowp)->wd_trim.tr_last_cookie = cookie;
4132         (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4133         ASSERT(cookie->dmac_size > trim_sz);
4134         (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4135         (*windowp)->wd_size -= trim_sz;
4136 
4137         /* save the buffer offsets for the next window */
4138         coffset = cookie->dmac_size - trim_sz;
4139         new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4140 
4141         /*
4142          * set this now in case this is the first window. all other cases are
4143          * set in dma_win()
4144          */
4145         cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4146 
4147         /*
4148          * initialize the next window using what's left over in the previous
4149          * cookie.
4150          */
4151         (*windowp)++;
4152         rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4153         (*windowp)->wd_cookie_cnt++;
4154         (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4155         (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4156         (*windowp)->wd_trim.tr_first_size = trim_sz;
4157         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4158                 (*windowp)->wd_dosync = B_TRUE;
4159         }
4160 
4161         /*
4162          * now go back to the current cookie and add it to the new window. set
4163          * the new window size to the what was left over from the previous
4164          * cookie and what's in the current cookie.
4165          */
4166         cookie++;
4167         (*windowp)->wd_cookie_cnt++;
4168         (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4169 
4170         /*
4171          * trim plus the next cookie could put us over maxxfer (a cookie can be
4172          * a max size of maxxfer). Handle that case.
4173          */
4174         if ((*windowp)->wd_size > dma->dp_maxxfer) {
4175                 /*
4176                  * maxxfer is already a whole multiple of granularity, and this
4177                  * trim will be <= the previous trim (since a cookie can't be
4178                  * larger than maxxfer). Make things simple here.
4179                  */
4180                 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4181                 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4182                 (*windowp)->wd_trim.tr_last_cookie = cookie;
4183                 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4184                 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4185                 (*windowp)->wd_size -= trim_sz;
4186                 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4187 
4188                 /* save the buffer offsets for the next window */
4189                 coffset = cookie->dmac_size - trim_sz;
4190                 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4191 
4192                 /* setup the next window */
4193                 (*windowp)++;
4194                 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4195                 (*windowp)->wd_cookie_cnt++;
4196                 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4197                 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4198                     coffset;
4199                 (*windowp)->wd_trim.tr_first_size = trim_sz;
4200         }
4201 
4202         return (DDI_SUCCESS);
4203 }
4204 
4205 
4206 /*
4207  * rootnex_copybuf_window_boundary()
4208  *    Called in bind slowpath when we get to a window boundary because we used
4209  *    up all the copy buffer that we have.
4210  */
4211 static int
4212 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4213     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4214     size_t *copybuf_used)
4215 {
4216         rootnex_sglinfo_t *sinfo;
4217         off_t new_offset;
4218         size_t trim_sz;
4219         paddr_t paddr;
4220         off_t coffset;
4221         uint_t pidx;
4222         off_t poff;
4223 
4224 
4225         sinfo = &dma->dp_sglinfo;
4226 
4227         /*
4228          * the copy buffer should be a whole multiple of page size. We know that
4229          * this cookie is <= MMU_PAGESIZE.
4230          */
4231         ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4232 
4233         /*
4234          * from now on, all new windows in this bind need to be re-mapped during
4235          * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4236          * space...
4237          */
4238 #if !defined(__amd64)
4239         dma->dp_cb_remaping = B_TRUE;
4240 #endif
4241 
4242         /* reset copybuf used */
4243         *copybuf_used = 0;
4244 
4245         /*
4246          * if we don't have to trim (since granularity is set to 1), go to the
4247          * next window and add the current cookie to it. We know the current
4248          * cookie uses the copy buffer since we're in this code path.
4249          */
4250         if (!dma->dp_trim_required) {
4251                 (*windowp)++;
4252                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4253 
4254                 /* Add this cookie to the new window */
4255                 (*windowp)->wd_cookie_cnt++;
4256                 (*windowp)->wd_size += cookie->dmac_size;
4257                 *copybuf_used += MMU_PAGESIZE;
4258                 return (DDI_SUCCESS);
4259         }
4260 
4261         /*
4262          * *** may need to trim, figure it out.
4263          */
4264 
4265         /* figure out how much we need to trim from the window */
4266         if (dma->dp_granularity_power_2) {
4267                 trim_sz = (*windowp)->wd_size &
4268                     (hp->dmai_attr.dma_attr_granular - 1);
4269         } else {
4270                 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4271         }
4272 
4273         /*
4274          * if the window's a whole multiple of granularity, go to the next
4275          * window, init it, then add in the current cookie. We know the current
4276          * cookie uses the copy buffer since we're in this code path.
4277          */
4278         if (trim_sz == 0) {
4279                 (*windowp)++;
4280                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4281 
4282                 /* Add this cookie to the new window */
4283                 (*windowp)->wd_cookie_cnt++;
4284                 (*windowp)->wd_size += cookie->dmac_size;
4285                 *copybuf_used += MMU_PAGESIZE;
4286                 return (DDI_SUCCESS);
4287         }
4288 
4289         /*
4290          * *** We figured it out, we definitly need to trim
4291          */
4292 
4293         /*
4294          * make sure the driver isn't making us do something bad...
4295          * Trimming and sgllen == 1 don't go together.
4296          */
4297         if (hp->dmai_attr.dma_attr_sgllen == 1) {
4298                 return (DDI_DMA_NOMAPPING);
4299         }
4300 
4301         /*
4302          * first, setup the current window to account for the trim. Need to go
4303          * back to the last cookie for this. Some of the last cookie will be in
4304          * the current window, and some of the last cookie will be in the new
4305          * window. All of the current cookie will be in the new window.
4306          */
4307         cookie--;
4308         (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4309         (*windowp)->wd_trim.tr_last_cookie = cookie;
4310         (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4311         ASSERT(cookie->dmac_size > trim_sz);
4312         (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4313         (*windowp)->wd_size -= trim_sz;
4314 
4315         /*
4316          * we're trimming the last cookie (not the current cookie). So that
4317          * last cookie may have or may not have been using the copy buffer (
4318          * we know the cookie passed in uses the copy buffer since we're in
4319          * this code path).
4320          *
4321          * If the last cookie doesn't use the copy buffer, nothing special to
4322          * do. However, if it does uses the copy buffer, it will be both the
4323          * last page in the current window and the first page in the next
4324          * window. Since we are reusing the copy buffer (and KVA space on the
4325          * 32-bit kernel), this page will use the end of the copy buffer in the
4326          * current window, and the start of the copy buffer in the next window.
4327          * Track that info... The cookie physical address was already set to
4328          * the copy buffer physical address in setup_cookie..
4329          */
4330         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4331                 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4332                     (*windowp)->wd_size) >> MMU_PAGESHIFT;
4333                 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4334                 (*windowp)->wd_trim.tr_last_pidx = pidx;
4335                 (*windowp)->wd_trim.tr_last_cbaddr =
4336                     dma->dp_pgmap[pidx].pm_cbaddr;
4337 #if !defined(__amd64)
4338                 (*windowp)->wd_trim.tr_last_kaddr =
4339                     dma->dp_pgmap[pidx].pm_kaddr;
4340 #endif
4341         }
4342 
4343         /* save the buffer offsets for the next window */
4344         coffset = cookie->dmac_size - trim_sz;
4345         new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4346 
4347         /*
4348          * set this now in case this is the first window. all other cases are
4349          * set in dma_win()
4350          */
4351         cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4352 
4353         /*
4354          * initialize the next window using what's left over in the previous
4355          * cookie.
4356          */
4357         (*windowp)++;
4358         rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4359         (*windowp)->wd_cookie_cnt++;
4360         (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4361         (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4362         (*windowp)->wd_trim.tr_first_size = trim_sz;
4363 
4364         /*
4365          * again, we're tracking if the last cookie uses the copy buffer.
4366          * read the comment above for more info on why we need to track
4367          * additional state.
4368          *
4369          * For the first cookie in the new window, we need reset the physical
4370          * address to DMA into to the start of the copy buffer plus any
4371          * initial page offset which may be present.
4372          */
4373         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4374                 (*windowp)->wd_dosync = B_TRUE;
4375                 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4376                 (*windowp)->wd_trim.tr_first_pidx = pidx;
4377                 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4378                 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4379 
4380                 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4381                     poff;
4382                 (*windowp)->wd_trim.tr_first_paddr =
4383                     ROOTNEX_PADDR_TO_RBASE(paddr);
4384 
4385 #if !defined(__amd64)
4386                 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4387 #endif
4388                 /* account for the cookie copybuf usage in the new window */
4389                 *copybuf_used += MMU_PAGESIZE;
4390 
4391                 /*
4392                  * every piece of code has to have a hack, and here is this
4393                  * ones :-)
4394                  *
4395                  * There is a complex interaction between setup_cookie and the
4396                  * copybuf window boundary. The complexity had to be in either
4397                  * the maxxfer window, or the copybuf window, and I chose the
4398                  * copybuf code.
4399                  *
4400                  * So in this code path, we have taken the last cookie,
4401                  * virtually broken it in half due to the trim, and it happens
4402                  * to use the copybuf which further complicates life. At the
4403                  * same time, we have already setup the current cookie, which
4404                  * is now wrong. More background info: the current cookie uses
4405                  * the copybuf, so it is only a page long max. So we need to
4406                  * fix the current cookies copy buffer address, physical
4407                  * address, and kva for the 32-bit kernel. We due this by
4408                  * bumping them by page size (of course, we can't due this on
4409                  * the physical address since the copy buffer may not be
4410                  * physically contiguous).
4411                  */
4412                 cookie++;
4413                 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4414                 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4415 
4416                 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4417                     dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4418                 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4419 
4420 #if !defined(__amd64)
4421                 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4422                 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4423 #endif
4424         } else {
4425                 /* go back to the current cookie */
4426                 cookie++;
4427         }
4428 
4429         /*
4430          * add the current cookie to the new window. set the new window size to
4431          * the what was left over from the previous cookie and what's in the
4432          * current cookie.
4433          */
4434         (*windowp)->wd_cookie_cnt++;
4435         (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4436         ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4437 
4438         /*
4439          * we know that the cookie passed in always uses the copy buffer. We
4440          * wouldn't be here if it didn't.
4441          */
4442         *copybuf_used += MMU_PAGESIZE;
4443 
4444         return (DDI_SUCCESS);
4445 }
4446 
4447 
4448 /*
4449  * rootnex_maxxfer_window_boundary()
4450  *    Called in bind slowpath when we get to a window boundary because we will
4451  *    go over maxxfer.
4452  */
4453 static int
4454 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4455     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4456 {
4457         size_t dmac_size;
4458         off_t new_offset;
4459         size_t trim_sz;
4460         off_t coffset;
4461 
4462 
4463         /*
4464          * calculate how much we have to trim off of the current cookie to equal
4465          * maxxfer. We don't have to account for granularity here since our
4466          * maxxfer already takes that into account.
4467          */
4468         trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4469         ASSERT(trim_sz <= cookie->dmac_size);
4470         ASSERT(trim_sz <= dma->dp_maxxfer);
4471 
4472         /* save cookie size since we need it later and we might change it */
4473         dmac_size = cookie->dmac_size;
4474 
4475         /*
4476          * if we're not trimming the entire cookie, setup the current window to
4477          * account for the trim.
4478          */
4479         if (trim_sz < cookie->dmac_size) {
4480                 (*windowp)->wd_cookie_cnt++;
4481                 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4482                 (*windowp)->wd_trim.tr_last_cookie = cookie;
4483                 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4484                 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4485                 (*windowp)->wd_size = dma->dp_maxxfer;
4486 
4487                 /*
4488                  * set the adjusted cookie size now in case this is the first
4489                  * window. All other windows are taken care of in get win
4490                  */
4491                 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4492         }
4493 
4494         /*
4495          * coffset is the current offset within the cookie, new_offset is the
4496          * current offset with the entire buffer.
4497          */
4498         coffset = dmac_size - trim_sz;
4499         new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4500 
4501         /* initialize the next window */
4502         (*windowp)++;
4503         rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4504         (*windowp)->wd_cookie_cnt++;
4505         (*windowp)->wd_size = trim_sz;
4506         if (trim_sz < dmac_size) {
4507                 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4508                 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4509                     coffset;
4510                 (*windowp)->wd_trim.tr_first_size = trim_sz;
4511         }
4512 
4513         return (DDI_SUCCESS);
4514 }
4515 
4516 
4517 /*ARGSUSED*/
4518 static int
4519 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4520     off_t off, size_t len, uint_t cache_flags)
4521 {
4522         rootnex_sglinfo_t *sinfo;
4523         rootnex_pgmap_t *cbpage;
4524         rootnex_window_t *win;
4525         ddi_dma_impl_t *hp;
4526         rootnex_dma_t *dma;
4527         caddr_t fromaddr;
4528         caddr_t toaddr;
4529         uint_t psize;
4530         off_t offset;
4531         uint_t pidx;
4532         size_t size;
4533         off_t poff;
4534         int e;
4535 
4536 
4537         hp = (ddi_dma_impl_t *)handle;
4538         dma = (rootnex_dma_t *)hp->dmai_private;
4539         sinfo = &dma->dp_sglinfo;
4540 
4541         /*
4542          * if we don't have any windows, we don't need to sync. A copybuf
4543          * will cause us to have at least one window.
4544          */
4545         if (dma->dp_window == NULL) {
4546                 return (DDI_SUCCESS);
4547         }
4548 
4549         /* This window may not need to be sync'd */
4550         win = &dma->dp_window[dma->dp_current_win];
4551         if (!win->wd_dosync) {
4552                 return (DDI_SUCCESS);
4553         }
4554 
4555         /* handle off and len special cases */
4556         if ((off == 0) || (rootnex_sync_ignore_params)) {
4557                 offset = win->wd_offset;
4558         } else {
4559                 offset = off;
4560         }
4561         if ((len == 0) || (rootnex_sync_ignore_params)) {
4562                 size = win->wd_size;
4563         } else {
4564                 size = len;
4565         }
4566 
4567         /* check the sync args to make sure they make a little sense */
4568         if (rootnex_sync_check_parms) {
4569                 e = rootnex_valid_sync_parms(hp, win, offset, size,
4570                     cache_flags);
4571                 if (e != DDI_SUCCESS) {
4572                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4573                         return (DDI_FAILURE);
4574                 }
4575         }
4576 
4577         /*
4578          * special case the first page to handle the offset into the page. The
4579          * offset to the current page for our buffer is the offset into the
4580          * first page of the buffer plus our current offset into the buffer
4581          * itself, masked of course.
4582          */
4583         poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4584         psize = MIN((MMU_PAGESIZE - poff), size);
4585 
4586         /* go through all the pages that we want to sync */
4587         while (size > 0) {
4588                 /*
4589                  * Calculate the page index relative to the start of the buffer.
4590                  * The index to the current page for our buffer is the offset
4591                  * into the first page of the buffer plus our current offset
4592                  * into the buffer itself, shifted of course...
4593                  */
4594                 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4595                 ASSERT(pidx < sinfo->si_max_pages);
4596 
4597                 /*
4598                  * if this page uses the copy buffer, we need to sync it,
4599                  * otherwise, go on to the next page.
4600                  */
4601                 cbpage = &dma->dp_pgmap[pidx];
4602                 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4603                     (cbpage->pm_uses_copybuf == B_FALSE));
4604                 if (cbpage->pm_uses_copybuf) {
4605                         /* cbaddr and kaddr should be page aligned */
4606                         ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4607                             MMU_PAGEOFFSET) == 0);
4608                         ASSERT(((uintptr_t)cbpage->pm_kaddr &
4609                             MMU_PAGEOFFSET) == 0);
4610 
4611                         /*
4612                          * if we're copying for the device, we are going to
4613                          * copy from the drivers buffer and to the rootnex
4614                          * allocated copy buffer.
4615                          */
4616                         if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4617                                 fromaddr = cbpage->pm_kaddr + poff;
4618                                 toaddr = cbpage->pm_cbaddr + poff;
4619                                 ROOTNEX_DPROBE2(rootnex__sync__dev,
4620                                     dev_info_t *, dma->dp_dip, size_t, psize);
4621 
4622                         /*
4623                          * if we're copying for the cpu/kernel, we are going to
4624                          * copy from the rootnex allocated copy buffer to the
4625                          * drivers buffer.
4626                          */
4627                         } else {
4628                                 fromaddr = cbpage->pm_cbaddr + poff;
4629                                 toaddr = cbpage->pm_kaddr + poff;
4630                                 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4631                                     dev_info_t *, dma->dp_dip, size_t, psize);
4632                         }
4633 
4634                         bcopy(fromaddr, toaddr, psize);
4635                 }
4636 
4637                 /*
4638                  * decrement size until we're done, update our offset into the
4639                  * buffer, and get the next page size.
4640                  */
4641                 size -= psize;
4642                 offset += psize;
4643                 psize = MIN(MMU_PAGESIZE, size);
4644 
4645                 /* page offset is zero for the rest of this loop */
4646                 poff = 0;
4647         }
4648 
4649         return (DDI_SUCCESS);
4650 }
4651 
4652 /*
4653  * rootnex_dma_sync()
4654  *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4655  *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4656  *    is set, ddi_dma_sync() returns immediately passing back success.
4657  */
4658 /*ARGSUSED*/
4659 static int
4660 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4661     off_t off, size_t len, uint_t cache_flags)
4662 {
4663 #if defined(__amd64) && !defined(__xpv)
4664         if (IOMMU_USED(rdip)) {
4665                 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4666                     cache_flags));
4667         }
4668 #endif
4669         return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4670             cache_flags));
4671 }
4672 
4673 /*
4674  * rootnex_valid_sync_parms()
4675  *    checks the parameters passed to sync to verify they are correct.
4676  */
4677 static int
4678 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4679     off_t offset, size_t size, uint_t cache_flags)
4680 {
4681         off_t woffset;
4682 
4683 
4684         /*
4685          * the first part of the test to make sure the offset passed in is
4686          * within the window.
4687          */
4688         if (offset < win->wd_offset) {
4689                 return (DDI_FAILURE);
4690         }
4691 
4692         /*
4693          * second and last part of the test to make sure the offset and length
4694          * passed in is within the window.
4695          */
4696         woffset = offset - win->wd_offset;
4697         if ((woffset + size) > win->wd_size) {
4698                 return (DDI_FAILURE);
4699         }
4700 
4701         /*
4702          * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4703          * be set too.
4704          */
4705         if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4706             (hp->dmai_rflags & DDI_DMA_WRITE)) {
4707                 return (DDI_SUCCESS);
4708         }
4709 
4710         /*
4711          * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4712          * should be set. Also DDI_DMA_READ should be set in the flags.
4713          */
4714         if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4715             (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4716             (hp->dmai_rflags & DDI_DMA_READ)) {
4717                 return (DDI_SUCCESS);
4718         }
4719 
4720         return (DDI_FAILURE);
4721 }
4722 
4723 
4724 /*ARGSUSED*/
4725 static int
4726 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4727     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4728     uint_t *ccountp)
4729 {
4730         rootnex_window_t *window;
4731         rootnex_trim_t *trim;
4732         ddi_dma_impl_t *hp;
4733         rootnex_dma_t *dma;
4734         ddi_dma_obj_t *dmao;
4735 #if !defined(__amd64)
4736         rootnex_sglinfo_t *sinfo;
4737         rootnex_pgmap_t *pmap;
4738         uint_t pidx;
4739         uint_t pcnt;
4740         off_t poff;
4741         int i;
4742 #endif
4743 
4744 
4745         hp = (ddi_dma_impl_t *)handle;
4746         dma = (rootnex_dma_t *)hp->dmai_private;
4747 #if !defined(__amd64)
4748         sinfo = &dma->dp_sglinfo;
4749 #endif
4750 
4751         /* If we try and get a window which doesn't exist, return failure */
4752         if (win >= hp->dmai_nwin) {
4753                 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4754                 return (DDI_FAILURE);
4755         }
4756 
4757         dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4758 
4759         /*
4760          * if we don't have any windows, and they're asking for the first
4761          * window, setup the cookie pointer to the first cookie in the bind.
4762          * setup our return values, then increment the cookie since we return
4763          * the first cookie on the stack.
4764          */
4765         if (dma->dp_window == NULL) {
4766                 if (win != 0) {
4767                         ROOTNEX_DPROF_INC(
4768                             &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4769                         return (DDI_FAILURE);
4770                 }
4771                 hp->dmai_cookie = dma->dp_cookies;
4772                 *offp = 0;
4773                 *lenp = dmao->dmao_size;
4774                 *ccountp = dma->dp_sglinfo.si_sgl_size;
4775                 *cookiep = hp->dmai_cookie[0];
4776                 hp->dmai_cookie++;
4777                 return (DDI_SUCCESS);
4778         }
4779 
4780         /* sync the old window before moving on to the new one */
4781         window = &dma->dp_window[dma->dp_current_win];
4782         if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4783                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4784                     DDI_DMA_SYNC_FORCPU);
4785         }
4786 
4787 #if !defined(__amd64)
4788         /*
4789          * before we move to the next window, if we need to re-map, unmap all
4790          * the pages in this window.
4791          */
4792         if (dma->dp_cb_remaping) {
4793                 /*
4794                  * If we switch to this window again, we'll need to map in
4795                  * on the fly next time.
4796                  */
4797                 window->wd_remap_copybuf = B_TRUE;
4798 
4799                 /*
4800                  * calculate the page index into the buffer where this window
4801                  * starts, and the number of pages this window takes up.
4802                  */
4803                 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4804                     MMU_PAGESHIFT;
4805                 poff = (sinfo->si_buf_offset + window->wd_offset) &
4806                     MMU_PAGEOFFSET;
4807                 pcnt = mmu_btopr(window->wd_size + poff);
4808                 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4809 
4810                 /* unmap pages which are currently mapped in this window */
4811                 for (i = 0; i < pcnt; i++) {
4812                         if (dma->dp_pgmap[pidx].pm_mapped) {
4813                                 hat_unload(kas.a_hat,
4814                                     dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4815                                     HAT_UNLOAD);
4816                                 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4817                         }
4818                         pidx++;
4819                 }
4820         }
4821 #endif
4822 
4823         /*
4824          * Move to the new window.
4825          * NOTE: current_win must be set for sync to work right
4826          */
4827         dma->dp_current_win = win;
4828         window = &dma->dp_window[win];
4829 
4830         /* if needed, adjust the first and/or last cookies for trim */
4831         trim = &window->wd_trim;
4832         if (trim->tr_trim_first) {
4833                 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4834                 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4835 #if !defined(__amd64)
4836                 window->wd_first_cookie->dmac_type =
4837                     (window->wd_first_cookie->dmac_type &
4838                     ROOTNEX_USES_COPYBUF) + window->wd_offset;
4839 #endif
4840                 if (trim->tr_first_copybuf_win) {
4841                         dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4842                             trim->tr_first_cbaddr;
4843 #if !defined(__amd64)
4844                         dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4845                             trim->tr_first_kaddr;
4846 #endif
4847                 }
4848         }
4849         if (trim->tr_trim_last) {
4850                 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4851                 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4852                 if (trim->tr_last_copybuf_win) {
4853                         dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4854                             trim->tr_last_cbaddr;
4855 #if !defined(__amd64)
4856                         dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4857                             trim->tr_last_kaddr;
4858 #endif
4859                 }
4860         }
4861 
4862         /*
4863          * setup the cookie pointer to the first cookie in the window. setup
4864          * our return values, then increment the cookie since we return the
4865          * first cookie on the stack.
4866          */
4867         hp->dmai_cookie = window->wd_first_cookie;
4868         *offp = window->wd_offset;
4869         *lenp = window->wd_size;
4870         *ccountp = window->wd_cookie_cnt;
4871         *cookiep = hp->dmai_cookie[0];
4872         hp->dmai_cookie++;
4873 
4874 #if !defined(__amd64)
4875         /* re-map copybuf if required for this window */
4876         if (dma->dp_cb_remaping) {
4877                 /*
4878                  * calculate the page index into the buffer where this
4879                  * window starts.
4880                  */
4881                 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4882                     MMU_PAGESHIFT;
4883                 ASSERT(pidx < sinfo->si_max_pages);
4884 
4885                 /*
4886                  * the first page can get unmapped if it's shared with the
4887                  * previous window. Even if the rest of this window is already
4888                  * mapped in, we need to still check this one.
4889                  */
4890                 pmap = &dma->dp_pgmap[pidx];
4891                 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4892                         if (pmap->pm_pp != NULL) {
4893                                 pmap->pm_mapped = B_TRUE;
4894                                 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4895                         } else if (pmap->pm_vaddr != NULL) {
4896                                 pmap->pm_mapped = B_TRUE;
4897                                 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4898                                     pmap->pm_kaddr);
4899                         }
4900                 }
4901                 pidx++;
4902 
4903                 /* map in the rest of the pages if required */
4904                 if (window->wd_remap_copybuf) {
4905                         window->wd_remap_copybuf = B_FALSE;
4906 
4907                         /* figure out many pages this window takes up */
4908                         poff = (sinfo->si_buf_offset + window->wd_offset) &
4909                             MMU_PAGEOFFSET;
4910                         pcnt = mmu_btopr(window->wd_size + poff);
4911                         ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4912 
4913                         /* map pages which require it */
4914                         for (i = 1; i < pcnt; i++) {
4915                                 pmap = &dma->dp_pgmap[pidx];
4916                                 if (pmap->pm_uses_copybuf) {
4917                                         ASSERT(pmap->pm_mapped == B_FALSE);
4918                                         if (pmap->pm_pp != NULL) {
4919                                                 pmap->pm_mapped = B_TRUE;
4920                                                 i86_pp_map(pmap->pm_pp,
4921                                                     pmap->pm_kaddr);
4922                                         } else if (pmap->pm_vaddr != NULL) {
4923                                                 pmap->pm_mapped = B_TRUE;
4924                                                 i86_va_map(pmap->pm_vaddr,
4925                                                     sinfo->si_asp,
4926                                                     pmap->pm_kaddr);
4927                                         }
4928                                 }
4929                                 pidx++;
4930                         }
4931                 }
4932         }
4933 #endif
4934 
4935         /* if the new window uses the copy buffer, sync it for the device */
4936         if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4937                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4938                     DDI_DMA_SYNC_FORDEV);
4939         }
4940 
4941         return (DDI_SUCCESS);
4942 }
4943 
4944 /*
4945  * rootnex_dma_win()
4946  *    called from ddi_dma_getwin()
4947  */
4948 /*ARGSUSED*/
4949 static int
4950 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4951     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4952     uint_t *ccountp)
4953 {
4954 #if defined(__amd64) && !defined(__xpv)
4955         if (IOMMU_USED(rdip)) {
4956                 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4957                     cookiep, ccountp));
4958         }
4959 #endif
4960 
4961         return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4962             cookiep, ccountp));
4963 }
4964 
4965 #if defined(__amd64) && !defined(__xpv)
4966 /*ARGSUSED*/
4967 static int
4968 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4969     ddi_dma_handle_t handle, void *v)
4970 {
4971         ddi_dma_impl_t *hp;
4972         rootnex_dma_t *dma;
4973 
4974         hp = (ddi_dma_impl_t *)handle;
4975         dma = (rootnex_dma_t *)hp->dmai_private;
4976         dma->dp_iommu_private = v;
4977 
4978         return (DDI_SUCCESS);
4979 }
4980 
4981 /*ARGSUSED*/
4982 static void *
4983 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4984     ddi_dma_handle_t handle)
4985 {
4986         ddi_dma_impl_t *hp;
4987         rootnex_dma_t *dma;
4988 
4989         hp = (ddi_dma_impl_t *)handle;
4990         dma = (rootnex_dma_t *)hp->dmai_private;
4991 
4992         return (dma->dp_iommu_private);
4993 }
4994 #endif
4995 
4996 /*
4997  * ************************
4998  *  obsoleted dma routines
4999  * ************************
5000  */
5001 
5002 /*
5003  * rootnex_dma_mctl()
5004  *
5005  * We don't support this legacy interface any more on x86.
5006  */
5007 /* ARGSUSED */
5008 static int
5009 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5010     enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5011     uint_t cache_flags)
5012 {
5013         /*
5014          * The only thing dma_mctl is usef for anymore is legacy SPARC
5015          * dvma and sbus-specific routines.
5016          */
5017         return (DDI_FAILURE);
5018 }
5019 
5020 /*
5021  * *********
5022  *  FMA Code
5023  * *********
5024  */
5025 
5026 /*
5027  * rootnex_fm_init()
5028  *    FMA init busop
5029  */
5030 /* ARGSUSED */
5031 static int
5032 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5033     ddi_iblock_cookie_t *ibc)
5034 {
5035         *ibc = rootnex_state->r_err_ibc;
5036 
5037         return (ddi_system_fmcap);
5038 }
5039 
5040 /*
5041  * rootnex_dma_check()
5042  *    Function called after a dma fault occurred to find out whether the
5043  *    fault address is associated with a driver that is able to handle faults
5044  *    and recover from faults.
5045  */
5046 /* ARGSUSED */
5047 static int
5048 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5049     const void *not_used)
5050 {
5051         rootnex_window_t *window;
5052         uint64_t start_addr;
5053         uint64_t fault_addr;
5054         ddi_dma_impl_t *hp;
5055         rootnex_dma_t *dma;
5056         uint64_t end_addr;
5057         size_t csize;
5058         int i;
5059         int j;
5060 
5061 
5062         /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5063         hp = (ddi_dma_impl_t *)handle;
5064         ASSERT(hp);
5065 
5066         dma = (rootnex_dma_t *)hp->dmai_private;
5067 
5068         /* Get the address that we need to search for */
5069         fault_addr = *(uint64_t *)addr;
5070 
5071         /*
5072          * if we don't have any windows, we can just walk through all the
5073          * cookies.
5074          */
5075         if (dma->dp_window == NULL) {
5076                 /* for each cookie */
5077                 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5078                         /*
5079                          * if the faulted address is within the physical address
5080                          * range of the cookie, return DDI_FM_NONFATAL.
5081                          */
5082                         if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5083                             (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5084                             dma->dp_cookies[i].dmac_size))) {
5085                                 return (DDI_FM_NONFATAL);
5086                         }
5087                 }
5088 
5089                 /* fault_addr not within this DMA handle */
5090                 return (DDI_FM_UNKNOWN);
5091         }
5092 
5093         /* we have mutiple windows, walk through each window */
5094         for (i = 0; i < hp->dmai_nwin; i++) {
5095                 window = &dma->dp_window[i];
5096 
5097                 /* Go through all the cookies in the window */
5098                 for (j = 0; j < window->wd_cookie_cnt; j++) {
5099 
5100                         start_addr = window->wd_first_cookie[j].dmac_laddress;
5101                         csize = window->wd_first_cookie[j].dmac_size;
5102 
5103                         /*
5104                          * if we are trimming the first cookie in the window,
5105                          * and this is the first cookie, adjust the start
5106                          * address and size of the cookie to account for the
5107                          * trim.
5108                          */
5109                         if (window->wd_trim.tr_trim_first && (j == 0)) {
5110                                 start_addr = window->wd_trim.tr_first_paddr;
5111                                 csize = window->wd_trim.tr_first_size;
5112                         }
5113 
5114                         /*
5115                          * if we are trimming the last cookie in the window,
5116                          * and this is the last cookie, adjust the start
5117                          * address and size of the cookie to account for the
5118                          * trim.
5119                          */
5120                         if (window->wd_trim.tr_trim_last &&
5121                             (j == (window->wd_cookie_cnt - 1))) {
5122                                 start_addr = window->wd_trim.tr_last_paddr;
5123                                 csize = window->wd_trim.tr_last_size;
5124                         }
5125 
5126                         end_addr = start_addr + csize;
5127 
5128                         /*
5129                          * if the faulted address is within the physical
5130                          * address of the cookie, return DDI_FM_NONFATAL.
5131                          */
5132                         if ((fault_addr >= start_addr) &&
5133                             (fault_addr <= end_addr)) {
5134                                 return (DDI_FM_NONFATAL);
5135                         }
5136                 }
5137         }
5138 
5139         /* fault_addr not within this DMA handle */
5140         return (DDI_FM_UNKNOWN);
5141 }
5142 
5143 /*ARGSUSED*/
5144 static int
5145 rootnex_quiesce(dev_info_t *dip)
5146 {
5147 #if defined(__amd64) && !defined(__xpv)
5148         return (immu_quiesce());
5149 #else
5150         return (DDI_SUCCESS);
5151 #endif
5152 }
5153 
5154 #if defined(__xpv)
5155 void
5156 immu_init(void)
5157 {
5158         ;
5159 }
5160 
5161 void
5162 immu_startup(void)
5163 {
5164         ;
5165 }
5166 /*ARGSUSED*/
5167 void
5168 immu_physmem_update(uint64_t addr, uint64_t size)
5169 {
5170         ;
5171 }
5172 #endif