1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  26  * Copyright (c) 2011 Bayard G. Bell.  All rights reserved.
  27  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
  28  * Copyright 2017 Joyent, Inc.
  29  */
  30 
  31 /*
  32  * x86 root nexus driver
  33  */
  34 
  35 #include <sys/sysmacros.h>
  36 #include <sys/conf.h>
  37 #include <sys/autoconf.h>
  38 #include <sys/sysmacros.h>
  39 #include <sys/debug.h>
  40 #include <sys/psw.h>
  41 #include <sys/ddidmareq.h>
  42 #include <sys/promif.h>
  43 #include <sys/devops.h>
  44 #include <sys/kmem.h>
  45 #include <sys/cmn_err.h>
  46 #include <vm/seg.h>
  47 #include <vm/seg_kmem.h>
  48 #include <vm/seg_dev.h>
  49 #include <sys/vmem.h>
  50 #include <sys/mman.h>
  51 #include <vm/hat.h>
  52 #include <vm/as.h>
  53 #include <vm/page.h>
  54 #include <sys/avintr.h>
  55 #include <sys/errno.h>
  56 #include <sys/modctl.h>
  57 #include <sys/ddi_impldefs.h>
  58 #include <sys/sunddi.h>
  59 #include <sys/sunndi.h>
  60 #include <sys/mach_intr.h>
  61 #include <sys/psm.h>
  62 #include <sys/ontrap.h>
  63 #include <sys/atomic.h>
  64 #include <sys/sdt.h>
  65 #include <sys/rootnex.h>
  66 #include <vm/hat_i86.h>
  67 #include <sys/ddifm.h>
  68 #include <sys/ddi_isa.h>
  69 #include <sys/apic.h>
  70 
  71 #ifdef __xpv
  72 #include <sys/bootinfo.h>
  73 #include <sys/hypervisor.h>
  74 #include <sys/bootconf.h>
  75 #include <vm/kboot_mmu.h>
  76 #endif
  77 
  78 #if defined(__amd64) && !defined(__xpv)
  79 #include <sys/immu.h>
  80 #endif
  81 
  82 
  83 /*
  84  * enable/disable extra checking of function parameters. Useful for debugging
  85  * drivers.
  86  */
  87 #ifdef  DEBUG
  88 int rootnex_alloc_check_parms = 1;
  89 int rootnex_bind_check_parms = 1;
  90 int rootnex_bind_check_inuse = 1;
  91 int rootnex_unbind_verify_buffer = 0;
  92 int rootnex_sync_check_parms = 1;
  93 #else
  94 int rootnex_alloc_check_parms = 0;
  95 int rootnex_bind_check_parms = 0;
  96 int rootnex_bind_check_inuse = 0;
  97 int rootnex_unbind_verify_buffer = 0;
  98 int rootnex_sync_check_parms = 0;
  99 #endif
 100 
 101 boolean_t rootnex_dmar_not_setup;
 102 
 103 /* Master Abort and Target Abort panic flag */
 104 int rootnex_fm_ma_ta_panic_flag = 0;
 105 
 106 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
 107 int rootnex_bind_fail = 1;
 108 int rootnex_bind_warn = 1;
 109 uint8_t *rootnex_warn_list;
 110 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
 111 #define ROOTNEX_BIND_WARNING    (0x1 << 0)
 112 
 113 /*
 114  * revert back to old broken behavior of always sync'ing entire copy buffer.
 115  * This is useful if be have a buggy driver which doesn't correctly pass in
 116  * the offset and size into ddi_dma_sync().
 117  */
 118 int rootnex_sync_ignore_params = 0;
 119 
 120 /*
 121  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
 122  * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
 123  * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
 124  * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
 125  * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
 126  * (< 8K). We will still need to allocate the copy buffer during bind though
 127  * (if we need one). These can only be modified in /etc/system before rootnex
 128  * attach.
 129  */
 130 #if defined(__amd64)
 131 int rootnex_prealloc_cookies = 65;
 132 int rootnex_prealloc_windows = 4;
 133 int rootnex_prealloc_copybuf = 2;
 134 #else
 135 int rootnex_prealloc_cookies = 33;
 136 int rootnex_prealloc_windows = 4;
 137 int rootnex_prealloc_copybuf = 2;
 138 #endif
 139 
 140 /* driver global state */
 141 static rootnex_state_t *rootnex_state;
 142 
 143 #ifdef DEBUG
 144 /* shortcut to rootnex counters */
 145 static uint64_t *rootnex_cnt;
 146 #endif
 147 
 148 /*
 149  * XXX - does x86 even need these or are they left over from the SPARC days?
 150  */
 151 /* statically defined integer/boolean properties for the root node */
 152 static rootnex_intprop_t rootnex_intprp[] = {
 153         { "PAGESIZE",                   PAGESIZE },
 154         { "MMU_PAGESIZE",               MMU_PAGESIZE },
 155         { "MMU_PAGEOFFSET",             MMU_PAGEOFFSET },
 156         { DDI_RELATIVE_ADDRESSING,      1 },
 157 };
 158 #define NROOT_INTPROPS  (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
 159 
 160 /*
 161  * If we're dom0, we're using a real device so we need to load
 162  * the cookies with MFNs instead of PFNs.
 163  */
 164 #ifdef __xpv
 165 typedef maddr_t rootnex_addr_t;
 166 #define ROOTNEX_PADDR_TO_RBASE(pa)      \
 167         (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
 168 #else
 169 typedef paddr_t rootnex_addr_t;
 170 #define ROOTNEX_PADDR_TO_RBASE(pa)      (pa)
 171 #endif
 172 
 173 static struct cb_ops rootnex_cb_ops = {
 174         nodev,          /* open */
 175         nodev,          /* close */
 176         nodev,          /* strategy */
 177         nodev,          /* print */
 178         nodev,          /* dump */
 179         nodev,          /* read */
 180         nodev,          /* write */
 181         nodev,          /* ioctl */
 182         nodev,          /* devmap */
 183         nodev,          /* mmap */
 184         nodev,          /* segmap */
 185         nochpoll,       /* chpoll */
 186         ddi_prop_op,    /* cb_prop_op */
 187         NULL,           /* struct streamtab */
 188         D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
 189         CB_REV,         /* Rev */
 190         nodev,          /* cb_aread */
 191         nodev           /* cb_awrite */
 192 };
 193 
 194 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
 195     off_t offset, off_t len, caddr_t *vaddrp);
 196 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
 197     struct hat *hat, struct seg *seg, caddr_t addr,
 198     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
 199 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
 200     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
 201     ddi_dma_handle_t *handlep);
 202 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
 203     ddi_dma_handle_t handle);
 204 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
 205     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
 206     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 207 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
 208     ddi_dma_handle_t handle);
 209 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
 210     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
 211 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
 212     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
 213     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 214 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
 215     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
 216     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
 217 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
 218     ddi_ctl_enum_t ctlop, void *arg, void *result);
 219 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
 220     ddi_iblock_cookie_t *ibc);
 221 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
 222     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
 223 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
 224     void *);
 225 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
 226 
 227 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
 228     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
 229     ddi_dma_handle_t *handlep);
 230 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
 231     ddi_dma_handle_t handle);
 232 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
 233     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
 234     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 235 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
 236     ddi_dma_handle_t handle);
 237 #if defined(__amd64) && !defined(__xpv)
 238 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
 239     ddi_dma_handle_t handle);
 240 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
 241     ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
 242 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
 243     ddi_dma_cookie_t *cookiep, uint_t ccount);
 244 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
 245     ddi_dma_handle_t handle);
 246 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
 247 #endif
 248 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
 249     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
 250 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
 251     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
 252     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
 253 
 254 #if defined(__amd64) && !defined(__xpv)
 255 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
 256     ddi_dma_handle_t handle, void *v);
 257 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
 258     ddi_dma_handle_t handle);
 259 #endif
 260 
 261 
 262 static struct bus_ops rootnex_bus_ops = {
 263         BUSO_REV,
 264         rootnex_map,
 265         NULL,
 266         NULL,
 267         NULL,
 268         rootnex_map_fault,
 269         0,
 270         rootnex_dma_allochdl,
 271         rootnex_dma_freehdl,
 272         rootnex_dma_bindhdl,
 273         rootnex_dma_unbindhdl,
 274         rootnex_dma_sync,
 275         rootnex_dma_win,
 276         rootnex_dma_mctl,
 277         rootnex_ctlops,
 278         ddi_bus_prop_op,
 279         i_ddi_rootnex_get_eventcookie,
 280         i_ddi_rootnex_add_eventcall,
 281         i_ddi_rootnex_remove_eventcall,
 282         i_ddi_rootnex_post_event,
 283         0,                      /* bus_intr_ctl */
 284         0,                      /* bus_config */
 285         0,                      /* bus_unconfig */
 286         rootnex_fm_init,        /* bus_fm_init */
 287         NULL,                   /* bus_fm_fini */
 288         NULL,                   /* bus_fm_access_enter */
 289         NULL,                   /* bus_fm_access_exit */
 290         NULL,                   /* bus_powr */
 291         rootnex_intr_ops        /* bus_intr_op */
 292 };
 293 
 294 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
 295 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
 296 static int rootnex_quiesce(dev_info_t *dip);
 297 
 298 static struct dev_ops rootnex_ops = {
 299         DEVO_REV,
 300         0,
 301         ddi_no_info,
 302         nulldev,
 303         nulldev,
 304         rootnex_attach,
 305         rootnex_detach,
 306         nulldev,
 307         &rootnex_cb_ops,
 308         &rootnex_bus_ops,
 309         NULL,
 310         rootnex_quiesce,                /* quiesce */
 311 };
 312 
 313 static struct modldrv rootnex_modldrv = {
 314         &mod_driverops,
 315         "i86pc root nexus",
 316         &rootnex_ops
 317 };
 318 
 319 static struct modlinkage rootnex_modlinkage = {
 320         MODREV_1,
 321         (void *)&rootnex_modldrv,
 322         NULL
 323 };
 324 
 325 #if defined(__amd64) && !defined(__xpv)
 326 static iommulib_nexops_t iommulib_nexops = {
 327         IOMMU_NEXOPS_VERSION,
 328         "Rootnex IOMMU ops Vers 1.1",
 329         NULL,
 330         rootnex_coredma_allochdl,
 331         rootnex_coredma_freehdl,
 332         rootnex_coredma_bindhdl,
 333         rootnex_coredma_unbindhdl,
 334         rootnex_coredma_reset_cookies,
 335         rootnex_coredma_get_cookies,
 336         rootnex_coredma_set_cookies,
 337         rootnex_coredma_clear_cookies,
 338         rootnex_coredma_get_sleep_flags,
 339         rootnex_coredma_sync,
 340         rootnex_coredma_win,
 341         rootnex_coredma_hdl_setprivate,
 342         rootnex_coredma_hdl_getprivate
 343 };
 344 #endif
 345 
 346 /*
 347  *  extern hacks
 348  */
 349 extern struct seg_ops segdev_ops;
 350 extern int ignore_hardware_nodes;       /* force flag from ddi_impl.c */
 351 #ifdef  DDI_MAP_DEBUG
 352 extern int ddi_map_debug_flag;
 353 #define ddi_map_debug   if (ddi_map_debug_flag) prom_printf
 354 #endif
 355 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
 356 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
 357 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
 358     psm_intr_op_t, int *);
 359 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
 360 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
 361 
 362 /*
 363  * Use device arena to use for device control register mappings.
 364  * Various kernel memory walkers (debugger, dtrace) need to know
 365  * to avoid this address range to prevent undesired device activity.
 366  */
 367 extern void *device_arena_alloc(size_t size, int vm_flag);
 368 extern void device_arena_free(void * vaddr, size_t size);
 369 
 370 
 371 /*
 372  *  Internal functions
 373  */
 374 static int rootnex_dma_init();
 375 static void rootnex_add_props(dev_info_t *);
 376 static int rootnex_ctl_reportdev(dev_info_t *dip);
 377 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
 378 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
 379 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
 380 static int rootnex_map_handle(ddi_map_req_t *mp);
 381 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
 382 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
 383 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
 384     ddi_dma_attr_t *attr);
 385 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
 386     rootnex_sglinfo_t *sglinfo);
 387 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
 388     ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
 389 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
 390     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
 391 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
 392     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
 393 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
 394 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
 395     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
 396 static void rootnex_teardown_windows(rootnex_dma_t *dma);
 397 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
 398     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
 399 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
 400     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
 401     size_t *copybuf_used, page_t **cur_pp);
 402 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
 403     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
 404     ddi_dma_attr_t *attr, off_t cur_offset);
 405 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
 406     rootnex_dma_t *dma, rootnex_window_t **windowp,
 407     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
 408 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
 409     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
 410 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
 411     off_t offset, size_t size, uint_t cache_flags);
 412 static int rootnex_verify_buffer(rootnex_dma_t *dma);
 413 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
 414     const void *comp_addr, const void *not_used);
 415 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
 416     rootnex_sglinfo_t *sglinfo);
 417 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
 418 
 419 /*
 420  * _init()
 421  *
 422  */
 423 int
 424 _init(void)
 425 {
 426 
 427         rootnex_state = NULL;
 428         return (mod_install(&rootnex_modlinkage));
 429 }
 430 
 431 
 432 /*
 433  * _info()
 434  *
 435  */
 436 int
 437 _info(struct modinfo *modinfop)
 438 {
 439         return (mod_info(&rootnex_modlinkage, modinfop));
 440 }
 441 
 442 
 443 /*
 444  * _fini()
 445  *
 446  */
 447 int
 448 _fini(void)
 449 {
 450         return (EBUSY);
 451 }
 452 
 453 
 454 /*
 455  * rootnex_attach()
 456  *
 457  */
 458 static int
 459 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 460 {
 461         int fmcap;
 462         int e;
 463 
 464         switch (cmd) {
 465         case DDI_ATTACH:
 466                 break;
 467         case DDI_RESUME:
 468 #if defined(__amd64) && !defined(__xpv)
 469                 return (immu_unquiesce());
 470 #else
 471                 return (DDI_SUCCESS);
 472 #endif
 473         default:
 474                 return (DDI_FAILURE);
 475         }
 476 
 477         /*
 478          * We should only have one instance of rootnex. Save it away since we
 479          * don't have an easy way to get it back later.
 480          */
 481         ASSERT(rootnex_state == NULL);
 482         rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
 483 
 484         rootnex_state->r_dip = dip;
 485         rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
 486         rootnex_state->r_reserved_msg_printed = B_FALSE;
 487 #ifdef DEBUG
 488         rootnex_cnt = &rootnex_state->r_counters[0];
 489 #endif
 490 
 491         /*
 492          * Set minimum fm capability level for i86pc platforms and then
 493          * initialize error handling. Since we're the rootnex, we don't
 494          * care what's returned in the fmcap field.
 495          */
 496         ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
 497             DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
 498         fmcap = ddi_system_fmcap;
 499         ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
 500 
 501         /* initialize DMA related state */
 502         e = rootnex_dma_init();
 503         if (e != DDI_SUCCESS) {
 504                 kmem_free(rootnex_state, sizeof (rootnex_state_t));
 505                 return (DDI_FAILURE);
 506         }
 507 
 508         /* Add static root node properties */
 509         rootnex_add_props(dip);
 510 
 511         /* since we can't call ddi_report_dev() */
 512         cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
 513 
 514         /* Initialize rootnex event handle */
 515         i_ddi_rootnex_init_events(dip);
 516 
 517 #if defined(__amd64) && !defined(__xpv)
 518         e = iommulib_nexus_register(dip, &iommulib_nexops,
 519             &rootnex_state->r_iommulib_handle);
 520 
 521         ASSERT(e == DDI_SUCCESS);
 522 #endif
 523 
 524         return (DDI_SUCCESS);
 525 }
 526 
 527 
 528 /*
 529  * rootnex_detach()
 530  *
 531  */
 532 /*ARGSUSED*/
 533 static int
 534 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 535 {
 536         switch (cmd) {
 537         case DDI_SUSPEND:
 538 #if defined(__amd64) && !defined(__xpv)
 539                 return (immu_quiesce());
 540 #else
 541                 return (DDI_SUCCESS);
 542 #endif
 543         default:
 544                 return (DDI_FAILURE);
 545         }
 546         /*NOTREACHED*/
 547 
 548 }
 549 
 550 
 551 /*
 552  * rootnex_dma_init()
 553  *
 554  */
 555 /*ARGSUSED*/
 556 static int
 557 rootnex_dma_init()
 558 {
 559         size_t bufsize;
 560 
 561 
 562         /*
 563          * size of our cookie/window/copybuf state needed in dma bind that we
 564          * pre-alloc in dma_alloc_handle
 565          */
 566         rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
 567         rootnex_state->r_prealloc_size =
 568             (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
 569             (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
 570             (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
 571 
 572         /*
 573          * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
 574          * allocate 16 extra bytes for struct pointer alignment
 575          * (p->dmai_private & dma->dp_prealloc_buffer)
 576          */
 577         bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
 578             rootnex_state->r_prealloc_size + 0x10;
 579         rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
 580             bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
 581         if (rootnex_state->r_dmahdl_cache == NULL) {
 582                 return (DDI_FAILURE);
 583         }
 584 
 585         /*
 586          * allocate array to track which major numbers we have printed warnings
 587          * for.
 588          */
 589         rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
 590             KM_SLEEP);
 591 
 592         return (DDI_SUCCESS);
 593 }
 594 
 595 
 596 /*
 597  * rootnex_add_props()
 598  *
 599  */
 600 static void
 601 rootnex_add_props(dev_info_t *dip)
 602 {
 603         rootnex_intprop_t *rpp;
 604         int i;
 605 
 606         /* Add static integer/boolean properties to the root node */
 607         rpp = rootnex_intprp;
 608         for (i = 0; i < NROOT_INTPROPS; i++) {
 609                 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
 610                     rpp[i].prop_name, rpp[i].prop_value);
 611         }
 612 }
 613 
 614 
 615 
 616 /*
 617  * *************************
 618  *  ctlops related routines
 619  * *************************
 620  */
 621 
 622 /*
 623  * rootnex_ctlops()
 624  *
 625  */
 626 /*ARGSUSED*/
 627 static int
 628 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
 629     void *arg, void *result)
 630 {
 631         int n, *ptr;
 632         struct ddi_parent_private_data *pdp;
 633 
 634         switch (ctlop) {
 635         case DDI_CTLOPS_DMAPMAPC:
 636                 /*
 637                  * Return 'partial' to indicate that dma mapping
 638                  * has to be done in the main MMU.
 639                  */
 640                 return (DDI_DMA_PARTIAL);
 641 
 642         case DDI_CTLOPS_BTOP:
 643                 /*
 644                  * Convert byte count input to physical page units.
 645                  * (byte counts that are not a page-size multiple
 646                  * are rounded down)
 647                  */
 648                 *(ulong_t *)result = btop(*(ulong_t *)arg);
 649                 return (DDI_SUCCESS);
 650 
 651         case DDI_CTLOPS_PTOB:
 652                 /*
 653                  * Convert size in physical pages to bytes
 654                  */
 655                 *(ulong_t *)result = ptob(*(ulong_t *)arg);
 656                 return (DDI_SUCCESS);
 657 
 658         case DDI_CTLOPS_BTOPR:
 659                 /*
 660                  * Convert byte count input to physical page units
 661                  * (byte counts that are not a page-size multiple
 662                  * are rounded up)
 663                  */
 664                 *(ulong_t *)result = btopr(*(ulong_t *)arg);
 665                 return (DDI_SUCCESS);
 666 
 667         case DDI_CTLOPS_INITCHILD:
 668                 return (impl_ddi_sunbus_initchild(arg));
 669 
 670         case DDI_CTLOPS_UNINITCHILD:
 671                 impl_ddi_sunbus_removechild(arg);
 672                 return (DDI_SUCCESS);
 673 
 674         case DDI_CTLOPS_REPORTDEV:
 675                 return (rootnex_ctl_reportdev(rdip));
 676 
 677         case DDI_CTLOPS_IOMIN:
 678                 /*
 679                  * Nothing to do here but reflect back..
 680                  */
 681                 return (DDI_SUCCESS);
 682 
 683         case DDI_CTLOPS_REGSIZE:
 684         case DDI_CTLOPS_NREGS:
 685                 break;
 686 
 687         case DDI_CTLOPS_SIDDEV:
 688                 if (ndi_dev_is_prom_node(rdip))
 689                         return (DDI_SUCCESS);
 690                 if (ndi_dev_is_persistent_node(rdip))
 691                         return (DDI_SUCCESS);
 692                 return (DDI_FAILURE);
 693 
 694         case DDI_CTLOPS_POWER:
 695                 return ((*pm_platform_power)((power_req_t *)arg));
 696 
 697         case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
 698         case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
 699         case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
 700         case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
 701         case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
 702         case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
 703                 if (!rootnex_state->r_reserved_msg_printed) {
 704                         rootnex_state->r_reserved_msg_printed = B_TRUE;
 705                         cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
 706                             "1 or more reserved/obsolete operations.");
 707                 }
 708                 return (DDI_FAILURE);
 709 
 710         default:
 711                 return (DDI_FAILURE);
 712         }
 713         /*
 714          * The rest are for "hardware" properties
 715          */
 716         if ((pdp = ddi_get_parent_data(rdip)) == NULL)
 717                 return (DDI_FAILURE);
 718 
 719         if (ctlop == DDI_CTLOPS_NREGS) {
 720                 ptr = (int *)result;
 721                 *ptr = pdp->par_nreg;
 722         } else {
 723                 off_t *size = (off_t *)result;
 724 
 725                 ptr = (int *)arg;
 726                 n = *ptr;
 727                 if (n >= pdp->par_nreg) {
 728                         return (DDI_FAILURE);
 729                 }
 730                 *size = (off_t)pdp->par_reg[n].regspec_size;
 731         }
 732         return (DDI_SUCCESS);
 733 }
 734 
 735 
 736 /*
 737  * rootnex_ctl_reportdev()
 738  *
 739  */
 740 static int
 741 rootnex_ctl_reportdev(dev_info_t *dev)
 742 {
 743         int i, n, len, f_len = 0;
 744         char *buf;
 745 
 746         buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
 747         f_len += snprintf(buf, REPORTDEV_BUFSIZE,
 748             "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
 749         len = strlen(buf);
 750 
 751         for (i = 0; i < sparc_pd_getnreg(dev); i++) {
 752 
 753                 struct regspec *rp = sparc_pd_getreg(dev, i);
 754 
 755                 if (i == 0)
 756                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 757                             ": ");
 758                 else
 759                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 760                             " and ");
 761                 len = strlen(buf);
 762 
 763                 switch (rp->regspec_bustype) {
 764 
 765                 case BTEISA:
 766                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 767                             "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
 768                         break;
 769 
 770                 case BTISA:
 771                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 772                             "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
 773                         break;
 774 
 775                 default:
 776                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 777                             "space %x offset %x",
 778                             rp->regspec_bustype, rp->regspec_addr);
 779                         break;
 780                 }
 781                 len = strlen(buf);
 782         }
 783         for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
 784                 int pri;
 785 
 786                 if (i != 0) {
 787                         f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 788                             ",");
 789                         len = strlen(buf);
 790                 }
 791                 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
 792                 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
 793                     " sparc ipl %d", pri);
 794                 len = strlen(buf);
 795         }
 796 #ifdef DEBUG
 797         if (f_len + 1 >= REPORTDEV_BUFSIZE) {
 798                 cmn_err(CE_NOTE, "next message is truncated: "
 799                     "printed length 1024, real length %d", f_len);
 800         }
 801 #endif /* DEBUG */
 802         cmn_err(CE_CONT, "?%s\n", buf);
 803         kmem_free(buf, REPORTDEV_BUFSIZE);
 804         return (DDI_SUCCESS);
 805 }
 806 
 807 
 808 /*
 809  * ******************
 810  *  map related code
 811  * ******************
 812  */
 813 
 814 /*
 815  * rootnex_map()
 816  *
 817  */
 818 static int
 819 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
 820     off_t len, caddr_t *vaddrp)
 821 {
 822         struct regspec *rp, tmp_reg;
 823         ddi_map_req_t mr = *mp;         /* Get private copy of request */
 824         int error;
 825 
 826         mp = &mr;
 827 
 828         switch (mp->map_op)  {
 829         case DDI_MO_MAP_LOCKED:
 830         case DDI_MO_UNMAP:
 831         case DDI_MO_MAP_HANDLE:
 832                 break;
 833         default:
 834 #ifdef  DDI_MAP_DEBUG
 835                 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
 836                     mp->map_op);
 837 #endif  /* DDI_MAP_DEBUG */
 838                 return (DDI_ME_UNIMPLEMENTED);
 839         }
 840 
 841         if (mp->map_flags & DDI_MF_USER_MAPPING)  {
 842 #ifdef  DDI_MAP_DEBUG
 843                 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
 844 #endif  /* DDI_MAP_DEBUG */
 845                 return (DDI_ME_UNIMPLEMENTED);
 846         }
 847 
 848         /*
 849          * First, if given an rnumber, convert it to a regspec...
 850          * (Presumably, this is on behalf of a child of the root node?)
 851          */
 852 
 853         if (mp->map_type == DDI_MT_RNUMBER)  {
 854 
 855                 int rnumber = mp->map_obj.rnumber;
 856 #ifdef  DDI_MAP_DEBUG
 857                 static char *out_of_range =
 858                     "rootnex_map: Out of range rnumber <%d>, device <%s>";
 859 #endif  /* DDI_MAP_DEBUG */
 860 
 861                 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
 862                 if (rp == NULL)  {
 863 #ifdef  DDI_MAP_DEBUG
 864                         cmn_err(CE_WARN, out_of_range, rnumber,
 865                             ddi_get_name(rdip));
 866 #endif  /* DDI_MAP_DEBUG */
 867                         return (DDI_ME_RNUMBER_RANGE);
 868                 }
 869 
 870                 /*
 871                  * Convert the given ddi_map_req_t from rnumber to regspec...
 872                  */
 873 
 874                 mp->map_type = DDI_MT_REGSPEC;
 875                 mp->map_obj.rp = rp;
 876         }
 877 
 878         /*
 879          * Adjust offset and length correspnding to called values...
 880          * XXX: A non-zero length means override the one in the regspec
 881          * XXX: (regardless of what's in the parent's range?)
 882          */
 883 
 884         tmp_reg = *(mp->map_obj.rp);         /* Preserve underlying data */
 885         rp = mp->map_obj.rp = &tmp_reg;          /* Use tmp_reg in request */
 886 
 887 #ifdef  DDI_MAP_DEBUG
 888         cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
 889             "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
 890             rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
 891             len, mp->map_handlep);
 892 #endif  /* DDI_MAP_DEBUG */
 893 
 894         /*
 895          * I/O or memory mapping:
 896          *
 897          *      <bustype=0, addr=x, len=x>: memory
 898          *      <bustype=1, addr=x, len=x>: i/o
 899          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
 900          */
 901 
 902         if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
 903                 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
 904                     " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
 905                     ddi_get_name(rdip), rp->regspec_bustype,
 906                     rp->regspec_addr, rp->regspec_size);
 907                 return (DDI_ME_INVAL);
 908         }
 909 
 910         if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
 911                 /*
 912                  * compatibility i/o mapping
 913                  */
 914                 rp->regspec_bustype += (uint_t)offset;
 915         } else {
 916                 /*
 917                  * Normal memory or i/o mapping
 918                  */
 919                 rp->regspec_addr += (uint_t)offset;
 920         }
 921 
 922         if (len != 0)
 923                 rp->regspec_size = (uint_t)len;
 924 
 925 #ifdef  DDI_MAP_DEBUG
 926         cmn_err(CE_CONT, "             <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
 927             "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
 928             rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
 929             offset, len, mp->map_handlep);
 930 #endif  /* DDI_MAP_DEBUG */
 931 
 932         /*
 933          * Apply any parent ranges at this level, if applicable.
 934          * (This is where nexus specific regspec translation takes place.
 935          * Use of this function is implicit agreement that translation is
 936          * provided via ddi_apply_range.)
 937          */
 938 
 939 #ifdef  DDI_MAP_DEBUG
 940         ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
 941             ddi_get_name(dip), ddi_get_name(rdip));
 942 #endif  /* DDI_MAP_DEBUG */
 943 
 944         if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
 945                 return (error);
 946 
 947         switch (mp->map_op)  {
 948         case DDI_MO_MAP_LOCKED:
 949 
 950                 /*
 951                  * Set up the locked down kernel mapping to the regspec...
 952                  */
 953 
 954                 return (rootnex_map_regspec(mp, vaddrp));
 955 
 956         case DDI_MO_UNMAP:
 957 
 958                 /*
 959                  * Release mapping...
 960                  */
 961 
 962                 return (rootnex_unmap_regspec(mp, vaddrp));
 963 
 964         case DDI_MO_MAP_HANDLE:
 965 
 966                 return (rootnex_map_handle(mp));
 967 
 968         default:
 969                 return (DDI_ME_UNIMPLEMENTED);
 970         }
 971 }
 972 
 973 
 974 /*
 975  * rootnex_map_fault()
 976  *
 977  *      fault in mappings for requestors
 978  */
 979 /*ARGSUSED*/
 980 static int
 981 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
 982     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
 983     uint_t lock)
 984 {
 985 
 986 #ifdef  DDI_MAP_DEBUG
 987         ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
 988         ddi_map_debug(" Seg <%s>\n",
 989             seg->s_ops == &segdev_ops ? "segdev" :
 990             seg == &kvseg ? "segkmem" : "NONE!");
 991 #endif  /* DDI_MAP_DEBUG */
 992 
 993         /*
 994          * This is all terribly broken, but it is a start
 995          *
 996          * XXX  Note that this test means that segdev_ops
 997          *      must be exported from seg_dev.c.
 998          * XXX  What about devices with their own segment drivers?
 999          */
1000         if (seg->s_ops == &segdev_ops) {
1001                 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1002 
1003                 if (hat == NULL) {
1004                         /*
1005                          * This is one plausible interpretation of
1006                          * a null hat i.e. use the first hat on the
1007                          * address space hat list which by convention is
1008                          * the hat of the system MMU.  At alternative
1009                          * would be to panic .. this might well be better ..
1010                          */
1011                         ASSERT(AS_READ_HELD(seg->s_as));
1012                         hat = seg->s_as->a_hat;
1013                         cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1014                 }
1015                 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1016                     (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1017         } else if (seg == &kvseg && dp == NULL) {
1018                 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1019                     HAT_LOAD_LOCK);
1020         } else
1021                 return (DDI_FAILURE);
1022         return (DDI_SUCCESS);
1023 }
1024 
1025 
1026 /*
1027  * rootnex_map_regspec()
1028  *     we don't support mapping of I/O cards above 4Gb
1029  */
1030 static int
1031 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1032 {
1033         rootnex_addr_t rbase;
1034         void *cvaddr;
1035         uint_t npages, pgoffset;
1036         struct regspec *rp;
1037         ddi_acc_hdl_t *hp;
1038         ddi_acc_impl_t *ap;
1039         uint_t  hat_acc_flags;
1040         paddr_t pbase;
1041 
1042         rp = mp->map_obj.rp;
1043         hp = mp->map_handlep;
1044 
1045 #ifdef  DDI_MAP_DEBUG
1046         ddi_map_debug(
1047             "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1048             rp->regspec_bustype, rp->regspec_addr,
1049             rp->regspec_size, mp->map_handlep);
1050 #endif  /* DDI_MAP_DEBUG */
1051 
1052         /*
1053          * I/O or memory mapping
1054          *
1055          *      <bustype=0, addr=x, len=x>: memory
1056          *      <bustype=1, addr=x, len=x>: i/o
1057          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
1058          */
1059 
1060         if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1061                 cmn_err(CE_WARN, "rootnex: invalid register spec"
1062                     " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1063                     rp->regspec_addr, rp->regspec_size);
1064                 return (DDI_FAILURE);
1065         }
1066 
1067         if (rp->regspec_bustype != 0) {
1068                 /*
1069                  * I/O space - needs a handle.
1070                  */
1071                 if (hp == NULL) {
1072                         return (DDI_FAILURE);
1073                 }
1074                 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1075                 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1076                 impl_acc_hdl_init(hp);
1077 
1078                 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1079 #ifdef  DDI_MAP_DEBUG
1080                         ddi_map_debug("rootnex_map_regspec: mmap() "
1081                             "to I/O space is not supported.\n");
1082 #endif  /* DDI_MAP_DEBUG */
1083                         return (DDI_ME_INVAL);
1084                 } else {
1085                         /*
1086                          * 1275-compliant vs. compatibility i/o mapping
1087                          */
1088                         *vaddrp =
1089                             (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1090                             ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1091                             ((caddr_t)(uintptr_t)rp->regspec_addr);
1092 #ifdef __xpv
1093                         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1094                                 hp->ah_pfn = xen_assign_pfn(
1095                                     mmu_btop((ulong_t)rp->regspec_addr &
1096                                     MMU_PAGEMASK));
1097                         } else {
1098                                 hp->ah_pfn = mmu_btop(
1099                                     (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1100                         }
1101 #else
1102                         hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1103                             MMU_PAGEMASK);
1104 #endif
1105                         hp->ah_pnum = mmu_btopr(rp->regspec_size +
1106                             (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1107                 }
1108 
1109 #ifdef  DDI_MAP_DEBUG
1110                 ddi_map_debug(
1111             "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1112                     rp->regspec_size, *vaddrp);
1113 #endif  /* DDI_MAP_DEBUG */
1114                 return (DDI_SUCCESS);
1115         }
1116 
1117         /*
1118          * Memory space
1119          */
1120 
1121         if (hp != NULL) {
1122                 /*
1123                  * hat layer ignores
1124                  * hp->ah_acc.devacc_attr_endian_flags.
1125                  */
1126                 switch (hp->ah_acc.devacc_attr_dataorder) {
1127                 case DDI_STRICTORDER_ACC:
1128                         hat_acc_flags = HAT_STRICTORDER;
1129                         break;
1130                 case DDI_UNORDERED_OK_ACC:
1131                         hat_acc_flags = HAT_UNORDERED_OK;
1132                         break;
1133                 case DDI_MERGING_OK_ACC:
1134                         hat_acc_flags = HAT_MERGING_OK;
1135                         break;
1136                 case DDI_LOADCACHING_OK_ACC:
1137                         hat_acc_flags = HAT_LOADCACHING_OK;
1138                         break;
1139                 case DDI_STORECACHING_OK_ACC:
1140                         hat_acc_flags = HAT_STORECACHING_OK;
1141                         break;
1142                 }
1143                 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1144                 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1145                 impl_acc_hdl_init(hp);
1146                 hp->ah_hat_flags = hat_acc_flags;
1147         } else {
1148                 hat_acc_flags = HAT_STRICTORDER;
1149         }
1150 
1151         rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1152 #ifdef __xpv
1153         /*
1154          * If we're dom0, we're using a real device so we need to translate
1155          * the MA to a PA.
1156          */
1157         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1158                 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1159         } else {
1160                 pbase = rbase;
1161         }
1162 #else
1163         pbase = rbase;
1164 #endif
1165         pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1166 
1167         if (rp->regspec_size == 0) {
1168 #ifdef  DDI_MAP_DEBUG
1169                 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1170 #endif  /* DDI_MAP_DEBUG */
1171                 return (DDI_ME_INVAL);
1172         }
1173 
1174         if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1175                 /* extra cast to make gcc happy */
1176                 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1177         } else {
1178                 npages = mmu_btopr(rp->regspec_size + pgoffset);
1179 
1180 #ifdef  DDI_MAP_DEBUG
1181                 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1182                     "physical %llx", npages, pbase);
1183 #endif  /* DDI_MAP_DEBUG */
1184 
1185                 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1186                 if (cvaddr == NULL)
1187                         return (DDI_ME_NORESOURCES);
1188 
1189                 /*
1190                  * Now map in the pages we've allocated...
1191                  */
1192                 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1193                     mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1194                     HAT_LOAD_LOCK);
1195                 *vaddrp = (caddr_t)cvaddr + pgoffset;
1196 
1197                 /* save away pfn and npages for FMA */
1198                 hp = mp->map_handlep;
1199                 if (hp) {
1200                         hp->ah_pfn = mmu_btop(pbase);
1201                         hp->ah_pnum = npages;
1202                 }
1203         }
1204 
1205 #ifdef  DDI_MAP_DEBUG
1206         ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1207 #endif  /* DDI_MAP_DEBUG */
1208         return (DDI_SUCCESS);
1209 }
1210 
1211 
1212 /*
1213  * rootnex_unmap_regspec()
1214  *
1215  */
1216 static int
1217 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1218 {
1219         caddr_t addr = (caddr_t)*vaddrp;
1220         uint_t npages, pgoffset;
1221         struct regspec *rp;
1222 
1223         if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1224                 return (0);
1225 
1226         rp = mp->map_obj.rp;
1227 
1228         if (rp->regspec_size == 0) {
1229 #ifdef  DDI_MAP_DEBUG
1230                 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1231 #endif  /* DDI_MAP_DEBUG */
1232                 return (DDI_ME_INVAL);
1233         }
1234 
1235         /*
1236          * I/O or memory mapping:
1237          *
1238          *      <bustype=0, addr=x, len=x>: memory
1239          *      <bustype=1, addr=x, len=x>: i/o
1240          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
1241          */
1242         if (rp->regspec_bustype != 0) {
1243                 /*
1244                  * This is I/O space, which requires no particular
1245                  * processing on unmap since it isn't mapped in the
1246                  * first place.
1247                  */
1248                 return (DDI_SUCCESS);
1249         }
1250 
1251         /*
1252          * Memory space
1253          */
1254         pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1255         npages = mmu_btopr(rp->regspec_size + pgoffset);
1256         hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1257         device_arena_free(addr - pgoffset, ptob(npages));
1258 
1259         /*
1260          * Destroy the pointer - the mapping has logically gone
1261          */
1262         *vaddrp = NULL;
1263 
1264         return (DDI_SUCCESS);
1265 }
1266 
1267 
1268 /*
1269  * rootnex_map_handle()
1270  *
1271  */
1272 static int
1273 rootnex_map_handle(ddi_map_req_t *mp)
1274 {
1275         rootnex_addr_t rbase;
1276         ddi_acc_hdl_t *hp;
1277         uint_t pgoffset;
1278         struct regspec *rp;
1279         paddr_t pbase;
1280 
1281         rp = mp->map_obj.rp;
1282 
1283 #ifdef  DDI_MAP_DEBUG
1284         ddi_map_debug(
1285             "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1286             rp->regspec_bustype, rp->regspec_addr,
1287             rp->regspec_size, mp->map_handlep);
1288 #endif  /* DDI_MAP_DEBUG */
1289 
1290         /*
1291          * I/O or memory mapping:
1292          *
1293          *      <bustype=0, addr=x, len=x>: memory
1294          *      <bustype=1, addr=x, len=x>: i/o
1295          *      <bustype>1, addr=0, len=x>: x86-compatibility i/o
1296          */
1297         if (rp->regspec_bustype != 0) {
1298                 /*
1299                  * This refers to I/O space, and we don't support "mapping"
1300                  * I/O space to a user.
1301                  */
1302                 return (DDI_FAILURE);
1303         }
1304 
1305         /*
1306          * Set up the hat_flags for the mapping.
1307          */
1308         hp = mp->map_handlep;
1309 
1310         switch (hp->ah_acc.devacc_attr_endian_flags) {
1311         case DDI_NEVERSWAP_ACC:
1312                 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1313                 break;
1314         case DDI_STRUCTURE_LE_ACC:
1315                 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1316                 break;
1317         case DDI_STRUCTURE_BE_ACC:
1318                 return (DDI_FAILURE);
1319         default:
1320                 return (DDI_REGS_ACC_CONFLICT);
1321         }
1322 
1323         switch (hp->ah_acc.devacc_attr_dataorder) {
1324         case DDI_STRICTORDER_ACC:
1325                 break;
1326         case DDI_UNORDERED_OK_ACC:
1327                 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1328                 break;
1329         case DDI_MERGING_OK_ACC:
1330                 hp->ah_hat_flags |= HAT_MERGING_OK;
1331                 break;
1332         case DDI_LOADCACHING_OK_ACC:
1333                 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1334                 break;
1335         case DDI_STORECACHING_OK_ACC:
1336                 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1337                 break;
1338         default:
1339                 return (DDI_FAILURE);
1340         }
1341 
1342         rbase = (rootnex_addr_t)rp->regspec_addr &
1343             (~(rootnex_addr_t)MMU_PAGEOFFSET);
1344         pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1345 
1346         if (rp->regspec_size == 0)
1347                 return (DDI_ME_INVAL);
1348 
1349 #ifdef __xpv
1350         /*
1351          * If we're dom0, we're using a real device so we need to translate
1352          * the MA to a PA.
1353          */
1354         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1355                 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1356                     (rbase & MMU_PAGEOFFSET);
1357         } else {
1358                 pbase = rbase;
1359         }
1360 #else
1361         pbase = rbase;
1362 #endif
1363 
1364         hp->ah_pfn = mmu_btop(pbase);
1365         hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1366 
1367         return (DDI_SUCCESS);
1368 }
1369 
1370 
1371 
1372 /*
1373  * ************************
1374  *  interrupt related code
1375  * ************************
1376  */
1377 
1378 /*
1379  * rootnex_intr_ops()
1380  *      bus_intr_op() function for interrupt support
1381  */
1382 /* ARGSUSED */
1383 static int
1384 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1385     ddi_intr_handle_impl_t *hdlp, void *result)
1386 {
1387         struct intrspec                 *ispec;
1388 
1389         DDI_INTR_NEXDBG((CE_CONT,
1390             "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1391             (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1392 
1393         /* Process the interrupt operation */
1394         switch (intr_op) {
1395         case DDI_INTROP_GETCAP:
1396                 /* First check with pcplusmp */
1397                 if (psm_intr_ops == NULL)
1398                         return (DDI_FAILURE);
1399 
1400                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1401                         *(int *)result = 0;
1402                         return (DDI_FAILURE);
1403                 }
1404                 break;
1405         case DDI_INTROP_SETCAP:
1406                 if (psm_intr_ops == NULL)
1407                         return (DDI_FAILURE);
1408 
1409                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1410                         return (DDI_FAILURE);
1411                 break;
1412         case DDI_INTROP_ALLOC:
1413                 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1414                 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1415         case DDI_INTROP_FREE:
1416                 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1417                 return (rootnex_free_intr_fixed(rdip, hdlp));
1418         case DDI_INTROP_GETPRI:
1419                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1420                         return (DDI_FAILURE);
1421                 *(int *)result = ispec->intrspec_pri;
1422                 break;
1423         case DDI_INTROP_SETPRI:
1424                 /* Validate the interrupt priority passed to us */
1425                 if (*(int *)result > LOCK_LEVEL)
1426                         return (DDI_FAILURE);
1427 
1428                 /* Ensure that PSM is all initialized and ispec is ok */
1429                 if ((psm_intr_ops == NULL) ||
1430                     ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1431                         return (DDI_FAILURE);
1432 
1433                 /* Change the priority */
1434                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1435                     PSM_FAILURE)
1436                         return (DDI_FAILURE);
1437 
1438                 /* update the ispec with the new priority */
1439                 ispec->intrspec_pri =  *(int *)result;
1440                 break;
1441         case DDI_INTROP_ADDISR:
1442                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1443                         return (DDI_FAILURE);
1444                 ispec->intrspec_func = hdlp->ih_cb_func;
1445                 break;
1446         case DDI_INTROP_REMISR:
1447                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1448                         return (DDI_FAILURE);
1449                 ispec->intrspec_func = (uint_t (*)()) 0;
1450                 break;
1451         case DDI_INTROP_ENABLE:
1452                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1453                         return (DDI_FAILURE);
1454 
1455                 /* Call psmi to translate irq with the dip */
1456                 if (psm_intr_ops == NULL)
1457                         return (DDI_FAILURE);
1458 
1459                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1460                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1461                     (int *)&hdlp->ih_vector) == PSM_FAILURE)
1462                         return (DDI_FAILURE);
1463 
1464                 /* Add the interrupt handler */
1465                 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1466                     hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1467                     hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1468                         return (DDI_FAILURE);
1469                 break;
1470         case DDI_INTROP_DISABLE:
1471                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1472                         return (DDI_FAILURE);
1473 
1474                 /* Call psm_ops() to translate irq with the dip */
1475                 if (psm_intr_ops == NULL)
1476                         return (DDI_FAILURE);
1477 
1478                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1479                 (void) (*psm_intr_ops)(rdip, hdlp,
1480                     PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1481 
1482                 /* Remove the interrupt handler */
1483                 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1484                     hdlp->ih_cb_func, hdlp->ih_vector);
1485                 break;
1486         case DDI_INTROP_SETMASK:
1487                 if (psm_intr_ops == NULL)
1488                         return (DDI_FAILURE);
1489 
1490                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1491                         return (DDI_FAILURE);
1492                 break;
1493         case DDI_INTROP_CLRMASK:
1494                 if (psm_intr_ops == NULL)
1495                         return (DDI_FAILURE);
1496 
1497                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1498                         return (DDI_FAILURE);
1499                 break;
1500         case DDI_INTROP_GETPENDING:
1501                 if (psm_intr_ops == NULL)
1502                         return (DDI_FAILURE);
1503 
1504                 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1505                     result)) {
1506                         *(int *)result = 0;
1507                         return (DDI_FAILURE);
1508                 }
1509                 break;
1510         case DDI_INTROP_NAVAIL:
1511         case DDI_INTROP_NINTRS:
1512                 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1513                 if (*(int *)result == 0) {
1514                         /*
1515                          * Special case for 'pcic' driver' only. This driver
1516                          * driver is a child of 'isa' and 'rootnex' drivers.
1517                          *
1518                          * See detailed comments on this in the function
1519                          * rootnex_get_ispec().
1520                          *
1521                          * Children of 'pcic' send 'NINITR' request all the
1522                          * way to rootnex driver. But, the 'pdp->par_nintr'
1523                          * field may not initialized. So, we fake it here
1524                          * to return 1 (a la what PCMCIA nexus does).
1525                          */
1526                         if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1527                                 *(int *)result = 1;
1528                         else
1529                                 return (DDI_FAILURE);
1530                 }
1531                 break;
1532         case DDI_INTROP_SUPPORTED_TYPES:
1533                 *(int *)result = DDI_INTR_TYPE_FIXED;   /* Always ... */
1534                 break;
1535         default:
1536                 return (DDI_FAILURE);
1537         }
1538 
1539         return (DDI_SUCCESS);
1540 }
1541 
1542 
1543 /*
1544  * rootnex_get_ispec()
1545  *      convert an interrupt number to an interrupt specification.
1546  *      The interrupt number determines which interrupt spec will be
1547  *      returned if more than one exists.
1548  *
1549  *      Look into the parent private data area of the 'rdip' to find out
1550  *      the interrupt specification.  First check to make sure there is
1551  *      one that matchs "inumber" and then return a pointer to it.
1552  *
1553  *      Return NULL if one could not be found.
1554  *
1555  *      NOTE: This is needed for rootnex_intr_ops()
1556  */
1557 static struct intrspec *
1558 rootnex_get_ispec(dev_info_t *rdip, int inum)
1559 {
1560         struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1561 
1562         /*
1563          * Special case handling for drivers that provide their own
1564          * intrspec structures instead of relying on the DDI framework.
1565          *
1566          * A broken hardware driver in ON could potentially provide its
1567          * own intrspec structure, instead of relying on the hardware.
1568          * If these drivers are children of 'rootnex' then we need to
1569          * continue to provide backward compatibility to them here.
1570          *
1571          * Following check is a special case for 'pcic' driver which
1572          * was found to have broken hardwre andby provides its own intrspec.
1573          *
1574          * Verbatim comments from this driver are shown here:
1575          * "Don't use the ddi_add_intr since we don't have a
1576          * default intrspec in all cases."
1577          *
1578          * Since an 'ispec' may not be always created for it,
1579          * check for that and create one if so.
1580          *
1581          * NOTE: Currently 'pcic' is the only driver found to do this.
1582          */
1583         if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1584                 pdp->par_nintr = 1;
1585                 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1586                     pdp->par_nintr, KM_SLEEP);
1587         }
1588 
1589         /* Validate the interrupt number */
1590         if (inum >= pdp->par_nintr)
1591                 return (NULL);
1592 
1593         /* Get the interrupt structure pointer and return that */
1594         return ((struct intrspec *)&pdp->par_intr[inum]);
1595 }
1596 
1597 /*
1598  * Allocate interrupt vector for FIXED (legacy) type.
1599  */
1600 static int
1601 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1602     void *result)
1603 {
1604         struct intrspec         *ispec;
1605         ddi_intr_handle_impl_t  info_hdl;
1606         int                     ret;
1607         int                     free_phdl = 0;
1608         apic_get_type_t         type_info;
1609 
1610         if (psm_intr_ops == NULL)
1611                 return (DDI_FAILURE);
1612 
1613         if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1614                 return (DDI_FAILURE);
1615 
1616         /*
1617          * If the PSM module is "APIX" then pass the request for it
1618          * to allocate the vector now.
1619          */
1620         bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1621         info_hdl.ih_private = &type_info;
1622         if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1623             PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1624                 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1625                         free_phdl = 1;
1626                         i_ddi_alloc_intr_phdl(hdlp);
1627                 }
1628                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1629                 ret = (*psm_intr_ops)(rdip, hdlp,
1630                     PSM_INTR_OP_ALLOC_VECTORS, result);
1631                 if (free_phdl) { /* free up the phdl structure */
1632                         free_phdl = 0;
1633                         i_ddi_free_intr_phdl(hdlp);
1634                         hdlp->ih_private = NULL;
1635                 }
1636         } else {
1637                 /*
1638                  * No APIX module; fall back to the old scheme where the
1639                  * interrupt vector is allocated during ddi_enable_intr() call.
1640                  */
1641                 hdlp->ih_pri = ispec->intrspec_pri;
1642                 *(int *)result = hdlp->ih_scratch1;
1643                 ret = DDI_SUCCESS;
1644         }
1645 
1646         return (ret);
1647 }
1648 
1649 /*
1650  * Free up interrupt vector for FIXED (legacy) type.
1651  */
1652 static int
1653 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1654 {
1655         struct intrspec                 *ispec;
1656         struct ddi_parent_private_data  *pdp;
1657         ddi_intr_handle_impl_t          info_hdl;
1658         int                             ret;
1659         apic_get_type_t                 type_info;
1660 
1661         if (psm_intr_ops == NULL)
1662                 return (DDI_FAILURE);
1663 
1664         /*
1665          * If the PSM module is "APIX" then pass the request for it
1666          * to free up the vector now.
1667          */
1668         bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1669         info_hdl.ih_private = &type_info;
1670         if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1671             PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1672                 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1673                         return (DDI_FAILURE);
1674                 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1675                 ret = (*psm_intr_ops)(rdip, hdlp,
1676                     PSM_INTR_OP_FREE_VECTORS, NULL);
1677         } else {
1678                 /*
1679                  * No APIX module; fall back to the old scheme where
1680                  * the interrupt vector was already freed during
1681                  * ddi_disable_intr() call.
1682                  */
1683                 ret = DDI_SUCCESS;
1684         }
1685 
1686         pdp = ddi_get_parent_data(rdip);
1687 
1688         /*
1689          * Special case for 'pcic' driver' only.
1690          * If an intrspec was created for it, clean it up here
1691          * See detailed comments on this in the function
1692          * rootnex_get_ispec().
1693          */
1694         if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1695                 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1696                     pdp->par_nintr);
1697                 /*
1698                  * Set it to zero; so that
1699                  * DDI framework doesn't free it again
1700                  */
1701                 pdp->par_intr = NULL;
1702                 pdp->par_nintr = 0;
1703         }
1704 
1705         return (ret);
1706 }
1707 
1708 
1709 /*
1710  * ******************
1711  *  dma related code
1712  * ******************
1713  */
1714 
1715 /*ARGSUSED*/
1716 static int
1717 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1718     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1719     ddi_dma_handle_t *handlep)
1720 {
1721         uint64_t maxsegmentsize_ll;
1722         uint_t maxsegmentsize;
1723         ddi_dma_impl_t *hp;
1724         rootnex_dma_t *dma;
1725         uint64_t count_max;
1726         uint64_t seg;
1727         int kmflag;
1728         int e;
1729 
1730 
1731         /* convert our sleep flags */
1732         if (waitfp == DDI_DMA_SLEEP) {
1733                 kmflag = KM_SLEEP;
1734         } else {
1735                 kmflag = KM_NOSLEEP;
1736         }
1737 
1738         /*
1739          * We try to do only one memory allocation here. We'll do a little
1740          * pointer manipulation later. If the bind ends up taking more than
1741          * our prealloc's space, we'll have to allocate more memory in the
1742          * bind operation. Not great, but much better than before and the
1743          * best we can do with the current bind interfaces.
1744          */
1745         hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1746         if (hp == NULL)
1747                 return (DDI_DMA_NORESOURCES);
1748 
1749         /* Do our pointer manipulation now, align the structures */
1750         hp->dmai_private = (void *)(((uintptr_t)hp +
1751             (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1752         dma = (rootnex_dma_t *)hp->dmai_private;
1753         dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1754             sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1755 
1756         /* setup the handle */
1757         rootnex_clean_dmahdl(hp);
1758         hp->dmai_error.err_fep = NULL;
1759         hp->dmai_error.err_cf = NULL;
1760         dma->dp_dip = rdip;
1761         dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1762         dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1763 
1764         /*
1765          * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1766          * is being used. Set the upper limit to the seg value.
1767          * There will be enough DVMA space to always get addresses
1768          * that will match the constraints.
1769          */
1770         if (IOMMU_USED(rdip) &&
1771             (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1772                 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1773                 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1774         } else
1775                 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1776 
1777         hp->dmai_minxfer = attr->dma_attr_minxfer;
1778         hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1779         hp->dmai_rdip = rdip;
1780         hp->dmai_attr = *attr;
1781 
1782         if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1783                 dma->dp_sglinfo.si_cancross = B_FALSE;
1784         else
1785                 dma->dp_sglinfo.si_cancross = B_TRUE;
1786 
1787         /* we don't need to worry about the SPL since we do a tryenter */
1788         mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1789 
1790         /*
1791          * Figure out our maximum segment size. If the segment size is greater
1792          * than 4G, we will limit it to (4G - 1) since the max size of a dma
1793          * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1794          * dma_attr_count_max are size-1 type values.
1795          *
1796          * Maximum segment size is the largest physically contiguous chunk of
1797          * memory that we can return from a bind (i.e. the maximum size of a
1798          * single cookie).
1799          */
1800 
1801         /* handle the rollover cases */
1802         seg = attr->dma_attr_seg + 1;
1803         if (seg < attr->dma_attr_seg) {
1804                 seg = attr->dma_attr_seg;
1805         }
1806         count_max = attr->dma_attr_count_max + 1;
1807         if (count_max < attr->dma_attr_count_max) {
1808                 count_max = attr->dma_attr_count_max;
1809         }
1810 
1811         /*
1812          * granularity may or may not be a power of two. If it isn't, we can't
1813          * use a simple mask.
1814          */
1815         if (!ISP2(attr->dma_attr_granular)) {
1816                 dma->dp_granularity_power_2 = B_FALSE;
1817         } else {
1818                 dma->dp_granularity_power_2 = B_TRUE;
1819         }
1820 
1821         /*
1822          * maxxfer should be a whole multiple of granularity. If we're going to
1823          * break up a window because we're greater than maxxfer, we might as
1824          * well make sure it's maxxfer is a whole multiple so we don't have to
1825          * worry about triming the window later on for this case.
1826          */
1827         if (attr->dma_attr_granular > 1) {
1828                 if (dma->dp_granularity_power_2) {
1829                         dma->dp_maxxfer = attr->dma_attr_maxxfer -
1830                             (attr->dma_attr_maxxfer &
1831                             (attr->dma_attr_granular - 1));
1832                 } else {
1833                         dma->dp_maxxfer = attr->dma_attr_maxxfer -
1834                             (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1835                 }
1836         } else {
1837                 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1838         }
1839 
1840         maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1841         maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1842         if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1843                 maxsegmentsize = 0xFFFFFFFF;
1844         } else {
1845                 maxsegmentsize = maxsegmentsize_ll;
1846         }
1847         dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1848         dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1849 
1850         /* check the ddi_dma_attr arg to make sure it makes a little sense */
1851         if (rootnex_alloc_check_parms) {
1852                 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1853                 if (e != DDI_SUCCESS) {
1854                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1855                         (void) rootnex_dma_freehdl(dip, rdip,
1856                             (ddi_dma_handle_t)hp);
1857                         return (e);
1858                 }
1859         }
1860 
1861         *handlep = (ddi_dma_handle_t)hp;
1862 
1863         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1864         ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1865             rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1866 
1867         return (DDI_SUCCESS);
1868 }
1869 
1870 
1871 /*
1872  * rootnex_dma_allochdl()
1873  *    called from ddi_dma_alloc_handle().
1874  */
1875 static int
1876 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1877     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1878 {
1879         int retval = DDI_SUCCESS;
1880 #if defined(__amd64) && !defined(__xpv)
1881 
1882         if (IOMMU_UNITIALIZED(rdip)) {
1883                 retval = iommulib_nex_open(dip, rdip);
1884 
1885                 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1886                         return (retval);
1887         }
1888 
1889         if (IOMMU_UNUSED(rdip)) {
1890                 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1891                     handlep);
1892         } else {
1893                 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1894                     waitfp, arg, handlep);
1895         }
1896 #else
1897         retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1898             handlep);
1899 #endif
1900         switch (retval) {
1901         case DDI_DMA_NORESOURCES:
1902                 if (waitfp != DDI_DMA_DONTWAIT) {
1903                         ddi_set_callback(waitfp, arg,
1904                             &rootnex_state->r_dvma_call_list_id);
1905                 }
1906                 break;
1907         case DDI_SUCCESS:
1908                 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1909                 break;
1910         default:
1911                 break;
1912         }
1913         return (retval);
1914 }
1915 
1916 /*ARGSUSED*/
1917 static int
1918 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1919     ddi_dma_handle_t handle)
1920 {
1921         ddi_dma_impl_t *hp;
1922         rootnex_dma_t *dma;
1923 
1924 
1925         hp = (ddi_dma_impl_t *)handle;
1926         dma = (rootnex_dma_t *)hp->dmai_private;
1927 
1928         /* unbind should have been called first */
1929         ASSERT(!dma->dp_inuse);
1930 
1931         mutex_destroy(&dma->dp_mutex);
1932         kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1933 
1934         ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1935         ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1936             rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1937 
1938         return (DDI_SUCCESS);
1939 }
1940 
1941 /*
1942  * rootnex_dma_freehdl()
1943  *    called from ddi_dma_free_handle().
1944  */
1945 static int
1946 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1947 {
1948         int ret;
1949 
1950         ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1951 #if defined(__amd64) && !defined(__xpv)
1952         if (IOMMU_USED(rdip))
1953                 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1954         else
1955 #endif
1956         ret = rootnex_coredma_freehdl(dip, rdip, handle);
1957 
1958         if (rootnex_state->r_dvma_call_list_id)
1959                 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1960 
1961         return (ret);
1962 }
1963 
1964 /*ARGSUSED*/
1965 static int
1966 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1967     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1968     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1969 {
1970         rootnex_sglinfo_t *sinfo;
1971         ddi_dma_obj_t *dmao;
1972 #if defined(__amd64) && !defined(__xpv)
1973         struct dvmaseg *dvs;
1974         ddi_dma_cookie_t *cookie;
1975 #endif
1976         ddi_dma_attr_t *attr;
1977         ddi_dma_impl_t *hp;
1978         rootnex_dma_t *dma;
1979         int kmflag;
1980         int e;
1981         uint_t ncookies;
1982 
1983         hp = (ddi_dma_impl_t *)handle;
1984         dma = (rootnex_dma_t *)hp->dmai_private;
1985         dmao = &dma->dp_dma;
1986         sinfo = &dma->dp_sglinfo;
1987         attr = &hp->dmai_attr;
1988 
1989         /* convert the sleep flags */
1990         if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1991                 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1992         } else {
1993                 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1994         }
1995 
1996         hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1997 
1998         /*
1999          * This is useful for debugging a driver. Not as useful in a production
2000          * system. The only time this will fail is if you have a driver bug.
2001          */
2002         if (rootnex_bind_check_inuse) {
2003                 /*
2004                  * No one else should ever have this lock unless someone else
2005                  * is trying to use this handle. So contention on the lock
2006                  * is the same as inuse being set.
2007                  */
2008                 e = mutex_tryenter(&dma->dp_mutex);
2009                 if (e == 0) {
2010                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2011                         return (DDI_DMA_INUSE);
2012                 }
2013                 if (dma->dp_inuse) {
2014                         mutex_exit(&dma->dp_mutex);
2015                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2016                         return (DDI_DMA_INUSE);
2017                 }
2018                 dma->dp_inuse = B_TRUE;
2019                 mutex_exit(&dma->dp_mutex);
2020         }
2021 
2022         /* check the ddi_dma_attr arg to make sure it makes a little sense */
2023         if (rootnex_bind_check_parms) {
2024                 e = rootnex_valid_bind_parms(dmareq, attr);
2025                 if (e != DDI_SUCCESS) {
2026                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2027                         rootnex_clean_dmahdl(hp);
2028                         return (e);
2029                 }
2030         }
2031 
2032         /* save away the original bind info */
2033         dma->dp_dma = dmareq->dmar_object;
2034 
2035 #if defined(__amd64) && !defined(__xpv)
2036         if (IOMMU_USED(rdip)) {
2037                 dmao = &dma->dp_dvma;
2038                 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2039                 switch (e) {
2040                 case DDI_SUCCESS:
2041                         if (sinfo->si_cancross ||
2042                             dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2043                             dmao->dmao_size > sinfo->si_max_cookie_size) {
2044                                 dma->dp_dvma_used = B_TRUE;
2045                                 break;
2046                         }
2047                         sinfo->si_sgl_size = 1;
2048                         hp->dmai_rflags |= DMP_NOSYNC;
2049 
2050                         dma->dp_dvma_used = B_TRUE;
2051                         dma->dp_need_to_free_cookie = B_FALSE;
2052 
2053                         dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2054                         cookie = hp->dmai_cookie = dma->dp_cookies =
2055                             (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2056                         cookie->dmac_laddress = dvs->dvs_start +
2057                             dmao->dmao_obj.dvma_obj.dv_off;
2058                         cookie->dmac_size = dvs->dvs_len;
2059                         cookie->dmac_type = 0;
2060 
2061                         ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2062                             rdip);
2063                         goto fast;
2064                 case DDI_ENOTSUP:
2065                         break;
2066                 default:
2067                         rootnex_clean_dmahdl(hp);
2068                         return (e);
2069                 }
2070         }
2071 #endif
2072 
2073         /*
2074          * Figure out a rough estimate of what maximum number of pages
2075          * this buffer could use (a high estimate of course).
2076          */
2077         sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2078 
2079         if (dma->dp_dvma_used) {
2080                 /*
2081                  * The number of physical pages is the worst case.
2082                  *
2083                  * For DVMA, the worst case is the length divided
2084                  * by the maximum cookie length, plus 1. Add to that
2085                  * the number of segment boundaries potentially crossed, and
2086                  * the additional number of DVMA segments that was returned.
2087                  *
2088                  * In the normal case, for modern devices, si_cancross will
2089                  * be false, and dv_nseg will be 1, and the fast path will
2090                  * have been taken above.
2091                  */
2092                 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2093                     + 1;
2094                 if (sinfo->si_cancross)
2095                         ncookies +=
2096                             (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2097                 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2098 
2099                 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2100         }
2101 
2102         /*
2103          * We'll use the pre-allocated cookies for any bind that will *always*
2104          * fit (more important to be consistent, we don't want to create
2105          * additional degenerate cases).
2106          */
2107         if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2108                 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2109                 dma->dp_need_to_free_cookie = B_FALSE;
2110                 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2111                     uint_t, sinfo->si_max_pages);
2112 
2113         /*
2114          * For anything larger than that, we'll go ahead and allocate the
2115          * maximum number of pages we expect to see. Hopefuly, we won't be
2116          * seeing this path in the fast path for high performance devices very
2117          * frequently.
2118          *
2119          * a ddi bind interface that allowed the driver to provide storage to
2120          * the bind interface would speed this case up.
2121          */
2122         } else {
2123                 /*
2124                  * Save away how much memory we allocated. If we're doing a
2125                  * nosleep, the alloc could fail...
2126                  */
2127                 dma->dp_cookie_size = sinfo->si_max_pages *
2128                     sizeof (ddi_dma_cookie_t);
2129                 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2130                 if (dma->dp_cookies == NULL) {
2131                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2132                         rootnex_clean_dmahdl(hp);
2133                         return (DDI_DMA_NORESOURCES);
2134                 }
2135                 dma->dp_need_to_free_cookie = B_TRUE;
2136                 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2137                     uint_t, sinfo->si_max_pages);
2138         }
2139         hp->dmai_cookie = dma->dp_cookies;
2140 
2141         /*
2142          * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2143          * looking at the constraints in the dma structure. It will then put
2144          * some additional state about the sgl in the dma struct (i.e. is
2145          * the sgl clean, or do we need to do some munging; how many pages
2146          * need to be copied, etc.)
2147          */
2148         if (dma->dp_dvma_used)
2149                 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2150         else
2151                 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2152 
2153 out:
2154         ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2155         /* if we don't need a copy buffer, we don't need to sync */
2156         if (sinfo->si_copybuf_req == 0) {
2157                 hp->dmai_rflags |= DMP_NOSYNC;
2158         }
2159 
2160         /*
2161          * if we don't need the copybuf and we don't need to do a partial,  we
2162          * hit the fast path. All the high performance devices should be trying
2163          * to hit this path. To hit this path, a device should be able to reach
2164          * all of memory, shouldn't try to bind more than it can transfer, and
2165          * the buffer shouldn't require more cookies than the driver/device can
2166          * handle [sgllen]).
2167          *
2168          * Note that negative values of dma_attr_sgllen are supposed
2169          * to mean unlimited, but we just cast them to mean a
2170          * "ridiculous large limit".  This saves some extra checks on
2171          * hot paths.
2172          */
2173         if ((sinfo->si_copybuf_req == 0) &&
2174             (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2175             (dmao->dmao_size < dma->dp_maxxfer)) {
2176 fast:
2177                 /*
2178                  * If the driver supports FMA, insert the handle in the FMA DMA
2179                  * handle cache.
2180                  */
2181                 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2182                         hp->dmai_error.err_cf = rootnex_dma_check;
2183 
2184                 /*
2185                  * copy out the first cookie and ccountp, set the cookie
2186                  * pointer to the second cookie. The first cookie is passed
2187                  * back on the stack. Additional cookies are accessed via
2188                  * ddi_dma_nextcookie()
2189                  */
2190                 *cookiep = dma->dp_cookies[0];
2191                 *ccountp = sinfo->si_sgl_size;
2192                 hp->dmai_cookie++;
2193                 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2194                 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2195                 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2196                     uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2197                     uint_t, dmao->dmao_size, uint_t, *ccountp);
2198 
2199 
2200                 return (DDI_DMA_MAPPED);
2201         }
2202 
2203         /*
2204          * go to the slow path, we may need to alloc more memory, create
2205          * multiple windows, and munge up a sgl to make the device happy.
2206          */
2207 
2208         /*
2209          * With the IOMMU mapobject method used, we should never hit
2210          * the slow path. If we do, something is seriously wrong.
2211          * Clean up and return an error.
2212          */
2213 
2214 #if defined(__amd64) && !defined(__xpv)
2215 
2216         if (dma->dp_dvma_used) {
2217                 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2218                     &dma->dp_dvma);
2219                 e = DDI_DMA_NOMAPPING;
2220         } else {
2221 #endif
2222                 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2223                     kmflag);
2224 #if defined(__amd64) && !defined(__xpv)
2225         }
2226 #endif
2227         if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2228                 if (dma->dp_need_to_free_cookie) {
2229                         kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2230                 }
2231                 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2232                 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2233                 return (e);
2234         }
2235 
2236         /*
2237          * If the driver supports FMA, insert the handle in the FMA DMA handle
2238          * cache.
2239          */
2240         if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2241                 hp->dmai_error.err_cf = rootnex_dma_check;
2242 
2243         /* if the first window uses the copy buffer, sync it for the device */
2244         if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2245             (hp->dmai_rflags & DDI_DMA_WRITE)) {
2246                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2247                     DDI_DMA_SYNC_FORDEV);
2248         }
2249 
2250         /*
2251          * copy out the first cookie and ccountp, set the cookie pointer to the
2252          * second cookie. Make sure the partial flag is set/cleared correctly.
2253          * If we have a partial map (i.e. multiple windows), the number of
2254          * cookies we return is the number of cookies in the first window.
2255          */
2256         if (e == DDI_DMA_MAPPED) {
2257                 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2258                 *ccountp = sinfo->si_sgl_size;
2259                 hp->dmai_nwin = 1;
2260         } else {
2261                 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2262                 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2263                 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2264         }
2265         *cookiep = dma->dp_cookies[0];
2266         hp->dmai_cookie++;
2267 
2268         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2269         ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2270             rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2271             dmao->dmao_size, uint_t, *ccountp);
2272         return (e);
2273 }
2274 
2275 /*
2276  * rootnex_dma_bindhdl()
2277  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2278  */
2279 static int
2280 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2281     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2282     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2283 {
2284         int ret;
2285 #if defined(__amd64) && !defined(__xpv)
2286         if (IOMMU_USED(rdip))
2287                 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2288                     cookiep, ccountp);
2289         else
2290 #endif
2291         ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2292             cookiep, ccountp);
2293 
2294         if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2295                 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2296                     &rootnex_state->r_dvma_call_list_id);
2297         }
2298 
2299         return (ret);
2300 }
2301 
2302 
2303 
2304 /*ARGSUSED*/
2305 static int
2306 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2307     ddi_dma_handle_t handle)
2308 {
2309         ddi_dma_impl_t *hp;
2310         rootnex_dma_t *dma;
2311         int e;
2312 
2313         hp = (ddi_dma_impl_t *)handle;
2314         dma = (rootnex_dma_t *)hp->dmai_private;
2315 
2316         /* make sure the buffer wasn't free'd before calling unbind */
2317         if (rootnex_unbind_verify_buffer) {
2318                 e = rootnex_verify_buffer(dma);
2319                 if (e != DDI_SUCCESS) {
2320                         ASSERT(0);
2321                         return (DDI_FAILURE);
2322                 }
2323         }
2324 
2325         /* sync the current window before unbinding the buffer */
2326         if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2327             (hp->dmai_rflags & DDI_DMA_READ)) {
2328                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2329                     DDI_DMA_SYNC_FORCPU);
2330         }
2331 
2332         /*
2333          * cleanup and copy buffer or window state. if we didn't use the copy
2334          * buffer or windows, there won't be much to do :-)
2335          */
2336         rootnex_teardown_copybuf(dma);
2337         rootnex_teardown_windows(dma);
2338 
2339 #if defined(__amd64) && !defined(__xpv)
2340         if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2341                 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2342                     &dma->dp_dvma);
2343 #endif
2344 
2345         /*
2346          * If we had to allocate space to for the worse case sgl (it didn't
2347          * fit into our pre-allocate buffer), free that up now
2348          */
2349         if (dma->dp_need_to_free_cookie) {
2350                 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2351         }
2352 
2353         /*
2354          * clean up the handle so it's ready for the next bind (i.e. if the
2355          * handle is reused).
2356          */
2357         rootnex_clean_dmahdl(hp);
2358         hp->dmai_error.err_cf = NULL;
2359 
2360         ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2361         ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2362             rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2363 
2364         return (DDI_SUCCESS);
2365 }
2366 
2367 /*
2368  * rootnex_dma_unbindhdl()
2369  *    called from ddi_dma_unbind_handle()
2370  */
2371 /*ARGSUSED*/
2372 static int
2373 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2374     ddi_dma_handle_t handle)
2375 {
2376         int ret;
2377 
2378 #if defined(__amd64) && !defined(__xpv)
2379         if (IOMMU_USED(rdip))
2380                 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2381         else
2382 #endif
2383         ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2384 
2385         if (rootnex_state->r_dvma_call_list_id)
2386                 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2387 
2388         return (ret);
2389 }
2390 
2391 #if defined(__amd64) && !defined(__xpv)
2392 
2393 static int
2394 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2395 {
2396         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2397         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2398 
2399         if (dma->dp_sleep_flags != KM_SLEEP &&
2400             dma->dp_sleep_flags != KM_NOSLEEP)
2401                 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2402         return (dma->dp_sleep_flags);
2403 }
2404 /*ARGSUSED*/
2405 static void
2406 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2407 {
2408         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2409         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2410         rootnex_window_t *window;
2411 
2412         if (dma->dp_window) {
2413                 window = &dma->dp_window[dma->dp_current_win];
2414                 hp->dmai_cookie = window->wd_first_cookie;
2415         } else {
2416                 hp->dmai_cookie = dma->dp_cookies;
2417         }
2418         hp->dmai_cookie++;
2419 }
2420 
2421 /*ARGSUSED*/
2422 static int
2423 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2424     ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2425 {
2426         int i;
2427         int km_flags;
2428         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2429         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2430         rootnex_window_t *window;
2431         ddi_dma_cookie_t *cp;
2432         ddi_dma_cookie_t *cookie;
2433 
2434         ASSERT(*cookiepp == NULL);
2435         ASSERT(*ccountp == 0);
2436 
2437         if (dma->dp_window) {
2438                 window = &dma->dp_window[dma->dp_current_win];
2439                 cp = window->wd_first_cookie;
2440                 *ccountp = window->wd_cookie_cnt;
2441         } else {
2442                 cp = dma->dp_cookies;
2443                 *ccountp = dma->dp_sglinfo.si_sgl_size;
2444         }
2445 
2446         km_flags = rootnex_coredma_get_sleep_flags(handle);
2447         cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2448         if (cookie == NULL) {
2449                 return (DDI_DMA_NORESOURCES);
2450         }
2451 
2452         for (i = 0; i < *ccountp; i++) {
2453                 cookie[i].dmac_notused = cp[i].dmac_notused;
2454                 cookie[i].dmac_type = cp[i].dmac_type;
2455                 cookie[i].dmac_address = cp[i].dmac_address;
2456                 cookie[i].dmac_size = cp[i].dmac_size;
2457         }
2458 
2459         *cookiepp = cookie;
2460 
2461         return (DDI_SUCCESS);
2462 }
2463 
2464 /*ARGSUSED*/
2465 static int
2466 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2467     ddi_dma_cookie_t *cookiep, uint_t ccount)
2468 {
2469         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2470         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2471         rootnex_window_t *window;
2472         ddi_dma_cookie_t *cur_cookiep;
2473 
2474         ASSERT(cookiep);
2475         ASSERT(ccount != 0);
2476         ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2477 
2478         if (dma->dp_window) {
2479                 window = &dma->dp_window[dma->dp_current_win];
2480                 dma->dp_saved_cookies = window->wd_first_cookie;
2481                 window->wd_first_cookie = cookiep;
2482                 ASSERT(ccount == window->wd_cookie_cnt);
2483                 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2484                     + window->wd_first_cookie;
2485         } else {
2486                 dma->dp_saved_cookies = dma->dp_cookies;
2487                 dma->dp_cookies = cookiep;
2488                 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2489                 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2490                     + dma->dp_cookies;
2491         }
2492 
2493         dma->dp_need_to_switch_cookies = B_TRUE;
2494         hp->dmai_cookie = cur_cookiep;
2495 
2496         return (DDI_SUCCESS);
2497 }
2498 
2499 /*ARGSUSED*/
2500 static int
2501 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2502 {
2503         ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2504         rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2505         rootnex_window_t *window;
2506         ddi_dma_cookie_t *cur_cookiep;
2507         ddi_dma_cookie_t *cookie_array;
2508         uint_t ccount;
2509 
2510         /* check if cookies have not been switched */
2511         if (dma->dp_need_to_switch_cookies == B_FALSE)
2512                 return (DDI_SUCCESS);
2513 
2514         ASSERT(dma->dp_saved_cookies);
2515 
2516         if (dma->dp_window) {
2517                 window = &dma->dp_window[dma->dp_current_win];
2518                 cookie_array = window->wd_first_cookie;
2519                 window->wd_first_cookie = dma->dp_saved_cookies;
2520                 dma->dp_saved_cookies = NULL;
2521                 ccount = window->wd_cookie_cnt;
2522                 cur_cookiep = (hp->dmai_cookie - cookie_array)
2523                     + window->wd_first_cookie;
2524         } else {
2525                 cookie_array = dma->dp_cookies;
2526                 dma->dp_cookies = dma->dp_saved_cookies;
2527                 dma->dp_saved_cookies = NULL;
2528                 ccount = dma->dp_sglinfo.si_sgl_size;
2529                 cur_cookiep = (hp->dmai_cookie - cookie_array)
2530                     + dma->dp_cookies;
2531         }
2532 
2533         kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2534 
2535         hp->dmai_cookie = cur_cookiep;
2536 
2537         dma->dp_need_to_switch_cookies = B_FALSE;
2538 
2539         return (DDI_SUCCESS);
2540 }
2541 
2542 #endif
2543 
2544 static struct as *
2545 rootnex_get_as(ddi_dma_obj_t *dmao)
2546 {
2547         struct as *asp;
2548 
2549         switch (dmao->dmao_type) {
2550         case DMA_OTYP_VADDR:
2551         case DMA_OTYP_BUFVADDR:
2552                 asp = dmao->dmao_obj.virt_obj.v_as;
2553                 if (asp == NULL)
2554                         asp = &kas;
2555                 break;
2556         default:
2557                 asp = NULL;
2558                 break;
2559         }
2560         return (asp);
2561 }
2562 
2563 /*
2564  * rootnex_verify_buffer()
2565  *   verify buffer wasn't free'd
2566  */
2567 static int
2568 rootnex_verify_buffer(rootnex_dma_t *dma)
2569 {
2570         page_t **pplist;
2571         caddr_t vaddr;
2572         uint_t pcnt;
2573         uint_t poff;
2574         page_t *pp;
2575         char b;
2576         int i;
2577 
2578         /* Figure out how many pages this buffer occupies */
2579         if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2580                 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2581         } else {
2582                 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2583                 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2584         }
2585         pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2586 
2587         switch (dma->dp_dma.dmao_type) {
2588         case DMA_OTYP_PAGES:
2589                 /*
2590                  * for a linked list of pp's walk through them to make sure
2591                  * they're locked and not free.
2592                  */
2593                 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2594                 for (i = 0; i < pcnt; i++) {
2595                         if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2596                                 return (DDI_FAILURE);
2597                         }
2598                         pp = pp->p_next;
2599                 }
2600                 break;
2601 
2602         case DMA_OTYP_VADDR:
2603         case DMA_OTYP_BUFVADDR:
2604                 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2605                 /*
2606                  * for an array of pp's walk through them to make sure they're
2607                  * not free. It's possible that they may not be locked.
2608                  */
2609                 if (pplist) {
2610                         for (i = 0; i < pcnt; i++) {
2611                                 if (PP_ISFREE(pplist[i])) {
2612                                         return (DDI_FAILURE);
2613                                 }
2614                         }
2615 
2616                 /* For a virtual address, try to peek at each page */
2617                 } else {
2618                         if (rootnex_get_as(&dma->dp_dma) == &kas) {
2619                                 for (i = 0; i < pcnt; i++) {
2620                                         if (ddi_peek8(NULL, vaddr, &b) ==
2621                                             DDI_FAILURE)
2622                                                 return (DDI_FAILURE);
2623                                         vaddr += MMU_PAGESIZE;
2624                                 }
2625                         }
2626                 }
2627                 break;
2628 
2629         default:
2630                 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2631                 break;
2632         }
2633 
2634         return (DDI_SUCCESS);
2635 }
2636 
2637 
2638 /*
2639  * rootnex_clean_dmahdl()
2640  *    Clean the dma handle. This should be called on a handle alloc and an
2641  *    unbind handle. Set the handle state to the default settings.
2642  */
2643 static void
2644 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2645 {
2646         rootnex_dma_t *dma;
2647 
2648 
2649         dma = (rootnex_dma_t *)hp->dmai_private;
2650 
2651         hp->dmai_nwin = 0;
2652         dma->dp_current_cookie = 0;
2653         dma->dp_copybuf_size = 0;
2654         dma->dp_window = NULL;
2655         dma->dp_cbaddr = NULL;
2656         dma->dp_inuse = B_FALSE;
2657         dma->dp_dvma_used = B_FALSE;
2658         dma->dp_need_to_free_cookie = B_FALSE;
2659         dma->dp_need_to_switch_cookies = B_FALSE;
2660         dma->dp_saved_cookies = NULL;
2661         dma->dp_sleep_flags = KM_PANIC;
2662         dma->dp_need_to_free_window = B_FALSE;
2663         dma->dp_partial_required = B_FALSE;
2664         dma->dp_trim_required = B_FALSE;
2665         dma->dp_sglinfo.si_copybuf_req = 0;
2666 #if !defined(__amd64)
2667         dma->dp_cb_remaping = B_FALSE;
2668         dma->dp_kva = NULL;
2669 #endif
2670 
2671         /* FMA related initialization */
2672         hp->dmai_fault = 0;
2673         hp->dmai_fault_check = NULL;
2674         hp->dmai_fault_notify = NULL;
2675         hp->dmai_error.err_ena = 0;
2676         hp->dmai_error.err_status = DDI_FM_OK;
2677         hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2678         hp->dmai_error.err_ontrap = NULL;
2679 }
2680 
2681 
2682 /*
2683  * rootnex_valid_alloc_parms()
2684  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2685  */
2686 static int
2687 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2688 {
2689         if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2690             (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2691             (attr->dma_attr_granular > MMU_PAGESIZE) ||
2692             (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2693                 return (DDI_DMA_BADATTR);
2694         }
2695 
2696         if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2697                 return (DDI_DMA_BADATTR);
2698         }
2699 
2700         if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2701             MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2702             attr->dma_attr_sgllen == 0) {
2703                 return (DDI_DMA_BADATTR);
2704         }
2705 
2706         /* We should be able to DMA into every byte offset in a page */
2707         if (maxsegmentsize < MMU_PAGESIZE) {
2708                 return (DDI_DMA_BADATTR);
2709         }
2710 
2711         /* if we're bouncing on seg, seg must be <= addr_hi */
2712         if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2713             (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2714                 return (DDI_DMA_BADATTR);
2715         }
2716         return (DDI_SUCCESS);
2717 }
2718 
2719 /*
2720  * rootnex_valid_bind_parms()
2721  *    Called in ddi_dma_*_bind_handle path to validate its parameters.
2722  */
2723 /* ARGSUSED */
2724 static int
2725 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2726 {
2727 #if !defined(__amd64)
2728         /*
2729          * we only support up to a 2G-1 transfer size on 32-bit kernels so
2730          * we can track the offset for the obsoleted interfaces.
2731          */
2732         if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2733                 return (DDI_DMA_TOOBIG);
2734         }
2735 #endif
2736 
2737         return (DDI_SUCCESS);
2738 }
2739 
2740 
2741 /*
2742  * rootnex_need_bounce_seg()
2743  *    check to see if the buffer lives on both side of the seg.
2744  */
2745 static boolean_t
2746 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2747 {
2748         ddi_dma_atyp_t buftype;
2749         rootnex_addr_t raddr;
2750         boolean_t lower_addr;
2751         boolean_t upper_addr;
2752         uint64_t offset;
2753         page_t **pplist;
2754         uint64_t paddr;
2755         uint32_t psize;
2756         uint32_t size;
2757         caddr_t vaddr;
2758         uint_t pcnt;
2759         page_t *pp;
2760 
2761 
2762         /* shortcuts */
2763         pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2764         vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2765         buftype = dmar_object->dmao_type;
2766         size = dmar_object->dmao_size;
2767 
2768         lower_addr = B_FALSE;
2769         upper_addr = B_FALSE;
2770         pcnt = 0;
2771 
2772         /*
2773          * Process the first page to handle the initial offset of the buffer.
2774          * We'll use the base address we get later when we loop through all
2775          * the pages.
2776          */
2777         if (buftype == DMA_OTYP_PAGES) {
2778                 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2779                 offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2780                     MMU_PAGEOFFSET;
2781                 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2782                 psize = MIN(size, (MMU_PAGESIZE - offset));
2783                 pp = pp->p_next;
2784                 sglinfo->si_asp = NULL;
2785         } else if (pplist != NULL) {
2786                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2787                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2788                 if (sglinfo->si_asp == NULL) {
2789                         sglinfo->si_asp = &kas;
2790                 }
2791                 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2792                 paddr += offset;
2793                 psize = MIN(size, (MMU_PAGESIZE - offset));
2794                 pcnt++;
2795         } else {
2796                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2797                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2798                 if (sglinfo->si_asp == NULL) {
2799                         sglinfo->si_asp = &kas;
2800                 }
2801                 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2802                 paddr += offset;
2803                 psize = MIN(size, (MMU_PAGESIZE - offset));
2804                 vaddr += psize;
2805         }
2806 
2807         raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2808 
2809         if ((raddr + psize) > sglinfo->si_segmask) {
2810                 upper_addr = B_TRUE;
2811         } else {
2812                 lower_addr = B_TRUE;
2813         }
2814         size -= psize;
2815 
2816         /*
2817          * Walk through the rest of the pages in the buffer. Track to see
2818          * if we have pages on both sides of the segment boundary.
2819          */
2820         while (size > 0) {
2821                 /* partial or full page */
2822                 psize = MIN(size, MMU_PAGESIZE);
2823 
2824                 if (buftype == DMA_OTYP_PAGES) {
2825                         /* get the paddr from the page_t */
2826                         ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2827                         paddr = pfn_to_pa(pp->p_pagenum);
2828                         pp = pp->p_next;
2829                 } else if (pplist != NULL) {
2830                         /* index into the array of page_t's to get the paddr */
2831                         ASSERT(!PP_ISFREE(pplist[pcnt]));
2832                         paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2833                         pcnt++;
2834                 } else {
2835                         /* call into the VM to get the paddr */
2836                         paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2837                             vaddr));
2838                         vaddr += psize;
2839                 }
2840 
2841                 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2842 
2843                 if ((raddr + psize) > sglinfo->si_segmask) {
2844                         upper_addr = B_TRUE;
2845                 } else {
2846                         lower_addr = B_TRUE;
2847                 }
2848                 /*
2849                  * if the buffer lives both above and below the segment
2850                  * boundary, or the current page is the page immediately
2851                  * after the segment, we will use a copy/bounce buffer for
2852                  * all pages > seg.
2853                  */
2854                 if ((lower_addr && upper_addr) ||
2855                     (raddr == (sglinfo->si_segmask + 1))) {
2856                         return (B_TRUE);
2857                 }
2858 
2859                 size -= psize;
2860         }
2861 
2862         return (B_FALSE);
2863 }
2864 
2865 /*
2866  * rootnex_get_sgl()
2867  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2868  *    with a call to the vm layer when vm2.0 comes around...
2869  */
2870 static void
2871 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2872     rootnex_sglinfo_t *sglinfo)
2873 {
2874         ddi_dma_atyp_t buftype;
2875         rootnex_addr_t raddr;
2876         uint64_t last_page;
2877         uint64_t offset;
2878         uint64_t addrhi;
2879         uint64_t addrlo;
2880         uint64_t maxseg;
2881         page_t **pplist;
2882         uint64_t paddr;
2883         uint32_t psize;
2884         uint32_t size;
2885         caddr_t vaddr;
2886         uint_t pcnt;
2887         page_t *pp;
2888         uint_t cnt;
2889 
2890 
2891         /* shortcuts */
2892         pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2893         vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2894         maxseg = sglinfo->si_max_cookie_size;
2895         buftype = dmar_object->dmao_type;
2896         addrhi = sglinfo->si_max_addr;
2897         addrlo = sglinfo->si_min_addr;
2898         size = dmar_object->dmao_size;
2899 
2900         pcnt = 0;
2901         cnt = 0;
2902 
2903 
2904         /*
2905          * check to see if we need to use the copy buffer for pages over
2906          * the segment attr.
2907          */
2908         sglinfo->si_bounce_on_seg = B_FALSE;
2909         if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2910                 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2911                     dmar_object, sglinfo);
2912         }
2913 
2914         /*
2915          * if we were passed down a linked list of pages, i.e. pointer to
2916          * page_t, use this to get our physical address and buf offset.
2917          */
2918         if (buftype == DMA_OTYP_PAGES) {
2919                 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2920                 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2921                 offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2922                     MMU_PAGEOFFSET;
2923                 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2924                 psize = MIN(size, (MMU_PAGESIZE - offset));
2925                 pp = pp->p_next;
2926                 sglinfo->si_asp = NULL;
2927 
2928         /*
2929          * We weren't passed down a linked list of pages, but if we were passed
2930          * down an array of pages, use this to get our physical address and buf
2931          * offset.
2932          */
2933         } else if (pplist != NULL) {
2934                 ASSERT((buftype == DMA_OTYP_VADDR) ||
2935                     (buftype == DMA_OTYP_BUFVADDR));
2936 
2937                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2938                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2939                 if (sglinfo->si_asp == NULL) {
2940                         sglinfo->si_asp = &kas;
2941                 }
2942 
2943                 ASSERT(!PP_ISFREE(pplist[pcnt]));
2944                 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2945                 paddr += offset;
2946                 psize = MIN(size, (MMU_PAGESIZE - offset));
2947                 pcnt++;
2948 
2949         /*
2950          * All we have is a virtual address, we'll need to call into the VM
2951          * to get the physical address.
2952          */
2953         } else {
2954                 ASSERT((buftype == DMA_OTYP_VADDR) ||
2955                     (buftype == DMA_OTYP_BUFVADDR));
2956 
2957                 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2958                 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2959                 if (sglinfo->si_asp == NULL) {
2960                         sglinfo->si_asp = &kas;
2961                 }
2962 
2963                 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2964                 paddr += offset;
2965                 psize = MIN(size, (MMU_PAGESIZE - offset));
2966                 vaddr += psize;
2967         }
2968 
2969         raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2970 
2971         /*
2972          * Setup the first cookie with the physical address of the page and the
2973          * size of the page (which takes into account the initial offset into
2974          * the page.
2975          */
2976         sgl[cnt].dmac_laddress = raddr;
2977         sgl[cnt].dmac_size = psize;
2978         sgl[cnt].dmac_type = 0;
2979 
2980         /*
2981          * Save away the buffer offset into the page. We'll need this later in
2982          * the copy buffer code to help figure out the page index within the
2983          * buffer and the offset into the current page.
2984          */
2985         sglinfo->si_buf_offset = offset;
2986 
2987         /*
2988          * If we are using the copy buffer for anything over the segment
2989          * boundary, and this page is over the segment boundary.
2990          *   OR
2991          * if the DMA engine can't reach the physical address.
2992          */
2993         if (((sglinfo->si_bounce_on_seg) &&
2994             ((raddr + psize) > sglinfo->si_segmask)) ||
2995             ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2996                 /*
2997                  * Increase how much copy buffer we use. We always increase by
2998                  * pagesize so we don't have to worry about converting offsets.
2999                  * Set a flag in the cookies dmac_type to indicate that it uses
3000                  * the copy buffer. If this isn't the last cookie, go to the
3001                  * next cookie (since we separate each page which uses the copy
3002                  * buffer in case the copy buffer is not physically contiguous.
3003                  */
3004                 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3005                 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3006                 if ((cnt + 1) < sglinfo->si_max_pages) {
3007                         cnt++;
3008                         sgl[cnt].dmac_laddress = 0;
3009                         sgl[cnt].dmac_size = 0;
3010                         sgl[cnt].dmac_type = 0;
3011                 }
3012         }
3013 
3014         /*
3015          * save this page's physical address so we can figure out if the next
3016          * page is physically contiguous. Keep decrementing size until we are
3017          * done with the buffer.
3018          */
3019         last_page = raddr & MMU_PAGEMASK;
3020         size -= psize;
3021 
3022         while (size > 0) {
3023                 /* Get the size for this page (i.e. partial or full page) */
3024                 psize = MIN(size, MMU_PAGESIZE);
3025 
3026                 if (buftype == DMA_OTYP_PAGES) {
3027                         /* get the paddr from the page_t */
3028                         ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3029                         paddr = pfn_to_pa(pp->p_pagenum);
3030                         pp = pp->p_next;
3031                 } else if (pplist != NULL) {
3032                         /* index into the array of page_t's to get the paddr */
3033                         ASSERT(!PP_ISFREE(pplist[pcnt]));
3034                         paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3035                         pcnt++;
3036                 } else {
3037                         /* call into the VM to get the paddr */
3038                         paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3039                             vaddr));
3040                         vaddr += psize;
3041                 }
3042 
3043                 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3044 
3045                 /*
3046                  * If we are using the copy buffer for anything over the
3047                  * segment boundary, and this page is over the segment
3048                  * boundary.
3049                  *   OR
3050                  * if the DMA engine can't reach the physical address.
3051                  */
3052                 if (((sglinfo->si_bounce_on_seg) &&
3053                     ((raddr + psize) > sglinfo->si_segmask)) ||
3054                     ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3055 
3056                         sglinfo->si_copybuf_req += MMU_PAGESIZE;
3057 
3058                         /*
3059                          * if there is something in the current cookie, go to
3060                          * the next one. We only want one page in a cookie which
3061                          * uses the copybuf since the copybuf doesn't have to
3062                          * be physically contiguous.
3063                          */
3064                         if (sgl[cnt].dmac_size != 0) {
3065                                 cnt++;
3066                         }
3067                         sgl[cnt].dmac_laddress = raddr;
3068                         sgl[cnt].dmac_size = psize;
3069 #if defined(__amd64)
3070                         sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3071 #else
3072                         /*
3073                          * save the buf offset for 32-bit kernel. used in the
3074                          * obsoleted interfaces.
3075                          */
3076                         sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3077                             (dmar_object->dmao_size - size);
3078 #endif
3079                         /* if this isn't the last cookie, go to the next one */
3080                         if ((cnt + 1) < sglinfo->si_max_pages) {
3081                                 cnt++;
3082                                 sgl[cnt].dmac_laddress = 0;
3083                                 sgl[cnt].dmac_size = 0;
3084                                 sgl[cnt].dmac_type = 0;
3085                         }
3086 
3087                 /*
3088                  * this page didn't need the copy buffer, if it's not physically
3089                  * contiguous, or it would put us over a segment boundary, or it
3090                  * puts us over the max cookie size, or the current sgl doesn't
3091                  * have anything in it.
3092                  */
3093                 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3094                     !(raddr & sglinfo->si_segmask) ||
3095                     ((sgl[cnt].dmac_size + psize) > maxseg) ||
3096                     (sgl[cnt].dmac_size == 0)) {
3097                         /*
3098                          * if we're not already in a new cookie, go to the next
3099                          * cookie.
3100                          */
3101                         if (sgl[cnt].dmac_size != 0) {
3102                                 cnt++;
3103                         }
3104 
3105                         /* save the cookie information */
3106                         sgl[cnt].dmac_laddress = raddr;
3107                         sgl[cnt].dmac_size = psize;
3108 #if defined(__amd64)
3109                         sgl[cnt].dmac_type = 0;
3110 #else
3111                         /*
3112                          * save the buf offset for 32-bit kernel. used in the
3113                          * obsoleted interfaces.
3114                          */
3115                         sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3116 #endif
3117 
3118                 /*
3119                  * this page didn't need the copy buffer, it is physically
3120                  * contiguous with the last page, and it's <= the max cookie
3121                  * size.
3122                  */
3123                 } else {
3124                         sgl[cnt].dmac_size += psize;
3125 
3126                         /*
3127                          * if this exactly ==  the maximum cookie size, and
3128                          * it isn't the last cookie, go to the next cookie.
3129                          */
3130                         if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3131                             ((cnt + 1) < sglinfo->si_max_pages)) {
3132                                 cnt++;
3133                                 sgl[cnt].dmac_laddress = 0;
3134                                 sgl[cnt].dmac_size = 0;
3135                                 sgl[cnt].dmac_type = 0;
3136                         }
3137                 }
3138 
3139                 /*
3140                  * save this page's physical address so we can figure out if the
3141                  * next page is physically contiguous. Keep decrementing size
3142                  * until we are done with the buffer.
3143                  */
3144                 last_page = raddr;
3145                 size -= psize;
3146         }
3147 
3148         /* we're done, save away how many cookies the sgl has */
3149         if (sgl[cnt].dmac_size == 0) {
3150                 ASSERT(cnt < sglinfo->si_max_pages);
3151                 sglinfo->si_sgl_size = cnt;
3152         } else {
3153                 sglinfo->si_sgl_size = cnt + 1;
3154         }
3155 }
3156 
3157 static void
3158 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3159     rootnex_sglinfo_t *sglinfo)
3160 {
3161         uint64_t offset;
3162         uint64_t maxseg;
3163         uint64_t dvaddr;
3164         struct dvmaseg *dvs;
3165         uint64_t paddr;
3166         uint32_t psize, ssize;
3167         uint32_t size;
3168         uint_t cnt;
3169         int physcontig;
3170 
3171         ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3172 
3173         /* shortcuts */
3174         maxseg = sglinfo->si_max_cookie_size;
3175         size = dmar_object->dmao_size;
3176 
3177         cnt = 0;
3178         sglinfo->si_bounce_on_seg = B_FALSE;
3179 
3180         dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3181         offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3182         ssize = dvs->dvs_len;
3183         paddr = dvs->dvs_start;
3184         paddr += offset;
3185         psize = MIN(ssize, (maxseg - offset));
3186         dvaddr = paddr + psize;
3187         ssize -= psize;
3188 
3189         sgl[cnt].dmac_laddress = paddr;
3190         sgl[cnt].dmac_size = psize;
3191         sgl[cnt].dmac_type = 0;
3192 
3193         size -= psize;
3194         while (size > 0) {
3195                 if (ssize == 0) {
3196                         dvs++;
3197                         ssize = dvs->dvs_len;
3198                         dvaddr = dvs->dvs_start;
3199                         physcontig = 0;
3200                 } else
3201                         physcontig = 1;
3202 
3203                 paddr = dvaddr;
3204                 psize = MIN(ssize, maxseg);
3205                 dvaddr += psize;
3206                 ssize -= psize;
3207 
3208                 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3209                     ((sgl[cnt].dmac_size + psize) > maxseg) ||
3210                     (sgl[cnt].dmac_size == 0)) {
3211                         /*
3212                          * if we're not already in a new cookie, go to the next
3213                          * cookie.
3214                          */
3215                         if (sgl[cnt].dmac_size != 0) {
3216                                 cnt++;
3217                         }
3218 
3219                         /* save the cookie information */
3220                         sgl[cnt].dmac_laddress = paddr;
3221                         sgl[cnt].dmac_size = psize;
3222                         sgl[cnt].dmac_type = 0;
3223                 } else {
3224                         sgl[cnt].dmac_size += psize;
3225 
3226                         /*
3227                          * if this exactly ==  the maximum cookie size, and
3228                          * it isn't the last cookie, go to the next cookie.
3229                          */
3230                         if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3231                             ((cnt + 1) < sglinfo->si_max_pages)) {
3232                                 cnt++;
3233                                 sgl[cnt].dmac_laddress = 0;
3234                                 sgl[cnt].dmac_size = 0;
3235                                 sgl[cnt].dmac_type = 0;
3236                         }
3237                 }
3238                 size -= psize;
3239         }
3240 
3241         /* we're done, save away how many cookies the sgl has */
3242         if (sgl[cnt].dmac_size == 0) {
3243                 sglinfo->si_sgl_size = cnt;
3244         } else {
3245                 sglinfo->si_sgl_size = cnt + 1;
3246         }
3247 }
3248 
3249 /*
3250  * rootnex_bind_slowpath()
3251  *    Call in the bind path if the calling driver can't use the sgl without
3252  *    modifying it. We either need to use the copy buffer and/or we will end up
3253  *    with a partial bind.
3254  */
3255 static int
3256 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3257     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3258 {
3259         rootnex_sglinfo_t *sinfo;
3260         rootnex_window_t *window;
3261         ddi_dma_cookie_t *cookie;
3262         size_t copybuf_used;
3263         size_t dmac_size;
3264         boolean_t partial;
3265         off_t cur_offset;
3266         page_t *cur_pp;
3267         major_t mnum;
3268         int e;
3269         int i;
3270 
3271 
3272         sinfo = &dma->dp_sglinfo;
3273         copybuf_used = 0;
3274         partial = B_FALSE;
3275 
3276         /*
3277          * If we're using the copybuf, set the copybuf state in dma struct.
3278          * Needs to be first since it sets the copy buffer size.
3279          */
3280         if (sinfo->si_copybuf_req != 0) {
3281                 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3282                 if (e != DDI_SUCCESS) {
3283                         return (e);
3284                 }
3285         } else {
3286                 dma->dp_copybuf_size = 0;
3287         }
3288 
3289         /*
3290          * Figure out if we need to do a partial mapping. If so, figure out
3291          * if we need to trim the buffers when we munge the sgl.
3292          */
3293         if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3294             (dmao->dmao_size > dma->dp_maxxfer) ||
3295             ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3296                 dma->dp_partial_required = B_TRUE;
3297                 if (attr->dma_attr_granular != 1) {
3298                         dma->dp_trim_required = B_TRUE;
3299                 }
3300         } else {
3301                 dma->dp_partial_required = B_FALSE;
3302                 dma->dp_trim_required = B_FALSE;
3303         }
3304 
3305         /* If we need to do a partial bind, make sure the driver supports it */
3306         if (dma->dp_partial_required &&
3307             !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3308 
3309                 mnum = ddi_driver_major(dma->dp_dip);
3310                 /*
3311                  * patchable which allows us to print one warning per major
3312                  * number.
3313                  */
3314                 if ((rootnex_bind_warn) &&
3315                     ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3316                         rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3317                         cmn_err(CE_WARN, "!%s: coding error detected, the "
3318                             "driver is using ddi_dma_attr(9S) incorrectly. "
3319                             "There is a small risk of data corruption in "
3320                             "particular with large I/Os. The driver should be "
3321                             "replaced with a corrected version for proper "
3322                             "system operation. To disable this warning, add "
3323                             "'set rootnex:rootnex_bind_warn=0' to "
3324                             "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3325                 }
3326                 return (DDI_DMA_TOOBIG);
3327         }
3328 
3329         /*
3330          * we might need multiple windows, setup state to handle them. In this
3331          * code path, we will have at least one window.
3332          */
3333         e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3334         if (e != DDI_SUCCESS) {
3335                 rootnex_teardown_copybuf(dma);
3336                 return (e);
3337         }
3338 
3339         window = &dma->dp_window[0];
3340         cookie = &dma->dp_cookies[0];
3341         cur_offset = 0;
3342         rootnex_init_win(hp, dma, window, cookie, cur_offset);
3343         if (dmao->dmao_type == DMA_OTYP_PAGES) {
3344                 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3345         }
3346 
3347         /* loop though all the cookies we got back from get_sgl() */
3348         for (i = 0; i < sinfo->si_sgl_size; i++) {
3349                 /*
3350                  * If we're using the copy buffer, check this cookie and setup
3351                  * its associated copy buffer state. If this cookie uses the
3352                  * copy buffer, make sure we sync this window during dma_sync.
3353                  */
3354                 if (dma->dp_copybuf_size > 0) {
3355                         rootnex_setup_cookie(dmao, dma, cookie,
3356                             cur_offset, &copybuf_used, &cur_pp);
3357                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3358                                 window->wd_dosync = B_TRUE;
3359                         }
3360                 }
3361 
3362                 /*
3363                  * save away the cookie size, since it could be modified in
3364                  * the windowing code.
3365                  */
3366                 dmac_size = cookie->dmac_size;
3367 
3368                 /* if we went over max copybuf size */
3369                 if (dma->dp_copybuf_size &&
3370                     (copybuf_used > dma->dp_copybuf_size)) {
3371                         partial = B_TRUE;
3372                         e = rootnex_copybuf_window_boundary(hp, dma, &window,
3373                             cookie, cur_offset, &copybuf_used);
3374                         if (e != DDI_SUCCESS) {
3375                                 rootnex_teardown_copybuf(dma);
3376                                 rootnex_teardown_windows(dma);
3377                                 return (e);
3378                         }
3379 
3380                         /*
3381                          * if the coookie uses the copy buffer, make sure the
3382                          * new window we just moved to is set to sync.
3383                          */
3384                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3385                                 window->wd_dosync = B_TRUE;
3386                         }
3387                         ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3388                             dma->dp_dip);
3389 
3390                 /* if the cookie cnt == max sgllen, move to the next window */
3391                 } else if (window->wd_cookie_cnt >=
3392                     (unsigned)attr->dma_attr_sgllen) {
3393                         partial = B_TRUE;
3394                         ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3395                         e = rootnex_sgllen_window_boundary(hp, dma, &window,
3396                             cookie, attr, cur_offset);
3397                         if (e != DDI_SUCCESS) {
3398                                 rootnex_teardown_copybuf(dma);
3399                                 rootnex_teardown_windows(dma);
3400                                 return (e);
3401                         }
3402 
3403                         /*
3404                          * if the coookie uses the copy buffer, make sure the
3405                          * new window we just moved to is set to sync.
3406                          */
3407                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3408                                 window->wd_dosync = B_TRUE;
3409                         }
3410                         ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3411                             dma->dp_dip);
3412 
3413                 /* else if we will be over maxxfer */
3414                 } else if ((window->wd_size + dmac_size) >
3415                     dma->dp_maxxfer) {
3416                         partial = B_TRUE;
3417                         e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3418                             cookie);
3419                         if (e != DDI_SUCCESS) {
3420                                 rootnex_teardown_copybuf(dma);
3421                                 rootnex_teardown_windows(dma);
3422                                 return (e);
3423                         }
3424 
3425                         /*
3426                          * if the coookie uses the copy buffer, make sure the
3427                          * new window we just moved to is set to sync.
3428                          */
3429                         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3430                                 window->wd_dosync = B_TRUE;
3431                         }
3432                         ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3433                             dma->dp_dip);
3434 
3435                 /* else this cookie fits in the current window */
3436                 } else {
3437                         window->wd_cookie_cnt++;
3438                         window->wd_size += dmac_size;
3439                 }
3440 
3441                 /* track our offset into the buffer, go to the next cookie */
3442                 ASSERT(dmac_size <= dmao->dmao_size);
3443                 ASSERT(cookie->dmac_size <= dmac_size);
3444                 cur_offset += dmac_size;
3445                 cookie++;
3446         }
3447 
3448         /* if we ended up with a zero sized window in the end, clean it up */
3449         if (window->wd_size == 0) {
3450                 hp->dmai_nwin--;
3451                 window--;
3452         }
3453 
3454         ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3455 
3456         if (!partial) {
3457                 return (DDI_DMA_MAPPED);
3458         }
3459 
3460         ASSERT(dma->dp_partial_required);
3461         return (DDI_DMA_PARTIAL_MAP);
3462 }
3463 
3464 /*
3465  * rootnex_setup_copybuf()
3466  *    Called in bind slowpath. Figures out if we're going to use the copy
3467  *    buffer, and if we do, sets up the basic state to handle it.
3468  */
3469 static int
3470 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3471     rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3472 {
3473         rootnex_sglinfo_t *sinfo;
3474         ddi_dma_attr_t lattr;
3475         size_t max_copybuf;
3476         int cansleep;
3477         int e;
3478 #if !defined(__amd64)
3479         int vmflag;
3480 #endif
3481 
3482         ASSERT(!dma->dp_dvma_used);
3483 
3484         sinfo = &dma->dp_sglinfo;
3485 
3486         /* read this first so it's consistent through the routine  */
3487         max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3488 
3489         /* We need to call into the rootnex on ddi_dma_sync() */
3490         hp->dmai_rflags &= ~DMP_NOSYNC;
3491 
3492         /* make sure the copybuf size <= the max size */
3493         dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3494         ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3495 
3496 #if !defined(__amd64)
3497         /*
3498          * if we don't have kva space to copy to/from, allocate the KVA space
3499          * now. We only do this for the 32-bit kernel. We use seg kpm space for
3500          * the 64-bit kernel.
3501          */
3502         if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3503             (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3504 
3505                 /* convert the sleep flags */
3506                 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3507                         vmflag = VM_SLEEP;
3508                 } else {
3509                         vmflag = VM_NOSLEEP;
3510                 }
3511 
3512                 /* allocate Kernel VA space that we can bcopy to/from */
3513                 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3514                     vmflag);
3515                 if (dma->dp_kva == NULL) {
3516                         return (DDI_DMA_NORESOURCES);
3517                 }
3518         }
3519 #endif
3520 
3521         /* convert the sleep flags */
3522         if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3523                 cansleep = 1;
3524         } else {
3525                 cansleep = 0;
3526         }
3527 
3528         /*
3529          * Allocate the actual copy buffer. This needs to fit within the DMA
3530          * engine limits, so we can't use kmem_alloc... We don't need
3531          * contiguous memory (sgllen) since we will be forcing windows on
3532          * sgllen anyway.
3533          */
3534         lattr = *attr;
3535         lattr.dma_attr_align = MMU_PAGESIZE;
3536         lattr.dma_attr_sgllen = -1;     /* no limit */
3537         /*
3538          * if we're using the copy buffer because of seg, use that for our
3539          * upper address limit.
3540          */
3541         if (sinfo->si_bounce_on_seg) {
3542                 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3543         }
3544         e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3545             0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3546         if (e != DDI_SUCCESS) {
3547 #if !defined(__amd64)
3548                 if (dma->dp_kva != NULL) {
3549                         vmem_free(heap_arena, dma->dp_kva,
3550                             dma->dp_copybuf_size);
3551                 }
3552 #endif
3553                 return (DDI_DMA_NORESOURCES);
3554         }
3555 
3556         ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3557             size_t, dma->dp_copybuf_size);
3558 
3559         return (DDI_SUCCESS);
3560 }
3561 
3562 
3563 /*
3564  * rootnex_setup_windows()
3565  *    Called in bind slowpath to setup the window state. We always have windows
3566  *    in the slowpath. Even if the window count = 1.
3567  */
3568 static int
3569 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3570     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3571 {
3572         rootnex_window_t *windowp;
3573         rootnex_sglinfo_t *sinfo;
3574         size_t copy_state_size;
3575         size_t win_state_size;
3576         size_t state_available;
3577         size_t space_needed;
3578         uint_t copybuf_win;
3579         uint_t maxxfer_win;
3580         size_t space_used;
3581         uint_t sglwin;
3582 
3583 
3584         sinfo = &dma->dp_sglinfo;
3585 
3586         dma->dp_current_win = 0;
3587         hp->dmai_nwin = 0;
3588 
3589         /* If we don't need to do a partial, we only have one window */
3590         if (!dma->dp_partial_required) {
3591                 dma->dp_max_win = 1;
3592 
3593         /*
3594          * we need multiple windows, need to figure out the worse case number
3595          * of windows.
3596          */
3597         } else {
3598                 /*
3599                  * if we need windows because we need more copy buffer that
3600                  * we allow, the worse case number of windows we could need
3601                  * here would be (copybuf space required / copybuf space that
3602                  * we have) plus one for remainder, and plus 2 to handle the
3603                  * extra pages on the trim for the first and last pages of the
3604                  * buffer (a page is the minimum window size so under the right
3605                  * attr settings, you could have a window for each page).
3606                  * The last page will only be hit here if the size is not a
3607                  * multiple of the granularity (which theoretically shouldn't
3608                  * be the case but never has been enforced, so we could have
3609                  * broken things without it).
3610                  */
3611                 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3612                         ASSERT(dma->dp_copybuf_size > 0);
3613                         copybuf_win = (sinfo->si_copybuf_req /
3614                             dma->dp_copybuf_size) + 1 + 2;
3615                 } else {
3616                         copybuf_win = 0;
3617                 }
3618 
3619                 /*
3620                  * if we need windows because we have more cookies than the H/W
3621                  * can handle, the number of windows we would need here would
3622                  * be (cookie count / cookies count H/W supports minus 1[for
3623                  * trim]) plus one for remainder.
3624                  */
3625                 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3626                         sglwin = (sinfo->si_sgl_size /
3627                             (attr->dma_attr_sgllen - 1)) + 1;
3628                 } else {
3629                         sglwin = 0;
3630                 }
3631 
3632                 /*
3633                  * if we need windows because we're binding more memory than the
3634                  * H/W can transfer at once, the number of windows we would need
3635                  * here would be (xfer count / max xfer H/W supports) plus one
3636                  * for remainder, and plus 2 to handle the extra pages on the
3637                  * trim (see above comment about trim)
3638                  */
3639                 if (dmao->dmao_size > dma->dp_maxxfer) {
3640                         maxxfer_win = (dmao->dmao_size /
3641                             dma->dp_maxxfer) + 1 + 2;
3642                 } else {
3643                         maxxfer_win = 0;
3644                 }
3645                 dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
3646                 ASSERT(dma->dp_max_win > 0);
3647         }
3648         win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3649 
3650         /*
3651          * Get space for window and potential copy buffer state. Before we
3652          * go and allocate memory, see if we can get away with using what's
3653          * left in the pre-allocted state or the dynamically allocated sgl.
3654          */
3655         space_used = (uintptr_t)(sinfo->si_sgl_size *
3656             sizeof (ddi_dma_cookie_t));
3657 
3658         /* if we dynamically allocated space for the cookies */
3659         if (dma->dp_need_to_free_cookie) {
3660                 /* if we have more space in the pre-allocted buffer, use it */
3661                 ASSERT(space_used <= dma->dp_cookie_size);
3662                 if ((dma->dp_cookie_size - space_used) <=
3663                     rootnex_state->r_prealloc_size) {
3664                         state_available = rootnex_state->r_prealloc_size;
3665                         windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3666 
3667                 /*
3668                  * else, we have more free space in the dynamically allocated
3669                  * buffer, i.e. the buffer wasn't worse case fragmented so we
3670                  * didn't need a lot of cookies.
3671                  */
3672                 } else {
3673                         state_available = dma->dp_cookie_size - space_used;
3674                         windowp = (rootnex_window_t *)
3675                             &dma->dp_cookies[sinfo->si_sgl_size];
3676                 }
3677 
3678         /* we used the pre-alloced buffer */
3679         } else {
3680                 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3681                 state_available = rootnex_state->r_prealloc_size - space_used;
3682                 windowp = (rootnex_window_t *)
3683                     &dma->dp_cookies[sinfo->si_sgl_size];
3684         }
3685 
3686         /*
3687          * figure out how much state we need to track the copy buffer. Add an
3688          * addition 8 bytes for pointer alignemnt later.
3689          */
3690         if (dma->dp_copybuf_size > 0) {
3691                 copy_state_size = sinfo->si_max_pages *
3692                     sizeof (rootnex_pgmap_t);
3693         } else {
3694                 copy_state_size = 0;
3695         }
3696         /* add an additional 8 bytes for pointer alignment */
3697         space_needed = win_state_size + copy_state_size + 0x8;
3698 
3699         /* if we have enough space already, use it */
3700         if (state_available >= space_needed) {
3701                 dma->dp_window = windowp;
3702                 dma->dp_need_to_free_window = B_FALSE;
3703 
3704         /* not enough space, need to allocate more. */
3705         } else {
3706                 dma->dp_window = kmem_alloc(space_needed, kmflag);
3707                 if (dma->dp_window == NULL) {
3708                         return (DDI_DMA_NORESOURCES);
3709                 }
3710                 dma->dp_need_to_free_window = B_TRUE;
3711                 dma->dp_window_size = space_needed;
3712                 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3713                     dma->dp_dip, size_t, space_needed);
3714         }
3715 
3716         /*
3717          * we allocate copy buffer state and window state at the same time.
3718          * setup our copy buffer state pointers. Make sure it's aligned.
3719          */
3720         if (dma->dp_copybuf_size > 0) {
3721                 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3722                     &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3723 
3724 #if !defined(__amd64)
3725                 /*
3726                  * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3727                  * false/NULL. Should be quicker to bzero vs loop and set.
3728                  */
3729                 bzero(dma->dp_pgmap, copy_state_size);
3730 #endif
3731         } else {
3732                 dma->dp_pgmap = NULL;
3733         }
3734 
3735         return (DDI_SUCCESS);
3736 }
3737 
3738 
3739 /*
3740  * rootnex_teardown_copybuf()
3741  *    cleans up after rootnex_setup_copybuf()
3742  */
3743 static void
3744 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3745 {
3746 #if !defined(__amd64)
3747         int i;
3748 
3749         /*
3750          * if we allocated kernel heap VMEM space, go through all the pages and
3751          * map out any of the ones that we're mapped into the kernel heap VMEM
3752          * arena. Then free the VMEM space.
3753          */
3754         if (dma->dp_kva != NULL) {
3755                 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3756                         if (dma->dp_pgmap[i].pm_mapped) {
3757                                 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3758                                     MMU_PAGESIZE, HAT_UNLOAD);
3759                                 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3760                         }
3761                 }
3762 
3763                 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3764         }
3765 
3766 #endif
3767 
3768         /* if we allocated a copy buffer, free it */
3769         if (dma->dp_cbaddr != NULL) {
3770                 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3771         }
3772 }
3773 
3774 
3775 /*
3776  * rootnex_teardown_windows()
3777  *    cleans up after rootnex_setup_windows()
3778  */
3779 static void
3780 rootnex_teardown_windows(rootnex_dma_t *dma)
3781 {
3782         /*
3783          * if we had to allocate window state on the last bind (because we
3784          * didn't have enough pre-allocated space in the handle), free it.
3785          */
3786         if (dma->dp_need_to_free_window) {
3787                 kmem_free(dma->dp_window, dma->dp_window_size);
3788         }
3789 }
3790 
3791 
3792 /*
3793  * rootnex_init_win()
3794  *    Called in bind slow path during creation of a new window. Initializes
3795  *    window state to default values.
3796  */
3797 /*ARGSUSED*/
3798 static void
3799 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3800     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3801 {
3802         hp->dmai_nwin++;
3803         window->wd_dosync = B_FALSE;
3804         window->wd_offset = cur_offset;
3805         window->wd_size = 0;
3806         window->wd_first_cookie = cookie;
3807         window->wd_cookie_cnt = 0;
3808         window->wd_trim.tr_trim_first = B_FALSE;
3809         window->wd_trim.tr_trim_last = B_FALSE;
3810         window->wd_trim.tr_first_copybuf_win = B_FALSE;
3811         window->wd_trim.tr_last_copybuf_win = B_FALSE;
3812 #if !defined(__amd64)
3813         window->wd_remap_copybuf = dma->dp_cb_remaping;
3814 #endif
3815 }
3816 
3817 
3818 /*
3819  * rootnex_setup_cookie()
3820  *    Called in the bind slow path when the sgl uses the copy buffer. If any of
3821  *    the sgl uses the copy buffer, we need to go through each cookie, figure
3822  *    out if it uses the copy buffer, and if it does, save away everything we'll
3823  *    need during sync.
3824  */
3825 static void
3826 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3827     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3828     page_t **cur_pp)
3829 {
3830         boolean_t copybuf_sz_power_2;
3831         rootnex_sglinfo_t *sinfo;
3832         paddr_t paddr;
3833         uint_t pidx;
3834         uint_t pcnt;
3835         off_t poff;
3836 #if defined(__amd64)
3837         pfn_t pfn;
3838 #else
3839         page_t **pplist;
3840 #endif
3841 
3842         ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3843 
3844         sinfo = &dma->dp_sglinfo;
3845 
3846         /*
3847          * Calculate the page index relative to the start of the buffer. The
3848          * index to the current page for our buffer is the offset into the
3849          * first page of the buffer plus our current offset into the buffer
3850          * itself, shifted of course...
3851          */
3852         pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3853         ASSERT(pidx < sinfo->si_max_pages);
3854 
3855         /* if this cookie uses the copy buffer */
3856         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3857                 /*
3858                  * NOTE: we know that since this cookie uses the copy buffer, it
3859                  * is <= MMU_PAGESIZE.
3860                  */
3861 
3862                 /*
3863                  * get the offset into the page. For the 64-bit kernel, get the
3864                  * pfn which we'll use with seg kpm.
3865                  */
3866                 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3867 #if defined(__amd64)
3868                 /* mfn_to_pfn() is a NOP on i86pc */
3869                 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3870 #endif /* __amd64 */
3871 
3872                 /* figure out if the copybuf size is a power of 2 */
3873                 if (!ISP2(dma->dp_copybuf_size)) {
3874                         copybuf_sz_power_2 = B_FALSE;
3875                 } else {
3876                         copybuf_sz_power_2 = B_TRUE;
3877                 }
3878 
3879                 /* This page uses the copy buffer */
3880                 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3881 
3882                 /*
3883                  * save the copy buffer KVA that we'll use with this page.
3884                  * if we still fit within the copybuf, it's a simple add.
3885                  * otherwise, we need to wrap over using & or % accordingly.
3886                  */
3887                 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3888                         dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3889                             *copybuf_used;
3890                 } else {
3891                         if (copybuf_sz_power_2) {
3892                                 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3893                                     (uintptr_t)dma->dp_cbaddr +
3894                                     (*copybuf_used &
3895                                     (dma->dp_copybuf_size - 1)));
3896                         } else {
3897                                 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3898                                     (uintptr_t)dma->dp_cbaddr +
3899                                     (*copybuf_used % dma->dp_copybuf_size));
3900                         }
3901                 }
3902 
3903                 /*
3904                  * over write the cookie physical address with the address of
3905                  * the physical address of the copy buffer page that we will
3906                  * use.
3907                  */
3908                 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3909                     dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3910 
3911                 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3912 
3913                 /* if we have a kernel VA, it's easy, just save that address */
3914                 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3915                     (sinfo->si_asp == &kas)) {
3916                         /*
3917                          * save away the page aligned virtual address of the
3918                          * driver buffer. Offsets are handled in the sync code.
3919                          */
3920                         dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3921                             dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3922                             & MMU_PAGEMASK);
3923 #if !defined(__amd64)
3924                         /*
3925                          * we didn't need to, and will never need to map this
3926                          * page.
3927                          */
3928                         dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3929 #endif
3930 
3931                 /* we don't have a kernel VA. We need one for the bcopy. */
3932                 } else {
3933 #if defined(__amd64)
3934                         /*
3935                          * for the 64-bit kernel, it's easy. We use seg kpm to
3936                          * get a Kernel VA for the corresponding pfn.
3937                          */
3938                         dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3939 #else
3940                         /*
3941                          * for the 32-bit kernel, this is a pain. First we'll
3942                          * save away the page_t or user VA for this page. This
3943                          * is needed in rootnex_dma_win() when we switch to a
3944                          * new window which requires us to re-map the copy
3945                          * buffer.
3946                          */
3947                         pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3948                         if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3949                                 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3950                                 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3951                         } else if (pplist != NULL) {
3952                                 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3953                                 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3954                         } else {
3955                                 dma->dp_pgmap[pidx].pm_pp = NULL;
3956                                 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3957                                     (((uintptr_t)
3958                                     dmar_object->dmao_obj.virt_obj.v_addr +
3959                                     cur_offset) & MMU_PAGEMASK);
3960                         }
3961 
3962                         /*
3963                          * save away the page aligned virtual address which was
3964                          * allocated from the kernel heap arena (taking into
3965                          * account if we need more copy buffer than we alloced
3966                          * and use multiple windows to handle this, i.e. &,%).
3967                          * NOTE: there isn't and physical memory backing up this
3968                          * virtual address space currently.
3969                          */
3970                         if ((*copybuf_used + MMU_PAGESIZE) <=
3971                             dma->dp_copybuf_size) {
3972                                 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3973                                     (((uintptr_t)dma->dp_kva + *copybuf_used) &
3974                                     MMU_PAGEMASK);
3975                         } else {
3976                                 if (copybuf_sz_power_2) {
3977                                         dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3978                                             (((uintptr_t)dma->dp_kva +
3979                                             (*copybuf_used &
3980                                             (dma->dp_copybuf_size - 1))) &
3981                                             MMU_PAGEMASK);
3982                                 } else {
3983                                         dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3984                                             (((uintptr_t)dma->dp_kva +
3985                                             (*copybuf_used %
3986                                             dma->dp_copybuf_size)) &
3987                                             MMU_PAGEMASK);
3988                                 }
3989                         }
3990 
3991                         /*
3992                          * if we haven't used up the available copy buffer yet,
3993                          * map the kva to the physical page.
3994                          */
3995                         if (!dma->dp_cb_remaping && ((*copybuf_used +
3996                             MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3997                                 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3998                                 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3999                                         i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
4000                                             dma->dp_pgmap[pidx].pm_kaddr);
4001                                 } else {
4002                                         i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4003                                             sinfo->si_asp,
4004                                             dma->dp_pgmap[pidx].pm_kaddr);
4005                                 }
4006 
4007                         /*
4008                          * we've used up the available copy buffer, this page
4009                          * will have to be mapped during rootnex_dma_win() when
4010                          * we switch to a new window which requires a re-map
4011                          * the copy buffer. (32-bit kernel only)
4012                          */
4013                         } else {
4014                                 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4015                         }
4016 #endif
4017                         /* go to the next page_t */
4018                         if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4019                                 *cur_pp = (*cur_pp)->p_next;
4020                         }
4021                 }
4022 
4023                 /* add to the copy buffer count */
4024                 *copybuf_used += MMU_PAGESIZE;
4025 
4026         /*
4027          * This cookie doesn't use the copy buffer. Walk through the pages this
4028          * cookie occupies to reflect this.
4029          */
4030         } else {
4031                 /*
4032                  * figure out how many pages the cookie occupies. We need to
4033                  * use the original page offset of the buffer and the cookies
4034                  * offset in the buffer to do this.
4035                  */
4036                 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4037                 pcnt = mmu_btopr(cookie->dmac_size + poff);
4038 
4039                 while (pcnt > 0) {
4040 #if !defined(__amd64)
4041                         /*
4042                          * the 32-bit kernel doesn't have seg kpm, so we need
4043                          * to map in the driver buffer (if it didn't come down
4044                          * with a kernel VA) on the fly. Since this page doesn't
4045                          * use the copy buffer, it's not, or will it ever, have
4046                          * to be mapped in.
4047                          */
4048                         dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4049 #endif
4050                         dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4051 
4052                         /*
4053                          * we need to update pidx and cur_pp or we'll loose
4054                          * track of where we are.
4055                          */
4056                         if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4057                                 *cur_pp = (*cur_pp)->p_next;
4058                         }
4059                         pidx++;
4060                         pcnt--;
4061                 }
4062         }
4063 }
4064 
4065 
4066 /*
4067  * rootnex_sgllen_window_boundary()
4068  *    Called in the bind slow path when the next cookie causes us to exceed (in
4069  *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4070  *    length supported by the DMA H/W.
4071  */
4072 static int
4073 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4074     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4075     off_t cur_offset)
4076 {
4077         off_t new_offset;
4078         size_t trim_sz;
4079         off_t coffset;
4080 
4081 
4082         /*
4083          * if we know we'll never have to trim, it's pretty easy. Just move to
4084          * the next window and init it. We're done.
4085          */
4086         if (!dma->dp_trim_required) {
4087                 (*windowp)++;
4088                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4089                 (*windowp)->wd_cookie_cnt++;
4090                 (*windowp)->wd_size = cookie->dmac_size;
4091                 return (DDI_SUCCESS);
4092         }
4093 
4094         /* figure out how much we need to trim from the window */
4095         ASSERT(attr->dma_attr_granular != 0);
4096         if (dma->dp_granularity_power_2) {
4097                 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4098         } else {
4099                 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4100         }
4101 
4102         /* The window's a whole multiple of granularity. We're done */
4103         if (trim_sz == 0) {
4104                 (*windowp)++;
4105                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4106                 (*windowp)->wd_cookie_cnt++;
4107                 (*windowp)->wd_size = cookie->dmac_size;
4108                 return (DDI_SUCCESS);
4109         }
4110 
4111         /*
4112          * The window's not a whole multiple of granularity, since we know this
4113          * is due to the sgllen, we need to go back to the last cookie and trim
4114          * that one, add the left over part of the old cookie into the new
4115          * window, and then add in the new cookie into the new window.
4116          */
4117 
4118         /*
4119          * make sure the driver isn't making us do something bad... Trimming and
4120          * sgllen == 1 don't go together.
4121          */
4122         if (attr->dma_attr_sgllen == 1) {
4123                 return (DDI_DMA_NOMAPPING);
4124         }
4125 
4126         /*
4127          * first, setup the current window to account for the trim. Need to go
4128          * back to the last cookie for this.
4129          */
4130         cookie--;
4131         (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4132         (*windowp)->wd_trim.tr_last_cookie = cookie;
4133         (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4134         ASSERT(cookie->dmac_size > trim_sz);
4135         (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4136         (*windowp)->wd_size -= trim_sz;
4137 
4138         /* save the buffer offsets for the next window */
4139         coffset = cookie->dmac_size - trim_sz;
4140         new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4141 
4142         /*
4143          * set this now in case this is the first window. all other cases are
4144          * set in dma_win()
4145          */
4146         cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4147 
4148         /*
4149          * initialize the next window using what's left over in the previous
4150          * cookie.
4151          */
4152         (*windowp)++;
4153         rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4154         (*windowp)->wd_cookie_cnt++;
4155         (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4156         (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4157         (*windowp)->wd_trim.tr_first_size = trim_sz;
4158         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4159                 (*windowp)->wd_dosync = B_TRUE;
4160         }
4161 
4162         /*
4163          * now go back to the current cookie and add it to the new window. set
4164          * the new window size to the what was left over from the previous
4165          * cookie and what's in the current cookie.
4166          */
4167         cookie++;
4168         (*windowp)->wd_cookie_cnt++;
4169         (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4170 
4171         /*
4172          * trim plus the next cookie could put us over maxxfer (a cookie can be
4173          * a max size of maxxfer). Handle that case.
4174          */
4175         if ((*windowp)->wd_size > dma->dp_maxxfer) {
4176                 /*
4177                  * maxxfer is already a whole multiple of granularity, and this
4178                  * trim will be <= the previous trim (since a cookie can't be
4179                  * larger than maxxfer). Make things simple here.
4180                  */
4181                 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4182                 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4183                 (*windowp)->wd_trim.tr_last_cookie = cookie;
4184                 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4185                 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4186                 (*windowp)->wd_size -= trim_sz;
4187                 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4188 
4189                 /* save the buffer offsets for the next window */
4190                 coffset = cookie->dmac_size - trim_sz;
4191                 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4192 
4193                 /* setup the next window */
4194                 (*windowp)++;
4195                 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4196                 (*windowp)->wd_cookie_cnt++;
4197                 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4198                 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4199                     coffset;
4200                 (*windowp)->wd_trim.tr_first_size = trim_sz;
4201         }
4202 
4203         return (DDI_SUCCESS);
4204 }
4205 
4206 
4207 /*
4208  * rootnex_copybuf_window_boundary()
4209  *    Called in bind slowpath when we get to a window boundary because we used
4210  *    up all the copy buffer that we have.
4211  */
4212 static int
4213 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4214     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4215     size_t *copybuf_used)
4216 {
4217         rootnex_sglinfo_t *sinfo;
4218         off_t new_offset;
4219         size_t trim_sz;
4220         paddr_t paddr;
4221         off_t coffset;
4222         uint_t pidx;
4223         off_t poff;
4224 
4225 
4226         sinfo = &dma->dp_sglinfo;
4227 
4228         /*
4229          * the copy buffer should be a whole multiple of page size. We know that
4230          * this cookie is <= MMU_PAGESIZE.
4231          */
4232         ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4233 
4234         /*
4235          * from now on, all new windows in this bind need to be re-mapped during
4236          * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4237          * space...
4238          */
4239 #if !defined(__amd64)
4240         dma->dp_cb_remaping = B_TRUE;
4241 #endif
4242 
4243         /* reset copybuf used */
4244         *copybuf_used = 0;
4245 
4246         /*
4247          * if we don't have to trim (since granularity is set to 1), go to the
4248          * next window and add the current cookie to it. We know the current
4249          * cookie uses the copy buffer since we're in this code path.
4250          */
4251         if (!dma->dp_trim_required) {
4252                 (*windowp)++;
4253                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4254 
4255                 /* Add this cookie to the new window */
4256                 (*windowp)->wd_cookie_cnt++;
4257                 (*windowp)->wd_size += cookie->dmac_size;
4258                 *copybuf_used += MMU_PAGESIZE;
4259                 return (DDI_SUCCESS);
4260         }
4261 
4262         /*
4263          * *** may need to trim, figure it out.
4264          */
4265 
4266         /* figure out how much we need to trim from the window */
4267         if (dma->dp_granularity_power_2) {
4268                 trim_sz = (*windowp)->wd_size &
4269                     (hp->dmai_attr.dma_attr_granular - 1);
4270         } else {
4271                 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4272         }
4273 
4274         /*
4275          * if the window's a whole multiple of granularity, go to the next
4276          * window, init it, then add in the current cookie. We know the current
4277          * cookie uses the copy buffer since we're in this code path.
4278          */
4279         if (trim_sz == 0) {
4280                 (*windowp)++;
4281                 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4282 
4283                 /* Add this cookie to the new window */
4284                 (*windowp)->wd_cookie_cnt++;
4285                 (*windowp)->wd_size += cookie->dmac_size;
4286                 *copybuf_used += MMU_PAGESIZE;
4287                 return (DDI_SUCCESS);
4288         }
4289 
4290         /*
4291          * *** We figured it out, we definitly need to trim
4292          */
4293 
4294         /*
4295          * make sure the driver isn't making us do something bad...
4296          * Trimming and sgllen == 1 don't go together.
4297          */
4298         if (hp->dmai_attr.dma_attr_sgllen == 1) {
4299                 return (DDI_DMA_NOMAPPING);
4300         }
4301 
4302         /*
4303          * first, setup the current window to account for the trim. Need to go
4304          * back to the last cookie for this. Some of the last cookie will be in
4305          * the current window, and some of the last cookie will be in the new
4306          * window. All of the current cookie will be in the new window.
4307          */
4308         cookie--;
4309         (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4310         (*windowp)->wd_trim.tr_last_cookie = cookie;
4311         (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4312         ASSERT(cookie->dmac_size > trim_sz);
4313         (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4314         (*windowp)->wd_size -= trim_sz;
4315 
4316         /*
4317          * we're trimming the last cookie (not the current cookie). So that
4318          * last cookie may have or may not have been using the copy buffer (
4319          * we know the cookie passed in uses the copy buffer since we're in
4320          * this code path).
4321          *
4322          * If the last cookie doesn't use the copy buffer, nothing special to
4323          * do. However, if it does uses the copy buffer, it will be both the
4324          * last page in the current window and the first page in the next
4325          * window. Since we are reusing the copy buffer (and KVA space on the
4326          * 32-bit kernel), this page will use the end of the copy buffer in the
4327          * current window, and the start of the copy buffer in the next window.
4328          * Track that info... The cookie physical address was already set to
4329          * the copy buffer physical address in setup_cookie..
4330          */
4331         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4332                 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4333                     (*windowp)->wd_size) >> MMU_PAGESHIFT;
4334                 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4335                 (*windowp)->wd_trim.tr_last_pidx = pidx;
4336                 (*windowp)->wd_trim.tr_last_cbaddr =
4337                     dma->dp_pgmap[pidx].pm_cbaddr;
4338 #if !defined(__amd64)
4339                 (*windowp)->wd_trim.tr_last_kaddr =
4340                     dma->dp_pgmap[pidx].pm_kaddr;
4341 #endif
4342         }
4343 
4344         /* save the buffer offsets for the next window */
4345         coffset = cookie->dmac_size - trim_sz;
4346         new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4347 
4348         /*
4349          * set this now in case this is the first window. all other cases are
4350          * set in dma_win()
4351          */
4352         cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4353 
4354         /*
4355          * initialize the next window using what's left over in the previous
4356          * cookie.
4357          */
4358         (*windowp)++;
4359         rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4360         (*windowp)->wd_cookie_cnt++;
4361         (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4362         (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4363         (*windowp)->wd_trim.tr_first_size = trim_sz;
4364 
4365         /*
4366          * again, we're tracking if the last cookie uses the copy buffer.
4367          * read the comment above for more info on why we need to track
4368          * additional state.
4369          *
4370          * For the first cookie in the new window, we need reset the physical
4371          * address to DMA into to the start of the copy buffer plus any
4372          * initial page offset which may be present.
4373          */
4374         if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4375                 (*windowp)->wd_dosync = B_TRUE;
4376                 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4377                 (*windowp)->wd_trim.tr_first_pidx = pidx;
4378                 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4379                 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4380 
4381                 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4382                     poff;
4383                 (*windowp)->wd_trim.tr_first_paddr =
4384                     ROOTNEX_PADDR_TO_RBASE(paddr);
4385 
4386 #if !defined(__amd64)
4387                 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4388 #endif
4389                 /* account for the cookie copybuf usage in the new window */
4390                 *copybuf_used += MMU_PAGESIZE;
4391 
4392                 /*
4393                  * every piece of code has to have a hack, and here is this
4394                  * ones :-)
4395                  *
4396                  * There is a complex interaction between setup_cookie and the
4397                  * copybuf window boundary. The complexity had to be in either
4398                  * the maxxfer window, or the copybuf window, and I chose the
4399                  * copybuf code.
4400                  *
4401                  * So in this code path, we have taken the last cookie,
4402                  * virtually broken it in half due to the trim, and it happens
4403                  * to use the copybuf which further complicates life. At the
4404                  * same time, we have already setup the current cookie, which
4405                  * is now wrong. More background info: the current cookie uses
4406                  * the copybuf, so it is only a page long max. So we need to
4407                  * fix the current cookies copy buffer address, physical
4408                  * address, and kva for the 32-bit kernel. We due this by
4409                  * bumping them by page size (of course, we can't due this on
4410                  * the physical address since the copy buffer may not be
4411                  * physically contiguous).
4412                  */
4413                 cookie++;
4414                 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4415                 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4416 
4417                 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4418                     dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4419                 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4420 
4421 #if !defined(__amd64)
4422                 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4423                 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4424 #endif
4425         } else {
4426                 /* go back to the current cookie */
4427                 cookie++;
4428         }
4429 
4430         /*
4431          * add the current cookie to the new window. set the new window size to
4432          * the what was left over from the previous cookie and what's in the
4433          * current cookie.
4434          */
4435         (*windowp)->wd_cookie_cnt++;
4436         (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4437         ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4438 
4439         /*
4440          * we know that the cookie passed in always uses the copy buffer. We
4441          * wouldn't be here if it didn't.
4442          */
4443         *copybuf_used += MMU_PAGESIZE;
4444 
4445         return (DDI_SUCCESS);
4446 }
4447 
4448 
4449 /*
4450  * rootnex_maxxfer_window_boundary()
4451  *    Called in bind slowpath when we get to a window boundary because we will
4452  *    go over maxxfer.
4453  */
4454 static int
4455 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4456     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4457 {
4458         size_t dmac_size;
4459         off_t new_offset;
4460         size_t trim_sz;
4461         off_t coffset;
4462 
4463 
4464         /*
4465          * calculate how much we have to trim off of the current cookie to equal
4466          * maxxfer. We don't have to account for granularity here since our
4467          * maxxfer already takes that into account.
4468          */
4469         trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4470         ASSERT(trim_sz <= cookie->dmac_size);
4471         ASSERT(trim_sz <= dma->dp_maxxfer);
4472 
4473         /* save cookie size since we need it later and we might change it */
4474         dmac_size = cookie->dmac_size;
4475 
4476         /*
4477          * if we're not trimming the entire cookie, setup the current window to
4478          * account for the trim.
4479          */
4480         if (trim_sz < cookie->dmac_size) {
4481                 (*windowp)->wd_cookie_cnt++;
4482                 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4483                 (*windowp)->wd_trim.tr_last_cookie = cookie;
4484                 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4485                 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4486                 (*windowp)->wd_size = dma->dp_maxxfer;
4487 
4488                 /*
4489                  * set the adjusted cookie size now in case this is the first
4490                  * window. All other windows are taken care of in get win
4491                  */
4492                 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4493         }
4494 
4495         /*
4496          * coffset is the current offset within the cookie, new_offset is the
4497          * current offset with the entire buffer.
4498          */
4499         coffset = dmac_size - trim_sz;
4500         new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4501 
4502         /* initialize the next window */
4503         (*windowp)++;
4504         rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4505         (*windowp)->wd_cookie_cnt++;
4506         (*windowp)->wd_size = trim_sz;
4507         if (trim_sz < dmac_size) {
4508                 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4509                 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4510                     coffset;
4511                 (*windowp)->wd_trim.tr_first_size = trim_sz;
4512         }
4513 
4514         return (DDI_SUCCESS);
4515 }
4516 
4517 
4518 /*ARGSUSED*/
4519 static int
4520 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4521     off_t off, size_t len, uint_t cache_flags)
4522 {
4523         rootnex_sglinfo_t *sinfo;
4524         rootnex_pgmap_t *cbpage;
4525         rootnex_window_t *win;
4526         ddi_dma_impl_t *hp;
4527         rootnex_dma_t *dma;
4528         caddr_t fromaddr;
4529         caddr_t toaddr;
4530         uint_t psize;
4531         off_t offset;
4532         uint_t pidx;
4533         size_t size;
4534         off_t poff;
4535         int e;
4536 
4537 
4538         hp = (ddi_dma_impl_t *)handle;
4539         dma = (rootnex_dma_t *)hp->dmai_private;
4540         sinfo = &dma->dp_sglinfo;
4541 
4542         /*
4543          * if we don't have any windows, we don't need to sync. A copybuf
4544          * will cause us to have at least one window.
4545          */
4546         if (dma->dp_window == NULL) {
4547                 return (DDI_SUCCESS);
4548         }
4549 
4550         /* This window may not need to be sync'd */
4551         win = &dma->dp_window[dma->dp_current_win];
4552         if (!win->wd_dosync) {
4553                 return (DDI_SUCCESS);
4554         }
4555 
4556         /* handle off and len special cases */
4557         if ((off == 0) || (rootnex_sync_ignore_params)) {
4558                 offset = win->wd_offset;
4559         } else {
4560                 offset = off;
4561         }
4562         if ((len == 0) || (rootnex_sync_ignore_params)) {
4563                 size = win->wd_size;
4564         } else {
4565                 size = len;
4566         }
4567 
4568         /* check the sync args to make sure they make a little sense */
4569         if (rootnex_sync_check_parms) {
4570                 e = rootnex_valid_sync_parms(hp, win, offset, size,
4571                     cache_flags);
4572                 if (e != DDI_SUCCESS) {
4573                         ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4574                         return (DDI_FAILURE);
4575                 }
4576         }
4577 
4578         /*
4579          * special case the first page to handle the offset into the page. The
4580          * offset to the current page for our buffer is the offset into the
4581          * first page of the buffer plus our current offset into the buffer
4582          * itself, masked of course.
4583          */
4584         poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4585         psize = MIN((MMU_PAGESIZE - poff), size);
4586 
4587         /* go through all the pages that we want to sync */
4588         while (size > 0) {
4589                 /*
4590                  * Calculate the page index relative to the start of the buffer.
4591                  * The index to the current page for our buffer is the offset
4592                  * into the first page of the buffer plus our current offset
4593                  * into the buffer itself, shifted of course...
4594                  */
4595                 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4596                 ASSERT(pidx < sinfo->si_max_pages);
4597 
4598                 /*
4599                  * if this page uses the copy buffer, we need to sync it,
4600                  * otherwise, go on to the next page.
4601                  */
4602                 cbpage = &dma->dp_pgmap[pidx];
4603                 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4604                     (cbpage->pm_uses_copybuf == B_FALSE));
4605                 if (cbpage->pm_uses_copybuf) {
4606                         /* cbaddr and kaddr should be page aligned */
4607                         ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4608                             MMU_PAGEOFFSET) == 0);
4609                         ASSERT(((uintptr_t)cbpage->pm_kaddr &
4610                             MMU_PAGEOFFSET) == 0);
4611 
4612                         /*
4613                          * if we're copying for the device, we are going to
4614                          * copy from the drivers buffer and to the rootnex
4615                          * allocated copy buffer.
4616                          */
4617                         if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4618                                 fromaddr = cbpage->pm_kaddr + poff;
4619                                 toaddr = cbpage->pm_cbaddr + poff;
4620                                 ROOTNEX_DPROBE2(rootnex__sync__dev,
4621                                     dev_info_t *, dma->dp_dip, size_t, psize);
4622 
4623                         /*
4624                          * if we're copying for the cpu/kernel, we are going to
4625                          * copy from the rootnex allocated copy buffer to the
4626                          * drivers buffer.
4627                          */
4628                         } else {
4629                                 fromaddr = cbpage->pm_cbaddr + poff;
4630                                 toaddr = cbpage->pm_kaddr + poff;
4631                                 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4632                                     dev_info_t *, dma->dp_dip, size_t, psize);
4633                         }
4634 
4635                         bcopy(fromaddr, toaddr, psize);
4636                 }
4637 
4638                 /*
4639                  * decrement size until we're done, update our offset into the
4640                  * buffer, and get the next page size.
4641                  */
4642                 size -= psize;
4643                 offset += psize;
4644                 psize = MIN(MMU_PAGESIZE, size);
4645 
4646                 /* page offset is zero for the rest of this loop */
4647                 poff = 0;
4648         }
4649 
4650         return (DDI_SUCCESS);
4651 }
4652 
4653 /*
4654  * rootnex_dma_sync()
4655  *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4656  *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4657  *    is set, ddi_dma_sync() returns immediately passing back success.
4658  */
4659 /*ARGSUSED*/
4660 static int
4661 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4662     off_t off, size_t len, uint_t cache_flags)
4663 {
4664 #if defined(__amd64) && !defined(__xpv)
4665         if (IOMMU_USED(rdip)) {
4666                 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4667                     cache_flags));
4668         }
4669 #endif
4670         return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4671             cache_flags));
4672 }
4673 
4674 /*
4675  * rootnex_valid_sync_parms()
4676  *    checks the parameters passed to sync to verify they are correct.
4677  */
4678 static int
4679 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4680     off_t offset, size_t size, uint_t cache_flags)
4681 {
4682         off_t woffset;
4683 
4684 
4685         /*
4686          * the first part of the test to make sure the offset passed in is
4687          * within the window.
4688          */
4689         if (offset < win->wd_offset) {
4690                 return (DDI_FAILURE);
4691         }
4692 
4693         /*
4694          * second and last part of the test to make sure the offset and length
4695          * passed in is within the window.
4696          */
4697         woffset = offset - win->wd_offset;
4698         if ((woffset + size) > win->wd_size) {
4699                 return (DDI_FAILURE);
4700         }
4701 
4702         /*
4703          * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4704          * be set too.
4705          */
4706         if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4707             (hp->dmai_rflags & DDI_DMA_WRITE)) {
4708                 return (DDI_SUCCESS);
4709         }
4710 
4711         /*
4712          * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4713          * should be set. Also DDI_DMA_READ should be set in the flags.
4714          */
4715         if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4716             (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4717             (hp->dmai_rflags & DDI_DMA_READ)) {
4718                 return (DDI_SUCCESS);
4719         }
4720 
4721         return (DDI_FAILURE);
4722 }
4723 
4724 
4725 /*ARGSUSED*/
4726 static int
4727 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4728     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4729     uint_t *ccountp)
4730 {
4731         rootnex_window_t *window;
4732         rootnex_trim_t *trim;
4733         ddi_dma_impl_t *hp;
4734         rootnex_dma_t *dma;
4735         ddi_dma_obj_t *dmao;
4736 #if !defined(__amd64)
4737         rootnex_sglinfo_t *sinfo;
4738         rootnex_pgmap_t *pmap;
4739         uint_t pidx;
4740         uint_t pcnt;
4741         off_t poff;
4742         int i;
4743 #endif
4744 
4745 
4746         hp = (ddi_dma_impl_t *)handle;
4747         dma = (rootnex_dma_t *)hp->dmai_private;
4748 #if !defined(__amd64)
4749         sinfo = &dma->dp_sglinfo;
4750 #endif
4751 
4752         /* If we try and get a window which doesn't exist, return failure */
4753         if (win >= hp->dmai_nwin) {
4754                 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4755                 return (DDI_FAILURE);
4756         }
4757 
4758         dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4759 
4760         /*
4761          * if we don't have any windows, and they're asking for the first
4762          * window, setup the cookie pointer to the first cookie in the bind.
4763          * setup our return values, then increment the cookie since we return
4764          * the first cookie on the stack.
4765          */
4766         if (dma->dp_window == NULL) {
4767                 if (win != 0) {
4768                         ROOTNEX_DPROF_INC(
4769                             &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4770                         return (DDI_FAILURE);
4771                 }
4772                 hp->dmai_cookie = dma->dp_cookies;
4773                 *offp = 0;
4774                 *lenp = dmao->dmao_size;
4775                 *ccountp = dma->dp_sglinfo.si_sgl_size;
4776                 *cookiep = hp->dmai_cookie[0];
4777                 hp->dmai_cookie++;
4778                 return (DDI_SUCCESS);
4779         }
4780 
4781         /* sync the old window before moving on to the new one */
4782         window = &dma->dp_window[dma->dp_current_win];
4783         if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4784                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4785                     DDI_DMA_SYNC_FORCPU);
4786         }
4787 
4788 #if !defined(__amd64)
4789         /*
4790          * before we move to the next window, if we need to re-map, unmap all
4791          * the pages in this window.
4792          */
4793         if (dma->dp_cb_remaping) {
4794                 /*
4795                  * If we switch to this window again, we'll need to map in
4796                  * on the fly next time.
4797                  */
4798                 window->wd_remap_copybuf = B_TRUE;
4799 
4800                 /*
4801                  * calculate the page index into the buffer where this window
4802                  * starts, and the number of pages this window takes up.
4803                  */
4804                 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4805                     MMU_PAGESHIFT;
4806                 poff = (sinfo->si_buf_offset + window->wd_offset) &
4807                     MMU_PAGEOFFSET;
4808                 pcnt = mmu_btopr(window->wd_size + poff);
4809                 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4810 
4811                 /* unmap pages which are currently mapped in this window */
4812                 for (i = 0; i < pcnt; i++) {
4813                         if (dma->dp_pgmap[pidx].pm_mapped) {
4814                                 hat_unload(kas.a_hat,
4815                                     dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4816                                     HAT_UNLOAD);
4817                                 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4818                         }
4819                         pidx++;
4820                 }
4821         }
4822 #endif
4823 
4824         /*
4825          * Move to the new window.
4826          * NOTE: current_win must be set for sync to work right
4827          */
4828         dma->dp_current_win = win;
4829         window = &dma->dp_window[win];
4830 
4831         /* if needed, adjust the first and/or last cookies for trim */
4832         trim = &window->wd_trim;
4833         if (trim->tr_trim_first) {
4834                 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4835                 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4836 #if !defined(__amd64)
4837                 window->wd_first_cookie->dmac_type =
4838                     (window->wd_first_cookie->dmac_type &
4839                     ROOTNEX_USES_COPYBUF) + window->wd_offset;
4840 #endif
4841                 if (trim->tr_first_copybuf_win) {
4842                         dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4843                             trim->tr_first_cbaddr;
4844 #if !defined(__amd64)
4845                         dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4846                             trim->tr_first_kaddr;
4847 #endif
4848                 }
4849         }
4850         if (trim->tr_trim_last) {
4851                 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4852                 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4853                 if (trim->tr_last_copybuf_win) {
4854                         dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4855                             trim->tr_last_cbaddr;
4856 #if !defined(__amd64)
4857                         dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4858                             trim->tr_last_kaddr;
4859 #endif
4860                 }
4861         }
4862 
4863         /*
4864          * setup the cookie pointer to the first cookie in the window. setup
4865          * our return values, then increment the cookie since we return the
4866          * first cookie on the stack.
4867          */
4868         hp->dmai_cookie = window->wd_first_cookie;
4869         *offp = window->wd_offset;
4870         *lenp = window->wd_size;
4871         *ccountp = window->wd_cookie_cnt;
4872         *cookiep = hp->dmai_cookie[0];
4873         hp->dmai_cookie++;
4874 
4875 #if !defined(__amd64)
4876         /* re-map copybuf if required for this window */
4877         if (dma->dp_cb_remaping) {
4878                 /*
4879                  * calculate the page index into the buffer where this
4880                  * window starts.
4881                  */
4882                 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4883                     MMU_PAGESHIFT;
4884                 ASSERT(pidx < sinfo->si_max_pages);
4885 
4886                 /*
4887                  * the first page can get unmapped if it's shared with the
4888                  * previous window. Even if the rest of this window is already
4889                  * mapped in, we need to still check this one.
4890                  */
4891                 pmap = &dma->dp_pgmap[pidx];
4892                 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4893                         if (pmap->pm_pp != NULL) {
4894                                 pmap->pm_mapped = B_TRUE;
4895                                 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4896                         } else if (pmap->pm_vaddr != NULL) {
4897                                 pmap->pm_mapped = B_TRUE;
4898                                 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4899                                     pmap->pm_kaddr);
4900                         }
4901                 }
4902                 pidx++;
4903 
4904                 /* map in the rest of the pages if required */
4905                 if (window->wd_remap_copybuf) {
4906                         window->wd_remap_copybuf = B_FALSE;
4907 
4908                         /* figure out many pages this window takes up */
4909                         poff = (sinfo->si_buf_offset + window->wd_offset) &
4910                             MMU_PAGEOFFSET;
4911                         pcnt = mmu_btopr(window->wd_size + poff);
4912                         ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4913 
4914                         /* map pages which require it */
4915                         for (i = 1; i < pcnt; i++) {
4916                                 pmap = &dma->dp_pgmap[pidx];
4917                                 if (pmap->pm_uses_copybuf) {
4918                                         ASSERT(pmap->pm_mapped == B_FALSE);
4919                                         if (pmap->pm_pp != NULL) {
4920                                                 pmap->pm_mapped = B_TRUE;
4921                                                 i86_pp_map(pmap->pm_pp,
4922                                                     pmap->pm_kaddr);
4923                                         } else if (pmap->pm_vaddr != NULL) {
4924                                                 pmap->pm_mapped = B_TRUE;
4925                                                 i86_va_map(pmap->pm_vaddr,
4926                                                     sinfo->si_asp,
4927                                                     pmap->pm_kaddr);
4928                                         }
4929                                 }
4930                                 pidx++;
4931                         }
4932                 }
4933         }
4934 #endif
4935 
4936         /* if the new window uses the copy buffer, sync it for the device */
4937         if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4938                 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4939                     DDI_DMA_SYNC_FORDEV);
4940         }
4941 
4942         return (DDI_SUCCESS);
4943 }
4944 
4945 /*
4946  * rootnex_dma_win()
4947  *    called from ddi_dma_getwin()
4948  */
4949 /*ARGSUSED*/
4950 static int
4951 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4952     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4953     uint_t *ccountp)
4954 {
4955 #if defined(__amd64) && !defined(__xpv)
4956         if (IOMMU_USED(rdip)) {
4957                 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4958                     cookiep, ccountp));
4959         }
4960 #endif
4961 
4962         return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4963             cookiep, ccountp));
4964 }
4965 
4966 #if defined(__amd64) && !defined(__xpv)
4967 /*ARGSUSED*/
4968 static int
4969 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4970     ddi_dma_handle_t handle, void *v)
4971 {
4972         ddi_dma_impl_t *hp;
4973         rootnex_dma_t *dma;
4974 
4975         hp = (ddi_dma_impl_t *)handle;
4976         dma = (rootnex_dma_t *)hp->dmai_private;
4977         dma->dp_iommu_private = v;
4978 
4979         return (DDI_SUCCESS);
4980 }
4981 
4982 /*ARGSUSED*/
4983 static void *
4984 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4985     ddi_dma_handle_t handle)
4986 {
4987         ddi_dma_impl_t *hp;
4988         rootnex_dma_t *dma;
4989 
4990         hp = (ddi_dma_impl_t *)handle;
4991         dma = (rootnex_dma_t *)hp->dmai_private;
4992 
4993         return (dma->dp_iommu_private);
4994 }
4995 #endif
4996 
4997 /*
4998  * ************************
4999  *  obsoleted dma routines
5000  * ************************
5001  */
5002 
5003 /*
5004  * rootnex_dma_mctl()
5005  *
5006  * We don't support this legacy interface any more on x86.
5007  */
5008 /* ARGSUSED */
5009 static int
5010 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5011     enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5012     uint_t cache_flags)
5013 {
5014         /*
5015          * The only thing dma_mctl is usef for anymore is legacy SPARC
5016          * dvma and sbus-specific routines.
5017          */
5018         return (DDI_FAILURE);
5019 }
5020 
5021 /*
5022  * *********
5023  *  FMA Code
5024  * *********
5025  */
5026 
5027 /*
5028  * rootnex_fm_init()
5029  *    FMA init busop
5030  */
5031 /* ARGSUSED */
5032 static int
5033 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5034     ddi_iblock_cookie_t *ibc)
5035 {
5036         *ibc = rootnex_state->r_err_ibc;
5037 
5038         return (ddi_system_fmcap);
5039 }
5040 
5041 /*
5042  * rootnex_dma_check()
5043  *    Function called after a dma fault occurred to find out whether the
5044  *    fault address is associated with a driver that is able to handle faults
5045  *    and recover from faults.
5046  */
5047 /* ARGSUSED */
5048 static int
5049 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5050     const void *not_used)
5051 {
5052         rootnex_window_t *window;
5053         uint64_t start_addr;
5054         uint64_t fault_addr;
5055         ddi_dma_impl_t *hp;
5056         rootnex_dma_t *dma;
5057         uint64_t end_addr;
5058         size_t csize;
5059         int i;
5060         int j;
5061 
5062 
5063         /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5064         hp = (ddi_dma_impl_t *)handle;
5065         ASSERT(hp);
5066 
5067         dma = (rootnex_dma_t *)hp->dmai_private;
5068 
5069         /* Get the address that we need to search for */
5070         fault_addr = *(uint64_t *)addr;
5071 
5072         /*
5073          * if we don't have any windows, we can just walk through all the
5074          * cookies.
5075          */
5076         if (dma->dp_window == NULL) {
5077                 /* for each cookie */
5078                 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5079                         /*
5080                          * if the faulted address is within the physical address
5081                          * range of the cookie, return DDI_FM_NONFATAL.
5082                          */
5083                         if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5084                             (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5085                             dma->dp_cookies[i].dmac_size))) {
5086                                 return (DDI_FM_NONFATAL);
5087                         }
5088                 }
5089 
5090                 /* fault_addr not within this DMA handle */
5091                 return (DDI_FM_UNKNOWN);
5092         }
5093 
5094         /* we have mutiple windows, walk through each window */
5095         for (i = 0; i < hp->dmai_nwin; i++) {
5096                 window = &dma->dp_window[i];
5097 
5098                 /* Go through all the cookies in the window */
5099                 for (j = 0; j < window->wd_cookie_cnt; j++) {
5100 
5101                         start_addr = window->wd_first_cookie[j].dmac_laddress;
5102                         csize = window->wd_first_cookie[j].dmac_size;
5103 
5104                         /*
5105                          * if we are trimming the first cookie in the window,
5106                          * and this is the first cookie, adjust the start
5107                          * address and size of the cookie to account for the
5108                          * trim.
5109                          */
5110                         if (window->wd_trim.tr_trim_first && (j == 0)) {
5111                                 start_addr = window->wd_trim.tr_first_paddr;
5112                                 csize = window->wd_trim.tr_first_size;
5113                         }
5114 
5115                         /*
5116                          * if we are trimming the last cookie in the window,
5117                          * and this is the last cookie, adjust the start
5118                          * address and size of the cookie to account for the
5119                          * trim.
5120                          */
5121                         if (window->wd_trim.tr_trim_last &&
5122                             (j == (window->wd_cookie_cnt - 1))) {
5123                                 start_addr = window->wd_trim.tr_last_paddr;
5124                                 csize = window->wd_trim.tr_last_size;
5125                         }
5126 
5127                         end_addr = start_addr + csize;
5128 
5129                         /*
5130                          * if the faulted address is within the physical
5131                          * address of the cookie, return DDI_FM_NONFATAL.
5132                          */
5133                         if ((fault_addr >= start_addr) &&
5134                             (fault_addr <= end_addr)) {
5135                                 return (DDI_FM_NONFATAL);
5136                         }
5137                 }
5138         }
5139 
5140         /* fault_addr not within this DMA handle */
5141         return (DDI_FM_UNKNOWN);
5142 }
5143 
5144 /*ARGSUSED*/
5145 static int
5146 rootnex_quiesce(dev_info_t *dip)
5147 {
5148 #if defined(__amd64) && !defined(__xpv)
5149         return (immu_quiesce());
5150 #else
5151         return (DDI_SUCCESS);
5152 #endif
5153 }
5154 
5155 #if defined(__xpv)
5156 void
5157 immu_init(void)
5158 {
5159         ;
5160 }
5161 
5162 void
5163 immu_startup(void)
5164 {
5165         ;
5166 }
5167 /*ARGSUSED*/
5168 void
5169 immu_physmem_update(uint64_t addr, uint64_t size)
5170 {
5171         ;
5172 }
5173 #endif