/*- * Copyright (c) 2018 VMware, Inc. * * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) */ /* These files are provided under a dual BSD-2 Clause/GPLv2 license. When using or redistributing this file, you may do so under either license. BSD-2 Clause License Copyright (c) 2018 VMware, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. GPL License Summary Copyright (c) 2018 VMware, Inc. This program is free software; you can redistribute it and/or modify it under the terms of version 2 of the GNU General Public License as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in the file called LICENSE.GPL. */ #include __KERNEL_RCSID(0, "$NetBSD: pvscsi.c,v 1.5 2025/09/06 02:56:52 riastradh Exp $"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pvscsi.h" #define PVSCSI_DEFAULT_NUM_PAGES_REQ_RING 8 #define PVSCSI_SENSE_LENGTH 256 #define PVSCSI_MAXPHYS MAXPHYS #define PVSCSI_MAXPHYS_SEGS ((PVSCSI_MAXPHYS / PAGE_SIZE) + 1) #define PVSCSI_CMD_PER_LUN 64 #define PVSCSI_MAX_LUN 8 #define PVSCSI_MAX_TARGET 16 //#define PVSCSI_DEBUG_LOGGING #ifdef PVSCSI_DEBUG_LOGGING #define DEBUG_PRINTF(level, dev, fmt, ...) \ do { \ if (pvscsi_log_level >= (level)) { \ aprint_normal_dev((dev), (fmt), ##__VA_ARGS__); \ } \ } while(0) #else #define DEBUG_PRINTF(level, dev, fmt, ...) #endif /* PVSCSI_DEBUG_LOGGING */ struct pvscsi_softc; struct pvscsi_hcb; struct pvscsi_dma; #define VMWARE_PVSCSI_DEVSTR "VMware Paravirtual SCSI Controller" static inline uint32_t pvscsi_reg_read(struct pvscsi_softc *sc, uint32_t offset); static inline void pvscsi_reg_write(struct pvscsi_softc *sc, uint32_t offset, uint32_t val); static inline uint32_t pvscsi_read_intr_status(struct pvscsi_softc *sc); static inline void pvscsi_write_intr_status(struct pvscsi_softc *sc, uint32_t val); static inline void pvscsi_intr_enable(struct pvscsi_softc *sc); static inline void pvscsi_intr_disable(struct pvscsi_softc *sc); static void pvscsi_kick_io(struct pvscsi_softc *sc, uint8_t cdb0); static void pvscsi_write_cmd(struct pvscsi_softc *sc, uint32_t cmd, void *data, uint32_t len); static uint32_t pvscsi_get_max_targets(struct pvscsi_softc *sc); static int pvscsi_setup_req_call(struct pvscsi_softc *sc, uint32_t enable); static void pvscsi_setup_rings(struct pvscsi_softc *sc); static void pvscsi_setup_msg_ring(struct pvscsi_softc *sc); static int pvscsi_hw_supports_msg(struct pvscsi_softc *sc); static void pvscsi_timeout(void *arg); static void pvscsi_adapter_reset(struct pvscsi_softc *sc); static void pvscsi_bus_reset(struct pvscsi_softc *sc); static void pvscsi_device_reset(struct pvscsi_softc *sc, uint32_t target); static void pvscsi_abort(struct pvscsi_softc *sc, uint32_t target, struct pvscsi_hcb *hcb); static void pvscsi_process_completion(struct pvscsi_softc *sc, struct pvscsi_ring_cmp_desc *e); static void pvscsi_process_cmp_ring(struct pvscsi_softc *sc); static void pvscsi_process_msg(struct pvscsi_softc *sc, struct pvscsi_ring_msg_desc *e); static void pvscsi_process_msg_ring(struct pvscsi_softc *sc); static void pvscsi_intr_locked(struct pvscsi_softc *sc); static int pvscsi_intr(void *xsc); static void pvscsi_scsipi_request(struct scsipi_channel *, scsipi_adapter_req_t, void *); static inline uint64_t pvscsi_hcb_to_context(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb); static inline struct pvscsi_hcb *pvscsi_context_to_hcb(struct pvscsi_softc *sc, uint64_t context); static struct pvscsi_hcb * pvscsi_hcb_get(struct pvscsi_softc *sc); static void pvscsi_hcb_put(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb); static void pvscsi_dma_free(struct pvscsi_softc *sc, struct pvscsi_dma *dma); static int pvscsi_dma_alloc(struct pvscsi_softc *sc, struct pvscsi_dma *dma, bus_size_t size, bus_size_t alignment); static int pvscsi_dma_alloc_ppns(struct pvscsi_softc *sc, struct pvscsi_dma *dma, uint64_t *ppn_list, uint32_t num_pages); static void pvscsi_dma_free_per_hcb(struct pvscsi_softc *sc, uint32_t hcbs_allocated); static int pvscsi_dma_alloc_per_hcb(struct pvscsi_softc *sc); static void pvscsi_free_rings(struct pvscsi_softc *sc); static int pvscsi_allocate_rings(struct pvscsi_softc *sc); static void pvscsi_free_interrupts(struct pvscsi_softc *sc); static int pvscsi_setup_interrupts(struct pvscsi_softc *sc, const struct pci_attach_args *); static void pvscsi_free_all(struct pvscsi_softc *sc); static void pvscsi_attach(device_t, device_t, void *); static int pvscsi_detach(device_t, int); static int pvscsi_probe(device_t, cfdata_t, void *); #define pvscsi_get_tunable(_sc, _name, _value) (_value) #ifdef PVSCSI_DEBUG_LOGGING static int pvscsi_log_level = 1; #endif #define TUNABLE_INT(__x, __d) \ err = sysctl_createv(clog, 0, &rnode, &cnode, \ CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, \ #__x, SYSCTL_DESCR(__d), \ NULL, 0, &(pvscsi_ ## __x), sizeof(pvscsi_ ## __x), \ CTL_CREATE, CTL_EOL); \ if (err) \ goto fail; static int pvscsi_request_ring_pages = 0; static int pvscsi_use_msg = 1; static int pvscsi_use_msi = 1; static int pvscsi_use_msix = 1; static int pvscsi_use_req_call_threshold = 0; static int pvscsi_max_queue_depth = 0; SYSCTL_SETUP(sysctl_hw_pvscsi_setup, "sysctl hw.pvscsi setup") { int err; const struct sysctlnode *rnode; const struct sysctlnode *cnode; err = sysctl_createv(clog, 0, NULL, &rnode, CTLFLAG_PERMANENT, CTLTYPE_NODE, "pvscsi", SYSCTL_DESCR("pvscsi global controls"), NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); if (err) goto fail; #ifdef PVSCSI_DEBUG_LOGGING TUNABLE_INT(log_level, "Enable debugging output"); #endif TUNABLE_INT(request_ring_pages, "No. of pages for the request ring"); TUNABLE_INT(use_msg, "Use message passing"); TUNABLE_INT(use_msi, "Use MSI interrupt"); TUNABLE_INT(use_msix, "Use MSXI interrupt"); TUNABLE_INT(use_req_call_threshold, "Use request limit"); TUNABLE_INT(max_queue_depth, "Maximum size of request queue"); return; fail: aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); } struct pvscsi_sg_list { struct pvscsi_sg_element sge[PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT]; }; #define PVSCSI_ABORT_TIMEOUT 2 #define PVSCSI_RESET_TIMEOUT 10 #define PVSCSI_HCB_NONE 0 #define PVSCSI_HCB_ABORT 1 #define PVSCSI_HCB_DEVICE_RESET 2 #define PVSCSI_HCB_BUS_RESET 3 struct pvscsi_hcb { struct scsipi_xfer *xs; struct pvscsi_softc *sc; struct pvscsi_ring_req_desc *e; int recovery; SLIST_ENTRY(pvscsi_hcb) links; bus_dmamap_t dma_map; bus_addr_t dma_map_offset; bus_size_t dma_map_size; void *sense_buffer; bus_addr_t sense_buffer_paddr; struct pvscsi_sg_list *sg_list; bus_addr_t sg_list_paddr; bus_addr_t sg_list_offset; }; struct pvscsi_dma { bus_dmamap_t map; void *vaddr; bus_addr_t paddr; bus_size_t size; bus_dma_segment_t seg[1]; }; struct pvscsi_softc { device_t dev; kmutex_t lock; device_t sc_scsibus_dv; struct scsipi_adapter sc_adapter; struct scsipi_channel sc_channel; struct pvscsi_rings_state *rings_state; struct pvscsi_ring_req_desc *req_ring; struct pvscsi_ring_cmp_desc *cmp_ring; struct pvscsi_ring_msg_desc *msg_ring; uint32_t hcb_cnt; struct pvscsi_hcb *hcbs; SLIST_HEAD(, pvscsi_hcb) free_list; bus_dma_tag_t sc_dmat; bus_space_tag_t sc_memt; bus_space_handle_t sc_memh; bus_size_t sc_mems; bool use_msg; uint32_t max_targets; int mm_rid; int irq_id; int use_req_call_threshold; pci_chipset_tag_t sc_pc; pci_intr_handle_t * sc_pihp; void *sc_ih; uint64_t rings_state_ppn; uint32_t req_ring_num_pages; uint64_t req_ring_ppn[PVSCSI_MAX_NUM_PAGES_REQ_RING]; uint32_t cmp_ring_num_pages; uint64_t cmp_ring_ppn[PVSCSI_MAX_NUM_PAGES_CMP_RING]; uint32_t msg_ring_num_pages; uint64_t msg_ring_ppn[PVSCSI_MAX_NUM_PAGES_MSG_RING]; struct pvscsi_dma rings_state_dma; struct pvscsi_dma req_ring_dma; struct pvscsi_dma cmp_ring_dma; struct pvscsi_dma msg_ring_dma; struct pvscsi_dma sg_list_dma; struct pvscsi_dma sense_buffer_dma; }; CFATTACH_DECL3_NEW(pvscsi, sizeof(struct pvscsi_softc), pvscsi_probe, pvscsi_attach, pvscsi_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN); #define PVSCSI_DMA_SYNC_STATE(sc, dma, structptr, member, ops) \ bus_dmamap_sync((sc)->sc_dmat, (dma)->map, \ /*offset*/offsetof(__typeof__(*(structptr)), member), \ /*length*/sizeof((structptr)->member), \ (ops)) #define PVSCSI_DMA_SYNC_RING(sc, dma, ring, idx, ops) \ bus_dmamap_sync((sc)->sc_dmat, (dma)->map, \ /*offset*/sizeof(*(ring)) * (idx), \ /*length*/sizeof(*(ring)), \ (ops)) static inline uint32_t pvscsi_reg_read(struct pvscsi_softc *sc, uint32_t offset) { return (bus_space_read_4(sc->sc_memt, sc->sc_memh, offset)); } static inline void pvscsi_reg_write(struct pvscsi_softc *sc, uint32_t offset, uint32_t val) { bus_space_write_4(sc->sc_memt, sc->sc_memh, offset, val); } static inline uint32_t pvscsi_read_intr_status(struct pvscsi_softc *sc) { return (pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_INTR_STATUS)); } static inline void pvscsi_write_intr_status(struct pvscsi_softc *sc, uint32_t val) { pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_STATUS, val); } static inline void pvscsi_intr_enable(struct pvscsi_softc *sc) { uint32_t mask; mask = PVSCSI_INTR_CMPL_MASK; if (sc->use_msg) { mask |= PVSCSI_INTR_MSG_MASK; } pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_MASK, mask); } static inline void pvscsi_intr_disable(struct pvscsi_softc *sc) { pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_MASK, 0); } static void pvscsi_kick_io(struct pvscsi_softc *sc, uint8_t cdb0) { struct pvscsi_dma *s_dma; struct pvscsi_rings_state *s; DEBUG_PRINTF(2, sc->dev, "%s: cdb0 %#x\n", __func__, cdb0); if (cdb0 == SCSI_READ_6_COMMAND || cdb0 == READ_10 || cdb0 == READ_12 || cdb0 == READ_16 || cdb0 == SCSI_WRITE_6_COMMAND || cdb0 == WRITE_10 || cdb0 == WRITE_12 || cdb0 == WRITE_16) { s_dma = &sc->rings_state_dma; s = sc->rings_state; /* * Ensure the command has been published before we read * req_cons_idx to test whether we need to kick the * host. */ paravirt_membar_sync(); PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, req_cons_idx, BUS_DMASYNC_POSTREAD); DEBUG_PRINTF(2, sc->dev, "%s req prod %d cons %d\n", __func__, s->req_prod_idx, s->req_cons_idx); if (!sc->use_req_call_threshold || (s->req_prod_idx - s->req_cons_idx) >= s->req_call_threshold) { pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_KICK_RW_IO, 0); DEBUG_PRINTF(2, sc->dev, "kicked\n"); } else { DEBUG_PRINTF(2, sc->dev, "wtf\n"); } PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, req_cons_idx, BUS_DMASYNC_PREREAD); } else { s = sc->rings_state; /* * XXX req_cons_idx in debug log might be stale, but no * need for DMA sync otherwise in this branch */ DEBUG_PRINTF(1, sc->dev, "%s req prod %d cons %d not checked\n", __func__, s->req_prod_idx, s->req_cons_idx); pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0); } } static void pvscsi_write_cmd(struct pvscsi_softc *sc, uint32_t cmd, void *data, uint32_t len) { uint32_t *data_ptr; int i; KASSERTMSG(len % sizeof(uint32_t) == 0, "command size not a multiple of 4"); data_ptr = data; len /= sizeof(uint32_t); pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, cmd); for (i = 0; i < len; ++i) { pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND_DATA, data_ptr[i]); } } static inline uint64_t pvscsi_hcb_to_context(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb) { /* Offset by 1 because context must not be 0 */ return (hcb - sc->hcbs + 1); } static inline struct pvscsi_hcb* pvscsi_context_to_hcb(struct pvscsi_softc *sc, uint64_t context) { return (sc->hcbs + (context - 1)); } static struct pvscsi_hcb * pvscsi_hcb_get(struct pvscsi_softc *sc) { struct pvscsi_hcb *hcb; KASSERT(mutex_owned(&sc->lock)); hcb = SLIST_FIRST(&sc->free_list); if (hcb) { SLIST_REMOVE_HEAD(&sc->free_list, links); } return (hcb); } static void pvscsi_hcb_put(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb) { KASSERT(mutex_owned(&sc->lock)); hcb->xs = NULL; hcb->e = NULL; hcb->recovery = PVSCSI_HCB_NONE; SLIST_INSERT_HEAD(&sc->free_list, hcb, links); } static uint32_t pvscsi_get_max_targets(struct pvscsi_softc *sc) { uint32_t max_targets; pvscsi_write_cmd(sc, PVSCSI_CMD_GET_MAX_TARGETS, NULL, 0); max_targets = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); if (max_targets == ~0) { max_targets = 16; } return (max_targets); } static int pvscsi_setup_req_call(struct pvscsi_softc *sc, uint32_t enable) { uint32_t status; struct pvscsi_cmd_desc_setup_req_call cmd; if (!pvscsi_get_tunable(sc, "pvscsi_use_req_call_threshold", pvscsi_use_req_call_threshold)) { return (0); } pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, PVSCSI_CMD_SETUP_REQCALLTHRESHOLD); status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); if (status != -1) { memset(&cmd, 0, sizeof(cmd)); cmd.enable = enable; pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_REQCALLTHRESHOLD, &cmd, sizeof(cmd)); status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); /* * After setup, sync req_call_threshold before use. * After this point it should be stable, so no need to * sync again during use. */ PVSCSI_DMA_SYNC_STATE(sc, &sc->rings_state_dma, sc->rings_state, req_call_threshold, BUS_DMASYNC_POSTREAD); return (status != 0); } else { return (0); } } static void pvscsi_dma_free(struct pvscsi_softc *sc, struct pvscsi_dma *dma) { bus_dmamap_unload(sc->sc_dmat, dma->map); bus_dmamem_unmap(sc->sc_dmat, dma->vaddr, dma->size); bus_dmamap_destroy(sc->sc_dmat, dma->map); bus_dmamem_free(sc->sc_dmat, dma->seg, __arraycount(dma->seg)); memset(dma, 0, sizeof(*dma)); } static int pvscsi_dma_alloc(struct pvscsi_softc *sc, struct pvscsi_dma *dma, bus_size_t size, bus_size_t alignment) { int error; int nsegs; memset(dma, 0, sizeof(*dma)); error = bus_dmamem_alloc(sc->sc_dmat, size, alignment, 0, dma->seg, __arraycount(dma->seg), &nsegs, BUS_DMA_WAITOK); if (error) { aprint_normal_dev(sc->dev, "error allocating dma mem, error %d\n", error); goto fail; } error = bus_dmamem_map(sc->sc_dmat, dma->seg, nsegs, size, &dma->vaddr, BUS_DMA_WAITOK); if (error != 0) { device_printf(sc->dev, "Failed to map DMA memory\n"); goto dmamemmap_fail; } error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, BUS_DMA_WAITOK, &dma->map); if (error != 0) { device_printf(sc->dev, "Failed to create DMA map\n"); goto dmamapcreate_fail; } error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->vaddr, size, NULL, BUS_DMA_WAITOK); if (error) { aprint_normal_dev(sc->dev, "error mapping dma mam, error %d\n", error); goto dmamapload_fail; } dma->paddr = dma->map->dm_segs[0].ds_addr; dma->size = size; return 0; dmamapload_fail: bus_dmamap_destroy(sc->sc_dmat, dma->map); dmamapcreate_fail: bus_dmamem_unmap(sc->sc_dmat, dma->vaddr, dma->size); dmamemmap_fail: bus_dmamem_free(sc->sc_dmat, dma->seg, __arraycount(dma->seg)); fail: return (error); } static int pvscsi_dma_alloc_ppns(struct pvscsi_softc *sc, struct pvscsi_dma *dma, uint64_t *ppn_list, uint32_t num_pages) { int error; uint32_t i; uint64_t ppn; error = pvscsi_dma_alloc(sc, dma, num_pages * PAGE_SIZE, PAGE_SIZE); if (error) { aprint_normal_dev(sc->dev, "Error allocating pages, error %d\n", error); return (error); } memset(dma->vaddr, 0, num_pages * PAGE_SIZE); bus_dmamap_sync(sc->sc_dmat, dma->map, 0, num_pages * PAGE_SIZE, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); ppn = dma->paddr >> PAGE_SHIFT; for (i = 0; i < num_pages; i++) { ppn_list[i] = ppn + i; } return (0); } static void pvscsi_dma_free_per_hcb(struct pvscsi_softc *sc, uint32_t hcbs_allocated) { int i; struct pvscsi_hcb *hcb; for (i = 0; i < hcbs_allocated; ++i) { hcb = sc->hcbs + i; bus_dmamap_destroy(sc->sc_dmat, hcb->dma_map); }; pvscsi_dma_free(sc, &sc->sense_buffer_dma); pvscsi_dma_free(sc, &sc->sg_list_dma); } static int pvscsi_dma_alloc_per_hcb(struct pvscsi_softc *sc) { int i; int error; struct pvscsi_hcb *hcb; i = 0; error = pvscsi_dma_alloc(sc, &sc->sg_list_dma, sizeof(struct pvscsi_sg_list) * sc->hcb_cnt, 1); if (error) { aprint_normal_dev(sc->dev, "Error allocation sg list DMA memory, error %d\n", error); goto fail; } error = pvscsi_dma_alloc(sc, &sc->sense_buffer_dma, PVSCSI_SENSE_LENGTH * sc->hcb_cnt, 1); if (error) { aprint_normal_dev(sc->dev, "Error allocation buffer DMA memory, error %d\n", error); goto fail; } for (i = 0; i < sc->hcb_cnt; ++i) { hcb = sc->hcbs + i; error = bus_dmamap_create(sc->sc_dmat, PVSCSI_MAXPHYS, PVSCSI_MAXPHYS_SEGS, PVSCSI_MAXPHYS, 0, BUS_DMA_WAITOK, &hcb->dma_map); if (error) { aprint_normal_dev(sc->dev, "Error creating dma map for hcb %d, error %d\n", i, error); goto fail; } hcb->sc = sc; hcb->dma_map_offset = PVSCSI_SENSE_LENGTH * i; hcb->dma_map_size = PVSCSI_SENSE_LENGTH; hcb->sense_buffer = (void *)((char *)sc->sense_buffer_dma.vaddr + PVSCSI_SENSE_LENGTH * i); hcb->sense_buffer_paddr = sc->sense_buffer_dma.paddr + PVSCSI_SENSE_LENGTH * i; hcb->sg_list = (struct pvscsi_sg_list *)((char *)sc->sg_list_dma.vaddr + sizeof(struct pvscsi_sg_list) * i); hcb->sg_list_paddr = sc->sg_list_dma.paddr + sizeof(struct pvscsi_sg_list) * i; hcb->sg_list_offset = sizeof(struct pvscsi_sg_list) * i; } SLIST_INIT(&sc->free_list); for (i = (sc->hcb_cnt - 1); i >= 0; --i) { hcb = sc->hcbs + i; SLIST_INSERT_HEAD(&sc->free_list, hcb, links); } fail: if (error) { pvscsi_dma_free_per_hcb(sc, i); } return (error); } static void pvscsi_free_rings(struct pvscsi_softc *sc) { bus_dmamap_sync(sc->sc_dmat, sc->rings_state_dma.map, 0, sc->rings_state_dma.size, BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(sc->sc_dmat, sc->req_ring_dma.map, 0, sc->req_ring_dma.size, BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); bus_dmamap_sync(sc->sc_dmat, sc->cmp_ring_dma.map, 0, sc->cmp_ring_dma.size, BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); pvscsi_dma_free(sc, &sc->rings_state_dma); pvscsi_dma_free(sc, &sc->req_ring_dma); pvscsi_dma_free(sc, &sc->cmp_ring_dma); if (sc->use_msg) { pvscsi_dma_free(sc, &sc->msg_ring_dma); } } static int pvscsi_allocate_rings(struct pvscsi_softc *sc) { int error; error = pvscsi_dma_alloc_ppns(sc, &sc->rings_state_dma, &sc->rings_state_ppn, 1); if (error) { aprint_normal_dev(sc->dev, "Error allocating rings state, error = %d\n", error); goto fail; } sc->rings_state = sc->rings_state_dma.vaddr; error = pvscsi_dma_alloc_ppns(sc, &sc->req_ring_dma, sc->req_ring_ppn, sc->req_ring_num_pages); if (error) { aprint_normal_dev(sc->dev, "Error allocating req ring pages, error = %d\n", error); goto fail; } sc->req_ring = sc->req_ring_dma.vaddr; error = pvscsi_dma_alloc_ppns(sc, &sc->cmp_ring_dma, sc->cmp_ring_ppn, sc->cmp_ring_num_pages); if (error) { aprint_normal_dev(sc->dev, "Error allocating cmp ring pages, error = %d\n", error); goto fail; } sc->cmp_ring = sc->cmp_ring_dma.vaddr; sc->msg_ring = NULL; if (sc->use_msg) { error = pvscsi_dma_alloc_ppns(sc, &sc->msg_ring_dma, sc->msg_ring_ppn, sc->msg_ring_num_pages); if (error) { aprint_normal_dev(sc->dev, "Error allocating cmp ring pages, error = %d\n", error); goto fail; } sc->msg_ring = sc->msg_ring_dma.vaddr; } fail: if (error) { pvscsi_free_rings(sc); } return (error); } static void pvscsi_setup_rings(struct pvscsi_softc *sc) { struct pvscsi_cmd_desc_setup_rings cmd; uint32_t i; memset(&cmd, 0, sizeof(cmd)); cmd.rings_state_ppn = sc->rings_state_ppn; cmd.req_ring_num_pages = sc->req_ring_num_pages; for (i = 0; i < sc->req_ring_num_pages; ++i) { cmd.req_ring_ppns[i] = sc->req_ring_ppn[i]; } cmd.cmp_ring_num_pages = sc->cmp_ring_num_pages; for (i = 0; i < sc->cmp_ring_num_pages; ++i) { cmd.cmp_ring_ppns[i] = sc->cmp_ring_ppn[i]; } pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof(cmd)); /* * After setup, sync *_num_entries_log2 before use. After this * point they should be stable, so no need to sync again during * use. */ PVSCSI_DMA_SYNC_STATE(sc, &sc->rings_state_dma, sc->rings_state, req_num_entries_log2, BUS_DMASYNC_POSTREAD); PVSCSI_DMA_SYNC_STATE(sc, &sc->rings_state_dma, sc->rings_state, cmp_num_entries_log2, BUS_DMASYNC_POSTREAD); } static int pvscsi_hw_supports_msg(struct pvscsi_softc *sc) { uint32_t status; pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, PVSCSI_CMD_SETUP_MSG_RING); status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); return (status != -1); } static void pvscsi_setup_msg_ring(struct pvscsi_softc *sc) { struct pvscsi_cmd_desc_setup_msg_ring cmd; uint32_t i; KASSERTMSG(sc->use_msg, "msg is not being used"); memset(&cmd, 0, sizeof(cmd)); cmd.num_pages = sc->msg_ring_num_pages; for (i = 0; i < sc->msg_ring_num_pages; ++i) { cmd.ring_ppns[i] = sc->msg_ring_ppn[i]; } pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_MSG_RING, &cmd, sizeof(cmd)); /* * After setup, sync msg_num_entries_log2 before use. After * this point it should be stable, so no need to sync again * during use. */ PVSCSI_DMA_SYNC_STATE(sc, &sc->rings_state_dma, sc->rings_state, msg_num_entries_log2, BUS_DMASYNC_POSTREAD); } static void pvscsi_adapter_reset(struct pvscsi_softc *sc) { aprint_normal_dev(sc->dev, "Adapter Reset\n"); pvscsi_write_cmd(sc, PVSCSI_CMD_ADAPTER_RESET, NULL, 0); #ifdef PVSCSI_DEBUG_LOGGING uint32_t val = #endif pvscsi_read_intr_status(sc); DEBUG_PRINTF(2, sc->dev, "adapter reset done: %u\n", val); } static void pvscsi_bus_reset(struct pvscsi_softc *sc) { aprint_normal_dev(sc->dev, "Bus Reset\n"); pvscsi_write_cmd(sc, PVSCSI_CMD_RESET_BUS, NULL, 0); pvscsi_process_cmp_ring(sc); DEBUG_PRINTF(2, sc->dev, "bus reset done\n"); } static void pvscsi_device_reset(struct pvscsi_softc *sc, uint32_t target) { struct pvscsi_cmd_desc_reset_device cmd; memset(&cmd, 0, sizeof(cmd)); cmd.target = target; aprint_normal_dev(sc->dev, "Device reset for target %u\n", target); pvscsi_write_cmd(sc, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof cmd); pvscsi_process_cmp_ring(sc); DEBUG_PRINTF(2, sc->dev, "device reset done\n"); } static void pvscsi_abort(struct pvscsi_softc *sc, uint32_t target, struct pvscsi_hcb *hcb) { struct pvscsi_cmd_desc_abort_cmd cmd; uint64_t context; pvscsi_process_cmp_ring(sc); if (hcb != NULL) { context = pvscsi_hcb_to_context(sc, hcb); memset(&cmd, 0, sizeof cmd); cmd.target = target; cmd.context = context; aprint_normal_dev(sc->dev, "Abort for target %u context %llx\n", target, (unsigned long long)context); pvscsi_write_cmd(sc, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd)); pvscsi_process_cmp_ring(sc); DEBUG_PRINTF(2, sc->dev, "abort done\n"); } else { DEBUG_PRINTF(1, sc->dev, "Target %u hcb %p not found for abort\n", target, hcb); } } static int pvscsi_probe(device_t dev, cfdata_t cf, void *aux) { const struct pci_attach_args *pa = aux; if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE && PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_PVSCSI) { return 1; } return 0; } static void pvscsi_timeout(void *arg) { struct pvscsi_hcb *hcb = arg; struct scsipi_xfer *xs = hcb->xs; if (xs == NULL) { /* Already completed */ return; } struct pvscsi_softc *sc = hcb->sc; mutex_enter(&sc->lock); scsipi_printaddr(xs->xs_periph); printf("command timeout, CDB: "); scsipi_print_cdb(xs->cmd); printf("\n"); switch (hcb->recovery) { case PVSCSI_HCB_NONE: hcb->recovery = PVSCSI_HCB_ABORT; pvscsi_abort(sc, hcb->e->target, hcb); callout_reset(&xs->xs_callout, mstohz(PVSCSI_ABORT_TIMEOUT * 1000), pvscsi_timeout, hcb); break; case PVSCSI_HCB_ABORT: hcb->recovery = PVSCSI_HCB_DEVICE_RESET; pvscsi_device_reset(sc, hcb->e->target); callout_reset(&xs->xs_callout, mstohz(PVSCSI_RESET_TIMEOUT * 1000), pvscsi_timeout, hcb); break; case PVSCSI_HCB_DEVICE_RESET: hcb->recovery = PVSCSI_HCB_BUS_RESET; pvscsi_bus_reset(sc); callout_reset(&xs->xs_callout, mstohz(PVSCSI_RESET_TIMEOUT * 1000), pvscsi_timeout, hcb); break; case PVSCSI_HCB_BUS_RESET: pvscsi_adapter_reset(sc); break; }; mutex_exit(&sc->lock); } static void pvscsi_process_completion(struct pvscsi_softc *sc, struct pvscsi_ring_cmp_desc *e) { struct pvscsi_hcb *hcb; struct scsipi_xfer *xs; uint32_t error = XS_NOERROR; uint32_t btstat; uint32_t sdstat; int op; hcb = pvscsi_context_to_hcb(sc, e->context); xs = hcb->xs; callout_stop(&xs->xs_callout); btstat = e->host_status; sdstat = e->scsi_status; xs->status = sdstat; xs->resid = xs->datalen - e->data_len; DEBUG_PRINTF(3, sc->dev, "command context %llx btstat %d (%#x) sdstat %d (%#x)\n", (unsigned long long)e->context, btstat, btstat, sdstat, sdstat); if ((xs->xs_control & XS_CTL_DATA_IN) == XS_CTL_DATA_IN) { op = BUS_DMASYNC_POSTREAD; } else { op = BUS_DMASYNC_POSTWRITE; } bus_dmamap_sync(sc->sc_dmat, sc->sense_buffer_dma.map, hcb->dma_map_offset, hcb->dma_map_size, op); if (btstat == BTSTAT_SUCCESS && sdstat == SCSI_OK) { DEBUG_PRINTF(3, sc->dev, "completing command context %llx success\n", (unsigned long long)e->context); xs->resid = 0; } else { switch (btstat) { case BTSTAT_SUCCESS: case BTSTAT_LINKED_COMMAND_COMPLETED: case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG: switch (sdstat) { case SCSI_OK: xs->resid = 0; error = XS_NOERROR; break; case SCSI_CHECK: error = XS_SENSE; xs->resid = 0; memset(&xs->sense, 0, sizeof(xs->sense)); memcpy(&xs->sense, hcb->sense_buffer, MIN(sizeof(xs->sense), e->sense_len)); break; case SCSI_BUSY: case SCSI_QUEUE_FULL: error = XS_NOERROR; break; case SCSI_TERMINATED: // case SCSI_STATUS_TASK_ABORTED: DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); error = XS_DRIVER_STUFFUP; break; default: DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); error = XS_DRIVER_STUFFUP; break; } break; case BTSTAT_SELTIMEO: error = XS_SELTIMEOUT; break; case BTSTAT_DATARUN: case BTSTAT_DATA_UNDERRUN: // xs->resid = xs->datalen - c->data_len; error = XS_NOERROR; break; case BTSTAT_ABORTQUEUE: case BTSTAT_HATIMEOUT: error = XS_NOERROR; break; case BTSTAT_NORESPONSE: case BTSTAT_SENTRST: case BTSTAT_RECVRST: case BTSTAT_BUSRESET: error = XS_RESET; break; case BTSTAT_SCSIPARITY: error = XS_DRIVER_STUFFUP; DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); break; case BTSTAT_BUSFREE: error = XS_DRIVER_STUFFUP; DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); break; case BTSTAT_INVPHASE: error = XS_DRIVER_STUFFUP; DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); break; case BTSTAT_SENSFAILED: error = XS_DRIVER_STUFFUP; DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); break; case BTSTAT_LUNMISMATCH: case BTSTAT_TAGREJECT: case BTSTAT_DISCONNECT: case BTSTAT_BADMSG: case BTSTAT_INVPARAM: error = XS_DRIVER_STUFFUP; DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); break; case BTSTAT_HASOFTWARE: case BTSTAT_HAHARDWARE: error = XS_DRIVER_STUFFUP; DEBUG_PRINTF(1, sc->dev, "xs: %p sdstat=0x%x\n", xs, sdstat); break; default: aprint_normal_dev(sc->dev, "unknown hba status: 0x%x\n", btstat); error = XS_DRIVER_STUFFUP; break; } DEBUG_PRINTF(3, sc->dev, "completing command context %llx btstat %x sdstat %x - error %x\n", (unsigned long long)e->context, btstat, sdstat, error); } xs->error = error; pvscsi_hcb_put(sc, hcb); mutex_exit(&sc->lock); scsipi_done(xs); mutex_enter(&sc->lock); } static void pvscsi_process_cmp_ring(struct pvscsi_softc *sc) { struct pvscsi_dma *ring_dma; struct pvscsi_ring_cmp_desc *ring; struct pvscsi_dma *s_dma; struct pvscsi_rings_state *s; struct pvscsi_ring_cmp_desc *e; uint32_t mask; KASSERT(mutex_owned(&sc->lock)); s_dma = &sc->rings_state_dma; s = sc->rings_state; ring_dma = &sc->cmp_ring_dma; ring = sc->cmp_ring; mask = MASK(s->cmp_num_entries_log2); for (;;) { PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, cmp_prod_idx, BUS_DMASYNC_POSTREAD); size_t crpidx = s->cmp_prod_idx; PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, cmp_prod_idx, BUS_DMASYNC_PREREAD); if (s->cmp_cons_idx == crpidx) break; size_t crcidx = s->cmp_cons_idx & mask; PVSCSI_DMA_SYNC_RING(sc, ring_dma, ring, crcidx, BUS_DMASYNC_POSTREAD); e = ring + crcidx; pvscsi_process_completion(sc, e); /* * ensure completion processing reads happen before write to * (increment of) cmp_cons_idx */ PVSCSI_DMA_SYNC_RING(sc, ring_dma, ring, crcidx, BUS_DMASYNC_PREREAD); /* * XXX Not actually sure the `device' does DMA for * s->cmp_cons_idx at all -- qemu doesn't. If not, we * can skip these DMA syncs. */ PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, cmp_cons_idx, BUS_DMASYNC_POSTWRITE); s->cmp_cons_idx++; PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, cmp_cons_idx, BUS_DMASYNC_PREWRITE); } } static void pvscsi_process_msg(struct pvscsi_softc *sc, struct pvscsi_ring_msg_desc *e) { struct pvscsi_ring_msg_dev_status_changed *desc; switch (e->type) { case PVSCSI_MSG_DEV_ADDED: case PVSCSI_MSG_DEV_REMOVED: { desc = (struct pvscsi_ring_msg_dev_status_changed *)e; struct scsibus_softc *ssc = device_private(sc->sc_scsibus_dv); aprint_normal_dev(sc->dev, "MSG: device %s at scsi%u:%u:%u\n", desc->type == PVSCSI_MSG_DEV_ADDED ? "addition" : "removal", desc->bus, desc->target, desc->lun[1]); if (desc->type == PVSCSI_MSG_DEV_ADDED) { if (scsi_probe_bus(ssc, desc->target, desc->lun[1]) != 0) { aprint_normal_dev(sc->dev, "Error creating path for dev change.\n"); break; } } else { if (scsipi_target_detach(ssc->sc_channel, desc->target, desc->lun[1], DETACH_FORCE) != 0) { aprint_normal_dev(sc->dev, "Error detaching target %d lun %d\n", desc->target, desc->lun[1]); }; } } break; default: aprint_normal_dev(sc->dev, "Unknown msg type 0x%x\n", e->type); }; } static void pvscsi_process_msg_ring(struct pvscsi_softc *sc) { struct pvscsi_dma *ring_dma; struct pvscsi_ring_msg_desc *ring; struct pvscsi_dma *s_dma; struct pvscsi_rings_state *s; struct pvscsi_ring_msg_desc *e; uint32_t mask; KASSERT(mutex_owned(&sc->lock)); s_dma = &sc->rings_state_dma; s = sc->rings_state; ring_dma = &sc->msg_ring_dma; ring = sc->msg_ring; mask = MASK(s->msg_num_entries_log2); for (;;) { PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, msg_prod_idx, BUS_DMASYNC_POSTREAD); size_t mpidx = s->msg_prod_idx; // dma read (device -> cpu) PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, msg_prod_idx, BUS_DMASYNC_PREREAD); if (s->msg_cons_idx == mpidx) break; size_t mcidx = s->msg_cons_idx & mask; PVSCSI_DMA_SYNC_RING(sc, ring_dma, ring, mcidx, BUS_DMASYNC_POSTREAD); e = ring + mcidx; pvscsi_process_msg(sc, e); /* * ensure message processing reads happen before write to * (increment of) msg_cons_idx */ PVSCSI_DMA_SYNC_RING(sc, ring_dma, ring, mcidx, BUS_DMASYNC_PREREAD); PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, msg_cons_idx, BUS_DMASYNC_POSTWRITE); s->msg_cons_idx++; PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, msg_cons_idx, BUS_DMASYNC_PREWRITE); } } static void pvscsi_intr_locked(struct pvscsi_softc *sc) { uint32_t val; KASSERT(mutex_owned(&sc->lock)); val = pvscsi_read_intr_status(sc); if ((val & PVSCSI_INTR_ALL_SUPPORTED) != 0) { pvscsi_write_intr_status(sc, val & PVSCSI_INTR_ALL_SUPPORTED); pvscsi_process_cmp_ring(sc); if (sc->use_msg) { pvscsi_process_msg_ring(sc); } } } static int pvscsi_intr(void *xsc) { struct pvscsi_softc *sc; sc = xsc; mutex_enter(&sc->lock); pvscsi_intr_locked(xsc); mutex_exit(&sc->lock); return 1; } static void pvscsi_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t request, void *arg) { struct pvscsi_softc *sc = device_private(chan->chan_adapter->adapt_dev); if (request == ADAPTER_REQ_SET_XFER_MODE) { struct scsipi_xfer_mode *xm = arg; xm->xm_mode = PERIPH_CAP_TQING; xm->xm_period = 0; xm->xm_offset = 0; scsipi_async_event(chan, ASYNC_EVENT_XFER_MODE, xm); return; } else if (request != ADAPTER_REQ_RUN_XFER) { DEBUG_PRINTF(1, sc->dev, "unhandled %d\n", request); return; } /* request is ADAPTER_REQ_RUN_XFER */ struct scsipi_xfer *xs = arg; struct scsipi_periph *periph = xs->xs_periph; #ifdef SCSIPI_DEBUG periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS; #endif uint32_t req_num_entries_log2; struct pvscsi_dma *ring_dma; struct pvscsi_ring_req_desc *ring; struct pvscsi_ring_req_desc *e; struct pvscsi_dma *s_dma; struct pvscsi_rings_state *s; struct pvscsi_hcb *hcb; if (xs->cmdlen < 0 || xs->cmdlen > sizeof(e->cdb)) { DEBUG_PRINTF(1, sc->dev, "bad cmdlen %zu > %zu\n", (size_t)xs->cmdlen, sizeof(e->cdb)); /* not a temporary condition */ xs->error = XS_DRIVER_STUFFUP; scsipi_done(xs); return; } ring_dma = &sc->req_ring_dma; ring = sc->req_ring; s_dma = &sc->rings_state_dma; s = sc->rings_state; hcb = NULL; req_num_entries_log2 = s->req_num_entries_log2; /* Protect against multiple senders */ mutex_enter(&sc->lock); if (s->req_prod_idx - s->cmp_cons_idx >= (1 << req_num_entries_log2)) { aprint_normal_dev(sc->dev, "Not enough room on completion ring.\n"); xs->error = XS_RESOURCE_SHORTAGE; goto finish_xs; } if (xs->cmdlen > sizeof(e->cdb)) { DEBUG_PRINTF(1, sc->dev, "cdb length %u too large\n", xs->cmdlen); xs->error = XS_DRIVER_STUFFUP; goto finish_xs; } hcb = pvscsi_hcb_get(sc); if (hcb == NULL) { aprint_normal_dev(sc->dev, "No free hcbs.\n"); xs->error = XS_RESOURCE_SHORTAGE; goto finish_xs; } hcb->xs = xs; const size_t rridx = s->req_prod_idx & MASK(req_num_entries_log2); PVSCSI_DMA_SYNC_RING(sc, ring_dma, ring, rridx, BUS_DMASYNC_POSTWRITE); e = ring + rridx; memset(e, 0, sizeof(*e)); e->bus = 0; e->target = periph->periph_target; e->lun[1] = periph->periph_lun; e->data_addr = 0; e->data_len = xs->datalen; e->vcpu_hint = cpu_index(curcpu()); e->flags = 0; e->cdb_len = xs->cmdlen; memcpy(e->cdb, xs->cmd, xs->cmdlen); e->sense_addr = 0; e->sense_len = sizeof(xs->sense); if (e->sense_len > 0) { e->sense_addr = hcb->sense_buffer_paddr; } //e->tag = xs->xs_tag_type; e->tag = MSG_SIMPLE_Q_TAG; switch (xs->xs_control & (XS_CTL_DATA_IN | XS_CTL_DATA_OUT)) { case XS_CTL_DATA_IN: e->flags |= PVSCSI_FLAG_CMD_DIR_TOHOST; break; case XS_CTL_DATA_OUT: e->flags |= PVSCSI_FLAG_CMD_DIR_TODEVICE; break; default: e->flags |= PVSCSI_FLAG_CMD_DIR_NONE; break; } e->context = pvscsi_hcb_to_context(sc, hcb); hcb->e = e; DEBUG_PRINTF(3, sc->dev, " queuing command %02x context %llx\n", e->cdb[0], (unsigned long long)e->context); int flags; flags = (xs->xs_control & XS_CTL_DATA_IN) ? BUS_DMA_READ : BUS_DMA_WRITE; flags |= (xs->xs_control & XS_CTL_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK; int error = bus_dmamap_load(sc->sc_dmat, hcb->dma_map, xs->data, xs->datalen, NULL, flags); if (error) { if (error == ENOMEM || error == EAGAIN) { xs->error = XS_RESOURCE_SHORTAGE; } else { xs->error = XS_DRIVER_STUFFUP; } DEBUG_PRINTF(1, sc->dev, "xs: %p load error %d data %p len %d", xs, error, xs->data, xs->datalen); goto error_load; } int op = (xs->xs_control & XS_CTL_DATA_IN) ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE; int nseg = hcb->dma_map->dm_nsegs; bus_dma_segment_t *segs = hcb->dma_map->dm_segs; if (nseg != 0) { if (nseg > 1) { struct pvscsi_sg_element *sge; KASSERTMSG(nseg <= PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT, "too many sg segments"); sge = hcb->sg_list->sge; e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST; for (size_t i = 0; i < nseg; ++i) { sge[i].addr = segs[i].ds_addr; sge[i].length = segs[i].ds_len; sge[i].flags = 0; } e->data_addr = hcb->sg_list_paddr; bus_dmamap_sync(sc->sc_dmat, sc->sg_list_dma.map, hcb->sg_list_offset, sizeof(*sge) * nseg, BUS_DMASYNC_PREWRITE); } else { e->data_addr = segs->ds_addr; } bus_dmamap_sync(sc->sc_dmat, hcb->dma_map, 0, xs->datalen, op); } else { e->data_addr = 0; } /* * Ensure request record writes happen before write to (increment of) * req_prod_idx. */ PVSCSI_DMA_SYNC_RING(sc, ring_dma, ring, rridx, BUS_DMASYNC_PREWRITE); uint8_t cdb0 = e->cdb[0]; /* handle timeout */ if ((xs->xs_control & XS_CTL_POLL) == 0) { int timeout = mstohz(xs->timeout); /* start expire timer */ if (timeout == 0) timeout = 1; callout_reset(&xs->xs_callout, timeout, pvscsi_timeout, hcb); } PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, req_prod_idx, BUS_DMASYNC_POSTWRITE); s->req_prod_idx++; /* * Ensure req_prod_idx write (increment) happens before * IO is kicked (via a write). */ PVSCSI_DMA_SYNC_STATE(sc, s_dma, s, req_prod_idx, BUS_DMASYNC_PREWRITE); pvscsi_kick_io(sc, cdb0); mutex_exit(&sc->lock); return; error_load: pvscsi_hcb_put(sc, hcb); finish_xs: mutex_exit(&sc->lock); scsipi_done(xs); } static void pvscsi_free_interrupts(struct pvscsi_softc *sc) { if (sc->sc_ih != NULL) { pci_intr_disestablish(sc->sc_pc, sc->sc_ih); sc->sc_ih = NULL; } if (sc->sc_pihp != NULL) { pci_intr_release(sc->sc_pc, sc->sc_pihp, 1); sc->sc_pihp = NULL; } } static int pvscsi_setup_interrupts(struct pvscsi_softc *sc, const struct pci_attach_args *pa) { int use_msix; int use_msi; int counts[PCI_INTR_TYPE_SIZE]; for (size_t i = 0; i < PCI_INTR_TYPE_SIZE; i++) { counts[i] = 1; } use_msix = pvscsi_get_tunable(sc, "use_msix", pvscsi_use_msix); use_msi = pvscsi_get_tunable(sc, "use_msi", pvscsi_use_msi); if (!use_msix) { counts[PCI_INTR_TYPE_MSIX] = 0; } if (!use_msi) { counts[PCI_INTR_TYPE_MSI] = 0; } /* Allocate and establish the interrupt. */ if (pci_intr_alloc(pa, &sc->sc_pihp, counts, PCI_INTR_TYPE_MSIX)) { aprint_error_dev(sc->dev, "can't allocate handler\n"); goto fail; } char intrbuf[PCI_INTRSTR_LEN]; const pci_chipset_tag_t pc = pa->pa_pc; char const *intrstr = pci_intr_string(pc, sc->sc_pihp[0], intrbuf, sizeof(intrbuf)); sc->sc_ih = pci_intr_establish_xname(pc, sc->sc_pihp[0], IPL_BIO, pvscsi_intr, sc, device_xname(sc->dev)); if (sc->sc_ih == NULL) { pci_intr_release(pc, sc->sc_pihp, 1); sc->sc_pihp = NULL; aprint_error_dev(sc->dev, "couldn't establish interrupt"); if (intrstr != NULL) aprint_error(" at %s", intrstr); aprint_error("\n"); goto fail; } pci_intr_setattr(pc, sc->sc_pihp, PCI_INTR_MPSAFE, true); aprint_normal_dev(sc->dev, "interrupting at %s\n", intrstr); return (0); fail: if (sc->sc_ih != NULL) { pci_intr_disestablish(sc->sc_pc, sc->sc_ih); sc->sc_ih = NULL; } if (sc->sc_pihp != NULL) { pci_intr_release(sc->sc_pc, sc->sc_pihp, 1); sc->sc_pihp = NULL; } if (sc->sc_mems) { bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); sc->sc_mems = 0; } return 1; } static void pvscsi_free_all(struct pvscsi_softc *sc) { pvscsi_dma_free_per_hcb(sc, sc->hcb_cnt); if (sc->hcbs) { kmem_free(sc->hcbs, sc->hcb_cnt * sizeof(*sc->hcbs)); } pvscsi_free_rings(sc); pvscsi_free_interrupts(sc); if (sc->sc_mems) { bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); sc->sc_mems = 0; } } static inline void pci_enable_busmaster(device_t dev, const pci_chipset_tag_t pc, const pcitag_t tag) { pcireg_t pci_cmd_word; pci_cmd_word = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); if (!(pci_cmd_word & PCI_COMMAND_MASTER_ENABLE)) { pci_cmd_word |= PCI_COMMAND_MASTER_ENABLE; pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, pci_cmd_word); } } static void pvscsi_attach(device_t parent, device_t dev, void *aux) { const struct pci_attach_args *pa = aux; struct pvscsi_softc *sc; int rid; int error; int max_queue_depth; int adapter_queue_size; sc = device_private(dev); sc->dev = dev; struct scsipi_adapter *adapt = &sc->sc_adapter; struct scsipi_channel *chan = &sc->sc_channel; mutex_init(&sc->lock, MUTEX_DEFAULT, IPL_BIO); sc->sc_pc = pa->pa_pc; pci_enable_busmaster(dev, pa->pa_pc, pa->pa_tag); pci_aprint_devinfo_fancy(pa, "virtual disk controller", VMWARE_PVSCSI_DEVSTR, true); /* * Map the device. All devices support memory-mapped acccess. */ bool memh_valid; bus_space_tag_t memt; bus_space_handle_t memh; bus_size_t mems; pcireg_t regt; for (rid = PCI_MAPREG_START; rid < PCI_MAPREG_END; rid += sizeof(regt)) { regt = pci_mapreg_type(pa->pa_pc, pa->pa_tag, rid); if (PCI_MAPREG_TYPE(regt) == PCI_MAPREG_TYPE_MEM) break; } if (rid >= PCI_MAPREG_END) { aprint_error_dev(dev, "unable to locate device registers\n"); } memh_valid = (pci_mapreg_map(pa, rid, regt, 0, &memt, &memh, NULL, &mems) == 0); if (!memh_valid) { aprint_error_dev(dev, "unable to map device registers\n"); return; } sc->sc_memt = memt; sc->sc_memh = memh; sc->sc_mems = mems; if (pci_dma64_available(pa)) { sc->sc_dmat = pa->pa_dmat64; aprint_verbose_dev(sc->dev, "64-bit DMA\n"); } else { aprint_verbose_dev(sc->dev, "32-bit DMA\n"); sc->sc_dmat = pa->pa_dmat; } error = pvscsi_setup_interrupts(sc, pa); if (error) { aprint_normal_dev(dev, "Interrupt setup failed\n"); pvscsi_free_all(sc); return; } sc->max_targets = pvscsi_get_max_targets(sc); sc->use_msg = pvscsi_get_tunable(sc, "use_msg", pvscsi_use_msg) && pvscsi_hw_supports_msg(sc); sc->msg_ring_num_pages = sc->use_msg ? 1 : 0; sc->req_ring_num_pages = pvscsi_get_tunable(sc, "request_ring_pages", pvscsi_request_ring_pages); if (sc->req_ring_num_pages <= 0) { if (sc->max_targets <= 16) { sc->req_ring_num_pages = PVSCSI_DEFAULT_NUM_PAGES_REQ_RING; } else { sc->req_ring_num_pages = PVSCSI_MAX_NUM_PAGES_REQ_RING; } } else if (sc->req_ring_num_pages > PVSCSI_MAX_NUM_PAGES_REQ_RING) { sc->req_ring_num_pages = PVSCSI_MAX_NUM_PAGES_REQ_RING; } sc->cmp_ring_num_pages = sc->req_ring_num_pages; max_queue_depth = pvscsi_get_tunable(sc, "max_queue_depth", pvscsi_max_queue_depth); adapter_queue_size = (sc->req_ring_num_pages * PAGE_SIZE) / sizeof(struct pvscsi_ring_req_desc); if (max_queue_depth > 0) { adapter_queue_size = MIN(adapter_queue_size, max_queue_depth); } adapter_queue_size = MIN(adapter_queue_size, PVSCSI_MAX_REQ_QUEUE_DEPTH); aprint_normal_dev(sc->dev, "Use Msg: %d\n", sc->use_msg); aprint_normal_dev(sc->dev, "Max targets: %d\n", sc->max_targets); aprint_normal_dev(sc->dev, "REQ num pages: %d\n", sc->req_ring_num_pages); aprint_normal_dev(sc->dev, "CMP num pages: %d\n", sc->cmp_ring_num_pages); aprint_normal_dev(sc->dev, "MSG num pages: %d\n", sc->msg_ring_num_pages); aprint_normal_dev(sc->dev, "Queue size: %d\n", adapter_queue_size); if (pvscsi_allocate_rings(sc)) { aprint_normal_dev(dev, "ring allocation failed\n"); pvscsi_free_all(sc); return; } sc->hcb_cnt = adapter_queue_size; sc->hcbs = kmem_zalloc(sc->hcb_cnt * sizeof(*sc->hcbs), KM_SLEEP); if (pvscsi_dma_alloc_per_hcb(sc)) { aprint_normal_dev(dev, "error allocating per hcb dma memory\n"); pvscsi_free_all(sc); return; } pvscsi_adapter_reset(sc); /* * Fill in the scsipi_adapter. */ memset(adapt, 0, sizeof(*adapt)); adapt->adapt_dev = sc->dev; adapt->adapt_nchannels = 1; adapt->adapt_openings = MIN(adapter_queue_size, PVSCSI_CMD_PER_LUN); adapt->adapt_max_periph = adapt->adapt_openings; adapt->adapt_request = pvscsi_scsipi_request; adapt->adapt_minphys = minphys; /* * Fill in the scsipi_channel. */ memset(chan, 0, sizeof(*chan)); chan->chan_adapter = adapt; chan->chan_bustype = &scsi_bustype; chan->chan_channel = 0; chan->chan_ntargets = MIN(PVSCSI_MAX_TARGET, 16); /* cap reasonably */ chan->chan_nluns = MIN(PVSCSI_MAX_LUN, 1024); /* cap reasonably */ chan->chan_id = PVSCSI_MAX_TARGET; chan->chan_flags = SCSIPI_CHAN_NOSETTLE; pvscsi_setup_rings(sc); if (sc->use_msg) { pvscsi_setup_msg_ring(sc); } sc->use_req_call_threshold = pvscsi_setup_req_call(sc, 1); pvscsi_intr_enable(sc); sc->sc_scsibus_dv = config_found(sc->dev, &sc->sc_channel, scsiprint, CFARGS_NONE); return; } static int pvscsi_detach(device_t dev, int flags) { struct pvscsi_softc *sc; sc = device_private(dev); pvscsi_intr_disable(sc); pvscsi_adapter_reset(sc); pvscsi_free_all(sc); mutex_destroy(&sc->lock); return (0); }