/* $NetBSD: if_vmx.c,v 1.17 2024/07/05 04:31:51 rin Exp $ */ /* $OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $ */ /* * Copyright (c) 2013 Tsubai Masanari * Copyright (c) 2013 Bryan Venteicher * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.17 2024/07/05 04:31:51 rin Exp $"); #ifdef _KERNEL_OPT #include "opt_if_vmx.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for */ #include /* for */ #include /* for struct ip */ #include /* for struct ip6_hdr */ #include /* for struct tcphdr */ #include /* for struct udphdr */ #include #include #include #include #define VMXNET3_DRIVER_VERSION 0x00010000 /* * Max descriptors per Tx packet. We must limit the size of the * any TSO packets based on the number of segments. */ #define VMXNET3_TX_MAXSEGS 32 #define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES) /* * Maximum support Tx segments size. The length field in the * Tx descriptor is 14 bits. */ #define VMXNET3_TX_MAXSEGSIZE (1 << 14) /* * The maximum number of Rx segments we accept. */ #define VMXNET3_MAX_RX_SEGS 0 /* no segments */ /* * Predetermined size of the multicast MACs filter table. If the * number of multicast addresses exceeds this size, then the * ALL_MULTI mode is use instead. */ #define VMXNET3_MULTICAST_MAX 32 /* * Our Tx watchdog timeout. */ #define VMXNET3_WATCHDOG_TIMEOUT 5 /* * Default value for vmx_intr_{rx,tx}_process_limit which is used for * max number of packets to process for interrupt handler */ #define VMXNET3_RX_INTR_PROCESS_LIMIT 0U #define VMXNET3_TX_INTR_PROCESS_LIMIT 256 /* * Default value for vmx_{rx,tx}_process_limit which is used for * max number of packets to process for deferred processing */ #define VMXNET3_RX_PROCESS_LIMIT 256 #define VMXNET3_TX_PROCESS_LIMIT 256 #define VMXNET3_WORKQUEUE_PRI PRI_SOFTNET /* * IP protocols that we can perform Tx checksum offloading of. */ #define VMXNET3_CSUM_OFFLOAD \ (M_CSUM_TCPv4 | M_CSUM_UDPv4) #define VMXNET3_CSUM_OFFLOAD_IPV6 \ (M_CSUM_TCPv6 | M_CSUM_UDPv6) #define VMXNET3_CSUM_ALL_OFFLOAD \ (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6) #define VMXNET3_RXRINGS_PERQ 2 #define VMXNET3_CORE_LOCK(_sc) mutex_enter((_sc)->vmx_mtx) #define VMXNET3_CORE_UNLOCK(_sc) mutex_exit((_sc)->vmx_mtx) #define VMXNET3_CORE_LOCK_ASSERT(_sc) mutex_owned((_sc)->vmx_mtx) #define VMXNET3_RXQ_LOCK(_rxq) mutex_enter((_rxq)->vxrxq_mtx) #define VMXNET3_RXQ_UNLOCK(_rxq) mutex_exit((_rxq)->vxrxq_mtx) #define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \ mutex_owned((_rxq)->vxrxq_mtx) #define VMXNET3_TXQ_LOCK(_txq) mutex_enter((_txq)->vxtxq_mtx) #define VMXNET3_TXQ_TRYLOCK(_txq) mutex_tryenter((_txq)->vxtxq_mtx) #define VMXNET3_TXQ_UNLOCK(_txq) mutex_exit((_txq)->vxtxq_mtx) #define VMXNET3_TXQ_LOCK_ASSERT(_txq) \ mutex_owned((_txq)->vxtxq_mtx) struct vmxnet3_dma_alloc { bus_addr_t dma_paddr; void *dma_vaddr; bus_dmamap_t dma_map; bus_size_t dma_size; bus_dma_segment_t dma_segs[1]; }; struct vmxnet3_txbuf { bus_dmamap_t vtxb_dmamap; struct mbuf *vtxb_m; }; struct vmxnet3_txring { struct vmxnet3_txbuf *vxtxr_txbuf; struct vmxnet3_txdesc *vxtxr_txd; u_int vxtxr_head; u_int vxtxr_next; u_int vxtxr_ndesc; int vxtxr_gen; struct vmxnet3_dma_alloc vxtxr_dma; }; struct vmxnet3_rxbuf { bus_dmamap_t vrxb_dmamap; struct mbuf *vrxb_m; }; struct vmxnet3_rxring { struct vmxnet3_rxbuf *vxrxr_rxbuf; struct vmxnet3_rxdesc *vxrxr_rxd; u_int vxrxr_fill; u_int vxrxr_ndesc; int vxrxr_gen; int vxrxr_rid; struct vmxnet3_dma_alloc vxrxr_dma; bus_dmamap_t vxrxr_spare_dmap; }; struct vmxnet3_comp_ring { union { struct vmxnet3_txcompdesc *txcd; struct vmxnet3_rxcompdesc *rxcd; } vxcr_u; u_int vxcr_next; u_int vxcr_ndesc; int vxcr_gen; struct vmxnet3_dma_alloc vxcr_dma; }; struct vmxnet3_txq_stats { uint64_t vmtxs_csum; uint64_t vmtxs_tso; uint64_t vmtxs_full; uint64_t vmtxs_offload_failed; }; struct vmxnet3_txqueue { kmutex_t *vxtxq_mtx; struct vmxnet3_softc *vxtxq_sc; int vxtxq_watchdog; pcq_t *vxtxq_interq; struct vmxnet3_txring vxtxq_cmd_ring; struct vmxnet3_comp_ring vxtxq_comp_ring; struct vmxnet3_txq_stats vxtxq_stats; struct vmxnet3_txq_shared *vxtxq_ts; char vxtxq_name[16]; void *vxtxq_si; struct evcnt vxtxq_intr; struct evcnt vxtxq_defer; struct evcnt vxtxq_deferreq; struct evcnt vxtxq_pcqdrop; struct evcnt vxtxq_transmitdef; struct evcnt vxtxq_watchdogto; struct evcnt vxtxq_defragged; struct evcnt vxtxq_defrag_failed; bool vxtxq_stopping; }; struct vmxnet3_rxqueue { kmutex_t *vxrxq_mtx; struct vmxnet3_softc *vxrxq_sc; struct mbuf *vxrxq_mhead; struct mbuf *vxrxq_mtail; struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ]; struct vmxnet3_comp_ring vxrxq_comp_ring; struct vmxnet3_rxq_shared *vxrxq_rs; char vxrxq_name[16]; struct evcnt vxrxq_intr; struct evcnt vxrxq_defer; struct evcnt vxrxq_deferreq; struct evcnt vxrxq_mgetcl_failed; struct evcnt vxrxq_mbuf_load_failed; bool vxrxq_stopping; }; struct vmxnet3_queue { int vxq_id; int vxq_intr_idx; struct vmxnet3_txqueue vxq_txqueue; struct vmxnet3_rxqueue vxq_rxqueue; void *vxq_si; bool vxq_workqueue; bool vxq_wq_enqueued; struct work vxq_wq_cookie; }; struct vmxnet3_softc { device_t vmx_dev; struct ethercom vmx_ethercom; struct ifmedia vmx_media; struct vmxnet3_driver_shared *vmx_ds; int vmx_flags; #define VMXNET3_FLAG_NO_MSIX (1 << 0) #define VMXNET3_FLAG_RSS (1 << 1) #define VMXNET3_FLAG_ATTACHED (1 << 2) struct vmxnet3_queue *vmx_queue; struct pci_attach_args *vmx_pa; pci_chipset_tag_t vmx_pc; bus_space_tag_t vmx_iot0; bus_space_tag_t vmx_iot1; bus_space_handle_t vmx_ioh0; bus_space_handle_t vmx_ioh1; bus_size_t vmx_ios0; bus_size_t vmx_ios1; bus_dma_tag_t vmx_dmat; int vmx_link_active; int vmx_ntxqueues; int vmx_nrxqueues; int vmx_ntxdescs; int vmx_nrxdescs; int vmx_max_rxsegs; struct evcnt vmx_event_intr; struct evcnt vmx_event_link; struct evcnt vmx_event_txqerror; struct evcnt vmx_event_rxqerror; struct evcnt vmx_event_dic; struct evcnt vmx_event_debug; int vmx_intr_type; int vmx_intr_mask_mode; int vmx_event_intr_idx; int vmx_nintrs; pci_intr_handle_t *vmx_intrs; /* legacy use vmx_intrs[0] */ void *vmx_ihs[VMXNET3_MAX_INTRS]; kmutex_t *vmx_mtx; int vmx_if_flags; bool vmx_promisc; bool vmx_mcastactive; uint8_t *vmx_mcast; void *vmx_qs; struct vmxnet3_rss_shared *vmx_rss; callout_t vmx_tick; struct vmxnet3_dma_alloc vmx_ds_dma; struct vmxnet3_dma_alloc vmx_qs_dma; struct vmxnet3_dma_alloc vmx_mcast_dma; struct vmxnet3_dma_alloc vmx_rss_dma; int vmx_max_ntxqueues; int vmx_max_nrxqueues; uint8_t vmx_lladdr[ETHER_ADDR_LEN]; u_int vmx_rx_intr_process_limit; u_int vmx_tx_intr_process_limit; u_int vmx_rx_process_limit; u_int vmx_tx_process_limit; struct sysctllog *vmx_sysctllog; bool vmx_txrx_workqueue; struct workqueue *vmx_queue_wq; struct workqueue *vmx_reset_wq; struct work vmx_reset_work; bool vmx_reset_pending; }; #define VMXNET3_STAT #ifdef VMXNET3_STAT struct { u_int txhead; u_int txdone; u_int maxtxlen; u_int rxdone; u_int rxfill; u_int intr; } vmxstat; #endif typedef enum { VMXNET3_BARRIER_RD, VMXNET3_BARRIER_WR, } vmxnet3_barrier_t; #define JUMBO_LEN (MCLBYTES - ETHER_ALIGN) /* XXX */ #define DMAADDR(map) ((map)->dm_segs[0].ds_addr) #define vtophys(va) 0 /* XXX ok? */ static int vmxnet3_match(device_t, cfdata_t, void *); static void vmxnet3_attach(device_t, device_t, void *); static int vmxnet3_detach(device_t, int); static int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *); static void vmxnet3_free_pci_resources(struct vmxnet3_softc *); static int vmxnet3_check_version(struct vmxnet3_softc *); static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); static void vmxnet3_free_interrupts(struct vmxnet3_softc *); static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); static int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *); static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *); static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); static int vmxnet3_setup_sysctl(struct vmxnet3_softc *); static int vmxnet3_setup_stats(struct vmxnet3_softc *); static void vmxnet3_teardown_stats(struct vmxnet3_softc *); static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); static int vmxnet3_init_txq(struct vmxnet3_softc *, int); static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); static void vmxnet3_free_shared_data(struct vmxnet3_softc *); static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); static void vmxnet3_free_txq_data(struct vmxnet3_softc *); static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); static void vmxnet3_free_queue_data(struct vmxnet3_softc *); static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); static void vmxnet3_free_mcast_table(struct vmxnet3_softc *); static void vmxnet3_init_shared_data(struct vmxnet3_softc *); static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); static int vmxnet3_alloc_data(struct vmxnet3_softc *); static void vmxnet3_free_data(struct vmxnet3_softc *); static int vmxnet3_setup_interface(struct vmxnet3_softc *); static void vmxnet3_evintr(struct vmxnet3_softc *); static bool vmxnet3_txq_eof(struct vmxnet3_txqueue *, u_int); static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxqueue *, struct vmxnet3_rxring *); static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, struct vmxnet3_rxring *, int); static void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *); static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); static void vmxnet3_rxq_input(struct vmxnet3_rxqueue *, struct vmxnet3_rxcompdesc *, struct mbuf *); static bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *, u_int); static int vmxnet3_legacy_intr(void *); static int vmxnet3_txrxq_intr(void *); static void vmxnet3_handle_queue(void *); static void vmxnet3_handle_queue_work(struct work *, void *); static int vmxnet3_event_intr(void *); static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); static void vmxnet3_stop_locked(struct vmxnet3_softc *); static void vmxnet3_stop_rendezvous(struct vmxnet3_softc *); static void vmxnet3_stop(struct ifnet *, int); static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); static int vmxnet3_reinit_queues(struct vmxnet3_softc *); static int vmxnet3_enable_device(struct vmxnet3_softc *); static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); static int vmxnet3_reinit(struct vmxnet3_softc *); static int vmxnet3_init_locked(struct vmxnet3_softc *); static int vmxnet3_init(struct ifnet *); static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *); static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t); static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); static void vmxnet3_start_locked(struct ifnet *); static void vmxnet3_start(struct ifnet *); static void vmxnet3_transmit_locked(struct ifnet *, struct vmxnet3_txqueue *); static int vmxnet3_transmit(struct ifnet *, struct mbuf *); static void vmxnet3_deferred_transmit(void *); static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); static int vmxnet3_ioctl(struct ifnet *, u_long, void *); static int vmxnet3_ifflags_cb(struct ethercom *); static int vmxnet3_watchdog(struct vmxnet3_txqueue *); static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); static void vmxnet3_tick(void *); static void vmxnet3_reset_work(struct work *, void *); static void vmxnet3_if_link_status(struct vmxnet3_softc *); static bool vmxnet3_cmd_link_status(struct ifnet *); static void vmxnet3_ifmedia_status(struct ifnet *, struct ifmediareq *); static int vmxnet3_ifmedia_change(struct ifnet *); static void vmxnet3_set_lladdr(struct vmxnet3_softc *); static void vmxnet3_get_lladdr(struct vmxnet3_softc *); static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t, struct vmxnet3_dma_alloc *); static void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *); CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc), vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0); /* round down to the nearest power of 2 */ static int vmxnet3_calc_queue_size(int n) { if (__predict_false(n <= 0)) return 1; return (1U << (fls32(n) - 1)); } static inline void vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) { bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); } static inline uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) { return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); } static inline void vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) { bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); } static inline void vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) { vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); } static inline uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) { vmxnet3_write_cmd(sc, cmd); return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); } static inline void vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) { vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); } static inline void vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) { vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); } static inline void vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr) { if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) { rxr->vxrxr_fill = 0; rxr->vxrxr_gen ^= 1; } } static inline int vmxnet3_txring_avail(struct vmxnet3_txring *txr) { int avail = txr->vxtxr_next - txr->vxtxr_head - 1; return (avail < 0 ? (int)txr->vxtxr_ndesc + avail : avail); } /* * Since this is a purely paravirtualized device, we do not have * to worry about DMA coherency. But at times, we must make sure * both the compiler and CPU do not reorder memory operations. */ static inline void vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) { switch (type) { case VMXNET3_BARRIER_RD: membar_consumer(); break; case VMXNET3_BARRIER_WR: membar_producer(); break; default: panic("%s: bad barrier type %d", __func__, type); } } static int vmxnet3_match(device_t parent, cfdata_t match, void *aux) { struct pci_attach_args *pa = (struct pci_attach_args *)aux; if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE && PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_VMXNET3) return 1; return 0; } static void vmxnet3_attach(device_t parent, device_t self, void *aux) { struct vmxnet3_softc *sc = device_private(self); struct pci_attach_args *pa = aux; pcireg_t preg; int error; int candidate; sc->vmx_dev = self; sc->vmx_pa = pa; sc->vmx_pc = pa->pa_pc; if (pci_dma64_available(pa)) sc->vmx_dmat = pa->pa_dmat64; else sc->vmx_dmat = pa->pa_dmat; pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1); preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); preg |= PCI_COMMAND_MASTER_ENABLE; pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg); sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); callout_init(&sc->vmx_tick, CALLOUT_MPSAFE); candidate = MIN(MIN(VMXNET3_MAX_TX_QUEUES, VMXNET3_MAX_RX_QUEUES), ncpu); sc->vmx_max_ntxqueues = sc->vmx_max_nrxqueues = vmxnet3_calc_queue_size(candidate); sc->vmx_ntxdescs = 512; sc->vmx_nrxdescs = 256; sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; error = vmxnet3_alloc_pci_resources(sc); if (error) return; error = vmxnet3_check_version(sc); if (error) return; error = vmxnet3_alloc_rxtx_queues(sc); if (error) return; error = vmxnet3_alloc_interrupts(sc); if (error) return; vmxnet3_check_multiqueue(sc); error = vmxnet3_alloc_data(sc); if (error) return; error = vmxnet3_setup_interface(sc); if (error) return; error = vmxnet3_setup_interrupts(sc); if (error) return; error = vmxnet3_setup_sysctl(sc); if (error) return; error = vmxnet3_setup_stats(sc); if (error) return; char buf[128]; snprintf(buf, sizeof(buf), "%s_reset", device_xname(sc->vmx_dev)); error = workqueue_create(&sc->vmx_reset_wq, "%s_reset", vmxnet3_reset_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_SOFTCLOCK, WQ_MPSAFE); if (error) { aprint_error_dev(sc->vmx_dev, "failed to create reset workqueue: %d\n", error); return; } sc->vmx_flags |= VMXNET3_FLAG_ATTACHED; } static int vmxnet3_detach(device_t self, int flags) { struct vmxnet3_softc *sc; struct ifnet *ifp; sc = device_private(self); ifp = &sc->vmx_ethercom.ec_if; if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) { VMXNET3_CORE_LOCK(sc); vmxnet3_stop_locked(sc); callout_halt(&sc->vmx_tick, sc->vmx_mtx); callout_destroy(&sc->vmx_tick); VMXNET3_CORE_UNLOCK(sc); ether_ifdetach(ifp); if_detach(ifp); ifmedia_fini(&sc->vmx_media); } vmxnet3_teardown_stats(sc); sysctl_teardown(&sc->vmx_sysctllog); vmxnet3_free_interrupts(sc); vmxnet3_free_data(sc); vmxnet3_free_pci_resources(sc); vmxnet3_free_rxtx_queues(sc); if (sc->vmx_mtx) mutex_obj_free(sc->vmx_mtx); return (0); } static int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc) { struct pci_attach_args *pa = sc->vmx_pa; pcireg_t memtype; memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0, NULL, &sc->vmx_ios0)) { aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n"); return (ENXIO); } memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1)); if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1, NULL, &sc->vmx_ios1)) { aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n"); return (ENXIO); } if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) { sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; return (0); } return (0); } static void vmxnet3_free_pci_resources(struct vmxnet3_softc *sc) { if (sc->vmx_ios0) { bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0); sc->vmx_ios0 = 0; } if (sc->vmx_ios1) { bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1); sc->vmx_ios1 = 0; } } static int vmxnet3_check_version(struct vmxnet3_softc *sc) { u_int ver; ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); if ((ver & 0x1) == 0) { aprint_error_dev(sc->vmx_dev, "unsupported hardware version 0x%x\n", ver); return (ENOTSUP); } vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); if ((ver & 0x1) == 0) { aprint_error_dev(sc->vmx_dev, "incompatible UPT version 0x%x\n", ver); return (ENOTSUP); } vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); return (0); } static void vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) { if (sc->vmx_intr_type != VMXNET3_IT_MSIX) goto out; /* Just use the maximum configured for now. */ sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; if (sc->vmx_nrxqueues > 1) sc->vmx_flags |= VMXNET3_FLAG_RSS; return; out: sc->vmx_ntxqueues = 1; sc->vmx_nrxqueues = 1; } static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) { int required; struct pci_attach_args *pa = sc->vmx_pa; if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) return (1); /* Allocate an additional vector for the events interrupt. */ required = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues) + 1; if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required) return (1); if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { sc->vmx_nintrs = required; return (0); } return (1); } static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) { int nmsi, required; struct pci_attach_args *pa = sc->vmx_pa; required = 1; nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag); if (nmsi < required) return (1); if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) { sc->vmx_nintrs = required; return (0); } return (1); } static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) { if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) { sc->vmx_nintrs = 1; return (0); } return (1); } static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) { u_int config; int error; config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); sc->vmx_intr_type = config & 0x03; sc->vmx_intr_mask_mode = (config >> 2) & 0x03; switch (sc->vmx_intr_type) { case VMXNET3_IT_AUTO: sc->vmx_intr_type = VMXNET3_IT_MSIX; /* FALLTHROUGH */ case VMXNET3_IT_MSIX: error = vmxnet3_alloc_msix_interrupts(sc); if (error == 0) break; sc->vmx_intr_type = VMXNET3_IT_MSI; /* FALLTHROUGH */ case VMXNET3_IT_MSI: error = vmxnet3_alloc_msi_interrupts(sc); if (error == 0) break; sc->vmx_intr_type = VMXNET3_IT_LEGACY; /* FALLTHROUGH */ case VMXNET3_IT_LEGACY: error = vmxnet3_alloc_legacy_interrupts(sc); if (error == 0) break; /* FALLTHROUGH */ default: sc->vmx_intr_type = -1; aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n"); return (ENXIO); } return (error); } static void vmxnet3_free_interrupts(struct vmxnet3_softc *sc) { pci_chipset_tag_t pc = sc->vmx_pc; int i; workqueue_destroy(sc->vmx_queue_wq); for (i = 0; i < sc->vmx_ntxqueues; i++) { struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; softint_disestablish(vmxq->vxq_si); vmxq->vxq_si = NULL; } for (i = 0; i < sc->vmx_nintrs; i++) { pci_intr_disestablish(pc, sc->vmx_ihs[i]); } pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs); } static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) { pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; struct vmxnet3_queue *vmxq; pci_intr_handle_t *intr; void **ihs; int intr_idx, i, use_queues, error; kcpuset_t *affinity; const char *intrstr; char intrbuf[PCI_INTRSTR_LEN]; char xnamebuf[32]; intr = sc->vmx_intrs; intr_idx = 0; ihs = sc->vmx_ihs; /* See vmxnet3_alloc_msix_interrupts() */ use_queues = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); for (i = 0; i < use_queues; i++, intr++, ihs++, intr_idx++) { snprintf(xnamebuf, 32, "%s: txrx %d", device_xname(sc->vmx_dev), i); vmxq = &sc->vmx_queue[i]; intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, vmxnet3_txrxq_intr, vmxq, xnamebuf); if (*ihs == NULL) { aprint_error_dev(sc->vmx_dev, "unable to establish txrx interrupt at %s\n", intrstr); return (-1); } aprint_normal_dev(sc->vmx_dev, "txrx interrupting at %s\n", intrstr); kcpuset_create(&affinity, true); kcpuset_set(affinity, intr_idx % ncpu); error = interrupt_distribute(*ihs, affinity, NULL); if (error) { aprint_normal_dev(sc->vmx_dev, "%s cannot be changed affinity, use default CPU\n", intrstr); } kcpuset_destroy(affinity); vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, vmxnet3_handle_queue, vmxq); if (vmxq->vxq_si == NULL) { aprint_error_dev(sc->vmx_dev, "softint_establish for vxq_si failed\n"); return (-1); } vmxq->vxq_intr_idx = intr_idx; } snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(sc->vmx_dev)); error = workqueue_create(&sc->vmx_queue_wq, xnamebuf, vmxnet3_handle_queue_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); if (error) { aprint_error_dev(sc->vmx_dev, "workqueue_create failed\n"); return (-1); } sc->vmx_txrx_workqueue = false; intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev)); pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, vmxnet3_event_intr, sc, xnamebuf); if (*ihs == NULL) { aprint_error_dev(sc->vmx_dev, "unable to establish event interrupt at %s\n", intrstr); return (-1); } aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr); sc->vmx_event_intr_idx = intr_idx; return (0); } static int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc) { pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; pci_intr_handle_t *intr; void **ihs; struct vmxnet3_queue *vmxq; int i; const char *intrstr; char intrbuf[PCI_INTRSTR_LEN]; char xnamebuf[32]; intr = &sc->vmx_intrs[0]; ihs = sc->vmx_ihs; vmxq = &sc->vmx_queue[0]; intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev)); pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, vmxnet3_legacy_intr, sc, xnamebuf); if (*ihs == NULL) { aprint_error_dev(sc->vmx_dev, "unable to establish interrupt at %s\n", intrstr); return (-1); } aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, vmxnet3_handle_queue, vmxq); if (vmxq->vxq_si == NULL) { aprint_error_dev(sc->vmx_dev, "softint_establish for vxq_si failed\n"); return (-1); } for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) sc->vmx_queue[i].vxq_intr_idx = 0; sc->vmx_event_intr_idx = 0; return (0); } static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) { pci_chipset_tag_t pc = sc->vmx_pa->pa_pc; pci_intr_handle_t *intr; void **ihs; struct vmxnet3_queue *vmxq; int i; const char *intrstr; char intrbuf[PCI_INTRSTR_LEN]; char xnamebuf[32]; intr = &sc->vmx_intrs[0]; ihs = sc->vmx_ihs; vmxq = &sc->vmx_queue[0]; intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev)); pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, vmxnet3_legacy_intr, sc, xnamebuf); if (*ihs == NULL) { aprint_error_dev(sc->vmx_dev, "unable to establish interrupt at %s\n", intrstr); return (-1); } aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr); vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, vmxnet3_handle_queue, vmxq); if (vmxq->vxq_si == NULL) { aprint_error_dev(sc->vmx_dev, "softint_establish for vxq_si failed\n"); return (-1); } for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++) sc->vmx_queue[i].vxq_intr_idx = 0; sc->vmx_event_intr_idx = 0; return (0); } static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) { struct vmxnet3_queue *vmxq; struct vmxnet3_txqueue *txq; struct vmxnet3_txq_shared *txs; struct vmxnet3_rxqueue *rxq; struct vmxnet3_rxq_shared *rxs; int i; sc->vmx_ds->evintr = sc->vmx_event_intr_idx; for (i = 0; i < sc->vmx_ntxqueues; i++) { vmxq = &sc->vmx_queue[i]; txq = &vmxq->vxq_txqueue; txs = txq->vxtxq_ts; txs->intr_idx = vmxq->vxq_intr_idx; } for (i = 0; i < sc->vmx_nrxqueues; i++) { vmxq = &sc->vmx_queue[i]; rxq = &vmxq->vxq_rxqueue; rxs = rxq->vxrxq_rs; rxs->intr_idx = vmxq->vxq_intr_idx; } } static int vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) { int error; switch (sc->vmx_intr_type) { case VMXNET3_IT_MSIX: error = vmxnet3_setup_msix_interrupts(sc); break; case VMXNET3_IT_MSI: error = vmxnet3_setup_msi_interrupt(sc); break; case VMXNET3_IT_LEGACY: error = vmxnet3_setup_legacy_interrupt(sc); break; default: panic("%s: invalid interrupt type %d", __func__, sc->vmx_intr_type); } if (error == 0) vmxnet3_set_interrupt_idx(sc); return (error); } static int vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) { struct vmxnet3_rxqueue *rxq; struct vmxnet3_rxring *rxr; int i; rxq = &sc->vmx_queue[q].vxq_rxqueue; snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", device_xname(sc->vmx_dev), q); rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); rxq->vxrxq_sc = sc; for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; rxr->vxrxr_rid = i; rxr->vxrxr_ndesc = sc->vmx_nrxdescs; rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf), KM_SLEEP); rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; } rxq->vxrxq_stopping = true; return (0); } static int vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) { struct vmxnet3_txqueue *txq; struct vmxnet3_txring *txr; txq = &sc->vmx_queue[q].vxq_txqueue; txr = &txq->vxtxq_cmd_ring; snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", device_xname(sc->vmx_dev), q); txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */); txq->vxtxq_sc = sc; txq->vxtxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, vmxnet3_deferred_transmit, txq); if (txq->vxtxq_si == NULL) { mutex_obj_free(txq->vxtxq_mtx); aprint_error_dev(sc->vmx_dev, "softint_establish for vxtxq_si failed\n"); return ENOMEM; } txr->vxtxr_ndesc = sc->vmx_ntxdescs; txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf), KM_SLEEP); txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; txq->vxtxq_interq = pcq_create(sc->vmx_ntxdescs, KM_SLEEP); txq->vxtxq_stopping = true; return (0); } static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) { int i, error, max_nqueues; KASSERT(!cpu_intr_p()); KASSERT(!cpu_softintr_p()); /* * Only attempt to create multiple queues if MSIX is available. * This check prevents us from allocating queue structures that * we will not use. * * FreeBSD: * MSIX is disabled by default because its apparently broken for * devices passed through by at least ESXi 5.1. * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX. */ if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { sc->vmx_max_nrxqueues = 1; sc->vmx_max_ntxqueues = 1; } max_nqueues = MAX(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues); sc->vmx_queue = kmem_zalloc(sizeof(struct vmxnet3_queue) * max_nqueues, KM_SLEEP); for (i = 0; i < max_nqueues; i++) { struct vmxnet3_queue *vmxq = &sc->vmx_queue[i]; vmxq->vxq_id = i; } for (i = 0; i < sc->vmx_max_nrxqueues; i++) { error = vmxnet3_init_rxq(sc, i); if (error) return (error); } for (i = 0; i < sc->vmx_max_ntxqueues; i++) { error = vmxnet3_init_txq(sc, i); if (error) return (error); } return (0); } static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) { struct vmxnet3_rxring *rxr; int i; rxq->vxrxq_sc = NULL; for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; if (rxr->vxrxr_rxbuf != NULL) { kmem_free(rxr->vxrxr_rxbuf, rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf)); rxr->vxrxr_rxbuf = NULL; } } if (rxq->vxrxq_mtx != NULL) mutex_obj_free(rxq->vxrxq_mtx); } static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) { struct vmxnet3_txring *txr; struct mbuf *m; txr = &txq->vxtxq_cmd_ring; txq->vxtxq_sc = NULL; softint_disestablish(txq->vxtxq_si); while ((m = pcq_get(txq->vxtxq_interq)) != NULL) m_freem(m); pcq_destroy(txq->vxtxq_interq); if (txr->vxtxr_txbuf != NULL) { kmem_free(txr->vxtxr_txbuf, txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf)); txr->vxtxr_txbuf = NULL; } if (txq->vxtxq_mtx != NULL) mutex_obj_free(txq->vxtxq_mtx); } static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) { int i; if (sc->vmx_queue != NULL) { int max_nqueues; for (i = 0; i < sc->vmx_max_nrxqueues; i++) vmxnet3_destroy_rxq(&sc->vmx_queue[i].vxq_rxqueue); for (i = 0; i < sc->vmx_max_ntxqueues; i++) vmxnet3_destroy_txq(&sc->vmx_queue[i].vxq_txqueue); max_nqueues = MAX(sc->vmx_max_nrxqueues, sc->vmx_max_ntxqueues); kmem_free(sc->vmx_queue, sizeof(struct vmxnet3_queue) * max_nqueues); } } static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) { device_t dev; uint8_t *kva; size_t size; int i, error; dev = sc->vmx_dev; size = sizeof(struct vmxnet3_driver_shared); error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); if (error) { device_printf(dev, "cannot alloc shared memory\n"); return (error); } sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); if (error) { device_printf(dev, "cannot alloc queue shared memory\n"); return (error); } sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; kva = sc->vmx_qs; for (i = 0; i < sc->vmx_ntxqueues; i++) { sc->vmx_queue[i].vxq_txqueue.vxtxq_ts = (struct vmxnet3_txq_shared *) kva; kva += sizeof(struct vmxnet3_txq_shared); } for (i = 0; i < sc->vmx_nrxqueues; i++) { sc->vmx_queue[i].vxq_rxqueue.vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; kva += sizeof(struct vmxnet3_rxq_shared); } if (sc->vmx_flags & VMXNET3_FLAG_RSS) { size = sizeof(struct vmxnet3_rss_shared); error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); if (error) { device_printf(dev, "cannot alloc rss shared memory\n"); return (error); } sc->vmx_rss = (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; } return (0); } static void vmxnet3_free_shared_data(struct vmxnet3_softc *sc) { if (sc->vmx_rss != NULL) { vmxnet3_dma_free(sc, &sc->vmx_rss_dma); sc->vmx_rss = NULL; } if (sc->vmx_qs != NULL) { vmxnet3_dma_free(sc, &sc->vmx_qs_dma); sc->vmx_qs = NULL; } if (sc->vmx_ds != NULL) { vmxnet3_dma_free(sc, &sc->vmx_ds_dma); sc->vmx_ds = NULL; } } static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) { device_t dev; struct vmxnet3_txqueue *txq; struct vmxnet3_txring *txr; struct vmxnet3_comp_ring *txc; size_t descsz, compsz; u_int i; int q, error; dev = sc->vmx_dev; for (q = 0; q < sc->vmx_ntxqueues; q++) { txq = &sc->vmx_queue[q].vxq_txqueue; txr = &txq->vxtxq_cmd_ring; txc = &txq->vxtxq_comp_ring; descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); if (error) { device_printf(dev, "cannot alloc Tx descriptors for " "queue %d error %d\n", q, error); return (error); } txr->vxtxr_txd = (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); if (error) { device_printf(dev, "cannot alloc Tx comp descriptors " "for queue %d error %d\n", q, error); return (error); } txc->vxcr_u.txcd = (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; for (i = 0; i < txr->vxtxr_ndesc; i++) { error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE, VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT, &txr->vxtxr_txbuf[i].vtxb_dmamap); if (error) { device_printf(dev, "unable to create Tx buf " "dmamap for queue %d idx %d\n", q, i); return (error); } } } return (0); } static void vmxnet3_free_txq_data(struct vmxnet3_softc *sc) { struct vmxnet3_txqueue *txq; struct vmxnet3_txring *txr; struct vmxnet3_comp_ring *txc; struct vmxnet3_txbuf *txb; u_int i; int q; for (q = 0; q < sc->vmx_ntxqueues; q++) { txq = &sc->vmx_queue[q].vxq_txqueue; txr = &txq->vxtxq_cmd_ring; txc = &txq->vxtxq_comp_ring; for (i = 0; i < txr->vxtxr_ndesc; i++) { txb = &txr->vxtxr_txbuf[i]; if (txb->vtxb_dmamap != NULL) { bus_dmamap_destroy(sc->vmx_dmat, txb->vtxb_dmamap); txb->vtxb_dmamap = NULL; } } if (txc->vxcr_u.txcd != NULL) { vmxnet3_dma_free(sc, &txc->vxcr_dma); txc->vxcr_u.txcd = NULL; } if (txr->vxtxr_txd != NULL) { vmxnet3_dma_free(sc, &txr->vxtxr_dma); txr->vxtxr_txd = NULL; } } } static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) { device_t dev; struct vmxnet3_rxqueue *rxq; struct vmxnet3_rxring *rxr; struct vmxnet3_comp_ring *rxc; int descsz, compsz; u_int i, j; int q, error; dev = sc->vmx_dev; for (q = 0; q < sc->vmx_nrxqueues; q++) { rxq = &sc->vmx_queue[q].vxq_rxqueue; rxc = &rxq->vxrxq_comp_ring; compsz = 0; for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; descsz = rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc); compsz += rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxcompdesc); error = vmxnet3_dma_malloc(sc, descsz, 512, &rxr->vxrxr_dma); if (error) { device_printf(dev, "cannot allocate Rx " "descriptors for queue %d/%d error %d\n", i, q, error); return (error); } rxr->vxrxr_rxd = (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; } error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); if (error) { device_printf(dev, "cannot alloc Rx comp descriptors " "for queue %d error %d\n", q, error); return (error); } rxc->vxcr_u.rxcd = (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, JUMBO_LEN, 0, BUS_DMA_NOWAIT, &rxr->vxrxr_spare_dmap); if (error) { device_printf(dev, "unable to create spare " "dmamap for queue %d/%d error %d\n", q, i, error); return (error); } for (j = 0; j < rxr->vxrxr_ndesc; j++) { error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1, JUMBO_LEN, 0, BUS_DMA_NOWAIT, &rxr->vxrxr_rxbuf[j].vrxb_dmamap); if (error) { device_printf(dev, "unable to create " "dmamap for queue %d/%d slot %d " "error %d\n", q, i, j, error); return (error); } } } } return (0); } static void vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) { struct vmxnet3_rxqueue *rxq; struct vmxnet3_rxring *rxr; struct vmxnet3_comp_ring *rxc; struct vmxnet3_rxbuf *rxb; u_int i, j; int q; for (q = 0; q < sc->vmx_nrxqueues; q++) { rxq = &sc->vmx_queue[q].vxq_rxqueue; rxc = &rxq->vxrxq_comp_ring; for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; if (rxr->vxrxr_spare_dmap != NULL) { bus_dmamap_destroy(sc->vmx_dmat, rxr->vxrxr_spare_dmap); rxr->vxrxr_spare_dmap = NULL; } for (j = 0; j < rxr->vxrxr_ndesc; j++) { rxb = &rxr->vxrxr_rxbuf[j]; if (rxb->vrxb_dmamap != NULL) { bus_dmamap_destroy(sc->vmx_dmat, rxb->vrxb_dmamap); rxb->vrxb_dmamap = NULL; } } } if (rxc->vxcr_u.rxcd != NULL) { vmxnet3_dma_free(sc, &rxc->vxcr_dma); rxc->vxcr_u.rxcd = NULL; } for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; if (rxr->vxrxr_rxd != NULL) { vmxnet3_dma_free(sc, &rxr->vxrxr_dma); rxr->vxrxr_rxd = NULL; } } } } static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) { int error; error = vmxnet3_alloc_txq_data(sc); if (error) return (error); error = vmxnet3_alloc_rxq_data(sc); if (error) return (error); return (0); } static void vmxnet3_free_queue_data(struct vmxnet3_softc *sc) { if (sc->vmx_queue != NULL) { vmxnet3_free_rxq_data(sc); vmxnet3_free_txq_data(sc); } } static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) { int error; error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 32, &sc->vmx_mcast_dma); if (error) device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); else sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; return (error); } static void vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) { if (sc->vmx_mcast != NULL) { vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); sc->vmx_mcast = NULL; } } static void vmxnet3_init_shared_data(struct vmxnet3_softc *sc) { struct vmxnet3_driver_shared *ds; struct vmxnet3_txqueue *txq; struct vmxnet3_txq_shared *txs; struct vmxnet3_rxqueue *rxq; struct vmxnet3_rxq_shared *rxs; int i; ds = sc->vmx_ds; /* * Initialize fields of the shared data that remains the same across * reinits. Note the shared data is zero'd when allocated. */ ds->magic = VMXNET3_REV1_MAGIC; /* DriverInfo */ ds->version = VMXNET3_DRIVER_VERSION; ds->guest = VMXNET3_GOS_FREEBSD | #ifdef __LP64__ VMXNET3_GOS_64BIT; #else VMXNET3_GOS_32BIT; #endif ds->vmxnet3_revision = 1; ds->upt_version = 1; /* Misc. conf */ ds->driver_data = vtophys(sc); ds->driver_data_len = sizeof(struct vmxnet3_softc); ds->queue_shared = sc->vmx_qs_dma.dma_paddr; ds->queue_shared_len = sc->vmx_qs_dma.dma_size; ds->nrxsg_max = sc->vmx_max_rxsegs; /* RSS conf */ if (sc->vmx_flags & VMXNET3_FLAG_RSS) { ds->rss.version = 1; ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; ds->rss.len = sc->vmx_rss_dma.dma_size; } /* Interrupt control. */ ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; ds->nintr = sc->vmx_nintrs; ds->evintr = sc->vmx_event_intr_idx; ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; for (i = 0; i < sc->vmx_nintrs; i++) ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; /* Receive filter. */ ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; /* Tx queues */ for (i = 0; i < sc->vmx_ntxqueues; i++) { txq = &sc->vmx_queue[i].vxq_txqueue; txs = txq->vxtxq_ts; txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; txs->driver_data = vtophys(txq); txs->driver_data_len = sizeof(struct vmxnet3_txqueue); } /* Rx queues */ for (i = 0; i < sc->vmx_nrxqueues; i++) { rxq = &sc->vmx_queue[i].vxq_rxqueue; rxs = rxq->vxrxq_rs; rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; rxs->driver_data = vtophys(rxq); rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); } } static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) { /* * Use the same key as the Linux driver until FreeBSD can do * RSS (presumably Toeplitz) in software. */ static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, }; struct vmxnet3_rss_shared *rss; int i; rss = sc->vmx_rss; rss->hash_type = UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) rss->ind_table[i] = i % sc->vmx_nrxqueues; } static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) { struct ifnet *ifp; struct vmxnet3_driver_shared *ds; ifp = &sc->vmx_ethercom.ec_if; ds = sc->vmx_ds; ds->mtu = ifp->if_mtu; ds->ntxqueue = sc->vmx_ntxqueues; ds->nrxqueue = sc->vmx_nrxqueues; ds->upt_features = 0; if (ifp->if_capenable & (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) ds->upt_features |= UPT1_F_CSUM; if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) ds->upt_features |= UPT1_F_VLAN; if (sc->vmx_flags & VMXNET3_FLAG_RSS) { ds->upt_features |= UPT1_F_RSS; vmxnet3_reinit_rss_shared_data(sc); } vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); } static int vmxnet3_alloc_data(struct vmxnet3_softc *sc) { int error; error = vmxnet3_alloc_shared_data(sc); if (error) return (error); error = vmxnet3_alloc_queue_data(sc); if (error) return (error); error = vmxnet3_alloc_mcast_table(sc); if (error) return (error); vmxnet3_init_shared_data(sc); return (0); } static void vmxnet3_free_data(struct vmxnet3_softc *sc) { vmxnet3_free_mcast_table(sc); vmxnet3_free_queue_data(sc); vmxnet3_free_shared_data(sc); } static int vmxnet3_setup_interface(struct vmxnet3_softc *sc) { struct ifnet *ifp = &sc->vmx_ethercom.ec_if; vmxnet3_get_lladdr(sc); aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n", ether_sprintf(sc->vmx_lladdr)); vmxnet3_set_lladdr(sc); strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ); ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; ifp->if_extflags = IFEF_MPSAFE; ifp->if_ioctl = vmxnet3_ioctl; ifp->if_start = vmxnet3_start; ifp->if_transmit = vmxnet3_transmit; ifp->if_watchdog = NULL; ifp->if_init = vmxnet3_init; ifp->if_stop = vmxnet3_stop; sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; ifp->if_capenable = ifp->if_capabilities; sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; sc->vmx_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING | ETHERCAP_JUMBO_MTU; sc->vmx_ethercom.ec_capenable |= ETHERCAP_VLAN_HWTAGGING; IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs); IFQ_SET_READY(&ifp->if_snd); /* Initialize ifmedia structures. */ sc->vmx_ethercom.ec_ifmedia = &sc->vmx_media; ifmedia_init_with_lock(&sc->vmx_media, IFM_IMASK, vmxnet3_ifmedia_change, vmxnet3_ifmedia_status, sc->vmx_mtx); ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL); ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T, 0, NULL); ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T, 0, NULL); ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); if_attach(ifp); if_deferred_start_init(ifp, NULL); ether_ifattach(ifp, sc->vmx_lladdr); ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb); vmxnet3_cmd_link_status(ifp); /* should set before setting interrupts */ sc->vmx_rx_intr_process_limit = VMXNET3_RX_INTR_PROCESS_LIMIT; sc->vmx_rx_process_limit = VMXNET3_RX_PROCESS_LIMIT; sc->vmx_tx_intr_process_limit = VMXNET3_TX_INTR_PROCESS_LIMIT; sc->vmx_tx_process_limit = VMXNET3_TX_PROCESS_LIMIT; return (0); } static int vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) { const char *devname; struct sysctllog **log; const struct sysctlnode *rnode, *rxnode, *txnode; int error; log = &sc->vmx_sysctllog; devname = device_xname(sc->vmx_dev); error = sysctl_createv(log, 0, NULL, &rnode, 0, CTLTYPE_NODE, devname, SYSCTL_DESCR("vmxnet3 information and settings"), NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rnode, NULL, CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), NULL, 0, &sc->vmx_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rnode, &rxnode, 0, CTLTYPE_NODE, "rx", SYSCTL_DESCR("vmxnet3 information and settings for Rx"), NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rxnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), NULL, 0, &sc->vmx_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rxnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), NULL, 0, &sc->vmx_rx_process_limit, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &rnode, &txnode, 0, CTLTYPE_NODE, "tx", SYSCTL_DESCR("vmxnet3 information and settings for Tx"), NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &txnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), NULL, 0, &sc->vmx_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); if (error) goto out; error = sysctl_createv(log, 0, &txnode, NULL, CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), NULL, 0, &sc->vmx_tx_process_limit, 0, CTL_CREATE, CTL_EOL); out: if (error) { aprint_error_dev(sc->vmx_dev, "unable to create sysctl node\n"); sysctl_teardown(log); } return error; } static int vmxnet3_setup_stats(struct vmxnet3_softc *sc) { struct vmxnet3_queue *vmxq; struct vmxnet3_txqueue *txq; struct vmxnet3_rxqueue *rxq; int i; for (i = 0; i < sc->vmx_ntxqueues; i++) { vmxq = &sc->vmx_queue[i]; txq = &vmxq->vxq_txqueue; evcnt_attach_dynamic(&txq->vxtxq_intr, EVCNT_TYPE_INTR, NULL, txq->vxtxq_name, "Interrupt on queue"); evcnt_attach_dynamic(&txq->vxtxq_defer, EVCNT_TYPE_MISC, NULL, txq->vxtxq_name, "Handled queue in softint/workqueue"); evcnt_attach_dynamic(&txq->vxtxq_deferreq, EVCNT_TYPE_MISC, NULL, txq->vxtxq_name, "Requested in softint/workqueue"); evcnt_attach_dynamic(&txq->vxtxq_pcqdrop, EVCNT_TYPE_MISC, NULL, txq->vxtxq_name, "Dropped in pcq"); evcnt_attach_dynamic(&txq->vxtxq_transmitdef, EVCNT_TYPE_MISC, NULL, txq->vxtxq_name, "Deferred transmit"); evcnt_attach_dynamic(&txq->vxtxq_watchdogto, EVCNT_TYPE_MISC, NULL, txq->vxtxq_name, "Watchdog timeout"); evcnt_attach_dynamic(&txq->vxtxq_defragged, EVCNT_TYPE_MISC, NULL, txq->vxtxq_name, "m_defrag successed"); evcnt_attach_dynamic(&txq->vxtxq_defrag_failed, EVCNT_TYPE_MISC, NULL, txq->vxtxq_name, "m_defrag failed"); } for (i = 0; i < sc->vmx_nrxqueues; i++) { vmxq = &sc->vmx_queue[i]; rxq = &vmxq->vxq_rxqueue; evcnt_attach_dynamic(&rxq->vxrxq_intr, EVCNT_TYPE_INTR, NULL, rxq->vxrxq_name, "Interrupt on queue"); evcnt_attach_dynamic(&rxq->vxrxq_defer, EVCNT_TYPE_MISC, NULL, rxq->vxrxq_name, "Handled queue in softint/workqueue"); evcnt_attach_dynamic(&rxq->vxrxq_deferreq, EVCNT_TYPE_MISC, NULL, rxq->vxrxq_name, "Requested in softint/workqueue"); evcnt_attach_dynamic(&rxq->vxrxq_mgetcl_failed, EVCNT_TYPE_MISC, NULL, rxq->vxrxq_name, "MCLGET failed"); evcnt_attach_dynamic(&rxq->vxrxq_mbuf_load_failed, EVCNT_TYPE_MISC, NULL, rxq->vxrxq_name, "bus_dmamap_load_mbuf failed"); } evcnt_attach_dynamic(&sc->vmx_event_intr, EVCNT_TYPE_INTR, NULL, device_xname(sc->vmx_dev), "Interrupt for other events"); evcnt_attach_dynamic(&sc->vmx_event_link, EVCNT_TYPE_MISC, NULL, device_xname(sc->vmx_dev), "Link status event"); evcnt_attach_dynamic(&sc->vmx_event_txqerror, EVCNT_TYPE_MISC, NULL, device_xname(sc->vmx_dev), "Tx queue error event"); evcnt_attach_dynamic(&sc->vmx_event_rxqerror, EVCNT_TYPE_MISC, NULL, device_xname(sc->vmx_dev), "Rx queue error event"); evcnt_attach_dynamic(&sc->vmx_event_dic, EVCNT_TYPE_MISC, NULL, device_xname(sc->vmx_dev), "Device impl change event"); evcnt_attach_dynamic(&sc->vmx_event_debug, EVCNT_TYPE_MISC, NULL, device_xname(sc->vmx_dev), "Debug event"); return 0; } static void vmxnet3_teardown_stats(struct vmxnet3_softc *sc) { struct vmxnet3_queue *vmxq; struct vmxnet3_txqueue *txq; struct vmxnet3_rxqueue *rxq; int i; for (i = 0; i < sc->vmx_ntxqueues; i++) { vmxq = &sc->vmx_queue[i]; txq = &vmxq->vxq_txqueue; evcnt_detach(&txq->vxtxq_intr); evcnt_detach(&txq->vxtxq_defer); evcnt_detach(&txq->vxtxq_deferreq); evcnt_detach(&txq->vxtxq_pcqdrop); evcnt_detach(&txq->vxtxq_transmitdef); evcnt_detach(&txq->vxtxq_watchdogto); evcnt_detach(&txq->vxtxq_defragged); evcnt_detach(&txq->vxtxq_defrag_failed); } for (i = 0; i < sc->vmx_nrxqueues; i++) { vmxq = &sc->vmx_queue[i]; rxq = &vmxq->vxq_rxqueue; evcnt_detach(&rxq->vxrxq_intr); evcnt_detach(&rxq->vxrxq_defer); evcnt_detach(&rxq->vxrxq_deferreq); evcnt_detach(&rxq->vxrxq_mgetcl_failed); evcnt_detach(&rxq->vxrxq_mbuf_load_failed); } evcnt_detach(&sc->vmx_event_intr); evcnt_detach(&sc->vmx_event_link); evcnt_detach(&sc->vmx_event_txqerror); evcnt_detach(&sc->vmx_event_rxqerror); evcnt_detach(&sc->vmx_event_dic); evcnt_detach(&sc->vmx_event_debug); } static void vmxnet3_evintr(struct vmxnet3_softc *sc) { device_t dev; struct vmxnet3_txq_shared *ts; struct vmxnet3_rxq_shared *rs; uint32_t event; int reset; dev = sc->vmx_dev; reset = 0; VMXNET3_CORE_LOCK(sc); /* Clear events. */ event = sc->vmx_ds->event; vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); if (event & VMXNET3_EVENT_LINK) { sc->vmx_event_link.ev_count++; vmxnet3_if_link_status(sc); if (sc->vmx_link_active != 0) if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); } if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { if (event & VMXNET3_EVENT_TQERROR) sc->vmx_event_txqerror.ev_count++; if (event & VMXNET3_EVENT_RQERROR) sc->vmx_event_rxqerror.ev_count++; reset = 1; vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); ts = sc->vmx_queue[0].vxq_txqueue.vxtxq_ts; if (ts->stopped != 0) device_printf(dev, "Tx queue error %#x\n", ts->error); rs = sc->vmx_queue[0].vxq_rxqueue.vxrxq_rs; if (rs->stopped != 0) device_printf(dev, "Rx queue error %#x\n", rs->error); device_printf(dev, "Rx/Tx queue error event ... resetting\n"); } if (event & VMXNET3_EVENT_DIC) { sc->vmx_event_dic.ev_count++; device_printf(dev, "device implementation change event\n"); } if (event & VMXNET3_EVENT_DEBUG) { sc->vmx_event_debug.ev_count++; device_printf(dev, "debug event\n"); } if (reset != 0) vmxnet3_init_locked(sc); VMXNET3_CORE_UNLOCK(sc); } static bool vmxnet3_txq_eof(struct vmxnet3_txqueue *txq, u_int limit) { struct vmxnet3_softc *sc; struct vmxnet3_txring *txr; struct vmxnet3_comp_ring *txc; struct vmxnet3_txcompdesc *txcd; struct vmxnet3_txbuf *txb; struct ifnet *ifp; struct mbuf *m; u_int sop; bool more = false; sc = txq->vxtxq_sc; txr = &txq->vxtxq_cmd_ring; txc = &txq->vxtxq_comp_ring; ifp = &sc->vmx_ethercom.ec_if; VMXNET3_TXQ_LOCK_ASSERT(txq); net_stat_ref_t nsr = IF_STAT_GETREF(ifp); for (;;) { if (limit-- == 0) { more = true; break; } txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; if (txcd->gen != txc->vxcr_gen) break; vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); if (++txc->vxcr_next == txc->vxcr_ndesc) { txc->vxcr_next = 0; txc->vxcr_gen ^= 1; } sop = txr->vxtxr_next; txb = &txr->vxtxr_txbuf[sop]; if ((m = txb->vtxb_m) != NULL) { bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 0, txb->vtxb_dmamap->dm_mapsize, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); if_statinc_ref(ifp, nsr, if_opackets); if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len); if (m->m_flags & M_MCAST) if_statinc_ref(ifp, nsr, if_omcasts); m_freem(m); txb->vtxb_m = NULL; } txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; } IF_STAT_PUTREF(ifp); if (txr->vxtxr_head == txr->vxtxr_next) txq->vxtxq_watchdog = 0; return more; } static int vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq, struct vmxnet3_rxring *rxr) { struct mbuf *m; struct vmxnet3_rxdesc *rxd; struct vmxnet3_rxbuf *rxb; bus_dma_tag_t tag; bus_dmamap_t dmap; int idx, btype, error; tag = sc->vmx_dmat; dmap = rxr->vxrxr_spare_dmap; idx = rxr->vxrxr_fill; rxd = &rxr->vxrxr_rxd[idx]; rxb = &rxr->vxrxr_rxbuf[idx]; /* Don't allocate buffers for ring 2 for now. */ if (rxr->vxrxr_rid != 0) return -1; btype = VMXNET3_BTYPE_HEAD; MGETHDR(m, M_DONTWAIT, MT_DATA); if (m == NULL) return (ENOBUFS); MCLGET(m, M_DONTWAIT); if ((m->m_flags & M_EXT) == 0) { rxq->vxrxq_mgetcl_failed.ev_count++; m_freem(m); return (ENOBUFS); } m->m_pkthdr.len = m->m_len = JUMBO_LEN; m_adj(m, ETHER_ALIGN); error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT); if (error) { m_freem(m); rxq->vxrxq_mbuf_load_failed.ev_count++; return (error); } if (rxb->vrxb_m != NULL) { bus_dmamap_sync(tag, rxb->vrxb_dmamap, 0, rxb->vrxb_dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(tag, rxb->vrxb_dmamap); } rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; rxb->vrxb_dmamap = dmap; rxb->vrxb_m = m; rxd->addr = DMAADDR(dmap); rxd->len = m->m_pkthdr.len; rxd->btype = btype; rxd->gen = rxr->vxrxr_gen; vmxnet3_rxr_increment_fill(rxr); return (0); } static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, struct vmxnet3_rxring *rxr, int idx) { struct vmxnet3_rxdesc *rxd; rxd = &rxr->vxrxr_rxd[idx]; rxd->gen = rxr->vxrxr_gen; vmxnet3_rxr_increment_fill(rxr); } static void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) { struct vmxnet3_softc *sc; struct vmxnet3_rxring *rxr; struct vmxnet3_comp_ring *rxc; struct vmxnet3_rxcompdesc *rxcd; int idx, eof; sc = rxq->vxrxq_sc; rxc = &rxq->vxrxq_comp_ring; do { rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; if (rxcd->gen != rxc->vxcr_gen) break; /* Not expected. */ vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); if (++rxc->vxcr_next == rxc->vxcr_ndesc) { rxc->vxcr_next = 0; rxc->vxcr_gen ^= 1; } idx = rxcd->rxd_idx; eof = rxcd->eop; if (rxcd->qid < sc->vmx_nrxqueues) rxr = &rxq->vxrxq_cmd_ring[0]; else rxr = &rxq->vxrxq_cmd_ring[1]; vmxnet3_rxq_eof_discard(rxq, rxr, idx); } while (!eof); } static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) { if (rxcd->no_csum) return; if (rxcd->ipv4) { m->m_pkthdr.csum_flags |= M_CSUM_IPv4; if (rxcd->ipcsum_ok == 0) m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD; } if (rxcd->fragment) return; if (rxcd->tcp) { m->m_pkthdr.csum_flags |= rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6; if ((rxcd->csum_ok) == 0) m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; } if (rxcd->udp) { m->m_pkthdr.csum_flags |= rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ; if ((rxcd->csum_ok) == 0) m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; } } static void vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) { struct vmxnet3_softc *sc; struct ifnet *ifp; sc = rxq->vxrxq_sc; ifp = &sc->vmx_ethercom.ec_if; if (rxcd->error) { if_statinc(ifp, if_ierrors); m_freem(m); return; } if (!rxcd->no_csum) vmxnet3_rx_csum(rxcd, m); if (rxcd->vlan) vlan_set_tag(m, rxcd->vtag); net_stat_ref_t nsr = IF_STAT_GETREF(ifp); if_statinc_ref(ifp, nsr, if_ipackets); if_statadd_ref(ifp, nsr, if_ibytes, m->m_pkthdr.len); IF_STAT_PUTREF(ifp); if_percpuq_enqueue(ifp->if_percpuq, m); } static bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq, u_int limit) { struct vmxnet3_softc *sc; struct ifnet *ifp; struct vmxnet3_rxring *rxr; struct vmxnet3_comp_ring *rxc; struct vmxnet3_rxdesc *rxd __diagused; struct vmxnet3_rxcompdesc *rxcd; struct mbuf *m, *m_head, *m_tail; u_int idx, length; bool more = false; sc = rxq->vxrxq_sc; ifp = &sc->vmx_ethercom.ec_if; rxc = &rxq->vxrxq_comp_ring; VMXNET3_RXQ_LOCK_ASSERT(rxq); if (rxq->vxrxq_stopping) return more; m_head = rxq->vxrxq_mhead; rxq->vxrxq_mhead = NULL; m_tail = rxq->vxrxq_mtail; rxq->vxrxq_mtail = NULL; KASSERT(m_head == NULL || m_tail != NULL); for (;;) { if (limit-- == 0) { more = true; break; } rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; if (rxcd->gen != rxc->vxcr_gen) { rxq->vxrxq_mhead = m_head; rxq->vxrxq_mtail = m_tail; break; } vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); if (++rxc->vxcr_next == rxc->vxcr_ndesc) { rxc->vxcr_next = 0; rxc->vxcr_gen ^= 1; } idx = rxcd->rxd_idx; length = rxcd->len; if (rxcd->qid < sc->vmx_nrxqueues) rxr = &rxq->vxrxq_cmd_ring[0]; else rxr = &rxq->vxrxq_cmd_ring[1]; rxd = &rxr->vxrxr_rxd[idx]; m = rxr->vxrxr_rxbuf[idx].vrxb_m; KASSERT(m != NULL); /* * The host may skip descriptors. We detect this when this * descriptor does not match the previous fill index. Catch * up with the host now. */ if (__predict_false(rxr->vxrxr_fill != idx)) { while (rxr->vxrxr_fill != idx) { rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = rxr->vxrxr_gen; vmxnet3_rxr_increment_fill(rxr); } } if (rxcd->sop) { /* start of frame w/o head buffer */ KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD); /* start of frame not in ring 0 */ KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]); /* duplicate start of frame? */ KASSERT(m_head == NULL); if (length == 0) { /* Just ignore this descriptor. */ vmxnet3_rxq_eof_discard(rxq, rxr, idx); goto nextp; } if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { if_statinc(ifp, if_iqdrops); vmxnet3_rxq_eof_discard(rxq, rxr, idx); if (!rxcd->eop) vmxnet3_rxq_discard_chain(rxq); goto nextp; } m_set_rcvif(m, ifp); m->m_pkthdr.len = m->m_len = length; m->m_pkthdr.csum_flags = 0; m_head = m_tail = m; } else { /* non start of frame w/o body buffer */ KASSERT(rxd->btype == VMXNET3_BTYPE_BODY); /* frame not started? */ KASSERT(m_head != NULL); if (vmxnet3_newbuf(sc, rxq, rxr) != 0) { if_statinc(ifp, if_iqdrops); vmxnet3_rxq_eof_discard(rxq, rxr, idx); if (!rxcd->eop) vmxnet3_rxq_discard_chain(rxq); m_freem(m_head); m_head = m_tail = NULL; goto nextp; } m->m_len = length; m_head->m_pkthdr.len += length; m_tail->m_next = m; m_tail = m; } if (rxcd->eop) { vmxnet3_rxq_input(rxq, rxcd, m_head); m_head = m_tail = NULL; /* Must recheck after dropping the Rx lock. */ if (rxq->vxrxq_stopping) break; } nextp: if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { int qid = rxcd->qid; bus_size_t r; idx = (idx + 1) % rxr->vxrxr_ndesc; if (qid >= sc->vmx_nrxqueues) { qid -= sc->vmx_nrxqueues; r = VMXNET3_BAR0_RXH2(qid); } else r = VMXNET3_BAR0_RXH1(qid); vmxnet3_write_bar0(sc, r, idx); } } return more; } static inline void vmxnet3_sched_handle_queue(struct vmxnet3_softc *sc, struct vmxnet3_queue *vmxq) { if (vmxq->vxq_workqueue) { /* * When this function is called, "vmxq" is owned by one CPU. * so, atomic operation is not required here. */ if (!vmxq->vxq_wq_enqueued) { vmxq->vxq_wq_enqueued = true; workqueue_enqueue(sc->vmx_queue_wq, &vmxq->vxq_wq_cookie, curcpu()); } } else { softint_schedule(vmxq->vxq_si); } } static int vmxnet3_legacy_intr(void *xsc) { struct vmxnet3_softc *sc; struct vmxnet3_queue *vmxq; struct vmxnet3_txqueue *txq; struct vmxnet3_rxqueue *rxq; u_int txlimit, rxlimit; bool txmore, rxmore; sc = xsc; vmxq = &sc->vmx_queue[0]; txq = &vmxq->vxq_txqueue; rxq = &vmxq->vxq_rxqueue; txlimit = sc->vmx_tx_intr_process_limit; rxlimit = sc->vmx_rx_intr_process_limit; if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) return (0); } if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) vmxnet3_disable_all_intrs(sc); if (sc->vmx_ds->event != 0) vmxnet3_evintr(sc); VMXNET3_TXQ_LOCK(txq); txmore = vmxnet3_txq_eof(txq, txlimit); VMXNET3_TXQ_UNLOCK(txq); VMXNET3_RXQ_LOCK(rxq); rxmore = vmxnet3_rxq_eof(rxq, rxlimit); VMXNET3_RXQ_UNLOCK(rxq); if (txmore || rxmore) vmxnet3_sched_handle_queue(sc, vmxq); else { if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); vmxnet3_enable_all_intrs(sc); } return (1); } static int vmxnet3_txrxq_intr(void *xvmxq) { struct vmxnet3_softc *sc; struct vmxnet3_queue *vmxq; struct vmxnet3_txqueue *txq; struct vmxnet3_rxqueue *rxq; u_int txlimit, rxlimit; bool txmore, rxmore; vmxq = xvmxq; txq = &vmxq->vxq_txqueue; rxq = &vmxq->vxq_rxqueue; sc = txq->vxtxq_sc; txlimit = sc->vmx_tx_intr_process_limit; rxlimit = sc->vmx_rx_intr_process_limit; vmxq->vxq_workqueue = sc->vmx_txrx_workqueue; if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) vmxnet3_disable_intr(sc, vmxq->vxq_intr_idx); VMXNET3_TXQ_LOCK(txq); txq->vxtxq_intr.ev_count++; txmore = vmxnet3_txq_eof(txq, txlimit); VMXNET3_TXQ_UNLOCK(txq); VMXNET3_RXQ_LOCK(rxq); rxq->vxrxq_intr.ev_count++; rxmore = vmxnet3_rxq_eof(rxq, rxlimit); VMXNET3_RXQ_UNLOCK(rxq); if (txmore || rxmore) vmxnet3_sched_handle_queue(sc, vmxq); else { /* for ALTQ */ if (vmxq->vxq_id == 0) if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); softint_schedule(txq->vxtxq_si); vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); } return (1); } static void vmxnet3_handle_queue(void *xvmxq) { struct vmxnet3_softc *sc; struct vmxnet3_queue *vmxq; struct vmxnet3_txqueue *txq; struct vmxnet3_rxqueue *rxq; u_int txlimit, rxlimit; bool txmore, rxmore; vmxq = xvmxq; txq = &vmxq->vxq_txqueue; rxq = &vmxq->vxq_rxqueue; sc = txq->vxtxq_sc; txlimit = sc->vmx_tx_process_limit; rxlimit = sc->vmx_rx_process_limit; VMXNET3_TXQ_LOCK(txq); txq->vxtxq_defer.ev_count++; txmore = vmxnet3_txq_eof(txq, txlimit); if (txmore) txq->vxtxq_deferreq.ev_count++; /* for ALTQ */ if (vmxq->vxq_id == 0) if_schedule_deferred_start(&sc->vmx_ethercom.ec_if); softint_schedule(txq->vxtxq_si); VMXNET3_TXQ_UNLOCK(txq); VMXNET3_RXQ_LOCK(rxq); rxq->vxrxq_defer.ev_count++; rxmore = vmxnet3_rxq_eof(rxq, rxlimit); if (rxmore) rxq->vxrxq_deferreq.ev_count++; VMXNET3_RXQ_UNLOCK(rxq); if (txmore || rxmore) vmxnet3_sched_handle_queue(sc, vmxq); else vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx); } static void vmxnet3_handle_queue_work(struct work *wk, void *context) { struct vmxnet3_queue *vmxq; vmxq = container_of(wk, struct vmxnet3_queue, vxq_wq_cookie); vmxq->vxq_wq_enqueued = false; vmxnet3_handle_queue(vmxq); } static int vmxnet3_event_intr(void *xsc) { struct vmxnet3_softc *sc; sc = xsc; if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); sc->vmx_event_intr.ev_count++; if (sc->vmx_ds->event != 0) vmxnet3_evintr(sc); vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); return (1); } static void vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) { struct vmxnet3_txring *txr; struct vmxnet3_txbuf *txb; u_int i; txr = &txq->vxtxq_cmd_ring; for (i = 0; i < txr->vxtxr_ndesc; i++) { txb = &txr->vxtxr_txbuf[i]; if (txb->vtxb_m == NULL) continue; bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap, 0, txb->vtxb_dmamap->dm_mapsize, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap); m_freem(txb->vtxb_m); txb->vtxb_m = NULL; } } static void vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) { struct vmxnet3_rxring *rxr; struct vmxnet3_rxbuf *rxb; u_int i, j; if (rxq->vxrxq_mhead != NULL) { m_freem(rxq->vxrxq_mhead); rxq->vxrxq_mhead = NULL; rxq->vxrxq_mtail = NULL; } for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; for (j = 0; j < rxr->vxrxr_ndesc; j++) { rxb = &rxr->vxrxr_rxbuf[j]; if (rxb->vrxb_m == NULL) continue; bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap, 0, rxb->vrxb_dmamap->dm_mapsize, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap); m_freem(rxb->vrxb_m); rxb->vrxb_m = NULL; } } } static void vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) { struct vmxnet3_rxqueue *rxq; struct vmxnet3_txqueue *txq; struct vmxnet3_queue *vmxq; int i; for (i = 0; i < sc->vmx_nrxqueues; i++) { rxq = &sc->vmx_queue[i].vxq_rxqueue; VMXNET3_RXQ_LOCK(rxq); rxq->vxrxq_stopping = true; VMXNET3_RXQ_UNLOCK(rxq); } for (i = 0; i < sc->vmx_ntxqueues; i++) { txq = &sc->vmx_queue[i].vxq_txqueue; VMXNET3_TXQ_LOCK(txq); txq->vxtxq_stopping = true; VMXNET3_TXQ_UNLOCK(txq); } for (i = 0; i < sc->vmx_nrxqueues; i++) { vmxq = &sc->vmx_queue[i]; workqueue_wait(sc->vmx_queue_wq, &vmxq->vxq_wq_cookie); } } static void vmxnet3_stop_locked(struct vmxnet3_softc *sc) { struct ifnet *ifp = &sc->vmx_ethercom.ec_if; int q; VMXNET3_CORE_LOCK_ASSERT(sc); KASSERT(IFNET_LOCKED(ifp)); vmxnet3_stop_rendezvous(sc); sc->vmx_mcastactive = false; sc->vmx_link_active = 0; callout_halt(&sc->vmx_tick, sc->vmx_mtx); ifp->if_flags &= ~IFF_RUNNING; /* Disable interrupts. */ vmxnet3_disable_all_intrs(sc); vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); for (q = 0; q < sc->vmx_ntxqueues; q++) vmxnet3_txstop(sc, &sc->vmx_queue[q].vxq_txqueue); for (q = 0; q < sc->vmx_nrxqueues; q++) vmxnet3_rxstop(sc, &sc->vmx_queue[q].vxq_rxqueue); vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); } static void vmxnet3_stop(struct ifnet *ifp, int disable) { struct vmxnet3_softc *sc = ifp->if_softc; KASSERT(IFNET_LOCKED(ifp)); VMXNET3_CORE_LOCK(sc); vmxnet3_stop_locked(sc); VMXNET3_CORE_UNLOCK(sc); } static void vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) { struct vmxnet3_txring *txr; struct vmxnet3_comp_ring *txc; txr = &txq->vxtxq_cmd_ring; txr->vxtxr_head = 0; txr->vxtxr_next = 0; txr->vxtxr_gen = VMXNET3_INIT_GEN; memset(txr->vxtxr_txd, 0, txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); txc = &txq->vxtxq_comp_ring; txc->vxcr_next = 0; txc->vxcr_gen = VMXNET3_INIT_GEN; memset(txc->vxcr_u.txcd, 0, txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); } static int vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) { struct vmxnet3_rxring *rxr; struct vmxnet3_comp_ring *rxc; u_int i, populate, idx; int error; /* LRO and jumbo frame is not supported yet */ populate = 1; for (i = 0; i < populate; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; rxr->vxrxr_fill = 0; rxr->vxrxr_gen = VMXNET3_INIT_GEN; memset(rxr->vxrxr_rxd, 0, rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { error = vmxnet3_newbuf(sc, rxq, rxr); if (error) return (error); } } for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { rxr = &rxq->vxrxq_cmd_ring[i]; rxr->vxrxr_fill = 0; rxr->vxrxr_gen = 0; memset(rxr->vxrxr_rxd, 0, rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); } rxc = &rxq->vxrxq_comp_ring; rxc->vxcr_next = 0; rxc->vxcr_gen = VMXNET3_INIT_GEN; memset(rxc->vxcr_u.rxcd, 0, rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); return (0); } static int vmxnet3_reinit_queues(struct vmxnet3_softc *sc) { device_t dev; int q, error; dev = sc->vmx_dev; for (q = 0; q < sc->vmx_ntxqueues; q++) vmxnet3_txinit(sc, &sc->vmx_queue[q].vxq_txqueue); for (q = 0; q < sc->vmx_nrxqueues; q++) { error = vmxnet3_rxinit(sc, &sc->vmx_queue[q].vxq_rxqueue); if (error) { device_printf(dev, "cannot populate Rx queue %d\n", q); return (error); } } return (0); } static int vmxnet3_enable_device(struct vmxnet3_softc *sc) { int q; if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { device_printf(sc->vmx_dev, "device enable command failed!\n"); return (1); } /* Reset the Rx queue heads. */ for (q = 0; q < sc->vmx_nrxqueues; q++) { vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); } return (0); } static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) { vmxnet3_set_rxfilter(sc); memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter)); vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); } static int vmxnet3_reinit(struct vmxnet3_softc *sc) { VMXNET3_CORE_LOCK_ASSERT(sc); vmxnet3_set_lladdr(sc); vmxnet3_reinit_shared_data(sc); if (vmxnet3_reinit_queues(sc) != 0) return (ENXIO); if (vmxnet3_enable_device(sc) != 0) return (ENXIO); vmxnet3_reinit_rxfilters(sc); return (0); } static int vmxnet3_init_locked(struct vmxnet3_softc *sc) { struct ifnet *ifp = &sc->vmx_ethercom.ec_if; int q; int error; KASSERT(IFNET_LOCKED(ifp)); VMXNET3_CORE_LOCK_ASSERT(sc); vmxnet3_stop_locked(sc); error = vmxnet3_reinit(sc); if (error) { vmxnet3_stop_locked(sc); return (error); } ifp->if_flags |= IFF_RUNNING; vmxnet3_if_link_status(sc); sc->vmx_mcastactive = true; vmxnet3_enable_all_intrs(sc); callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); for (q = 0; q < sc->vmx_ntxqueues; q++) { VMXNET3_TXQ_LOCK(&sc->vmx_queue[q].vxq_txqueue); sc->vmx_queue[q].vxq_txqueue.vxtxq_stopping = false; VMXNET3_TXQ_UNLOCK(&sc->vmx_queue[q].vxq_txqueue); } for (q = 0; q < sc->vmx_nrxqueues; q++) { VMXNET3_RXQ_LOCK(&sc->vmx_queue[q].vxq_rxqueue); sc->vmx_queue[q].vxq_rxqueue.vxrxq_stopping = false; VMXNET3_RXQ_UNLOCK(&sc->vmx_queue[q].vxq_rxqueue); } return (0); } static int vmxnet3_init(struct ifnet *ifp) { struct vmxnet3_softc *sc = ifp->if_softc; int error; KASSERT(IFNET_LOCKED(ifp)); VMXNET3_CORE_LOCK(sc); error = vmxnet3_init_locked(sc); VMXNET3_CORE_UNLOCK(sc); return (error); } static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, int *start, int *csum_start) { struct ether_header *eh; struct mbuf *mp; int offset, csum_off, iphl, offp; bool v4; eh = mtod(m, struct ether_header *); switch (htons(eh->ether_type)) { case ETHERTYPE_IP: case ETHERTYPE_IPV6: offset = ETHER_HDR_LEN; break; case ETHERTYPE_VLAN: offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; break; default: m_freem(m); return (EINVAL); } if ((m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_UDPv4 | M_CSUM_TCPv4)) != 0) { iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); v4 = true; } else { iphl = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); v4 = false; } *start = offset + iphl; if (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) { csum_off = offsetof(struct tcphdr, th_sum); } else { csum_off = offsetof(struct udphdr, uh_sum); } *csum_start = *start + csum_off; mp = m_pulldown(m, 0, *csum_start + 2, &offp); if (!mp) { /* m is already freed */ return ENOBUFS; } if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { struct tcphdr *tcp; txq->vxtxq_stats.vmtxs_tso++; tcp = (void *)(mtod(mp, char *) + offp + *start); if (v4) { struct ip *ip; ip = (void *)(mtod(mp, char *) + offp + offset); tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(IPPROTO_TCP)); } else { struct ip6_hdr *ip6; ip6 = (void *)(mtod(mp, char *) + offp + offset); tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); } /* * For TSO, the size of the protocol header is also * included in the descriptor header size. */ *start += (tcp->th_off << 2); } else txq->vxtxq_stats.vmtxs_csum++; return (0); } static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, bus_dmamap_t dmap) { struct mbuf *m; bus_dma_tag_t tag; int error; m = *m0; tag = txq->vxtxq_sc->vmx_dmat; error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); if (error == 0 || error != EFBIG) return (error); m = m_defrag(m, M_NOWAIT); if (m != NULL) { *m0 = m; error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT); } else error = ENOBUFS; if (error) { m_freem(*m0); *m0 = NULL; txq->vxtxq_defrag_failed.ev_count++; } else txq->vxtxq_defragged.ev_count++; return (error); } static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) { bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap); } static int vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) { struct vmxnet3_softc *sc; struct vmxnet3_txring *txr; struct vmxnet3_txdesc *txd, *sop; struct mbuf *m; bus_dmamap_t dmap; bus_dma_segment_t *segs; int i, gen, start, csum_start, nsegs, error; sc = txq->vxtxq_sc; start = 0; txd = NULL; txr = &txq->vxtxq_cmd_ring; dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; csum_start = 0; /* GCC */ error = vmxnet3_txq_load_mbuf(txq, m0, dmap); if (error) return (error); nsegs = dmap->dm_nsegs; segs = dmap->dm_segs; m = *m0; KASSERT(m->m_flags & M_PKTHDR); KASSERT(nsegs <= VMXNET3_TX_MAXSEGS); if (vmxnet3_txring_avail(txr) < nsegs) { txq->vxtxq_stats.vmtxs_full++; vmxnet3_txq_unload_mbuf(txq, dmap); return (ENOSPC); } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start); if (error) { /* m is already freed */ txq->vxtxq_stats.vmtxs_offload_failed++; vmxnet3_txq_unload_mbuf(txq, dmap); *m0 = NULL; return (error); } } txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; sop = &txr->vxtxr_txd[txr->vxtxr_head]; gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ for (i = 0; i < nsegs; i++) { txd = &txr->vxtxr_txd[txr->vxtxr_head]; txd->addr = segs[i].ds_addr; txd->len = segs[i].ds_len; txd->gen = gen; txd->dtype = 0; txd->offload_mode = VMXNET3_OM_NONE; txd->offload_pos = 0; txd->hlen = 0; txd->eop = 0; txd->compreq = 0; txd->vtag_mode = 0; txd->vtag = 0; if (++txr->vxtxr_head == txr->vxtxr_ndesc) { txr->vxtxr_head = 0; txr->vxtxr_gen ^= 1; } gen = txr->vxtxr_gen; } txd->eop = 1; txd->compreq = 1; if (vlan_has_tag(m)) { sop->vtag_mode = 1; sop->vtag = vlan_get_tag(m); } if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { sop->offload_mode = VMXNET3_OM_TSO; sop->hlen = start; sop->offload_pos = m->m_pkthdr.segsz; } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6)) { sop->offload_mode = VMXNET3_OM_CSUM; sop->hlen = start; sop->offload_pos = csum_start; } /* Finally, change the ownership. */ vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); sop->gen ^= 1; txq->vxtxq_ts->npending += nsegs; if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { struct vmxnet3_queue *vmxq; vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); txq->vxtxq_ts->npending = 0; vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(vmxq->vxq_id), txr->vxtxr_head); } return (0); } #define VMXNET3_TX_START 1 #define VMXNET3_TX_TRANSMIT 2 static inline void vmxnet3_tx_common_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq, int txtype) { struct vmxnet3_softc *sc; struct vmxnet3_txring *txr; struct mbuf *m_head; int tx; sc = ifp->if_softc; txr = &txq->vxtxq_cmd_ring; tx = 0; VMXNET3_TXQ_LOCK_ASSERT(txq); if (txq->vxtxq_stopping || sc->vmx_link_active == 0) return; for (;;) { if (txtype == VMXNET3_TX_START) IFQ_POLL(&ifp->if_snd, m_head); else m_head = pcq_peek(txq->vxtxq_interq); if (m_head == NULL) break; if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS) break; if (txtype == VMXNET3_TX_START) IFQ_DEQUEUE(&ifp->if_snd, m_head); else m_head = pcq_get(txq->vxtxq_interq); if (m_head == NULL) break; if (vmxnet3_txq_encap(txq, &m_head) != 0) { m_freem(m_head); break; } tx++; bpf_mtap(ifp, m_head, BPF_D_OUT); } if (tx > 0) txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; } static void vmxnet3_start_locked(struct ifnet *ifp) { struct vmxnet3_softc *sc; struct vmxnet3_txqueue *txq; sc = ifp->if_softc; txq = &sc->vmx_queue[0].vxq_txqueue; vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_START); } void vmxnet3_start(struct ifnet *ifp) { struct vmxnet3_softc *sc; struct vmxnet3_txqueue *txq; sc = ifp->if_softc; txq = &sc->vmx_queue[0].vxq_txqueue; VMXNET3_TXQ_LOCK(txq); vmxnet3_start_locked(ifp); VMXNET3_TXQ_UNLOCK(txq); } static int vmxnet3_select_txqueue(struct ifnet *ifp, struct mbuf *m __unused) { struct vmxnet3_softc *sc; u_int cpuid; sc = ifp->if_softc; cpuid = cpu_index(curcpu()); /* * Future work * We should select txqueue to even up the load even if ncpu is * different from sc->vmx_ntxqueues. Currently, the load is not * even, that is, when ncpu is six and ntxqueues is four, the load * of vmx_queue[0] and vmx_queue[1] is higher than vmx_queue[2] and * vmx_queue[3] because CPU#4 always uses vmx_queue[0] and CPU#5 always * uses vmx_queue[1]. * Furthermore, we should not use random value to select txqueue to * avoid reordering. We should use flow information of mbuf. */ return cpuid % sc->vmx_ntxqueues; } static void vmxnet3_transmit_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq) { vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_TRANSMIT); } static int vmxnet3_transmit(struct ifnet *ifp, struct mbuf *m) { struct vmxnet3_softc *sc; struct vmxnet3_txqueue *txq; int qid; qid = vmxnet3_select_txqueue(ifp, m); sc = ifp->if_softc; txq = &sc->vmx_queue[qid].vxq_txqueue; if (__predict_false(!pcq_put(txq->vxtxq_interq, m))) { VMXNET3_TXQ_LOCK(txq); txq->vxtxq_pcqdrop.ev_count++; VMXNET3_TXQ_UNLOCK(txq); m_freem(m); return ENOBUFS; } #ifdef VMXNET3_ALWAYS_TXDEFER kpreempt_disable(); softint_schedule(txq->vxtxq_si); kpreempt_enable(); #else if (VMXNET3_TXQ_TRYLOCK(txq)) { vmxnet3_transmit_locked(ifp, txq); VMXNET3_TXQ_UNLOCK(txq); } else { kpreempt_disable(); softint_schedule(txq->vxtxq_si); kpreempt_enable(); } #endif return 0; } static void vmxnet3_deferred_transmit(void *arg) { struct vmxnet3_txqueue *txq = arg; struct vmxnet3_softc *sc = txq->vxtxq_sc; struct ifnet *ifp = &sc->vmx_ethercom.ec_if; VMXNET3_TXQ_LOCK(txq); txq->vxtxq_transmitdef.ev_count++; if (pcq_peek(txq->vxtxq_interq) != NULL) vmxnet3_transmit_locked(ifp, txq); VMXNET3_TXQ_UNLOCK(txq); } static void vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) { struct ethercom *ec = &sc->vmx_ethercom; struct vmxnet3_driver_shared *ds = sc->vmx_ds; struct ether_multi *enm; struct ether_multistep step; u_int mode; uint8_t *p; VMXNET3_CORE_LOCK_ASSERT(sc); ds->mcast_tablelen = 0; ETHER_LOCK(ec); CLR(ec->ec_flags, ETHER_F_ALLMULTI); ETHER_UNLOCK(ec); /* * Always accept broadcast frames. * Always accept frames destined to our station address. */ mode = VMXNET3_RXMODE_BCAST | VMXNET3_RXMODE_UCAST; ETHER_LOCK(ec); if (sc->vmx_promisc || ec->ec_multicnt > VMXNET3_MULTICAST_MAX) goto allmulti; p = sc->vmx_mcast; ETHER_FIRST_MULTI(step, ec, enm); while (enm != NULL) { if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { /* * We must listen to a range of multicast addresses. * For now, just accept all multicasts, rather than * trying to set only those filter bits needed to match * the range. (At this time, the only use of address * ranges is for IP multicast routing, for which the * range is big enough to require all bits set.) */ goto allmulti; } memcpy(p, enm->enm_addrlo, ETHER_ADDR_LEN); p += ETHER_ADDR_LEN; ETHER_NEXT_MULTI(step, enm); } if (ec->ec_multicnt > 0) { SET(mode, VMXNET3_RXMODE_MCAST); ds->mcast_tablelen = p - sc->vmx_mcast; } ETHER_UNLOCK(ec); goto setit; allmulti: SET(ec->ec_flags, ETHER_F_ALLMULTI); ETHER_UNLOCK(ec); SET(mode, (VMXNET3_RXMODE_ALLMULTI | VMXNET3_RXMODE_MCAST)); if (sc->vmx_promisc) SET(mode, VMXNET3_RXMODE_PROMISC); setit: vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); ds->rxmode = mode; vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); } static int vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data) { struct vmxnet3_softc *sc = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; int s, error = 0; switch (cmd) { case SIOCADDMULTI: case SIOCDELMULTI: break; default: KASSERT(IFNET_LOCKED(ifp)); } switch (cmd) { case SIOCSIFMTU: { int nmtu = ifr->ifr_mtu; if (nmtu < VMXNET3_MIN_MTU || nmtu > VMXNET3_MAX_MTU) { error = EINVAL; break; } if (ifp->if_mtu != (uint64_t)nmtu) { s = splnet(); error = ether_ioctl(ifp, cmd, data); splx(s); if (error == ENETRESET) error = vmxnet3_init(ifp); } break; } default: s = splnet(); error = ether_ioctl(ifp, cmd, data); splx(s); } if (error == ENETRESET) { VMXNET3_CORE_LOCK(sc); if (sc->vmx_mcastactive) vmxnet3_set_rxfilter(sc); VMXNET3_CORE_UNLOCK(sc); error = 0; } return error; } static int vmxnet3_ifflags_cb(struct ethercom *ec) { struct ifnet *ifp = &ec->ec_if; struct vmxnet3_softc *sc = ifp->if_softc; int error = 0; KASSERT(IFNET_LOCKED(ifp)); VMXNET3_CORE_LOCK(sc); const unsigned short changed = ifp->if_flags ^ sc->vmx_if_flags; if ((changed & ~(IFF_CANTCHANGE | IFF_DEBUG)) == 0) { sc->vmx_if_flags = ifp->if_flags; if (changed & IFF_PROMISC) { sc->vmx_promisc = ifp->if_flags & IFF_PROMISC; error = ENETRESET; } } else { error = ENETRESET; } VMXNET3_CORE_UNLOCK(sc); vmxnet3_if_link_status(sc); return error; } static int vmxnet3_watchdog(struct vmxnet3_txqueue *txq) { struct vmxnet3_softc *sc; struct vmxnet3_queue *vmxq; sc = txq->vxtxq_sc; vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue); VMXNET3_TXQ_LOCK(txq); if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { VMXNET3_TXQ_UNLOCK(txq); return (0); } txq->vxtxq_watchdogto.ev_count++; VMXNET3_TXQ_UNLOCK(txq); device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n", vmxq->vxq_id); return (1); } static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) { vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); } static void vmxnet3_tick(void *xsc) { struct vmxnet3_softc *sc; int i, timedout; sc = xsc; timedout = 0; VMXNET3_CORE_LOCK(sc); vmxnet3_refresh_host_stats(sc); for (i = 0; i < sc->vmx_ntxqueues; i++) timedout |= vmxnet3_watchdog(&sc->vmx_queue[i].vxq_txqueue); if (timedout != 0) { if (!sc->vmx_reset_pending) { sc->vmx_reset_pending = true; workqueue_enqueue(sc->vmx_reset_wq, &sc->vmx_reset_work, NULL); } } else { callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); } VMXNET3_CORE_UNLOCK(sc); } static void vmxnet3_reset_work(struct work *work, void *arg) { struct vmxnet3_softc *sc = arg; struct ifnet *ifp = &sc->vmx_ethercom.ec_if; VMXNET3_CORE_LOCK(sc); KASSERT(sc->vmx_reset_pending); sc->vmx_reset_pending = false; VMXNET3_CORE_UNLOCK(sc); IFNET_LOCK(ifp); (void)vmxnet3_init(ifp); IFNET_UNLOCK(ifp); } /* * update link state of ifnet and softc */ static void vmxnet3_if_link_status(struct vmxnet3_softc *sc) { struct ifnet *ifp = &sc->vmx_ethercom.ec_if; u_int link; bool up; up = vmxnet3_cmd_link_status(ifp); if (up) { sc->vmx_link_active = 1; link = LINK_STATE_UP; } else { sc->vmx_link_active = 0; link = LINK_STATE_DOWN; } if_link_state_change(ifp, link); } /* * check vmx(4) state by VMXNET3_CMD and update ifp->if_baudrate * returns * - true: link up * - false: link down */ static bool vmxnet3_cmd_link_status(struct ifnet *ifp) { struct vmxnet3_softc *sc = ifp->if_softc; u_int x, speed; x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); if ((x & 1) == 0) return false; speed = x >> 16; ifp->if_baudrate = IF_Mbps(speed); return true; } static void vmxnet3_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr) { bool up; ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; up = vmxnet3_cmd_link_status(ifp); if (!up) return; ifmr->ifm_status |= IFM_ACTIVE; if (ifp->if_baudrate >= IF_Gbps(10ULL)) ifmr->ifm_active |= IFM_10G_T | IFM_FDX; } static int vmxnet3_ifmedia_change(struct ifnet *ifp) { return 0; } static void vmxnet3_set_lladdr(struct vmxnet3_softc *sc) { uint32_t ml, mh; ml = sc->vmx_lladdr[0]; ml |= sc->vmx_lladdr[1] << 8; ml |= sc->vmx_lladdr[2] << 16; ml |= sc->vmx_lladdr[3] << 24; vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); mh = sc->vmx_lladdr[4]; mh |= sc->vmx_lladdr[5] << 8; vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); } static void vmxnet3_get_lladdr(struct vmxnet3_softc *sc) { uint32_t ml, mh; ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); sc->vmx_lladdr[0] = ml; sc->vmx_lladdr[1] = ml >> 8; sc->vmx_lladdr[2] = ml >> 16; sc->vmx_lladdr[3] = ml >> 24; sc->vmx_lladdr[4] = mh; sc->vmx_lladdr[5] = mh >> 8; } static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) { int i; sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; for (i = 0; i < sc->vmx_nintrs; i++) vmxnet3_enable_intr(sc, i); } static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) { int i; sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; for (i = 0; i < sc->vmx_nintrs; i++) vmxnet3_disable_intr(sc, i); } static int vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, struct vmxnet3_dma_alloc *dma) { bus_dma_tag_t t = sc->vmx_dmat; bus_dma_segment_t *segs = dma->dma_segs; int n, error; memset(dma, 0, sizeof(*dma)); error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT); if (error) { aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error); goto fail1; } KASSERT(n == 1); error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT); if (error) { aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error); goto fail2; } error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map); if (error) { aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error); goto fail3; } error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL, BUS_DMA_NOWAIT); if (error) { aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error); goto fail4; } memset(dma->dma_vaddr, 0, size); dma->dma_paddr = DMAADDR(dma->dma_map); dma->dma_size = size; return (0); fail4: bus_dmamap_destroy(t, dma->dma_map); fail3: bus_dmamem_unmap(t, dma->dma_vaddr, size); fail2: bus_dmamem_free(t, segs, 1); fail1: return (error); } static void vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) { bus_dma_tag_t t = sc->vmx_dmat; bus_dmamap_unload(t, dma->dma_map); bus_dmamap_destroy(t, dma->dma_map); bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size); bus_dmamem_free(t, dma->dma_segs, 1); memset(dma, 0, sizeof(*dma)); } MODULE(MODULE_CLASS_DRIVER, if_vmx, "pci"); #ifdef _MODULE #include "ioconf.c" #endif static int if_vmx_modcmd(modcmd_t cmd, void *opaque) { int error = 0; switch (cmd) { case MODULE_CMD_INIT: #ifdef _MODULE error = config_init_component(cfdriver_ioconf_if_vmx, cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); #endif return error; case MODULE_CMD_FINI: #ifdef _MODULE error = config_fini_component(cfdriver_ioconf_if_vmx, cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx); #endif return error; default: return ENOTTY; } }