/* $NetBSD: dwc_gmac.c,v 1.97 2025/10/04 04:44:20 thorpej Exp $ */ /*- * Copyright (c) 2013, 2014 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Matt Thomas of 3am Software Foundry and Martin Husemann. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * This driver supports the Synopsis Designware GMAC core, as found * on Allwinner A20 cores and others. * * Real documentation seems to not be available, the marketing product * documents could be found here: * * http://www.synopsys.com/dw/ipdir.php?ds=dwc_ether_mac10_100_1000_unive */ /* * Lock order: * * IFNET_LOCK -> sc_mcast_lock * IFNET_LOCK -> sc_intr_lock -> {sc_txq.t_mtx, sc_rxq.r_mtx} */ #include __KERNEL_RCSID(1, "$NetBSD: dwc_gmac.c,v 1.97 2025/10/04 04:44:20 thorpej Exp $"); /* #define DWC_GMAC_DEBUG 1 */ #ifdef _KERNEL_OPT #include "opt_inet.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET #include #endif #include #include #include static int dwc_gmac_miibus_read_reg(device_t, int, int, uint16_t *); static int dwc_gmac_miibus_write_reg(device_t, int, int, uint16_t); static void dwc_gmac_miibus_statchg(struct ifnet *); static int dwc_gmac_reset(struct dwc_gmac_softc *); static void dwc_gmac_write_hwaddr(struct dwc_gmac_softc *, uint8_t[ETHER_ADDR_LEN]); static int dwc_gmac_alloc_dma_rings(struct dwc_gmac_softc *); static void dwc_gmac_free_dma_rings(struct dwc_gmac_softc *); static int dwc_gmac_alloc_rx_ring(struct dwc_gmac_softc *, struct dwc_gmac_rx_ring *); static void dwc_gmac_reset_rx_ring(struct dwc_gmac_softc *, struct dwc_gmac_rx_ring *); static void dwc_gmac_free_rx_ring(struct dwc_gmac_softc *, struct dwc_gmac_rx_ring *); static int dwc_gmac_alloc_tx_ring(struct dwc_gmac_softc *, struct dwc_gmac_tx_ring *); static void dwc_gmac_reset_tx_ring(struct dwc_gmac_softc *, struct dwc_gmac_tx_ring *); static void dwc_gmac_free_tx_ring(struct dwc_gmac_softc *, struct dwc_gmac_tx_ring *); static void dwc_gmac_txdesc_sync(struct dwc_gmac_softc *, int, int, int); static int dwc_gmac_init(struct ifnet *); static void dwc_gmac_stop(struct ifnet *, int); static void dwc_gmac_start(struct ifnet *); static void dwc_gmac_start_locked(struct ifnet *); static int dwc_gmac_queue(struct dwc_gmac_softc *, struct mbuf *); static int dwc_gmac_ioctl(struct ifnet *, u_long, void *); static void dwc_gmac_tx_intr(struct dwc_gmac_softc *); static void dwc_gmac_rx_intr(struct dwc_gmac_softc *); static void dwc_gmac_setmulti(struct dwc_gmac_softc *); static int dwc_gmac_ifflags_cb(struct ethercom *); static void dwc_gmac_desc_set_owned_by_dev(struct dwc_gmac_dev_dmadesc *); static int dwc_gmac_desc_is_owned_by_dev(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_std_set_len(struct dwc_gmac_dev_dmadesc *, int); static uint32_t dwc_gmac_desc_std_get_len(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_std_tx_init_flags(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_std_tx_set_first_frag(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_std_tx_set_last_frag(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_std_rx_init_flags(struct dwc_gmac_dev_dmadesc *); static int dwc_gmac_desc_std_rx_has_error(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_enh_set_len(struct dwc_gmac_dev_dmadesc *, int); static uint32_t dwc_gmac_desc_enh_get_len(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_enh_tx_init_flags(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_enh_tx_set_first_frag(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_enh_tx_set_last_frag(struct dwc_gmac_dev_dmadesc *); static void dwc_gmac_desc_enh_rx_init_flags(struct dwc_gmac_dev_dmadesc *); static int dwc_gmac_desc_enh_rx_has_error(struct dwc_gmac_dev_dmadesc *); static const struct dwc_gmac_desc_methods desc_methods_standard = { .tx_init_flags = dwc_gmac_desc_std_tx_init_flags, .tx_set_owned_by_dev = dwc_gmac_desc_set_owned_by_dev, .tx_is_owned_by_dev = dwc_gmac_desc_is_owned_by_dev, .tx_set_len = dwc_gmac_desc_std_set_len, .tx_set_first_frag = dwc_gmac_desc_std_tx_set_first_frag, .tx_set_last_frag = dwc_gmac_desc_std_tx_set_last_frag, .rx_init_flags = dwc_gmac_desc_std_rx_init_flags, .rx_set_owned_by_dev = dwc_gmac_desc_set_owned_by_dev, .rx_is_owned_by_dev = dwc_gmac_desc_is_owned_by_dev, .rx_set_len = dwc_gmac_desc_std_set_len, .rx_get_len = dwc_gmac_desc_std_get_len, .rx_has_error = dwc_gmac_desc_std_rx_has_error }; static const struct dwc_gmac_desc_methods desc_methods_enhanced = { .tx_init_flags = dwc_gmac_desc_enh_tx_init_flags, .tx_set_owned_by_dev = dwc_gmac_desc_set_owned_by_dev, .tx_is_owned_by_dev = dwc_gmac_desc_is_owned_by_dev, .tx_set_len = dwc_gmac_desc_enh_set_len, .tx_set_first_frag = dwc_gmac_desc_enh_tx_set_first_frag, .tx_set_last_frag = dwc_gmac_desc_enh_tx_set_last_frag, .rx_init_flags = dwc_gmac_desc_enh_rx_init_flags, .rx_set_owned_by_dev = dwc_gmac_desc_set_owned_by_dev, .rx_is_owned_by_dev = dwc_gmac_desc_is_owned_by_dev, .rx_set_len = dwc_gmac_desc_enh_set_len, .rx_get_len = dwc_gmac_desc_enh_get_len, .rx_has_error = dwc_gmac_desc_enh_rx_has_error }; #define TX_DESC_OFFSET(N) ((AWGE_RX_RING_COUNT + (N)) \ * sizeof(struct dwc_gmac_dev_dmadesc)) #define TX_NEXT(N) (((N) + 1) & (AWGE_TX_RING_COUNT - 1)) #define RX_DESC_OFFSET(N) ((N) * sizeof(struct dwc_gmac_dev_dmadesc)) #define RX_NEXT(N) (((N) + 1) & (AWGE_RX_RING_COUNT - 1)) #define GMAC_DEF_DMA_INT_MASK (GMAC_DMA_INT_TIE | GMAC_DMA_INT_RIE | \ GMAC_DMA_INT_NIE | GMAC_DMA_INT_AIE | \ GMAC_DMA_INT_FBE | GMAC_DMA_INT_UNE) #define GMAC_DMA_INT_ERRORS (GMAC_DMA_INT_AIE | GMAC_DMA_INT_ERE | \ GMAC_DMA_INT_FBE | \ GMAC_DMA_INT_RWE | GMAC_DMA_INT_RUE | \ GMAC_DMA_INT_UNE | GMAC_DMA_INT_OVE | \ GMAC_DMA_INT_TJE) #define AWIN_DEF_MAC_INTRMASK \ (AWIN_GMAC_MAC_INT_TSI | AWIN_GMAC_MAC_INT_ANEG | \ AWIN_GMAC_MAC_INT_LINKCHG) #ifdef DWC_GMAC_DEBUG static void dwc_gmac_dump_dma(struct dwc_gmac_softc *); static void dwc_gmac_dump_tx_desc(struct dwc_gmac_softc *); static void dwc_gmac_dump_rx_desc(struct dwc_gmac_softc *); static void dwc_dump_and_abort(struct dwc_gmac_softc *, const char *); static void dwc_dump_status(struct dwc_gmac_softc *); static void dwc_gmac_dump_ffilt(struct dwc_gmac_softc *, uint32_t); #endif int dwc_gmac_attach(struct dwc_gmac_softc *sc, int phy_id, uint32_t mii_clk) { uint8_t enaddr[ETHER_ADDR_LEN]; uint32_t maclo, machi, hwft; struct mii_data * const mii = &sc->sc_mii; struct ifnet * const ifp = &sc->sc_ec.ec_if; mutex_init(&sc->sc_mdio_lock, MUTEX_DEFAULT, IPL_NET); sc->sc_mii_clk = mii_clk & 7; if (! ether_getaddr(sc->sc_dev, enaddr)) { /* * If we did not get an externally configure address, * try to read one from the current filter setup, * before resetting the chip. */ maclo = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_ADDR0LO); machi = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_ADDR0HI); if (maclo == 0xffffffff && (machi & 0xffff) == 0xffff) { /* fake MAC address */ maclo = 0x00f2 | (cprng_strong32() << 16); machi = cprng_strong32(); } enaddr[0] = maclo & 0x0ff; enaddr[1] = (maclo >> 8) & 0x0ff; enaddr[2] = (maclo >> 16) & 0x0ff; enaddr[3] = (maclo >> 24) & 0x0ff; enaddr[4] = machi & 0x0ff; enaddr[5] = (machi >> 8) & 0x0ff; } const uint32_t ver = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_VERSION); const uint32_t snpsver = __SHIFTOUT(ver, AWIN_GMAC_MAC_VERSION_SNPSVER_MASK); aprint_normal_dev(sc->sc_dev, "Core version: %08x\n", snpsver); /* * Init chip and do initial setup */ if (dwc_gmac_reset(sc) != 0) return ENXIO; /* not much to cleanup, haven't attached yet */ dwc_gmac_write_hwaddr(sc, enaddr); aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", ether_sprintf(enaddr)); hwft = 0; if (snpsver >= 0x35) { hwft = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_HWFEATURES); aprint_normal_dev(sc->sc_dev, "HW feature mask: %x\n", hwft); } if (sizeof(bus_addr_t) > 4) { int error = bus_dmatag_subregion(sc->sc_dmat, 0, __MASK(32), &sc->sc_dmat, BUS_DMA_WAITOK); if (error != 0) { aprint_error_dev(sc->sc_dev, "failed to create DMA subregion\n"); return ENOMEM; } } if (hwft & GMAC_DMA_FEAT_ENHANCED_DESC) { aprint_normal_dev(sc->sc_dev, "Using enhanced descriptor format\n"); sc->sc_descm = &desc_methods_enhanced; } else { sc->sc_descm = &desc_methods_standard; } if (hwft & GMAC_DMA_FEAT_RMON) { uint32_t val; /* Mask all MMC interrupts */ val = 0xffffffff; bus_space_write_4(sc->sc_bst, sc->sc_bsh, GMAC_MMC_RX_INT_MSK, val); bus_space_write_4(sc->sc_bst, sc->sc_bsh, GMAC_MMC_TX_INT_MSK, val); } /* * Allocate Tx and Rx rings */ if (dwc_gmac_alloc_dma_rings(sc) != 0) { aprint_error_dev(sc->sc_dev, "could not allocate DMA rings\n"); goto fail; } if (dwc_gmac_alloc_tx_ring(sc, &sc->sc_txq) != 0) { aprint_error_dev(sc->sc_dev, "could not allocate Tx ring\n"); goto fail; } if (dwc_gmac_alloc_rx_ring(sc, &sc->sc_rxq) != 0) { aprint_error_dev(sc->sc_dev, "could not allocate Rx ring\n"); goto fail; } sc->sc_stopping = false; sc->sc_txbusy = false; sc->sc_mcast_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); sc->sc_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); mutex_init(&sc->sc_txq.t_mtx, MUTEX_DEFAULT, IPL_NET); mutex_init(&sc->sc_rxq.r_mtx, MUTEX_DEFAULT, IPL_NET); /* * Prepare interface data */ ifp->if_softc = sc; strlcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_extflags = IFEF_MPSAFE; ifp->if_ioctl = dwc_gmac_ioctl; ifp->if_start = dwc_gmac_start; ifp->if_init = dwc_gmac_init; ifp->if_stop = dwc_gmac_stop; IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); IFQ_SET_READY(&ifp->if_snd); /* * Attach MII subdevices */ sc->sc_ec.ec_mii = &sc->sc_mii; ifmedia_init(&mii->mii_media, 0, ether_mediachange, ether_mediastatus); mii->mii_ifp = ifp; mii->mii_readreg = dwc_gmac_miibus_read_reg; mii->mii_writereg = dwc_gmac_miibus_write_reg; mii->mii_statchg = dwc_gmac_miibus_statchg; mii_attach(sc->sc_dev, mii, 0xffffffff, phy_id, MII_OFFSET_ANY, MIIF_DOPAUSE); if (LIST_EMPTY(&mii->mii_phys)) { aprint_error_dev(sc->sc_dev, "no PHY found!\n"); ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_MANUAL, 0, NULL); ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_MANUAL); } else { ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO); } /* * We can support 802.1Q VLAN-sized frames. */ sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; /* * Ready, attach interface */ /* Attach the interface. */ if_initialize(ifp); sc->sc_ipq = if_percpuq_create(&sc->sc_ec.ec_if); if_deferred_start_init(ifp, NULL); ether_ifattach(ifp, enaddr); ether_set_ifflags_cb(&sc->sc_ec, dwc_gmac_ifflags_cb); if_register(ifp); rnd_attach_source(&sc->rnd_source, device_xname(sc->sc_dev), RND_TYPE_NET, RND_FLAG_DEFAULT); /* * Enable interrupts */ mutex_enter(sc->sc_intr_lock); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_INTMASK, AWIN_DEF_MAC_INTRMASK); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_INTENABLE, GMAC_DEF_DMA_INT_MASK); mutex_exit(sc->sc_intr_lock); return 0; fail: dwc_gmac_free_rx_ring(sc, &sc->sc_rxq); dwc_gmac_free_tx_ring(sc, &sc->sc_txq); dwc_gmac_free_dma_rings(sc); mutex_destroy(&sc->sc_mdio_lock); return ENXIO; } static int dwc_gmac_reset(struct dwc_gmac_softc *sc) { size_t cnt; bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_BUSMODE, bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_BUSMODE) | GMAC_BUSMODE_RESET); for (cnt = 0; cnt < 30000; cnt++) { if ((bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_BUSMODE) & GMAC_BUSMODE_RESET) == 0) return 0; delay(10); } aprint_error_dev(sc->sc_dev, "reset timed out\n"); return EIO; } static void dwc_gmac_write_hwaddr(struct dwc_gmac_softc *sc, uint8_t enaddr[ETHER_ADDR_LEN]) { uint32_t hi, lo; hi = enaddr[4] | (enaddr[5] << 8); lo = enaddr[0] | (enaddr[1] << 8) | (enaddr[2] << 16) | ((uint32_t)enaddr[3] << 24); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_ADDR0HI, hi); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_ADDR0LO, lo); } static int dwc_gmac_miibus_read_reg(device_t self, int phy, int reg, uint16_t *val) { struct dwc_gmac_softc * const sc = device_private(self); uint16_t mii; size_t cnt; mii = __SHIFTIN(phy, GMAC_MII_PHY_MASK) | __SHIFTIN(reg, GMAC_MII_REG_MASK) | __SHIFTIN(sc->sc_mii_clk, GMAC_MII_CLKMASK) | GMAC_MII_BUSY; mutex_enter(&sc->sc_mdio_lock); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_MIIADDR, mii); for (cnt = 0; cnt < 1000; cnt++) { if (!(bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_MIIADDR) & GMAC_MII_BUSY)) { *val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_MIIDATA); break; } delay(10); } mutex_exit(&sc->sc_mdio_lock); if (cnt >= 1000) return ETIMEDOUT; return 0; } static int dwc_gmac_miibus_write_reg(device_t self, int phy, int reg, uint16_t val) { struct dwc_gmac_softc * const sc = device_private(self); uint16_t mii; size_t cnt; mii = __SHIFTIN(phy, GMAC_MII_PHY_MASK) | __SHIFTIN(reg, GMAC_MII_REG_MASK) | __SHIFTIN(sc->sc_mii_clk, GMAC_MII_CLKMASK) | GMAC_MII_BUSY | GMAC_MII_WRITE; mutex_enter(&sc->sc_mdio_lock); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_MIIDATA, val); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_MIIADDR, mii); for (cnt = 0; cnt < 1000; cnt++) { if (!(bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_MIIADDR) & GMAC_MII_BUSY)) break; delay(10); } mutex_exit(&sc->sc_mdio_lock); if (cnt >= 1000) return ETIMEDOUT; return 0; } static int dwc_gmac_alloc_rx_ring(struct dwc_gmac_softc *sc, struct dwc_gmac_rx_ring *ring) { struct dwc_gmac_rx_data *data; bus_addr_t physaddr; const size_t rxringsz = AWGE_RX_RING_COUNT * sizeof(*ring->r_desc); int error, i, next; ring->r_cur = ring->r_next = 0; memset(ring->r_desc, 0, rxringsz); /* * Pre-allocate Rx buffers and populate Rx ring. */ for (i = 0; i < AWGE_RX_RING_COUNT; i++) { struct dwc_gmac_dev_dmadesc *desc; data = &sc->sc_rxq.r_data[i]; MGETHDR(data->rd_m, M_DONTWAIT, MT_DATA); if (data->rd_m == NULL) { aprint_error_dev(sc->sc_dev, "could not allocate rx mbuf #%d\n", i); error = ENOMEM; goto fail; } error = bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0, BUS_DMA_NOWAIT, &data->rd_map); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not create DMA map\n"); data->rd_map = NULL; goto fail; } MCLGET(data->rd_m, M_DONTWAIT); if (!(data->rd_m->m_flags & M_EXT)) { aprint_error_dev(sc->sc_dev, "could not allocate mbuf cluster #%d\n", i); error = ENOMEM; goto fail; } data->rd_m->m_len = data->rd_m->m_pkthdr.len = data->rd_m->m_ext.ext_size; m_adj(data->rd_m, ETHER_ALIGN); if (data->rd_m->m_len > AWGE_MAX_PACKET) { data->rd_m->m_len = data->rd_m->m_pkthdr.len = AWGE_MAX_PACKET; } error = bus_dmamap_load_mbuf(sc->sc_dmat, data->rd_map, data->rd_m, BUS_DMA_READ | BUS_DMA_NOWAIT); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not load rx buf DMA map #%d", i); goto fail; } bus_dmamap_sync(sc->sc_dmat, data->rd_map, 0, data->rd_map->dm_mapsize, BUS_DMASYNC_PREREAD); physaddr = data->rd_map->dm_segs[0].ds_addr; desc = &sc->sc_rxq.r_desc[i]; desc->ddesc_data = htole32(physaddr); next = RX_NEXT(i); desc->ddesc_next = htole32(ring->r_physaddr + next * sizeof(*desc)); sc->sc_descm->rx_init_flags(desc); sc->sc_descm->rx_set_len(desc, data->rd_m->m_len); sc->sc_descm->rx_set_owned_by_dev(desc); } bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, RX_DESC_OFFSET(0), AWGE_RX_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_RX_ADDR, ring->r_physaddr); return 0; fail: dwc_gmac_free_rx_ring(sc, ring); return error; } static void dwc_gmac_reset_rx_ring(struct dwc_gmac_softc *sc, struct dwc_gmac_rx_ring *ring) { struct dwc_gmac_dev_dmadesc *desc; struct dwc_gmac_rx_data *data; int i; mutex_enter(&ring->r_mtx); for (i = 0; i < AWGE_RX_RING_COUNT; i++) { desc = &sc->sc_rxq.r_desc[i]; data = &sc->sc_rxq.r_data[i]; sc->sc_descm->rx_init_flags(desc); sc->sc_descm->rx_set_len(desc, data->rd_m->m_len); sc->sc_descm->rx_set_owned_by_dev(desc); } bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, 0, AWGE_RX_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); ring->r_cur = ring->r_next = 0; /* reset DMA address to start of ring */ bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_RX_ADDR, sc->sc_rxq.r_physaddr); mutex_exit(&ring->r_mtx); } static int dwc_gmac_alloc_dma_rings(struct dwc_gmac_softc *sc) { const size_t ringsize = AWGE_TOTAL_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc); int error, nsegs; void *rings; error = bus_dmamap_create(sc->sc_dmat, ringsize, 1, ringsize, 0, BUS_DMA_NOWAIT, &sc->sc_dma_ring_map); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not create desc DMA map\n"); sc->sc_dma_ring_map = NULL; goto fail; } error = bus_dmamem_alloc(sc->sc_dmat, ringsize, PAGE_SIZE, 0, &sc->sc_dma_ring_seg, 1, &nsegs, BUS_DMA_NOWAIT |BUS_DMA_COHERENT); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not map DMA memory\n"); goto fail; } error = bus_dmamem_map(sc->sc_dmat, &sc->sc_dma_ring_seg, nsegs, ringsize, &rings, BUS_DMA_NOWAIT | BUS_DMA_COHERENT); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not allocate DMA memory\n"); goto fail; } error = bus_dmamap_load(sc->sc_dmat, sc->sc_dma_ring_map, rings, ringsize, NULL, BUS_DMA_NOWAIT | BUS_DMA_COHERENT); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not load desc DMA map\n"); goto fail; } /* give first AWGE_RX_RING_COUNT to the RX side */ sc->sc_rxq.r_desc = rings; sc->sc_rxq.r_physaddr = sc->sc_dma_ring_map->dm_segs[0].ds_addr; /* and next rings to the TX side */ sc->sc_txq.t_desc = sc->sc_rxq.r_desc + AWGE_RX_RING_COUNT; sc->sc_txq.t_physaddr = sc->sc_rxq.r_physaddr + AWGE_RX_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc); return 0; fail: dwc_gmac_free_dma_rings(sc); return error; } static void dwc_gmac_free_dma_rings(struct dwc_gmac_softc *sc) { bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, 0, sc->sc_dma_ring_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->sc_dmat, sc->sc_dma_ring_map); bus_dmamem_unmap(sc->sc_dmat, sc->sc_rxq.r_desc, AWGE_TOTAL_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc)); bus_dmamem_free(sc->sc_dmat, &sc->sc_dma_ring_seg, 1); } static void dwc_gmac_free_rx_ring(struct dwc_gmac_softc *sc, struct dwc_gmac_rx_ring *ring) { struct dwc_gmac_rx_data *data; int i; if (ring->r_desc == NULL) return; for (i = 0; i < AWGE_RX_RING_COUNT; i++) { data = &ring->r_data[i]; if (data->rd_map != NULL) { bus_dmamap_sync(sc->sc_dmat, data->rd_map, 0, AWGE_RX_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc), BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->sc_dmat, data->rd_map); bus_dmamap_destroy(sc->sc_dmat, data->rd_map); } m_freem(data->rd_m); } } static int dwc_gmac_alloc_tx_ring(struct dwc_gmac_softc *sc, struct dwc_gmac_tx_ring *ring) { int i, error = 0; ring->t_queued = 0; ring->t_cur = ring->t_next = 0; memset(ring->t_desc, 0, AWGE_TX_RING_COUNT * sizeof(*ring->t_desc)); bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, TX_DESC_OFFSET(0), AWGE_TX_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); for (i = 0; i < AWGE_TX_RING_COUNT; i++) { error = bus_dmamap_create(sc->sc_dmat, MCLBYTES, AWGE_TX_RING_COUNT, MCLBYTES, 0, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &ring->t_data[i].td_map); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not create TX DMA map #%d\n", i); ring->t_data[i].td_map = NULL; goto fail; } ring->t_desc[i].ddesc_next = htole32( ring->t_physaddr + sizeof(struct dwc_gmac_dev_dmadesc) * TX_NEXT(i)); } bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, TX_DESC_OFFSET(0), AWGE_TX_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); return 0; fail: dwc_gmac_free_tx_ring(sc, ring); return error; } static void dwc_gmac_txdesc_sync(struct dwc_gmac_softc *sc, int start, int end, int ops) { /* 'end' is pointing one descriptor beyond the last we want to sync */ if (end > start) { bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, TX_DESC_OFFSET(start), TX_DESC_OFFSET(end) - TX_DESC_OFFSET(start), ops); return; } /* sync from 'start' to end of ring */ bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, TX_DESC_OFFSET(start), TX_DESC_OFFSET(AWGE_TX_RING_COUNT) - TX_DESC_OFFSET(start), ops); if (TX_DESC_OFFSET(end) - TX_DESC_OFFSET(0) > 0) { /* sync from start of ring to 'end' */ bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, TX_DESC_OFFSET(0), TX_DESC_OFFSET(end) - TX_DESC_OFFSET(0), ops); } } static void dwc_gmac_reset_tx_ring(struct dwc_gmac_softc *sc, struct dwc_gmac_tx_ring *ring) { int i; mutex_enter(&ring->t_mtx); for (i = 0; i < AWGE_TX_RING_COUNT; i++) { struct dwc_gmac_tx_data *data = &ring->t_data[i]; if (data->td_m != NULL) { bus_dmamap_sync(sc->sc_dmat, data->td_active, 0, data->td_active->dm_mapsize, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->sc_dmat, data->td_active); m_freem(data->td_m); data->td_m = NULL; } } bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, TX_DESC_OFFSET(0), AWGE_TX_RING_COUNT * sizeof(struct dwc_gmac_dev_dmadesc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_TX_ADDR, sc->sc_txq.t_physaddr); ring->t_queued = 0; ring->t_cur = ring->t_next = 0; mutex_exit(&ring->t_mtx); } static void dwc_gmac_free_tx_ring(struct dwc_gmac_softc *sc, struct dwc_gmac_tx_ring *ring) { int i; /* unload the maps */ for (i = 0; i < AWGE_TX_RING_COUNT; i++) { struct dwc_gmac_tx_data *data = &ring->t_data[i]; if (data->td_m != NULL) { bus_dmamap_sync(sc->sc_dmat, data->td_active, 0, data->td_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->sc_dmat, data->td_active); m_freem(data->td_m); data->td_m = NULL; } } /* and actually free them */ for (i = 0; i < AWGE_TX_RING_COUNT; i++) { struct dwc_gmac_tx_data *data = &ring->t_data[i]; bus_dmamap_destroy(sc->sc_dmat, data->td_map); } } static void dwc_gmac_miibus_statchg(struct ifnet *ifp) { struct dwc_gmac_softc * const sc = ifp->if_softc; struct mii_data * const mii = &sc->sc_mii; uint32_t conf, flow; /* * Set MII or GMII interface based on the speed * negotiated by the PHY. */ conf = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_CONF); conf &= ~(AWIN_GMAC_MAC_CONF_FES100 | AWIN_GMAC_MAC_CONF_MIISEL | AWIN_GMAC_MAC_CONF_FULLDPLX); conf |= AWIN_GMAC_MAC_CONF_FRAMEBURST | AWIN_GMAC_MAC_CONF_DISABLERXOWN | AWIN_GMAC_MAC_CONF_DISABLEJABBER | AWIN_GMAC_MAC_CONF_RXENABLE | AWIN_GMAC_MAC_CONF_TXENABLE; switch (IFM_SUBTYPE(mii->mii_media_active)) { case IFM_10_T: conf |= AWIN_GMAC_MAC_CONF_MIISEL; break; case IFM_100_TX: conf |= AWIN_GMAC_MAC_CONF_FES100 | AWIN_GMAC_MAC_CONF_MIISEL; break; case IFM_1000_T: break; } if (sc->sc_set_speed) sc->sc_set_speed(sc, IFM_SUBTYPE(mii->mii_media_active)); flow = 0; if (IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) { conf |= AWIN_GMAC_MAC_CONF_FULLDPLX; flow |= __SHIFTIN(0x200, AWIN_GMAC_MAC_FLOWCTRL_PAUSE); } if (mii->mii_media_active & IFM_ETH_TXPAUSE) { flow |= AWIN_GMAC_MAC_FLOWCTRL_TFE; } if (mii->mii_media_active & IFM_ETH_RXPAUSE) { flow |= AWIN_GMAC_MAC_FLOWCTRL_RFE; } bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_FLOWCTRL, flow); #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "setting MAC conf register: %08x\n", conf); #endif bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_CONF, conf); } static int dwc_gmac_init(struct ifnet *ifp) { struct dwc_gmac_softc * const sc = ifp->if_softc; uint32_t ffilt; ASSERT_SLEEPABLE(); KASSERT(IFNET_LOCKED(ifp)); KASSERT(ifp == &sc->sc_ec.ec_if); dwc_gmac_stop(ifp, 0); /* * Configure DMA burst/transfer mode and RX/TX priorities. * XXX - the GMAC_BUSMODE_PRIORXTX bits are undocumented. */ bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_BUSMODE, GMAC_BUSMODE_FIXEDBURST | GMAC_BUSMODE_4PBL | __SHIFTIN(2, GMAC_BUSMODE_RPBL) | __SHIFTIN(2, GMAC_BUSMODE_PBL)); /* * Set up address filter */ ffilt = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_FFILT); if (ifp->if_flags & IFF_PROMISC) { ffilt |= AWIN_GMAC_MAC_FFILT_PR; } else { ffilt &= ~AWIN_GMAC_MAC_FFILT_PR; } if (ifp->if_flags & IFF_BROADCAST) { ffilt &= ~AWIN_GMAC_MAC_FFILT_DBF; } else { ffilt |= AWIN_GMAC_MAC_FFILT_DBF; } bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_FFILT, ffilt); /* * Set up multicast filter */ mutex_enter(sc->sc_mcast_lock); dwc_gmac_setmulti(sc); mutex_exit(sc->sc_mcast_lock); /* * Set up dma pointer for RX and TX ring */ bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_RX_ADDR, sc->sc_rxq.r_physaddr); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_TX_ADDR, sc->sc_txq.t_physaddr); /* * Start RX/TX part */ uint32_t opmode = GMAC_DMA_OP_RXSTART | GMAC_DMA_OP_TXSTART; if ((sc->sc_flags & DWC_GMAC_FORCE_THRESH_DMA_MODE) == 0) { opmode |= GMAC_DMA_OP_RXSTOREFORWARD | GMAC_DMA_OP_TXSTOREFORWARD; } bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_OPMODE, opmode); #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "setting DMA opmode register: %08x\n", opmode); #endif ifp->if_flags |= IFF_RUNNING; sc->sc_if_flags = ifp->if_flags; mutex_enter(sc->sc_intr_lock); sc->sc_stopping = false; mutex_exit(sc->sc_intr_lock); mutex_enter(&sc->sc_txq.t_mtx); sc->sc_txbusy = false; mutex_exit(&sc->sc_txq.t_mtx); return 0; } static void dwc_gmac_start(struct ifnet *ifp) { struct dwc_gmac_softc * const sc = ifp->if_softc; KASSERT(if_is_mpsafe(ifp)); mutex_enter(sc->sc_intr_lock); if (!sc->sc_stopping) { dwc_gmac_start_locked(ifp); } mutex_exit(sc->sc_intr_lock); } static void dwc_gmac_start_locked(struct ifnet *ifp) { struct dwc_gmac_softc * const sc = ifp->if_softc; int old = sc->sc_txq.t_queued; int start = sc->sc_txq.t_cur; struct mbuf *m0; KASSERT(mutex_owned(sc->sc_intr_lock)); mutex_enter(&sc->sc_txq.t_mtx); if (sc->sc_txbusy) { mutex_exit(&sc->sc_txq.t_mtx); return; } for (;;) { IFQ_POLL(&ifp->if_snd, m0); if (m0 == NULL) break; if (dwc_gmac_queue(sc, m0) != 0) { sc->sc_txbusy = true; break; } IFQ_DEQUEUE(&ifp->if_snd, m0); bpf_mtap(ifp, m0, BPF_D_OUT); if (sc->sc_txq.t_queued == AWGE_TX_RING_COUNT) { sc->sc_txbusy = true; break; } } if (sc->sc_txq.t_queued != old) { /* packets have been queued, kick it off */ dwc_gmac_txdesc_sync(sc, start, sc->sc_txq.t_cur, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); #ifdef DWC_GMAC_DEBUG dwc_dump_status(sc); #endif bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_TXPOLL, ~0U); } mutex_exit(&sc->sc_txq.t_mtx); } static void dwc_gmac_stop(struct ifnet *ifp, int disable) { struct dwc_gmac_softc * const sc = ifp->if_softc; ASSERT_SLEEPABLE(); KASSERT(IFNET_LOCKED(ifp)); ifp->if_flags &= ~IFF_RUNNING; mutex_enter(sc->sc_mcast_lock); sc->sc_if_flags = ifp->if_flags; mutex_exit(sc->sc_mcast_lock); mutex_enter(sc->sc_intr_lock); sc->sc_stopping = true; mutex_exit(sc->sc_intr_lock); mutex_enter(&sc->sc_txq.t_mtx); sc->sc_txbusy = false; mutex_exit(&sc->sc_txq.t_mtx); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_OPMODE, bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_OPMODE) & ~(GMAC_DMA_OP_TXSTART | GMAC_DMA_OP_RXSTART)); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_OPMODE, bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_OPMODE) | GMAC_DMA_OP_FLUSHTX); mii_down(&sc->sc_mii); dwc_gmac_reset_tx_ring(sc, &sc->sc_txq); dwc_gmac_reset_rx_ring(sc, &sc->sc_rxq); } /* * Add m0 to the TX ring */ static int dwc_gmac_queue(struct dwc_gmac_softc *sc, struct mbuf *m0) { struct dwc_gmac_dev_dmadesc *desc = NULL; struct dwc_gmac_tx_data *data = NULL; bus_dmamap_t map; int error, i, first; #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "dwc_gmac_queue: adding mbuf chain %p\n", m0); #endif first = sc->sc_txq.t_cur; map = sc->sc_txq.t_data[first].td_map; error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m0, BUS_DMA_WRITE | BUS_DMA_NOWAIT); if (error != 0) { aprint_error_dev(sc->sc_dev, "could not map mbuf " "(len: %d, error %d)\n", m0->m_pkthdr.len, error); return error; } if (sc->sc_txq.t_queued + map->dm_nsegs > AWGE_TX_RING_COUNT) { bus_dmamap_unload(sc->sc_dmat, map); return ENOBUFS; } for (i = 0; i < map->dm_nsegs; i++) { data = &sc->sc_txq.t_data[sc->sc_txq.t_cur]; desc = &sc->sc_txq.t_desc[sc->sc_txq.t_cur]; desc->ddesc_data = htole32(map->dm_segs[i].ds_addr); #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "enqueuing desc #%d data %08lx " "len %lu\n", sc->sc_txq.t_cur, (unsigned long)map->dm_segs[i].ds_addr, (unsigned long)map->dm_segs[i].ds_len); #endif sc->sc_descm->tx_init_flags(desc); sc->sc_descm->tx_set_len(desc, map->dm_segs[i].ds_len); if (i == 0) sc->sc_descm->tx_set_first_frag(desc); /* * Defer passing ownership of the first descriptor * until we are done. */ if (i != 0) sc->sc_descm->tx_set_owned_by_dev(desc); sc->sc_txq.t_queued++; sc->sc_txq.t_cur = TX_NEXT(sc->sc_txq.t_cur); } sc->sc_descm->tx_set_last_frag(desc); data->td_m = m0; data->td_active = map; /* sync the packet buffer */ bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, BUS_DMASYNC_PREWRITE); /* sync the new descriptors - ownership not transferred yet */ dwc_gmac_txdesc_sync(sc, first, sc->sc_txq.t_cur, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* Pass first to device */ sc->sc_descm->tx_set_owned_by_dev(&sc->sc_txq.t_desc[first]); return 0; } /* * If the interface is up and running, only modify the receive * filter when setting promiscuous or debug mode. Otherwise fall * through to ether_ioctl, which will reset the chip. */ static int dwc_gmac_ifflags_cb(struct ethercom *ec) { struct ifnet * const ifp = &ec->ec_if; struct dwc_gmac_softc * const sc = ifp->if_softc; int ret = 0; KASSERT(IFNET_LOCKED(ifp)); mutex_enter(sc->sc_mcast_lock); u_short change = ifp->if_flags ^ sc->sc_if_flags; sc->sc_if_flags = ifp->if_flags; if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { ret = ENETRESET; } else if ((change & IFF_PROMISC) != 0) { dwc_gmac_setmulti(sc); } mutex_exit(sc->sc_mcast_lock); return ret; } static int dwc_gmac_ioctl(struct ifnet *ifp, u_long cmd, void *data) { struct dwc_gmac_softc * const sc = ifp->if_softc; int error = 0; switch (cmd) { case SIOCADDMULTI: case SIOCDELMULTI: break; default: KASSERT(IFNET_LOCKED(ifp)); } const int s = splnet(); error = ether_ioctl(ifp, cmd, data); splx(s); if (error == ENETRESET) { error = 0; if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) { mutex_enter(sc->sc_mcast_lock); if (sc->sc_if_flags & IFF_RUNNING) { /* * Multicast list has changed; set the hardware * filter accordingly. */ dwc_gmac_setmulti(sc); } mutex_exit(sc->sc_mcast_lock); } } return error; } static void dwc_gmac_tx_intr(struct dwc_gmac_softc *sc) { struct ifnet * const ifp = &sc->sc_ec.ec_if; struct dwc_gmac_tx_data *data; struct dwc_gmac_dev_dmadesc *desc; int i, nsegs; mutex_enter(&sc->sc_txq.t_mtx); for (i = sc->sc_txq.t_next; sc->sc_txq.t_queued > 0; i = TX_NEXT(i)) { #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "%s: checking desc #%d (t_queued: %d)\n", __func__, i, sc->sc_txq.t_queued); #endif /* * i + 1 does not need to be a valid descriptor, * this is just a special notion to just sync * a single tx descriptor (i) */ dwc_gmac_txdesc_sync(sc, i, i + 1, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); desc = &sc->sc_txq.t_desc[i]; if (sc->sc_descm->tx_is_owned_by_dev(desc)) break; data = &sc->sc_txq.t_data[i]; if (data->td_m == NULL) continue; if_statinc(ifp, if_opackets); nsegs = data->td_active->dm_nsegs; bus_dmamap_sync(sc->sc_dmat, data->td_active, 0, data->td_active->dm_mapsize, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->sc_dmat, data->td_active); #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "%s: done with packet at desc #%d, freeing mbuf %p\n", __func__, i, data->td_m); #endif m_freem(data->td_m); data->td_m = NULL; sc->sc_txq.t_queued -= nsegs; } sc->sc_txq.t_next = i; if (sc->sc_txq.t_queued < AWGE_TX_RING_COUNT) { sc->sc_txbusy = false; } mutex_exit(&sc->sc_txq.t_mtx); } static void dwc_gmac_rx_intr(struct dwc_gmac_softc *sc) { struct ifnet * const ifp = &sc->sc_ec.ec_if; struct dwc_gmac_dev_dmadesc *desc; struct dwc_gmac_rx_data *data; bus_addr_t physaddr; struct mbuf *m, *mnew; int i, len, error; mutex_enter(&sc->sc_rxq.r_mtx); for (i = sc->sc_rxq.r_cur; ; i = RX_NEXT(i)) { #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "%s: checking desc #%d\n", __func__, i); #endif bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, RX_DESC_OFFSET(i), sizeof(*desc), BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); desc = &sc->sc_rxq.r_desc[i]; data = &sc->sc_rxq.r_data[i]; if (sc->sc_descm->rx_is_owned_by_dev(desc)) break; if (sc->sc_descm->rx_has_error(desc)) { #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "%s: RX error: status %08x, skipping\n", __func__, le32toh(desc->ddesc_status0)); #endif if_statinc(ifp, if_ierrors); goto skip; } len = sc->sc_descm->rx_get_len(desc); #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "%s: device is done with descriptor #%d, len: %d\n", __func__, i, len); #endif /* * Try to get a new mbuf before passing this one * up, if that fails, drop the packet and reuse * the existing one. */ MGETHDR(mnew, M_DONTWAIT, MT_DATA); if (mnew == NULL) { if_statinc(ifp, if_ierrors); goto skip; } MCLGET(mnew, M_DONTWAIT); if ((mnew->m_flags & M_EXT) == 0) { m_freem(mnew); if_statinc(ifp, if_ierrors); goto skip; } mnew->m_len = mnew->m_pkthdr.len = mnew->m_ext.ext_size; m_adj(mnew, ETHER_ALIGN); if (mnew->m_len > AWGE_MAX_PACKET) { mnew->m_len = mnew->m_pkthdr.len = AWGE_MAX_PACKET; } /* unload old DMA map */ bus_dmamap_sync(sc->sc_dmat, data->rd_map, 0, data->rd_map->dm_mapsize, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->sc_dmat, data->rd_map); /* and reload with new mbuf */ error = bus_dmamap_load_mbuf(sc->sc_dmat, data->rd_map, mnew, BUS_DMA_READ | BUS_DMA_NOWAIT); if (error != 0) { m_freem(mnew); /* try to reload old mbuf */ error = bus_dmamap_load_mbuf(sc->sc_dmat, data->rd_map, data->rd_m, BUS_DMA_READ | BUS_DMA_NOWAIT); if (error != 0) { panic("%s: could not load old rx mbuf", device_xname(sc->sc_dev)); } if_statinc(ifp, if_ierrors); goto skip; } physaddr = data->rd_map->dm_segs[0].ds_addr; #ifdef DWC_GMAC_DEBUG aprint_normal_dev(sc->sc_dev, "%s: receiving packet at desc #%d, using mbuf %p\n", __func__, i, data->rd_m); #endif /* * New mbuf loaded, update RX ring and continue */ m = data->rd_m; data->rd_m = mnew; desc->ddesc_data = htole32(physaddr); /* finalize mbuf */ m->m_pkthdr.len = m->m_len = len; m_set_rcvif(m, ifp); m->m_flags |= M_HASFCS; if_percpuq_enqueue(sc->sc_ipq, m); skip: bus_dmamap_sync(sc->sc_dmat, data->rd_map, 0, data->rd_map->dm_mapsize, BUS_DMASYNC_PREREAD); sc->sc_descm->rx_init_flags(desc); sc->sc_descm->rx_set_len(desc, data->rd_m->m_len); bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, RX_DESC_OFFSET(i), sizeof(*desc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); sc->sc_descm->rx_set_owned_by_dev(desc); bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, RX_DESC_OFFSET(i), sizeof(*desc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } /* update RX pointer */ sc->sc_rxq.r_cur = i; mutex_exit(&sc->sc_rxq.r_mtx); } static void dwc_gmac_setmulti(struct dwc_gmac_softc *sc) { struct ether_multi *enm; struct ether_multistep step; struct ethercom *ec = &sc->sc_ec; uint32_t hashes[2] = { 0, 0 }; uint32_t ffilt, h; int mcnt; KASSERT(mutex_owned(sc->sc_mcast_lock)); ffilt = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_FFILT); if (sc->sc_if_flags & IFF_PROMISC) { ffilt |= AWIN_GMAC_MAC_FFILT_PR; goto special_filter; } ffilt &= ~(AWIN_GMAC_MAC_FFILT_PM | AWIN_GMAC_MAC_FFILT_PR); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_HTLOW, 0); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_HTHIGH, 0); ETHER_LOCK(ec); ec->ec_flags &= ~ETHER_F_ALLMULTI; ETHER_FIRST_MULTI(step, ec, enm); mcnt = 0; while (enm != NULL) { if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN) != 0) { ffilt |= AWIN_GMAC_MAC_FFILT_PM; ec->ec_flags |= ETHER_F_ALLMULTI; ETHER_UNLOCK(ec); goto special_filter; } h = ~ether_crc32_be(enm->enm_addrlo, ETHER_ADDR_LEN) >> 26; hashes[h >> 5] |= (1 << (h & 0x1f)); mcnt++; ETHER_NEXT_MULTI(step, enm); } ETHER_UNLOCK(ec); if (mcnt) ffilt |= AWIN_GMAC_MAC_FFILT_HMC; else ffilt &= ~AWIN_GMAC_MAC_FFILT_HMC; bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_FFILT, ffilt); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_HTLOW, hashes[0]); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_HTHIGH, hashes[1]); #ifdef DWC_GMAC_DEBUG dwc_gmac_dump_ffilt(sc, ffilt); #endif return; special_filter: #ifdef DWC_GMAC_DEBUG dwc_gmac_dump_ffilt(sc, ffilt); #endif /* no MAC hashes, ALLMULTI or PROMISC */ bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_FFILT, ffilt); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_HTLOW, 0xffffffff); bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_HTHIGH, 0xffffffff); } int dwc_gmac_intr(struct dwc_gmac_softc *sc) { uint32_t status, dma_status; int rv = 0; mutex_enter(sc->sc_intr_lock); if (sc->sc_stopping) { mutex_exit(sc->sc_intr_lock); return 0; } status = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_INTR); if (status & AWIN_GMAC_MII_IRQ) { (void)bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MII_STATUS); rv = 1; mii_pollstat(&sc->sc_mii); } dma_status = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_STATUS); if (dma_status & (GMAC_DMA_INT_NIE | GMAC_DMA_INT_AIE)) rv = 1; if (dma_status & GMAC_DMA_INT_TIE) dwc_gmac_tx_intr(sc); if (dma_status & GMAC_DMA_INT_RIE) dwc_gmac_rx_intr(sc); /* * Check error conditions */ if (dma_status & GMAC_DMA_INT_ERRORS) { if_statinc(&sc->sc_ec.ec_if, if_oerrors); #ifdef DWC_GMAC_DEBUG dwc_dump_and_abort(sc, "interrupt error condition"); #endif } rnd_add_uint32(&sc->rnd_source, dma_status); /* ack interrupt */ if (dma_status) bus_space_write_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_STATUS, dma_status & GMAC_DMA_INT_MASK); /* * Get more packets */ if (rv) if_schedule_deferred_start(&sc->sc_ec.ec_if); mutex_exit(sc->sc_intr_lock); return rv; } static void dwc_gmac_desc_set_owned_by_dev(struct dwc_gmac_dev_dmadesc *desc) { desc->ddesc_status0 |= htole32(DDESC_STATUS_OWNEDBYDEV); } static int dwc_gmac_desc_is_owned_by_dev(struct dwc_gmac_dev_dmadesc *desc) { return !!(le32toh(desc->ddesc_status0) & DDESC_STATUS_OWNEDBYDEV); } static void dwc_gmac_desc_std_set_len(struct dwc_gmac_dev_dmadesc *desc, int len) { uint32_t cntl = le32toh(desc->ddesc_cntl1); desc->ddesc_cntl1 = htole32((cntl & ~DDESC_CNTL_SIZE1MASK) | __SHIFTIN(len, DDESC_CNTL_SIZE1MASK)); } static uint32_t dwc_gmac_desc_std_get_len(struct dwc_gmac_dev_dmadesc *desc) { return __SHIFTOUT(le32toh(desc->ddesc_status0), DDESC_STATUS_FRMLENMSK); } static void dwc_gmac_desc_std_tx_init_flags(struct dwc_gmac_dev_dmadesc *desc) { desc->ddesc_status0 = 0; desc->ddesc_cntl1 = htole32(DDESC_CNTL_TXCHAIN); } static void dwc_gmac_desc_std_tx_set_first_frag(struct dwc_gmac_dev_dmadesc *desc) { uint32_t cntl = le32toh(desc->ddesc_cntl1); desc->ddesc_cntl1 = htole32(cntl | DDESC_CNTL_TXFIRST); } static void dwc_gmac_desc_std_tx_set_last_frag(struct dwc_gmac_dev_dmadesc *desc) { uint32_t cntl = le32toh(desc->ddesc_cntl1); desc->ddesc_cntl1 = htole32(cntl | DDESC_CNTL_TXLAST | DDESC_CNTL_TXINT); } static void dwc_gmac_desc_std_rx_init_flags(struct dwc_gmac_dev_dmadesc *desc) { desc->ddesc_status0 = 0; desc->ddesc_cntl1 = htole32(DDESC_CNTL_TXCHAIN); } static int dwc_gmac_desc_std_rx_has_error(struct dwc_gmac_dev_dmadesc *desc) { return !!(le32toh(desc->ddesc_status0) & (DDESC_STATUS_RXERROR | DDESC_STATUS_RXTRUNCATED)); } static void dwc_gmac_desc_enh_set_len(struct dwc_gmac_dev_dmadesc *desc, int len) { uint32_t tdes1 = le32toh(desc->ddesc_cntl1); desc->ddesc_cntl1 = htole32((tdes1 & ~DDESC_DES1_SIZE1MASK) | __SHIFTIN(len, DDESC_DES1_SIZE1MASK)); } static uint32_t dwc_gmac_desc_enh_get_len(struct dwc_gmac_dev_dmadesc *desc) { return __SHIFTOUT(le32toh(desc->ddesc_status0), DDESC_RDES0_FL); } static void dwc_gmac_desc_enh_tx_init_flags(struct dwc_gmac_dev_dmadesc *desc) { desc->ddesc_status0 = htole32(DDESC_TDES0_TCH); desc->ddesc_cntl1 = 0; } static void dwc_gmac_desc_enh_tx_set_first_frag(struct dwc_gmac_dev_dmadesc *desc) { uint32_t tdes0 = le32toh(desc->ddesc_status0); desc->ddesc_status0 = htole32(tdes0 | DDESC_TDES0_FS); } static void dwc_gmac_desc_enh_tx_set_last_frag(struct dwc_gmac_dev_dmadesc *desc) { uint32_t tdes0 = le32toh(desc->ddesc_status0); desc->ddesc_status0 = htole32(tdes0 | DDESC_TDES0_LS | DDESC_TDES0_IC); } static void dwc_gmac_desc_enh_rx_init_flags(struct dwc_gmac_dev_dmadesc *desc) { desc->ddesc_status0 = 0; desc->ddesc_cntl1 = htole32(DDESC_RDES1_RCH); } static int dwc_gmac_desc_enh_rx_has_error(struct dwc_gmac_dev_dmadesc *desc) { return !!(le32toh(desc->ddesc_status0) & (DDESC_RDES0_ES | DDESC_RDES0_LE)); } #ifdef DWC_GMAC_DEBUG static void dwc_gmac_dump_dma(struct dwc_gmac_softc *sc) { aprint_normal_dev(sc->sc_dev, "busmode: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_BUSMODE)); aprint_normal_dev(sc->sc_dev, "tx poll: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_TXPOLL)); aprint_normal_dev(sc->sc_dev, "rx poll: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_RXPOLL)); aprint_normal_dev(sc->sc_dev, "rx descriptors: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_RX_ADDR)); aprint_normal_dev(sc->sc_dev, "tx descriptors: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_TX_ADDR)); aprint_normal_dev(sc->sc_dev, " status: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_STATUS)); aprint_normal_dev(sc->sc_dev, "op mode: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_OPMODE)); aprint_normal_dev(sc->sc_dev, "int en.: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_INTENABLE)); aprint_normal_dev(sc->sc_dev, " cur tx: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_CUR_TX_DESC)); aprint_normal_dev(sc->sc_dev, " cur rx: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_CUR_RX_DESC)); aprint_normal_dev(sc->sc_dev, "cur txb: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_CUR_TX_BUFADDR)); aprint_normal_dev(sc->sc_dev, "cur rxb: %08x\n", bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_CUR_RX_BUFADDR)); } static void dwc_gmac_dump_tx_desc(struct dwc_gmac_softc *sc) { const size_t descsz = sizeof(struct dwc_gmac_dev_dmadesc); aprint_normal_dev(sc->sc_dev, "TX queue: cur=%d, next=%d, queued=%d\n", sc->sc_txq.t_cur, sc->sc_txq.t_next, sc->sc_txq.t_queued); aprint_normal_dev(sc->sc_dev, "TX DMA descriptors:\n"); bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, TX_DESC_OFFSET(0), AWGE_TX_RING_COUNT * descsz, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); for (size_t i = 0; i < AWGE_TX_RING_COUNT; i++) { struct dwc_gmac_dev_dmadesc *desc = &sc->sc_txq.t_desc[i]; aprint_normal("#%3zu (%08lx): status: %08x cntl: %08x " "data: %08x next: %08x\n", i, sc->sc_txq.t_physaddr + i * descsz, le32toh(desc->ddesc_status0), le32toh(desc->ddesc_cntl1), le32toh(desc->ddesc_data), le32toh(desc->ddesc_next)); } } static void dwc_gmac_dump_rx_desc(struct dwc_gmac_softc *sc) { const size_t descsz = sizeof(struct dwc_gmac_dev_dmadesc); aprint_normal_dev(sc->sc_dev, "RX queue: cur=%d, next=%d\n", sc->sc_rxq.r_cur, sc->sc_rxq.r_next); aprint_normal_dev(sc->sc_dev, "RX DMA descriptors:\n"); bus_dmamap_sync(sc->sc_dmat, sc->sc_dma_ring_map, RX_DESC_OFFSET(0), AWGE_RX_RING_COUNT * descsz, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); for (size_t i = 0; i < AWGE_RX_RING_COUNT; i++) { struct dwc_gmac_dev_dmadesc *desc = &sc->sc_rxq.r_desc[i]; char buf[200]; if (!sc->sc_descm->rx_is_owned_by_dev(desc)) { /* print interrupt state */ snprintb(buf, sizeof(buf), "\177\20" "b\x1e" "daff\0" "f\x10\xe" "frlen\0" "b\x0f" "error\0" "b\x0e" "rxtrunc\0" /* descriptor error? */ "b\x0d" "saff\0" "b\x0c" "giantframe\0" /* length error? */ "b\x0b" "damaged\0" "b\x0a" "vlan\0" "b\x09" "first\0" "b\x08" "last\0" "b\x07" "giant\0" "b\x06" "collison\0" "b\x05" "ether\0" "b\x04" "watchdog\0" "b\x03" "miierror\0" "b\x02" "dribbling\0" "b\x01" "crc\0" "\0", le32toh(desc->ddesc_status0)); } aprint_normal("#%3zu (%08lx): status: %08x cntl: %08x " "data: %08x next: %08x %s\n", i, sc->sc_rxq.r_physaddr + i * descsz, le32toh(desc->ddesc_status0), le32toh(desc->ddesc_cntl1), le32toh(desc->ddesc_data), le32toh(desc->ddesc_next), sc->sc_descm->rx_is_owned_by_dev(desc) ? "" : buf); } } static void dwc_dump_status(struct dwc_gmac_softc *sc) { uint32_t status = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_INTR); uint32_t dma_status = bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_DMA_STATUS); char buf[200]; /* print interrupt state */ snprintb(buf, sizeof(buf), "\177\20" "b\x1c" "GPI\0" "b\x1b" "GMC\0" "b\x1a" "GLI\0" "f\x17\x3" "EB\0" "f\x14\x3" "TPS\0" "f\x11\x3" "RPS\0" "b\x10" "NI\0" "b\x0f" "AI\0" "b\x0e" "ER\0" "b\x0d" "FB\0" "b\x0a" "ET\0" "b\x09" "RW\0" "b\x08" "RS\0" "b\x07" "RU\0" "b\x06" "RI\0" "b\x05" "UN\0" "b\x04" "OV\0" "b\x03" "TJ\0" "b\x02" "TU\0" "b\x01" "TS\0" "b\x00" "TI\0" "\0", dma_status); aprint_normal_dev(sc->sc_dev, "INTR status: %08x, DMA status: %s\n", status, buf); } static void dwc_dump_and_abort(struct dwc_gmac_softc *sc, const char *msg) { dwc_dump_status(sc); dwc_gmac_dump_ffilt(sc, bus_space_read_4(sc->sc_bst, sc->sc_bsh, AWIN_GMAC_MAC_FFILT)); dwc_gmac_dump_dma(sc); dwc_gmac_dump_tx_desc(sc); dwc_gmac_dump_rx_desc(sc); panic("%s", msg); } static void dwc_gmac_dump_ffilt(struct dwc_gmac_softc *sc, uint32_t ffilt) { char buf[200]; /* print filter setup */ snprintb(buf, sizeof(buf), "\177\20" "b\x1f""RA\0" "b\x0a""HPF\0" "b\x09""SAF\0" "b\x08""SAIF\0" "b\x05""DBF\0" "b\x04""PM\0" "b\x03""DAIF\0" "b\x02""HMC\0" "b\x01""HUC\0" "b\x00""PR\0" "\0", ffilt); aprint_normal_dev(sc->sc_dev, "FFILT: %s\n", buf); } #endif