/* * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #define BAR0_SIZE SZ_4M enum bar0_amap_type { META_DATA, SIMPLE_IRQ, DMA_IRQ = SIMPLE_IRQ, EP_MEM, HOST_MEM, HOST_DMA, EP_RX_BUF, AMAP_MAX, }; struct bar0_amap { int size; struct page *page; void *virt; dma_addr_t iova; dma_addr_t phy; }; struct irqsp_data { struct nvhost_interrupt_syncpt *is; struct work_struct reprime_work; struct device *dev; }; struct pci_epf_tvnet { struct pci_epf *epf; struct device *fdev; struct pci_epf_header header; void __iomem *dma_base; struct bar0_amap bar0_amap[AMAP_MAX]; struct bar_md *bar_md; dma_addr_t bar0_iova; struct net_device *ndev; struct napi_struct napi; bool pcie_link_status; struct ep_ring_buf ep_ring_buf; struct host_ring_buf host_ring_buf; enum dir_link_state tx_link_state; enum dir_link_state rx_link_state; enum os_link_state os_link_state; /* To synchronize network link state machine*/ struct mutex link_state_lock; wait_queue_head_t link_state_wq; struct list_head h2ep_empty_list; #if ENABLE_DMA struct dma_desc_cnt desc_cnt; #endif /* To protect h2ep empty list */ spinlock_t h2ep_empty_lock; dma_addr_t rx_buf_iova; unsigned long *rx_buf_bitmap; int rx_num_pages; void __iomem *tx_dst_va; phys_addr_t tx_dst_pci_addr; void *ep_dma_virt; dma_addr_t ep_dma_iova; struct irqsp_data *ctrl_irqsp; struct irqsp_data *data_irqsp; struct tvnet_counter h2ep_ctrl; struct tvnet_counter ep2h_ctrl; struct tvnet_counter h2ep_empty; struct tvnet_counter h2ep_full; struct tvnet_counter ep2h_empty; struct tvnet_counter ep2h_full; }; static void tvnet_ep_read_ctrl_msg(struct pci_epf_tvnet *tvnet, struct ctrl_msg *msg) { struct host_ring_buf *host_ring_buf = &tvnet->host_ring_buf; struct ctrl_msg *ctrl_msg = host_ring_buf->h2ep_ctrl_msgs; u32 idx; if (tvnet_ivc_empty(&tvnet->h2ep_ctrl)) { dev_dbg(tvnet->fdev, "%s: H2EP ctrl ring empty\n", __func__); return; } idx = tvnet_ivc_get_rd_cnt(&tvnet->h2ep_ctrl) % RING_COUNT; memcpy(msg, &ctrl_msg[idx], sizeof(*msg)); tvnet_ivc_advance_rd(&tvnet->h2ep_ctrl); } /* TODO Handle error case */ static int tvnet_ep_write_ctrl_msg(struct pci_epf_tvnet *tvnet, struct ctrl_msg *msg) { struct ep_ring_buf *ep_ring_buf = &tvnet->ep_ring_buf; struct ctrl_msg *ctrl_msg = ep_ring_buf->ep2h_ctrl_msgs; struct pci_epc *epc = tvnet->epf->epc; u32 idx; if (tvnet_ivc_full(&tvnet->ep2h_ctrl)) { /* Raise an interrupt to let host process EP2H ring */ pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 0); dev_dbg(tvnet->fdev, "%s: EP2H ctrl ring full\n", __func__); return -EAGAIN; } idx = tvnet_ivc_get_wr_cnt(&tvnet->ep2h_ctrl) % RING_COUNT; memcpy(&ctrl_msg[idx], msg, sizeof(*msg)); tvnet_ivc_advance_wr(&tvnet->ep2h_ctrl); pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 0); return 0; } #if !ENABLE_DMA static dma_addr_t tvnet_ivoa_alloc(struct pci_epf_tvnet *tvnet) { dma_addr_t iova; int pageno; pageno = bitmap_find_free_region(tvnet->rx_buf_bitmap, tvnet->rx_num_pages, 0); if (pageno < 0) { dev_err(tvnet->fdev, "%s: Rx iova alloc fail, page: %d\n", __func__, pageno); return DMA_ERROR_CODE; } iova = tvnet->rx_buf_iova + (pageno << PAGE_SHIFT); return iova; } static void tvnet_ep_iova_dealloc(struct pci_epf_tvnet *tvnet, dma_addr_t iova) { int pageno; pageno = (iova - tvnet->rx_buf_iova) >> PAGE_SHIFT; bitmap_release_region(tvnet->rx_buf_bitmap, pageno, 0); } #endif static void tvnet_ep_alloc_empty_buffers(struct pci_epf_tvnet *tvnet) { struct ep_ring_buf *ep_ring_buf = &tvnet->ep_ring_buf; struct pci_epc *epc = tvnet->epf->epc; struct device *cdev = epc->dev.parent; struct data_msg *h2ep_empty_msg = ep_ring_buf->h2ep_empty_msgs; struct h2ep_empty_list *h2ep_empty_ptr; #if ENABLE_DMA struct net_device *ndev = tvnet->ndev; #else struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); int ret = 0; #endif while (!tvnet_ivc_full(&tvnet->h2ep_empty)) { dma_addr_t iova; #if ENABLE_DMA struct sk_buff *skb; int len = ndev->mtu + ETH_HLEN; #else struct page *page; void *virt; #endif u32 idx; unsigned long flags; #if ENABLE_DMA skb = netdev_alloc_skb(ndev, len); if (!skb) { pr_err("%s: alloc skb failed\n", __func__); break; } iova = dma_map_single(cdev, skb->data, len, DMA_FROM_DEVICE); if (dma_mapping_error(cdev, iova)) { pr_err("%s: dma map failed\n", __func__); dev_kfree_skb_any(skb); break; } #else iova = tvnet_ivoa_alloc(tvnet); if (iova == DMA_ERROR_CODE) { dev_err(tvnet->fdev, "%s: iova alloc failed\n", __func__); break; } page = alloc_pages(GFP_KERNEL, 1); if (!page) { dev_err(tvnet->fdev, "%s: alloc_pages() failed\n", __func__); tvnet_ep_iova_dealloc(tvnet, iova); break; } ret = iommu_map(domain, iova, page_to_phys(page), PAGE_SIZE, IOMMU_CACHE | IOMMU_READ | IOMMU_WRITE); if (ret < 0) { dev_err(tvnet->fdev, "%s: iommu_map(RAM) failed: %d\n", __func__, ret); __free_pages(page, 1); tvnet_ep_iova_dealloc(tvnet, iova); break; } virt = vmap(&page, 1, VM_MAP, PAGE_KERNEL); if (!virt) { dev_err(tvnet->fdev, "%s: vmap() failed\n", __func__); iommu_unmap(domain, iova, PAGE_SIZE); __free_pages(page, 1); tvnet_ep_iova_dealloc(tvnet, iova); break; } #endif h2ep_empty_ptr = kmalloc(sizeof(*h2ep_empty_ptr), GFP_KERNEL); if (!h2ep_empty_ptr) { #if ENABLE_DMA dma_unmap_single(cdev, iova, len, DMA_FROM_DEVICE); dev_kfree_skb_any(skb); #else vunmap(virt); iommu_unmap(domain, iova, PAGE_SIZE); __free_pages(page, 1); tvnet_ep_iova_dealloc(tvnet, iova); #endif break; } #if ENABLE_DMA h2ep_empty_ptr->skb = skb; h2ep_empty_ptr->size = len; #else h2ep_empty_ptr->page = page; h2ep_empty_ptr->virt = virt; h2ep_empty_ptr->size = PAGE_SIZE; #endif h2ep_empty_ptr->iova = iova; spin_lock_irqsave(&tvnet->h2ep_empty_lock, flags); list_add_tail(&h2ep_empty_ptr->list, &tvnet->h2ep_empty_list); spin_unlock_irqrestore(&tvnet->h2ep_empty_lock, flags); idx = tvnet_ivc_get_wr_cnt(&tvnet->h2ep_empty) % RING_COUNT; h2ep_empty_msg[idx].u.empty_buffer.pcie_address = iova; h2ep_empty_msg[idx].u.empty_buffer.buffer_len = PAGE_SIZE; tvnet_ivc_advance_wr(&tvnet->h2ep_empty); pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 0); } } static void tvnet_ep_free_empty_buffers(struct pci_epf_tvnet *tvnet) { struct pci_epf *epf = tvnet->epf; struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; #if !ENABLE_DMA struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); #endif struct h2ep_empty_list *h2ep_empty_ptr, *temp; unsigned long flags; spin_lock_irqsave(&tvnet->h2ep_empty_lock, flags); list_for_each_entry_safe(h2ep_empty_ptr, temp, &tvnet->h2ep_empty_list, list) { list_del(&h2ep_empty_ptr->list); #if ENABLE_DMA dma_unmap_single(cdev, h2ep_empty_ptr->iova, h2ep_empty_ptr->size, DMA_FROM_DEVICE); dev_kfree_skb_any(h2ep_empty_ptr->skb); #else vunmap(h2ep_empty_ptr->virt); iommu_unmap(domain, h2ep_empty_ptr->iova, PAGE_SIZE); __free_pages(h2ep_empty_ptr->page, 1); tvnet_ep_iova_dealloc(tvnet, h2ep_empty_ptr->iova); #endif kfree(h2ep_empty_ptr); } spin_unlock_irqrestore(&tvnet->h2ep_empty_lock, flags); } static void tvnet_ep_stop_tx_queue(struct pci_epf_tvnet *tvnet) { struct net_device *ndev = tvnet->ndev; netif_stop_queue(ndev); /* Get tx lock to make sure that there is no ongoing xmit */ netif_tx_lock(ndev); netif_tx_unlock(ndev); } static void tvnet_ep_stop_rx_work(struct pci_epf_tvnet *tvnet) { /* TODO wait for syncpoint interrupt handlers */ } static void tvnet_ep_clear_data_msg_counters(struct pci_epf_tvnet *tvnet) { struct host_ring_buf *host_ring_buf = &tvnet->host_ring_buf; struct host_own_cnt *host_cnt = host_ring_buf->host_cnt; struct ep_ring_buf *ep_ring_buf = &tvnet->ep_ring_buf; struct ep_own_cnt *ep_cnt = ep_ring_buf->ep_cnt; host_cnt->h2ep_empty_rd_cnt = 0; ep_cnt->h2ep_empty_wr_cnt = 0; ep_cnt->ep2h_full_wr_cnt = 0; host_cnt->ep2h_full_rd_cnt = 0; } static void tvnet_ep_update_link_state(struct net_device *ndev, enum os_link_state state) { if (state == OS_LINK_STATE_UP) { netif_start_queue(ndev); netif_carrier_on(ndev); } else if (state == OS_LINK_STATE_DOWN) { netif_carrier_off(ndev); netif_stop_queue(ndev); } else { pr_err("%s: invalid sate: %d\n", __func__, state); } } /* OS link state machine */ static void tvnet_ep_update_link_sm(struct pci_epf_tvnet *tvnet) { struct net_device *ndev = tvnet->ndev; enum os_link_state old_state = tvnet->os_link_state; if ((tvnet->rx_link_state == DIR_LINK_STATE_UP) && (tvnet->tx_link_state == DIR_LINK_STATE_UP)) tvnet->os_link_state = OS_LINK_STATE_UP; else tvnet->os_link_state = OS_LINK_STATE_DOWN; if (tvnet->os_link_state != old_state) tvnet_ep_update_link_state(ndev, tvnet->os_link_state); } /* One way link state machine */ static void tvnet_ep_user_link_up_req(struct pci_epf_tvnet *tvnet) { struct ctrl_msg msg; tvnet_ep_clear_data_msg_counters(tvnet); tvnet_ep_alloc_empty_buffers(tvnet); msg.msg_id = CTRL_MSG_LINK_UP; tvnet_ep_write_ctrl_msg(tvnet, &msg); tvnet->rx_link_state = DIR_LINK_STATE_UP; tvnet_ep_update_link_sm(tvnet); } static void tvnet_ep_user_link_down_req(struct pci_epf_tvnet *tvnet) { struct ctrl_msg msg; tvnet->rx_link_state = DIR_LINK_STATE_SENT_DOWN; msg.msg_id = CTRL_MSG_LINK_DOWN; tvnet_ep_write_ctrl_msg(tvnet, &msg); tvnet_ep_update_link_sm(tvnet); } static void tvnet_ep_rcv_link_up_msg(struct pci_epf_tvnet *tvnet) { tvnet->tx_link_state = DIR_LINK_STATE_UP; tvnet_ep_update_link_sm(tvnet); } static void tvnet_ep_rcv_link_down_msg(struct pci_epf_tvnet *tvnet) { struct ctrl_msg msg; /* Stop using empty buffers of remote system */ tvnet_ep_stop_tx_queue(tvnet); msg.msg_id = CTRL_MSG_LINK_DOWN_ACK; tvnet_ep_write_ctrl_msg(tvnet, &msg); tvnet->tx_link_state = DIR_LINK_STATE_DOWN; tvnet_ep_update_link_sm(tvnet); } static void tvnet_ep_rcv_link_down_ack(struct pci_epf_tvnet *tvnet) { /* Stop using empty buffers(which are full in rx) of local system */ tvnet_ep_stop_rx_work(tvnet); tvnet_ep_free_empty_buffers(tvnet); tvnet->rx_link_state = DIR_LINK_STATE_DOWN; wake_up_interruptible(&tvnet->link_state_wq); tvnet_ep_update_link_sm(tvnet); } static int tvnet_ep_open(struct net_device *ndev) { struct device *fdev = ndev->dev.parent; struct pci_epf_tvnet *tvnet = dev_get_drvdata(fdev); if (!tvnet->pcie_link_status) { dev_err(fdev, "%s: PCIe link is not up\n", __func__); return -ENODEV; } mutex_lock(&tvnet->link_state_lock); if (tvnet->rx_link_state == DIR_LINK_STATE_DOWN) tvnet_ep_user_link_up_req(tvnet); napi_enable(&tvnet->napi); mutex_unlock(&tvnet->link_state_lock); return 0; } static int tvnet_ep_close(struct net_device *ndev) { struct device *fdev = ndev->dev.parent; struct pci_epf_tvnet *tvnet = dev_get_drvdata(fdev); int ret = 0; mutex_lock(&tvnet->link_state_lock); napi_disable(&tvnet->napi); if (tvnet->rx_link_state == DIR_LINK_STATE_UP) tvnet_ep_user_link_down_req(tvnet); ret = wait_event_interruptible_timeout(tvnet->link_state_wq, (tvnet->rx_link_state == DIR_LINK_STATE_DOWN), msecs_to_jiffies(LINK_TIMEOUT)); ret = (ret > 0) ? 0 : -ETIMEDOUT; if (ret < 0) { pr_err("%s: link state machine failed: tx_state: %d rx_state: %d err: %d\n", __func__, tvnet->tx_link_state, tvnet->rx_link_state, ret); tvnet->rx_link_state = DIR_LINK_STATE_UP; } mutex_unlock(&tvnet->link_state_lock); return 0; } static int tvnet_ep_change_mtu(struct net_device *ndev, int new_mtu) { bool set_down = false; if (new_mtu > TVNET_MAX_MTU || new_mtu < TVNET_MIN_MTU) { pr_err("MTU range is %d to %d\n", TVNET_MIN_MTU, TVNET_MAX_MTU); return -EINVAL; } if (netif_running(ndev)) { set_down = true; tvnet_ep_close(ndev); } pr_info("changing MTU from %d to %d\n", ndev->mtu, new_mtu); ndev->mtu = new_mtu; if (set_down) tvnet_ep_open(ndev); return 0; } static netdev_tx_t tvnet_ep_start_xmit(struct sk_buff *skb, struct net_device *ndev) { struct device *fdev = ndev->dev.parent; struct pci_epf_tvnet *tvnet = dev_get_drvdata(fdev); struct host_ring_buf *host_ring_buf = &tvnet->host_ring_buf; struct ep_ring_buf *ep_ring_buf = &tvnet->ep_ring_buf; struct data_msg *ep2h_full_msg = ep_ring_buf->ep2h_full_msgs; struct skb_shared_info *info = skb_shinfo(skb); struct data_msg *ep2h_empty_msg = host_ring_buf->ep2h_empty_msgs; struct pci_epf *epf = tvnet->epf; struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; #if ENABLE_DMA struct dma_desc_cnt *desc_cnt = &tvnet->desc_cnt; struct tvnet_dma_desc *ep_dma_virt = (struct tvnet_dma_desc *)tvnet->ep_dma_virt; u32 desc_widx, desc_ridx, val, ctrl_d; unsigned long timeout; #endif dma_addr_t src_iova; u32 rd_idx, wr_idx; u64 dst_masked, dst_off, dst_iova; int ret, dst_len, len; /*TODO Not expecting skb frags, remove this after testing */ WARN_ON(info->nr_frags); /* Check if EP2H_EMPTY_BUF available to read */ if (!tvnet_ivc_rd_available(&tvnet->ep2h_empty)) { pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 0); dev_dbg(fdev, "%s: No EP2H empty msg, stop tx\n", __func__); netif_stop_queue(ndev); return NETDEV_TX_BUSY; } /* Check if EP2H_FULL_BUF available to write */ if (tvnet_ivc_full(&tvnet->ep2h_full)) { pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 1); dev_dbg(fdev, "%s: No EP2H full buf, stop tx\n", __func__); netif_stop_queue(ndev); return NETDEV_TX_BUSY; } #if ENABLE_DMA /* Check if dma desc available */ if ((desc_cnt->wr_cnt - desc_cnt->rd_cnt) >= DMA_DESC_COUNT) { dev_dbg(fdev, "%s: dma descs are not available\n", __func__); netif_stop_queue(ndev); return NETDEV_TX_BUSY; } #endif len = skb_headlen(skb); src_iova = dma_map_single(cdev, skb->data, len, DMA_TO_DEVICE); if (dma_mapping_error(cdev, src_iova)) { dev_err(fdev, "%s: dma_map_single failed\n", __func__); dev_kfree_skb_any(skb); return NETDEV_TX_OK; } /* Get EP2H empty msg */ rd_idx = tvnet_ivc_get_rd_cnt(&tvnet->ep2h_empty) % RING_COUNT; dst_iova = ep2h_empty_msg[rd_idx].u.empty_buffer.pcie_address; dst_len = ep2h_empty_msg[rd_idx].u.empty_buffer.buffer_len; /* * Map host dst mem to local PCIe address range. * PCIe address range is SZ_64K aligned. */ dst_masked = (dst_iova & ~(SZ_64K - 1)); dst_off = (dst_iova & (SZ_64K - 1)); ret = pci_epc_map_addr(epc, tvnet->tx_dst_pci_addr, dst_masked, dst_len); if (ret < 0) { dev_err(fdev, "failed to map dst addr to PCIe addr range\n"); dma_unmap_single(cdev, src_iova, len, DMA_TO_DEVICE); dev_kfree_skb_any(skb); return NETDEV_TX_OK; } /* * Advance read count after all failure cases completed, to avoid * dangling buffer at host. */ tvnet_ivc_advance_rd(&tvnet->ep2h_empty); /* Raise an interrupt to let host populate EP2H_EMPTY_BUF ring */ pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 0); #if ENABLE_DMA /* Trigger DMA write from src_iova to dst_iova */ desc_widx = desc_cnt->wr_cnt % DMA_DESC_COUNT; ep_dma_virt[desc_widx].size = len; ep_dma_virt[desc_widx].sar_low = lower_32_bits(src_iova); ep_dma_virt[desc_widx].sar_high = upper_32_bits(src_iova); ep_dma_virt[desc_widx].dar_low = lower_32_bits(dst_iova); ep_dma_virt[desc_widx].dar_high = upper_32_bits(dst_iova); /* CB bit should be set at the end */ mb(); ctrl_d = DMA_CH_CONTROL1_OFF_WRCH_LIE; ctrl_d |= DMA_CH_CONTROL1_OFF_WRCH_CB; ep_dma_virt[desc_widx].ctrl_reg.ctrl_d = ctrl_d; /* DMA write should not go out of order wrt CB bit set */ mb(); timeout = jiffies + msecs_to_jiffies(1000); dma_common_wr8(tvnet->dma_base, DMA_WR_DATA_CH, DMA_WRITE_DOORBELL_OFF); desc_cnt->wr_cnt++; while (true) { val = dma_common_rd(tvnet->dma_base, DMA_WRITE_INT_STATUS_OFF); if (val == BIT(DMA_WR_DATA_CH)) { dma_common_wr(tvnet->dma_base, val, DMA_WRITE_INT_CLEAR_OFF); break; } if (time_after(jiffies, timeout)) { dev_err(fdev, "dma took more time, reset dma engine\n"); dma_common_wr(tvnet->dma_base, DMA_WRITE_ENGINE_EN_OFF_DISABLE, DMA_WRITE_ENGINE_EN_OFF); mdelay(1); dma_common_wr(tvnet->dma_base, DMA_WRITE_ENGINE_EN_OFF_ENABLE, DMA_WRITE_ENGINE_EN_OFF); desc_cnt->wr_cnt--; pci_epc_unmap_addr(epc, tvnet->tx_dst_pci_addr); dma_unmap_single(cdev, src_iova, len, DMA_TO_DEVICE); return NETDEV_TX_BUSY; } } desc_ridx = tvnet->desc_cnt.rd_cnt % DMA_DESC_COUNT; /* Clear DMA cycle bit and increment rd_cnt */ ep_dma_virt[desc_ridx].ctrl_reg.ctrl_e.cb = 0; mb(); tvnet->desc_cnt.rd_cnt++; #else /* Copy skb->data to host dst address, use CPU virt addr */ memcpy((void *)(tvnet->tx_dst_va + dst_off), skb->data, len); /* * tx_dst_va is ioremap_wc() mem, add mb to make sure complete skb->data * written to dst before adding it to full buffer */ mb(); #endif /* Push dst to EP2H full ring */ wr_idx = tvnet_ivc_get_wr_cnt(&tvnet->ep2h_full) % RING_COUNT; ep2h_full_msg[wr_idx].u.full_buffer.packet_size = len; ep2h_full_msg[wr_idx].u.full_buffer.pcie_address = dst_iova; tvnet_ivc_advance_wr(&tvnet->ep2h_full); pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 1); /* Free temp src and skb */ pci_epc_unmap_addr(epc, tvnet->tx_dst_pci_addr); dma_unmap_single(cdev, src_iova, len, DMA_TO_DEVICE); dev_kfree_skb_any(skb); return NETDEV_TX_OK; } static const struct net_device_ops tvnet_netdev_ops = { .ndo_open = tvnet_ep_open, .ndo_stop = tvnet_ep_close, .ndo_start_xmit = tvnet_ep_start_xmit, .ndo_change_mtu = tvnet_ep_change_mtu, }; static void tvnet_ep_process_ctrl_msg(struct pci_epf_tvnet *tvnet) { struct ctrl_msg msg; while (tvnet_ivc_rd_available(&tvnet->h2ep_ctrl)) { tvnet_ep_read_ctrl_msg(tvnet, &msg); if (msg.msg_id == CTRL_MSG_LINK_UP) tvnet_ep_rcv_link_up_msg(tvnet); else if (msg.msg_id == CTRL_MSG_LINK_DOWN) tvnet_ep_rcv_link_down_msg(tvnet); else if (msg.msg_id == CTRL_MSG_LINK_DOWN_ACK) tvnet_ep_rcv_link_down_ack(tvnet); } } static int tvnet_ep_process_h2ep_msg(struct pci_epf_tvnet *tvnet) { struct host_ring_buf *host_ring_buf = &tvnet->host_ring_buf; struct data_msg *data_msg = host_ring_buf->h2ep_full_msgs; struct pci_epf *epf = tvnet->epf; struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; struct h2ep_empty_list *h2ep_empty_ptr; struct net_device *ndev = tvnet->ndev; #if !ENABLE_DMA struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); #endif int count = 0; while ((count < TVNET_NAPI_WEIGHT) && tvnet_ivc_rd_available(&tvnet->h2ep_full)) { struct sk_buff *skb; int idx, found = 0; u32 len; u64 pcie_address; unsigned long flags; /* Read H2EP full msg */ idx = tvnet_ivc_get_rd_cnt(&tvnet->h2ep_full) % RING_COUNT; len = data_msg[idx].u.full_buffer.packet_size; pcie_address = data_msg[idx].u.full_buffer.pcie_address; /* Get H2EP msg pointer from saved list */ spin_lock_irqsave(&tvnet->h2ep_empty_lock, flags); list_for_each_entry(h2ep_empty_ptr, &tvnet->h2ep_empty_list, list) { if (h2ep_empty_ptr->iova == pcie_address) { found = 1; break; } } WARN_ON(!found); list_del(&h2ep_empty_ptr->list); spin_unlock_irqrestore(&tvnet->h2ep_empty_lock, flags); /* Advance H2EP full buffer after search in local list */ tvnet_ivc_advance_rd(&tvnet->h2ep_full); /* * If H2EP network queue is stopped due to lack of H2EP_FULL * queue, raising ctrl irq will help. */ pci_epc_raise_irq(epc, PCI_EPC_IRQ_MSIX, 0); #if ENABLE_DMA dma_unmap_single(cdev, pcie_address, ndev->mtu, DMA_FROM_DEVICE); skb = h2ep_empty_ptr->skb; skb_put(skb, len); skb->protocol = eth_type_trans(skb, ndev); napi_gro_receive(&tvnet->napi, skb); #else /* Alloc new skb and copy data from full buffer */ skb = netdev_alloc_skb(ndev, len); memcpy(skb->data, h2ep_empty_ptr->virt, len); skb_put(skb, len); skb->protocol = eth_type_trans(skb, ndev); napi_gro_receive(&tvnet->napi, skb); /* Free H2EP dst msg */ vunmap(h2ep_empty_ptr->virt); iommu_unmap(domain, h2ep_empty_ptr->iova, PAGE_SIZE); __free_pages(h2ep_empty_ptr->page, 1); tvnet_ep_iova_dealloc(tvnet, h2ep_empty_ptr->iova); #endif kfree(h2ep_empty_ptr); count++; } return count; } #if ENABLE_DMA static void tvnet_ep_setup_dma(struct pci_epf_tvnet *tvnet) { dma_addr_t iova = tvnet->bar0_amap[HOST_DMA].iova; struct dma_desc_cnt *desc_cnt = &tvnet->desc_cnt; u32 val; desc_cnt->rd_cnt = desc_cnt->wr_cnt = 0; /* Enable linked list mode and set CCS for write channel-0 */ val = dma_channel_rd(tvnet->dma_base, DMA_WR_DATA_CH, DMA_CH_CONTROL1_OFF_WRCH); val |= DMA_CH_CONTROL1_OFF_WRCH_LLE; val |= DMA_CH_CONTROL1_OFF_WRCH_CCS; dma_channel_wr(tvnet->dma_base, DMA_WR_DATA_CH, val, DMA_CH_CONTROL1_OFF_WRCH); /* Unmask write channel-0 done irq to enable LIE */ val = dma_common_rd(tvnet->dma_base, DMA_WRITE_INT_MASK_OFF); val &= ~0x1; dma_common_wr(tvnet->dma_base, val, DMA_WRITE_INT_MASK_OFF); /* Enable write channel-0 local abort irq */ val = dma_common_rd(tvnet->dma_base, DMA_WRITE_LINKED_LIST_ERR_EN_OFF); val |= (0x1 << 16); dma_common_wr(tvnet->dma_base, val, DMA_WRITE_LINKED_LIST_ERR_EN_OFF); /* Program DMA write linked list base address to DMA LLP register */ dma_channel_wr(tvnet->dma_base, DMA_WR_DATA_CH, lower_32_bits(tvnet->ep_dma_iova), DMA_LLP_LOW_OFF_WRCH); dma_channel_wr(tvnet->dma_base, DMA_WR_DATA_CH, upper_32_bits(tvnet->ep_dma_iova), DMA_LLP_HIGH_OFF_WRCH); /* Enable DMA write engine */ dma_common_wr(tvnet->dma_base, DMA_WRITE_ENGINE_EN_OFF_ENABLE, DMA_WRITE_ENGINE_EN_OFF); /* Enable linked list mode and set CCS for read channel-0 */ val = dma_channel_rd(tvnet->dma_base, DMA_RD_DATA_CH, DMA_CH_CONTROL1_OFF_RDCH); val |= DMA_CH_CONTROL1_OFF_RDCH_LLE; val |= DMA_CH_CONTROL1_OFF_RDCH_CCS; dma_channel_wr(tvnet->dma_base, DMA_RD_DATA_CH, val, DMA_CH_CONTROL1_OFF_RDCH); /* Mask read channel-0 done irq to enable RIE */ val = dma_common_rd(tvnet->dma_base, DMA_READ_INT_MASK_OFF); val |= 0x1; dma_common_wr(tvnet->dma_base, val, DMA_READ_INT_MASK_OFF); val = dma_common_rd(tvnet->dma_base, DMA_READ_LINKED_LIST_ERR_EN_OFF); /* Enable read channel-0 remote abort irq */ val |= 0x1; dma_common_wr(tvnet->dma_base, val, DMA_READ_LINKED_LIST_ERR_EN_OFF); /* Program DMA read linked list base address to DMA LLP register */ dma_channel_wr(tvnet->dma_base, DMA_RD_DATA_CH, lower_32_bits(iova), DMA_LLP_LOW_OFF_RDCH); dma_channel_wr(tvnet->dma_base, DMA_RD_DATA_CH, upper_32_bits(iova), DMA_LLP_HIGH_OFF_RDCH); /* Enable DMA read engine */ dma_common_wr(tvnet->dma_base, DMA_READ_ENGINE_EN_OFF_ENABLE, DMA_READ_ENGINE_EN_OFF); } #endif static void tvnet_ep_ctrl_irqsp_reprime_work(struct work_struct *work) { struct irqsp_data *data_irqsp = container_of(work, struct irqsp_data, reprime_work); nvhost_interrupt_syncpt_prime(data_irqsp->is); } static void tvnet_ep_ctrl_irqsp_callback(void *private_data) { struct irqsp_data *data_irqsp = private_data; struct pci_epf_tvnet *tvnet = dev_get_drvdata(data_irqsp->dev); struct net_device *ndev = tvnet->ndev; if (netif_queue_stopped(ndev)) { if ((tvnet->os_link_state == OS_LINK_STATE_UP) && tvnet_ivc_rd_available(&tvnet->ep2h_empty) && !tvnet_ivc_full(&tvnet->ep2h_full)) { netif_wake_queue(ndev); } } if (tvnet_ivc_rd_available(&tvnet->h2ep_ctrl)) tvnet_ep_process_ctrl_msg(tvnet); if (!tvnet_ivc_full(&tvnet->h2ep_empty) && (tvnet->os_link_state == OS_LINK_STATE_UP)) tvnet_ep_alloc_empty_buffers(tvnet); schedule_work(&data_irqsp->reprime_work); } static void tvnet_ep_data_irqsp_reprime_work(struct work_struct *work) { struct irqsp_data *data_irqsp = container_of(work, struct irqsp_data, reprime_work); nvhost_interrupt_syncpt_prime(data_irqsp->is); } static void tvnet_ep_data_irqsp_callback(void *private_data) { struct irqsp_data *data_irqsp = private_data; struct pci_epf_tvnet *tvnet = dev_get_drvdata(data_irqsp->dev); if (tvnet_ivc_rd_available(&tvnet->h2ep_full)) napi_schedule(&tvnet->napi); else schedule_work(&data_irqsp->reprime_work); } static int tvnet_ep_poll(struct napi_struct *napi, int budget) { struct pci_epf_tvnet *tvnet = container_of(napi, struct pci_epf_tvnet, napi); struct irqsp_data *data_irqsp = tvnet->data_irqsp; int work_done; work_done = tvnet_ep_process_h2ep_msg(tvnet); if (work_done < budget) { napi_complete(napi); schedule_work(&data_irqsp->reprime_work); } return work_done; } static int tvnet_ep_pci_epf_setup_irqsp(struct pci_epf_tvnet *tvnet) { struct bar0_amap *amap = &tvnet->bar0_amap[SIMPLE_IRQ]; struct irqsp_data *ctrl_irqsp, *data_irqsp; struct pci_epf *epf = tvnet->epf; struct device *fdev = tvnet->fdev; struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); struct irq_md *irq; phys_addr_t syncpt_addr; int ret; ctrl_irqsp = devm_kzalloc(fdev, sizeof(*ctrl_irqsp), GFP_KERNEL); if (!ctrl_irqsp) { ret = -ENOMEM; goto fail; } ctrl_irqsp->is = nvhost_interrupt_syncpt_get(cdev->of_node, tvnet_ep_ctrl_irqsp_callback, ctrl_irqsp); if (IS_ERR(ctrl_irqsp->is)) { ret = PTR_ERR(ctrl_irqsp->is); dev_err(fdev, "failed to get ctrl syncpt irq: %d\n", ret); goto fail; } ctrl_irqsp->dev = fdev; INIT_WORK(&ctrl_irqsp->reprime_work, tvnet_ep_ctrl_irqsp_reprime_work); tvnet->ctrl_irqsp = ctrl_irqsp; data_irqsp = devm_kzalloc(fdev, sizeof(*data_irqsp), GFP_KERNEL); if (!data_irqsp) { ret = -ENOMEM; goto free_ctrl_sp; } data_irqsp->is = nvhost_interrupt_syncpt_get(cdev->of_node, tvnet_ep_data_irqsp_callback, data_irqsp); if (IS_ERR(data_irqsp->is)) { ret = PTR_ERR(data_irqsp->is); dev_err(fdev, "failed to get data syncpt irq: %d\n", ret); goto free_ctrl_sp; } data_irqsp->dev = fdev; INIT_WORK(&data_irqsp->reprime_work, tvnet_ep_data_irqsp_reprime_work); tvnet->data_irqsp = data_irqsp; syncpt_addr = nvhost_interrupt_syncpt_get_syncpt_addr(ctrl_irqsp->is); ret = iommu_map(domain, amap->iova, syncpt_addr, PAGE_SIZE, IOMMU_READ | IOMMU_WRITE); if (ret < 0) { dev_err(fdev, "%s: iommu_map of ctrlsp mem failed: %d\n", __func__, ret); goto free_data_sp; } irq = &tvnet->bar_md->irq_ctrl; irq->irq_addr = PAGE_SIZE; irq->irq_type = IRQ_SIMPLE; syncpt_addr = nvhost_interrupt_syncpt_get_syncpt_addr(data_irqsp->is); ret = iommu_map(domain, amap->iova + PAGE_SIZE, syncpt_addr, PAGE_SIZE, IOMMU_READ | IOMMU_WRITE); if (ret < 0) { dev_err(fdev, "%s: iommu_map of datasp mem failed: %d\n", __func__, ret); goto free_ctrl_ivoa; } irq = &tvnet->bar_md->irq_data; irq->irq_addr = 2 * PAGE_SIZE; irq->irq_type = IRQ_SIMPLE; return 0; free_ctrl_ivoa: iommu_unmap(domain, amap->iova, PAGE_SIZE); free_data_sp: nvhost_interrupt_syncpt_free(data_irqsp->is); free_ctrl_sp: nvhost_interrupt_syncpt_free(ctrl_irqsp->is); fail: return ret; } static void tvnet_ep_pci_epf_destroy_irqsp(struct pci_epf_tvnet *tvnet) { struct pci_epf *epf = tvnet->epf; struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); iommu_unmap(domain, tvnet->bar0_amap[SIMPLE_IRQ].iova + PAGE_SIZE, PAGE_SIZE); iommu_unmap(domain, tvnet->bar0_amap[SIMPLE_IRQ].iova, PAGE_SIZE); nvhost_interrupt_syncpt_free(tvnet->data_irqsp->is); nvhost_interrupt_syncpt_free(tvnet->ctrl_irqsp->is); } static int tvnet_ep_alloc_single_page_bar0_mem(struct pci_epf *epf, enum bar0_amap_type type) { struct pci_epf_tvnet *tvnet = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); struct bar0_amap *amap = &tvnet->bar0_amap[type]; int ret = 0; amap->page = alloc_pages(GFP_KERNEL, 1); if (!amap->page) { dev_err(tvnet->fdev, "%s: type: %d alloc_pages() failed\n", __func__, type); ret = -ENOMEM; goto fail; } ret = iommu_map(domain, amap->iova, page_to_phys(amap->page), PAGE_SIZE, IOMMU_READ | IOMMU_WRITE); if (ret < 0) { dev_err(tvnet->fdev, "%s: type: %d iommu_map(RAM) failed: %d\n", __func__, type, ret); goto fail_free_pages; } amap->virt = vmap(&amap->page, 1, VM_MAP, PAGE_KERNEL); if (!amap->virt) { dev_err(tvnet->fdev, "%s: type: %d vmap() failed\n", __func__, type); ret = -ENOMEM; goto fail_unmap_iova; } return 0; fail_unmap_iova: iommu_unmap(domain, amap->iova, PAGE_SIZE); fail_free_pages: __free_pages(amap->page, 1); fail: return ret; } static void tvnet_ep_free_single_page_bar0_mem(struct pci_epf *epf, enum bar0_amap_type type) { struct pci_epf_tvnet *tvnet = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); struct bar0_amap *amap = &tvnet->bar0_amap[type]; vunmap(amap->virt); iommu_unmap(domain, amap->iova, PAGE_SIZE); __free_pages(amap->page, 1); } static int tvnet_ep_alloc_multi_page_bar0_mem(struct pci_epf *epf, enum bar0_amap_type type) { struct pci_epf_tvnet *tvnet = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); struct bar0_amap *amap = &tvnet->bar0_amap[type]; struct page **map; int ret = 0, page_count, order, i; page_count = amap->size >> PAGE_SHIFT; order = get_order(amap->size); map = kmalloc(sizeof(struct page *) << order, GFP_KERNEL); if (!map) return -ENOMEM; amap->page = alloc_pages(GFP_KERNEL, page_count); if (!amap->page) { dev_err(tvnet->fdev, "%s: alloc_pages() failed\n", __func__); ret = -ENOMEM; goto fail; } split_page(amap->page, order); order = 1 << order; map[0] = amap->page; for (i = 1; i < page_count; i++) map[i] = amap->page + i; for (; i < order; i++) __free_page(amap->page + i); amap->virt = vmap(map, page_count, VM_MAP, PAGE_KERNEL); if (!amap->virt) { dev_err(tvnet->fdev, "%s: vmap() failed\n", __func__); ret = -ENOMEM; goto fail_free_pages; } kfree(map); ret = iommu_map(domain, amap->iova, page_to_phys(amap->page), amap->size, IOMMU_READ | IOMMU_WRITE); if (ret < 0) { dev_err(tvnet->fdev, "%s: iommu_map(RAM) failed: %d\n", __func__, ret); goto fail_vunmap; } return 0; fail_vunmap: vunmap(amap->virt); fail_free_pages: __free_pages(amap->page, page_count); fail: kfree(map); return ret; } static void tvnet_ep_free_multi_page_bar0_mem(struct pci_epf *epf, enum bar0_amap_type type) { struct pci_epf_tvnet *tvnet = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); struct bar0_amap *amap = &tvnet->bar0_amap[type]; int page_count = amap->size >> PAGE_SHIFT; iommu_unmap(domain, amap->iova, page_count); vfree(amap->virt); } static int tvnet_ep_pci_epf_bind(struct pci_epf *epf) { struct pci_epf_tvnet *tvnet = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; struct pci_epf_header *header = epf->header; struct device *fdev = &epf->dev; struct device *cdev = epc->dev.parent; struct iommu_domain *domain = iommu_get_domain_for_dev(cdev); struct platform_device *pdev = of_find_device_by_node(cdev->of_node); struct ep_ring_buf *ep_ring_buf = &tvnet->ep_ring_buf; struct host_ring_buf *host_ring_buf = &tvnet->host_ring_buf; struct net_device *ndev; struct bar_md *bar_md; struct resource *res; struct bar0_amap *amap; struct tvnet_dma_desc *dma_desc; int ret, size, bitmap_size; if (!domain) { dev_err(fdev, "IOMMU domain not found\n"); ret = -ENXIO; goto fail; } ret = pci_epc_write_header(epc, header); if (ret < 0) { dev_err(fdev, "pci_epc_write_header() failed: %d\n", ret); goto fail; } res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu_dma"); if (!res) { dev_err(fdev, "missing atu_dma resource in DT\n"); ret = PTR_ERR(res); goto fail; } tvnet->dma_base = devm_ioremap(fdev, res->start + DMA_OFFSET, resource_size(res) - DMA_OFFSET); if (IS_ERR(tvnet->dma_base)) { ret = PTR_ERR(tvnet->dma_base); dev_err(fdev, "dma region map failed: %d\n", ret); goto fail; } tvnet->bar0_iova = iommu_dma_alloc_iova(cdev, BAR0_SIZE, cdev->coherent_dma_mask); if (!tvnet->bar0_iova) { dev_err(fdev, "iommu_dma_alloc_iova() failed\n"); ret = -ENOMEM; goto fail; } pr_debug("BAR0 IOVA: 0x%08llx\n", tvnet->bar0_iova); /* BAR0 metadata memory allocation */ tvnet->bar0_amap[META_DATA].iova = tvnet->bar0_iova; tvnet->bar0_amap[META_DATA].size = PAGE_SIZE; ret = tvnet_ep_alloc_single_page_bar0_mem(epf, META_DATA); if (ret < 0) { dev_err(fdev, "BAR0 metadata alloc failed: %d\n", ret); goto free_iova; } tvnet->bar_md = (struct bar_md *)tvnet->bar0_amap[META_DATA].virt; bar_md = tvnet->bar_md; /* BAR0 SIMPLE_IRQ setup: two interrupts required two pages */ amap = &tvnet->bar0_amap[SIMPLE_IRQ]; amap->iova = tvnet->bar0_amap[META_DATA].iova + tvnet->bar0_amap[META_DATA].size; amap->size = 2 * PAGE_SIZE; ret = tvnet_ep_pci_epf_setup_irqsp(tvnet); if (ret < 0) { dev_err(fdev, "irqsp setup failed: %d\n", ret); goto free_bar0_md; } /* BAR0 EP memory allocation */ amap = &tvnet->bar0_amap[EP_MEM]; amap->iova = tvnet->bar0_amap[SIMPLE_IRQ].iova + tvnet->bar0_amap[SIMPLE_IRQ].size; size = sizeof(struct ep_own_cnt) + (RING_COUNT * (sizeof(struct ctrl_msg) + 2 * sizeof(struct data_msg))); amap->size = PAGE_ALIGN(size); ret = tvnet_ep_alloc_multi_page_bar0_mem(epf, EP_MEM); if (ret < 0) { dev_err(fdev, "BAR0 EP mem alloc failed: %d\n", ret); goto free_irqsp; } ep_ring_buf->ep_cnt = (struct ep_own_cnt *)amap->virt; ep_ring_buf->ep2h_ctrl_msgs = (struct ctrl_msg *) (ep_ring_buf->ep_cnt + 1); ep_ring_buf->ep2h_full_msgs = (struct data_msg *) (ep_ring_buf->ep2h_ctrl_msgs + RING_COUNT); ep_ring_buf->h2ep_empty_msgs = (struct data_msg *) (ep_ring_buf->ep2h_full_msgs + RING_COUNT); /* Clear EP counters */ memset(ep_ring_buf->ep_cnt, 0, sizeof(struct ep_own_cnt)); /* BAR0 host memory allocation */ amap = &tvnet->bar0_amap[HOST_MEM]; amap->iova = tvnet->bar0_amap[EP_MEM].iova + tvnet->bar0_amap[EP_MEM].size; size = (sizeof(struct host_own_cnt)) + (RING_COUNT * (sizeof(struct ctrl_msg) + 2 * sizeof(struct data_msg))); amap->size = PAGE_ALIGN(size); ret = tvnet_ep_alloc_multi_page_bar0_mem(epf, HOST_MEM); if (ret < 0) { dev_err(fdev, "BAR0 host mem alloc failed: %d\n", ret); goto free_ep_mem; } host_ring_buf->host_cnt = (struct host_own_cnt *)amap->virt; host_ring_buf->h2ep_ctrl_msgs = (struct ctrl_msg *) (host_ring_buf->host_cnt + 1); host_ring_buf->ep2h_empty_msgs = (struct data_msg *) (host_ring_buf->h2ep_ctrl_msgs + RING_COUNT); host_ring_buf->h2ep_full_msgs = (struct data_msg *) (host_ring_buf->ep2h_empty_msgs + RING_COUNT); /* Clear host counters */ memset(host_ring_buf->host_cnt, 0, sizeof(struct host_own_cnt)); /* * Allocate local memory for DMA read link list elements. * This is exposed through BAR0 to initiate DMA read from host. */ amap = &tvnet->bar0_amap[HOST_DMA]; amap->iova = tvnet->bar0_amap[HOST_MEM].iova + tvnet->bar0_amap[HOST_MEM].size; size = ((DMA_DESC_COUNT + 1) * sizeof(struct tvnet_dma_desc)); amap->size = PAGE_ALIGN(size); ret = tvnet_ep_alloc_multi_page_bar0_mem(epf, HOST_DMA); if (ret < 0) { dev_err(fdev, "BAR0 host dma mem alloc failed: %d\n", ret); goto free_host_mem; } /* Set link list pointer to create a dma desc ring */ memset(amap->virt, 0, amap->size); dma_desc = (struct tvnet_dma_desc *)amap->virt; dma_desc[DMA_DESC_COUNT].sar_low = (amap->iova & 0xffffffff); dma_desc[DMA_DESC_COUNT].sar_high = ((amap->iova >> 32) & 0xffffffff); dma_desc[DMA_DESC_COUNT].ctrl_reg.ctrl_e.llp = 1; /* Update BAR metadata region with offsets */ /* EP owned memory */ bar_md->ep_own_cnt_offset = tvnet->bar0_amap[META_DATA].size + tvnet->bar0_amap[SIMPLE_IRQ].size; bar_md->ctrl_md.ep2h_offset = bar_md->ep_own_cnt_offset + sizeof(struct ep_own_cnt); bar_md->ctrl_md.ep2h_size = RING_COUNT; bar_md->ep2h_md.ep2h_offset = bar_md->ctrl_md.ep2h_offset + (RING_COUNT * sizeof(struct ctrl_msg)); bar_md->ep2h_md.ep2h_size = RING_COUNT; bar_md->h2ep_md.ep2h_offset = bar_md->ep2h_md.ep2h_offset + (RING_COUNT * sizeof(struct data_msg)); bar_md->h2ep_md.ep2h_size = RING_COUNT; /* Host owned memory */ bar_md->host_own_cnt_offset = bar_md->ep_own_cnt_offset + tvnet->bar0_amap[EP_MEM].size; bar_md->ctrl_md.h2ep_offset = bar_md->host_own_cnt_offset + sizeof(struct host_own_cnt); bar_md->ctrl_md.h2ep_size = RING_COUNT; bar_md->ep2h_md.h2ep_offset = bar_md->ctrl_md.h2ep_offset + (RING_COUNT * sizeof(struct ctrl_msg)); bar_md->ep2h_md.h2ep_size = RING_COUNT; bar_md->h2ep_md.h2ep_offset = bar_md->ep2h_md.h2ep_offset + (RING_COUNT * sizeof(struct data_msg)); bar_md->h2ep_md.h2ep_size = RING_COUNT; tvnet->h2ep_ctrl.rd = &ep_ring_buf->ep_cnt->h2ep_ctrl_rd_cnt; tvnet->h2ep_ctrl.wr = &host_ring_buf->host_cnt->h2ep_ctrl_wr_cnt; tvnet->ep2h_ctrl.rd = &host_ring_buf->host_cnt->ep2h_ctrl_rd_cnt; tvnet->ep2h_ctrl.wr = &ep_ring_buf->ep_cnt->ep2h_ctrl_wr_cnt; tvnet->h2ep_empty.rd = &host_ring_buf->host_cnt->h2ep_empty_rd_cnt; tvnet->h2ep_empty.wr = &ep_ring_buf->ep_cnt->h2ep_empty_wr_cnt; tvnet->h2ep_full.rd = &ep_ring_buf->ep_cnt->h2ep_full_rd_cnt; tvnet->h2ep_full.wr = &host_ring_buf->host_cnt->h2ep_full_wr_cnt; tvnet->ep2h_empty.rd = &ep_ring_buf->ep_cnt->ep2h_empty_rd_cnt; tvnet->ep2h_empty.wr = &host_ring_buf->host_cnt->ep2h_empty_wr_cnt; tvnet->ep2h_full.rd = &host_ring_buf->host_cnt->ep2h_full_rd_cnt; tvnet->ep2h_full.wr = &ep_ring_buf->ep_cnt->ep2h_full_wr_cnt; /* RAM region for use by host when programming EP DMA controller */ bar_md->host_dma_offset = bar_md->host_own_cnt_offset + tvnet->bar0_amap[HOST_MEM].size; bar_md->host_dma_size = tvnet->bar0_amap[HOST_DMA].size; /* EP Rx pkt IOVA range */ tvnet->rx_buf_iova = tvnet->bar0_amap[HOST_DMA].iova + tvnet->bar0_amap[HOST_DMA].size; bar_md->bar0_base_phy = tvnet->bar0_iova; bar_md->ep_rx_pkt_offset = bar_md->host_dma_offset + tvnet->bar0_amap[HOST_DMA].size; bar_md->ep_rx_pkt_size = BAR0_SIZE - tvnet->bar0_amap[META_DATA].size - tvnet->bar0_amap[SIMPLE_IRQ].size - tvnet->bar0_amap[EP_MEM].size - tvnet->bar0_amap[HOST_MEM].size - tvnet->bar0_amap[HOST_DMA].size; /* Create bitmap for allocating RX buffers */ tvnet->rx_num_pages = (bar_md->ep_rx_pkt_size >> PAGE_SHIFT); bitmap_size = BITS_TO_LONGS(tvnet->rx_num_pages) * sizeof(long); tvnet->rx_buf_bitmap = devm_kzalloc(fdev, bitmap_size, GFP_KERNEL); if (!tvnet->rx_buf_bitmap) { dev_err(fdev, "rx_bitmap mem alloc failed\n"); ret = -ENOMEM; goto free_host_dma; } /* Allocate PCIe memory for RP's dst address during xmit */ tvnet->tx_dst_va = pci_epc_wc_mem_alloc_addr(epc, &tvnet->tx_dst_pci_addr, SZ_64K); if (!tvnet->tx_dst_va) { dev_err(fdev, "failed to allocate dst PCIe address\n"); ret = -ENOMEM; goto free_host_dma; } /* Register network device */ ndev = alloc_etherdev(0); if (!ndev) { dev_err(fdev, "alloc_etherdev() failed\n"); ret = -ENOMEM; goto free_pci_mem; } eth_hw_addr_random(ndev); tvnet->ndev = ndev; SET_NETDEV_DEV(ndev, fdev); ndev->netdev_ops = &tvnet_netdev_ops; netif_napi_add(ndev, &tvnet->napi, tvnet_ep_poll, TVNET_NAPI_WEIGHT); ndev->mtu = TVNET_DEFAULT_MTU; ret = register_netdev(ndev); if (ret < 0) { dev_err(fdev, "register_netdev() failed: %d\n", ret); goto fail_free_netdev; } netif_carrier_off(ndev); tvnet->rx_link_state = DIR_LINK_STATE_DOWN; tvnet->tx_link_state = DIR_LINK_STATE_DOWN; tvnet->os_link_state = OS_LINK_STATE_DOWN; mutex_init(&tvnet->link_state_lock); init_waitqueue_head(&tvnet->link_state_wq); INIT_LIST_HEAD(&tvnet->h2ep_empty_list); spin_lock_init(&tvnet->h2ep_empty_lock); /* TODO Update it to 64-bit prefetch type */ ret = pci_epc_set_bar(epc, BAR_0, tvnet->bar0_iova, BAR0_SIZE, PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_32); if (ret < 0) { dev_err(fdev, "pci_epc_set_bar() failed: %d\n", ret); goto fail_unreg_netdev; } ret = pci_epc_set_msi(epc, epf->msi_interrupts); if (ret) { dev_err(fdev, "pci_epc_set_msi() failed: %d\n", ret); goto fail_clear_bar; } /* Allocate local memory for DMA write link list elements */ size = ((DMA_DESC_COUNT + 1) * sizeof(struct tvnet_dma_desc)); tvnet->ep_dma_virt = dma_alloc_coherent(cdev, size, &tvnet->ep_dma_iova, GFP_KERNEL); if (!tvnet->ep_dma_virt) { dev_err(fdev, "%s ep dma mem alloc failed\n", __func__); ret = -ENOMEM; goto fail_clear_bar; } /* Set link list pointer to create a dma desc ring */ memset(tvnet->ep_dma_virt, 0, size); dma_desc = (struct tvnet_dma_desc *)tvnet->ep_dma_virt; dma_desc[DMA_DESC_COUNT].sar_low = (tvnet->ep_dma_iova & 0xffffffff); dma_desc[DMA_DESC_COUNT].sar_high = ((tvnet->ep_dma_iova >> 32) & 0xffffffff); dma_desc[DMA_DESC_COUNT].ctrl_reg.ctrl_e.llp = 1; nvhost_interrupt_syncpt_prime(tvnet->ctrl_irqsp->is); nvhost_interrupt_syncpt_prime(tvnet->data_irqsp->is); return 0; fail_clear_bar: pci_epc_clear_bar(epc, BAR_0); fail_unreg_netdev: unregister_netdev(ndev); fail_free_netdev: netif_napi_del(&tvnet->napi); free_netdev(ndev); free_pci_mem: pci_epc_mem_free_addr(epc, tvnet->tx_dst_pci_addr, tvnet->tx_dst_va, SZ_64K); free_host_dma: tvnet_ep_free_multi_page_bar0_mem(epf, HOST_DMA); free_host_mem: tvnet_ep_free_multi_page_bar0_mem(epf, HOST_MEM); free_ep_mem: tvnet_ep_free_multi_page_bar0_mem(epf, EP_MEM); free_irqsp: tvnet_ep_pci_epf_destroy_irqsp(tvnet); free_bar0_md: tvnet_ep_free_single_page_bar0_mem(epf, META_DATA); free_iova: iommu_dma_free_iova(cdev, tvnet->bar0_iova, BAR0_SIZE); fail: return ret; } static void tvnet_ep_pci_epf_unbind(struct pci_epf *epf) { struct pci_epf_tvnet *tvnet = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; struct device *cdev = epc->dev.parent; cancel_work_sync(&tvnet->ctrl_irqsp->reprime_work); cancel_work_sync(&tvnet->data_irqsp->reprime_work); pci_epc_stop(epc); pci_epc_clear_bar(epc, BAR_0); dma_free_coherent(cdev, ((RING_COUNT + 1) * sizeof(struct tvnet_dma_desc)), tvnet->ep_dma_virt, tvnet->ep_dma_iova); unregister_netdev(tvnet->ndev); netif_napi_del(&tvnet->napi); free_netdev(tvnet->ndev); pci_epc_mem_free_addr(epc, tvnet->tx_dst_pci_addr, tvnet->tx_dst_va, SZ_64K); tvnet_ep_free_multi_page_bar0_mem(epf, HOST_DMA); tvnet_ep_free_multi_page_bar0_mem(epf, HOST_MEM); tvnet_ep_free_multi_page_bar0_mem(epf, EP_MEM); tvnet_ep_pci_epf_destroy_irqsp(tvnet); tvnet_ep_free_single_page_bar0_mem(epf, META_DATA); iommu_dma_free_iova(cdev, tvnet->bar0_iova, BAR0_SIZE); } static void tvnet_ep_pci_epf_linkup(struct pci_epf *epf) { struct pci_epf_tvnet *tvnet = epf_get_drvdata(epf); #if ENABLE_DMA tvnet_ep_setup_dma(tvnet); #endif /* * If host goes through a suspend resume, it recycles EP2H empty buffer. * Clear any pending EP2H full buffer by setting "wr_cnt = rd_cnt". */ tvnet_ivc_set_wr(&tvnet->ep2h_full, tvnet_ivc_get_rd_cnt(&tvnet->ep2h_full)); tvnet->pcie_link_status = true; } static const struct pci_epf_device_id tvnet_ep_epf_tvnet_ids[] = { { .name = "pci_epf_tvnet", }, { }, }; int tvnet_ep_epf_tvnet_probe(struct pci_epf *epf) { struct device *fdev = &epf->dev; struct pci_epf_tvnet *tvnet; tvnet = devm_kzalloc(fdev, sizeof(*tvnet), GFP_KERNEL); if (!tvnet) return -ENOMEM; epf_set_drvdata(epf, tvnet); tvnet->fdev = fdev; tvnet->epf = epf; tvnet->header.vendorid = PCI_VENDOR_ID_NVIDIA; tvnet->header.deviceid = PCI_DEVICE_ID_NVIDIA_JETSON_AGX_NETWORK; tvnet->header.revid = 0x0; tvnet->header.baseclass_code = PCI_BASE_CLASS_NETWORK; tvnet->header.subclass_code = (PCI_CLASS_NETWORK_OTHER & 0xff); tvnet->header.subsys_vendor_id = PCI_VENDOR_ID_NVIDIA; tvnet->header.subsys_id = 0x0; tvnet->header.interrupt_pin = PCI_INTERRUPT_INTA; epf->header = &tvnet->header; return 0; } static struct pci_epf_ops tvnet_ep_ops = { .bind = tvnet_ep_pci_epf_bind, .unbind = tvnet_ep_pci_epf_unbind, .linkup = tvnet_ep_pci_epf_linkup, }; static struct pci_epf_driver tvnet_driver = { .driver.name = "pci_epf_tvnet", .probe = tvnet_ep_epf_tvnet_probe, .id_table = tvnet_ep_epf_tvnet_ids, .ops = &tvnet_ep_ops, .owner = THIS_MODULE, }; static int __init tvnet_ep_pci_epf_init(void) { int ret; ret = pci_epf_register_driver(&tvnet_driver); if (ret < 0) { pr_err("Failed to register EPF Tegra vnet driver: %d\n", ret); return ret; } return 0; } module_init(tvnet_ep_pci_epf_init); static void __exit tvnet_ep_pci_epf_exit(void) { pci_epf_unregister_driver(&tvnet_driver); } module_exit(tvnet_ep_pci_epf_exit); MODULE_DESCRIPTION("PCI EPF TEGRA VIRTUAL NETWORK DRIVER"); MODULE_AUTHOR("Manikanta Maddireddy "); MODULE_LICENSE("GPL v2");