/* * t19x-nvlink-endpt.c: * This is the NVLINK endpoint driver for the Tegra NVLINK controller. * * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "t19x-nvlink-endpt.h" #include "nvlink-hw.h" #define PLLNVHS_FREQ_150MHZ (150 * 1000 * 1000) /* NVLINK TOM is the top of the NVLINK aperture */ #define NVLINK_TOM_GB 512 /* Convert the NVLINK TOM value from GB to MB for register programming */ #define NVLINK_TOM_MB (((NVLINK_TOM_GB) * 1024) - 1) static struct of_device_id t19x_nvlink_controller_of_match[] = { { .compatible = "nvidia,t19x-nvlink-controller", }, { }, }; MODULE_DEVICE_TABLE(of, t19x_nvlink_controller_of_match); u32 nvlw_tioctrl_readl(struct tnvlink_dev *tdev, u32 reg) { return readl(tdev->nvlw_tioctrl_base + reg); } void nvlw_tioctrl_writel(struct tnvlink_dev *tdev, u32 reg, u32 val) { writel(val, tdev->nvlw_tioctrl_base + reg); } u32 nvlw_nvlipt_readl(struct tnvlink_dev *tdev, u32 reg) { return readl(tdev->nvlw_nvlipt_base + reg); } void nvlw_nvlipt_writel(struct tnvlink_dev *tdev, u32 reg, u32 val) { writel(val, tdev->nvlw_nvlipt_base + reg); } u32 nvlw_minion_readl(struct tnvlink_dev *tdev, u32 reg) { return readl(tdev->nvlw_minion_base + reg); } void nvlw_minion_writel(struct tnvlink_dev *tdev, u32 reg, u32 val) { writel(val, tdev->nvlw_minion_base + reg); } u32 nvlw_nvl_readl(struct tnvlink_dev *tdev, u32 reg) { return readl(tdev->tlink.nvlw_nvl_base + reg); } void nvlw_nvl_writel(struct tnvlink_dev *tdev, u32 reg, u32 val) { writel(val, tdev->tlink.nvlw_nvl_base + reg); } u32 nvlw_sync2x_readl(struct tnvlink_dev *tdev, u32 reg) { return readl(tdev->nvlw_sync2x_base + reg); } void nvlw_sync2x_writel(struct tnvlink_dev *tdev, u32 reg, u32 val) { writel(val, tdev->nvlw_sync2x_base + reg); } u32 nvlw_nvltlc_readl(struct tnvlink_dev *tdev, u32 reg) { return readl(tdev->tlink.nvlw_nvltlc_base + reg); } void nvlw_nvltlc_writel(struct tnvlink_dev *tdev, u32 reg, u32 val) { writel(val, tdev->tlink.nvlw_nvltlc_base + reg); } static inline u32 mssnvlink_0_readl(struct tnvlink_dev *tdev, u32 reg) { return readl(tdev->mssnvlink_0_base + reg); } static inline void mssnvlink_0_writel(struct tnvlink_dev *tdev, u32 reg, u32 val) { writel(val, tdev->mssnvlink_0_base + reg); } /* TODO: Remove all non-NVLINK MMIO register writes from the driver */ static inline void non_nvlink_writel(u32 reg, u32 val) { void __iomem *ptr = ioremap(reg, 0x4); __raw_writel(val, ptr); iounmap(ptr); } /* * Wait for a bit to be set or cleared in an NVLINK register. If the desired bit * condition doesn't happen in a certain amount of time, a timeout will happen. */ int wait_for_reg_cond_nvlink( struct tnvlink_dev *tdev, u32 reg, u32 bit, bool check_for_bit_set, char *bit_name, u32 (*reg_readl)(struct tnvlink_dev *, u32), u32 *reg_val, u32 timeout_us) { u32 elapsed_us = 0; do { usleep_range(DEFAULT_LOOP_SLEEP_US, DEFAULT_LOOP_SLEEP_US*2); elapsed_us += DEFAULT_LOOP_SLEEP_US; *reg_val = reg_readl(tdev, reg); if ((check_for_bit_set && (*reg_val & BIT(bit))) || (!check_for_bit_set && ((*reg_val & BIT(bit)) == 0))) break; } while (elapsed_us < timeout_us); if (elapsed_us >= timeout_us) { if (check_for_bit_set) { nvlink_err("Timeout waiting for the %s bit to get set", bit_name); } else { nvlink_err( "Timeout waiting for the %s bit to get cleared", bit_name); } return -1; } return 0; } /* * tegra_nvlink_car_init(): initializes UPHY mgmt and sys clk * clears the resets to uphy */ static int tegra_nvlink_car_enable(struct tnvlink_dev *tdev) { int ret; unsigned long clk_rate = 0; u32 reg_val = 0; /* enable the Management clock */ ret = clk_prepare_enable(tdev->clk_nvhs_pll0_mgmt); if (ret < 0) { nvlink_err("nvlink mgmt clock enable failed : %d", ret); goto fail; } /* Clear reset for UPHY PM */ reset_control_deassert(tdev->rst_nvhs_uphy_pm); /* Enable clock for nvlink_sys */ ret = clk_prepare_enable(tdev->clk_nvlink_sys); if (ret < 0) { nvlink_err("nvlink sys clock enable failed : %d", ret); goto nvlink_sys_enable_fail; } ret = clk_set_parent(tdev->clk_nvlink_sys, tdev->clk_pllrefe_vcoout_gated); if (ret < 0) { nvlink_err("nvlink sys clock set parent failed : %d", ret); goto nvlink_sys_set_parent_fail; } reset_control_reset(tdev->rst_nvlink); /* Take link out of reset */ reg_val = nvlw_tioctrl_readl(tdev, NVLW_RESET) | BIT(NVLW_RESET_LINKRESET); nvlw_tioctrl_writel(tdev, NVLW_RESET, reg_val); udelay(NVLW_POST_RESET_DELAY_US); /* Reset persistent HW state for this link */ reg_val = nvlw_tioctrl_readl(tdev, NVLW_DEBUG_RESET) & ~BIT(NVLW_DEBUG_RESET_LINK) & ~BIT(NVLW_DEBUG_RESET_COMMON); nvlw_tioctrl_writel(tdev, NVLW_DEBUG_RESET, reg_val); udelay(NVLW_POST_RESET_DELAY_US); reg_val = nvlw_tioctrl_readl(tdev, NVLW_DEBUG_RESET) | BIT(NVLW_DEBUG_RESET_LINK) | BIT(NVLW_DEBUG_RESET_COMMON); nvlw_tioctrl_writel(tdev, NVLW_DEBUG_RESET, reg_val); udelay(NVLW_POST_RESET_DELAY_US); reset_control_deassert(tdev->rst_nvhs_uphy_l0); reset_control_deassert(tdev->rst_nvhs_uphy_l1); reset_control_deassert(tdev->rst_nvhs_uphy_l2); reset_control_deassert(tdev->rst_nvhs_uphy_l3); reset_control_deassert(tdev->rst_nvhs_uphy_l4); reset_control_deassert(tdev->rst_nvhs_uphy_l5); reset_control_deassert(tdev->rst_nvhs_uphy_l6); reset_control_deassert(tdev->rst_nvhs_uphy_l7); reset_control_deassert(tdev->rst_nvhs_uphy_pll0); reset_control_deassert(tdev->rst_nvhs_uphy); if (tdev->refclk == NVLINK_REFCLK_150) { ret = clk_set_rate(tdev->clk_pllnvhs, PLLNVHS_FREQ_150MHZ); if (ret < 0) { nvlink_err("nvlink pllnvhs setrate failed : %d", ret); goto pllnvhs_fail; } else { clk_rate = clk_get_rate(tdev->clk_pllnvhs); if (clk_rate != PLLNVHS_FREQ_150MHZ) { nvlink_err("clk_pllnvhs rate = %lu", clk_rate); ret = -EINVAL; goto pllnvhs_fail; } } ret = clk_prepare_enable(tdev->clk_pllnvhs); if (ret < 0) { nvlink_err("pllnvhs clock enable failed : %d", ret); goto pllnvhs_fail; } } /* De-assert MSSNVLINK0 reset */ reset_control_deassert(tdev->rst_mssnvl); return ret; pllnvhs_fail: nvlink_sys_set_parent_fail: clk_disable_unprepare(tdev->clk_nvlink_sys); nvlink_sys_enable_fail: clk_disable_unprepare(tdev->clk_nvhs_pll0_mgmt); fail: return ret; } static void init_tlc_buffers(struct tnvlink_dev *tdev) { nvlink_dbg("Initializing TLC buffers"); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC0, 0x10000C0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC1, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC2, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC3, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC4, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC5, 0x10000C0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC6, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_CREDITS_VC7, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC0, 0xff00bf); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC1, 0xff00bf); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC2, 0xff00bf); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC3, 0xff00bf); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC4, 0xff00bf); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC5, 0x1ff017f); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC6, 0x1ff017f); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_SZ_VC7, 0x1ff017f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC0, 0x800040); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC1, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC2, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC3, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC4, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC5, 0x800040); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC6, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_CREDITS_VC7, 0x0); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC0, 0x7f003f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC1, 0x7f003f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC2, 0x7f003f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC3, 0x7f003f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC4, 0x7f003f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC5, 0xff007f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC6, 0xff007f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_SZ_VC7, 0xff007f); nvlw_nvltlc_writel(tdev, NVLTLC_TX_ERR_LOG_EN_0, 0x3ffffff); nvlw_nvltlc_writel(tdev, NVLTLC_TX_ERR_REPORT_EN_0, 0x3ffffff); nvlw_nvltlc_writel(tdev, NVLTLC_TX_ERR_CONTAIN_EN_0, 0x3ffffff); nvlw_nvltlc_writel(tdev, NVLTLC_RX_ERR_LOG_EN_0, 0xffffff); nvlw_nvltlc_writel(tdev, NVLTLC_RX_ERR_REPORT_EN_0, 0xffffff); nvlw_nvltlc_writel(tdev, NVLTLC_RX_ERR_CONTAIN_EN_0, 0xffffff); nvlw_nvltlc_writel(tdev, NVLTLC_RX_ERR_LOG_EN_1, 0x3fffff); nvlw_nvltlc_writel(tdev, NVLTLC_RX_ERR_REPORT_EN_1, 0x3fffff); nvlw_nvltlc_writel(tdev, NVLTLC_RX_ERR_CONTAIN_EN_1, 0x3fffff); nvlw_nvltlc_writel(tdev, NVLTLC_TX_CTRL_BUFFER_READY, 0x1); nvlw_nvltlc_writel(tdev, NVLTLC_RX_CTRL_BUFFER_READY, 0x1); } static void init_tlc(struct tnvlink_dev *tdev) { struct nvlink_device *ndev = tdev->ndev; init_tlc_buffers(tdev); if (ndev->link.is_sl_supported) init_single_lane_params(tdev); } /* * Enable the MSSNVLINK core master and slave clocks. */ static void disable_nvlink_slcg(struct tnvlink_dev *tdev) { u32 val; val = mssnvlink_0_readl(tdev, MSSNVLINK_CLK_SLCG); val |= BIT(MSSNVLINK_CLK_SLCG_MCF_MASTER_CLK_ENBL); val |= BIT(MSSNVLINK_CLK_SLCG_MCF_SLAVE_CLK_ENBL); val |= BIT(MSSNVLINK_CLK_SLCG_CORE_CLK_UPDATE_ENBL); mssnvlink_0_writel(tdev, MSSNVLINK_CLK_SLCG, val); } static void mssnvlink_lock_ila(struct tnvlink_dev *tdev) { u32 val; /* Enable PM clocks */ val = mssnvlink_0_readl(tdev, MSSNVLINK_CLK_CONTROL); val |= BIT(MSSNVLINK_CLK_CONTROL_PM_CLK_ENBL); mssnvlink_0_writel(tdev, MSSNVLINK_CLK_CONTROL, val); /* Set ILA LOCK bit */ val = mssnvlink_0_readl(tdev, MSSNVLINK_PM_MODE); val |= BIT(MSSNVLINK_PM_MODE_LOCK_ILA); mssnvlink_0_writel(tdev, MSSNVLINK_PM_MODE, val); /* Disable PM clocks */ val = mssnvlink_0_readl(tdev, MSSNVLINK_CLK_CONTROL); val &= ~BIT(MSSNVLINK_CLK_CONTROL_PM_CLK_ENBL); mssnvlink_0_writel(tdev, MSSNVLINK_CLK_CONTROL, val); } /* * mssnvlink_init: * Do the folllwing to initialize MSSNVLINK. This initialization is required to * allow traffic to flow between the NVLINK controller and MSSNVLINK: * - Enable the MSSNVLINK core master and slave clocks. * - Program the upper limit of the NVLINK aperture in MSSNVLINK. The bottom * of the aperture is fixed at 128 GB. So we don't need to program that. * - Release MSSNVLINK header and data credits to the NVLINK controller. * * TODO: Convert the magic values being programmed below into something that's * more understandable. */ static void mssnvlink_init(struct tnvlink_dev *tdev) { disable_nvlink_slcg(tdev); /* Program the upper limit of the NVLINK aperture in MSSNVLINK */ nvlink_dbg("Programming MSSNVLINK_TOM to %u GB", NVLINK_TOM_GB); non_nvlink_writel(MCB_BASE + MC_MSSNVLINK_TOM, NVLINK_TOM_MB); non_nvlink_writel(MCB_BASE + MC_MSSNVLINK_REG_CTRL, 0x1); /* MSSNVLINK credit programming */ mssnvlink_0_writel(tdev, MSSNVLINK_MASTER_CREDIT_TRANSINFO, 0x15455000); mssnvlink_0_writel(tdev, MSSNVLINK_MASTER_CREDIT_INGR_DATA, 0x8020000); mssnvlink_0_writel(tdev, MSSNVLINK_SLAVE_CREDIT_TRANSINFO, 0x14050000); mssnvlink_0_writel(tdev, MSSNVLINK_SLAVE_CREDIT_INGR_DATA, 0x300c0000); /* * Performance settings for balancing request and response bandwidth * across NVLINK */ non_nvlink_writel(MCB_BASE + MC_MCF_IREQX_VCARB_CONFIG, 0x8f0); non_nvlink_writel(MCB_BASE + MC_MCF_OREQX_VCARB_CONFIG, 0x8f0); mssnvlink_lock_ila(tdev); } /* * Program the upper limit of the NVLINK aperture in SCF. * The bottom of the aperture is fixed at 128 GB. So we don't need to program * that. */ static inline void program_scf_tom(void) { u32 reg_val = SCF_NVLINK_CFG_TOM_MB_F(NVLINK_TOM_MB) | BIT(SCF_NVLINK_CFG_EN); nvlink_dbg("Programming SCF TOM to %u GB", NVLINK_TOM_GB); asm volatile("msr s3_0_c15_c0_3, %0" : : "r" (reg_val)); } /* * Performs device level initialization like setting up the clocks and * resets, booting the minion and configuring the device level interrupts. */ int t19x_nvlink_dev_early_init(struct nvlink_device *ndev) { struct tnvlink_dev *tdev = NULL; int ret = 0; bool fuse_clk_enabled = false; if (!ndev) { nvlink_err("Invalid device struct pointer"); return -EINVAL; } tdev = (struct tnvlink_dev *)ndev->priv; if (!tdev) { nvlink_err("Invalid tnvlink_dev struct pointer"); return -EINVAL; } /* WAR for Bug 2301575: enable fuse clocks */ ret = tegra_fuse_clock_enable(); if (ret < 0) { nvlink_err("failed to enable fuse clocks"); goto fail; } else { fuse_clk_enabled = true; nvlink_dbg("fuse clocks are turned ON"); } ret = tegra_nvlink_car_enable(tdev); if (ret < 0) goto fail; /* * Make sure clocks and resets are enabled before installing * interrupt handler. It will prevent from accessing clock * gated block in case of spurious interrupt. */ ret = devm_request_threaded_irq(tdev->dev, tdev->irq, NULL, t19x_nvlink_endpt_isr, IRQF_ONESHOT | IRQF_TRIGGER_HIGH, dev_name(tdev->dev), tdev); if (ret < 0) { nvlink_err("Failed to register irq %d", tdev->irq); goto fail; } ret = minion_boot(tdev); if (ret < 0) goto fail; nvlink_config_common_intr(tdev); nvlink_dbg("Device early init done for dev%u", ndev->device_id); goto success; fail: nvlink_err("Device early init failed for dev%u", ndev->device_id); if (fuse_clk_enabled) { if (tegra_fuse_clock_disable()) { nvlink_err("failed to disable fuse clocks"); } else { nvlink_dbg("fuse clocks are turned OFF"); } } success: return ret; } /* * Performs link level initialization like phy_init, setting up the link * interrupts and enabling the AN0 packets. */ int t19x_nvlink_link_early_init(struct nvlink_device *ndev) { struct tnvlink_dev *tdev = NULL; int ret = 0; if (!ndev) { nvlink_err("Invalid device struct pointer"); return -EINVAL; } tdev = (struct tnvlink_dev *)ndev->priv; nvlink_enable_AN0_packets(tdev); ret = init_nvhs_phy(tdev); if (ret < 0) goto fail; nvlink_enable_link_interrupts(tdev); init_tlc(tdev); /* INITRXTERM is required if connected to nvlink 2.2 device. * For RXDET functionality to succeed on 2.2 devices, the opposite * endpoint must initialize the RX termination.It does no harm when * connected to 2.0 device. */ ret = minion_send_cmd(tdev, MINION_NVLINK_DL_CMD_COMMAND_INITRXTERM, 0); if (ret < 0) { nvlink_err("Error sending INITRXTERM command to MINION"); minion_dump_pc_trace(tdev); goto fail; } nvlink_dbg("Link early init done for dev%u", ndev->device_id); goto success; fail: nvlink_err("link early init failed for dev%u", ndev->device_id); success: return ret; } /* * Performs memory interface initialization */ int t19x_nvlink_dev_interface_init(struct nvlink_device *ndev) { struct tnvlink_dev *tdev = NULL; int ret = 0; if (!ndev) { nvlink_err("Invalid device struct pointer"); return -EINVAL; } tdev = (struct tnvlink_dev *)ndev->priv; mssnvlink_init(tdev); program_scf_tom(); ret = t19x_nvlink_config_tp_counters(tdev); if (ret < 0) return -EINVAL; ret = t19x_nvlink_reset_tp_counters(tdev); if (ret < 0) return -EINVAL; nvlink_dbg("Link interface init done for dev%u", ndev->device_id); return ret; } /* * Reg_init programs the prod-setting if any. */ int t19x_nvlink_dev_reg_init(struct nvlink_device *ndev) { struct tnvlink_dev *tdev = NULL; int ret = 0; if (!ndev) { nvlink_err("Invalid device struct pointer"); return -EINVAL; } tdev = (struct tnvlink_dev *)ndev->priv; if (tdev->prod_list) { ret = tegra_prod_set_by_name(&tdev->mssnvlink_0_base, "prod", tdev->prod_list); if (ret < 0) { /* prod setting failures should not stop nvlink init */ ret = 0; goto fail; } } nvlink_dbg("Device reg init done for dev%u", ndev->device_id); goto success; fail: nvlink_err("Device reg init failed for dev%u", ndev->device_id); success: return ret; } /* Disable the NVLINK aperture in SCF blocks */ static void disable_nvlink_aperture(void) { u32 reg_val; /* Read SCF TOM value */ asm volatile("mrs %0, s3_0_c15_c0_3" : "=r"(reg_val)); /* Clear the enable bit */ reg_val &= ~BIT(SCF_NVLINK_CFG_EN); /* Write SCF TOM value */ asm volatile("msr s3_0_c15_c0_3, %0" : : "r" (reg_val)); } /* Check there are no pending requests on NVLink */ static bool is_nvlink_idle(struct tnvlink_dev *tdev) { u32 reg; reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_ACT_TRANSINFO); if (reg & MSSNVLINK_MASTER_ACT_TRANSINFO_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_ACT_TRANSINFO_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_ACT_TRANSINFO_RMW); if (reg & MSSNVLINK_MASTER_ACT_TRANSINFO_RMW_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_ACT_TRANSINFO_RMW_CNT_CURRENT is" " not zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_ACT_INGR_DATA); if (reg & MSSNVLINK_MASTER_ACT_INGR_DATA_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_ACT_INGR_DATA_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_ACT_EGR_DATA); if (reg & MSSNVLINK_MASTER_ACT_EGR_DATA_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_ACT_EGR_DATA_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_RESERVED_EGR_DATA); if (reg & MSSNVLINK_MASTER_RESERVED_EGR_DATA_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_RESERVED_EGR_DATA_CNT_CURRENT is" " not zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_ACT_TRANSDONE); if (reg & MSSNVLINK_MASTER_ACT_TRANSDONE_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_ACT_TRANSDONE_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_INGR_CMD_QUEUE); if (reg & MSSNVLINK_MASTER_INGR_CMD_QUEUE_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_INGR_CMD_QUEUE_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_MASTER_EGR_CMD_QUEUE); if (reg & MSSNVLINK_MASTER_EGR_CMD_QUEUE_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_MASTER_EGR_CMD_QUEUE_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_SLAVE_ACT_TRANSINFO); if (reg & MSSNVLINK_SLAVE_ACT_TRANSINFO_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_SLAVE_ACT_TRANSINFO_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_SLAVE_ACT_INGR_DATA); if (reg & MSSNVLINK_SLAVE_ACT_INGR_DATA_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_SLAVE_ACT_INGR_DATA_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_SLAVE_ACT_EGR_DATA); if (reg & MSSNVLINK_SLAVE_ACT_EGR_DATA_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_SLAVE_ACT_EGR_DATA_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_SLAVE_ACT_TAG_TABLE); if (reg & MSSNVLINK_SLAVE_ACT_TAG_TABLE_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_SLAVE_ACT_TAG_TABLE_CNT_CURRENT is not" " zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_SLAVE_TRANSDONE_MCF_REQ); if (reg & MSSNVLINK_SLAVE_TRANSDONE_MCF_REQ_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_SLAVE_TRANSDONE_MCF_REQ_CNT_CURRENT is" " not zero and nvlink is not idle"); return false; } reg = mssnvlink_0_readl(tdev, MSSNVLINK_SLAVE_TRANSDONE_MCF_DAT); if (reg & MSSNVLINK_SLAVE_TRANSDONE_MCF_DAT_CNT_CURRENT_MASK) { nvlink_dbg("MSSNVLINK_SLAVE_TRANSDONE_MCF_DAT_CNT_CURRENT is" " not zero and nvlink is not idle"); return false; } return true; } static void enable_nvlink_slcg(struct tnvlink_dev *tdev) { u32 val; val = mssnvlink_0_readl(tdev, MSSNVLINK_CLK_SLCG); val &= ~(BIT(MSSNVLINK_CLK_SLCG_MCF_MASTER_CLK_ENBL)); val &= ~(BIT(MSSNVLINK_CLK_SLCG_MCF_SLAVE_CLK_ENBL)); val |= BIT(MSSNVLINK_CLK_SLCG_CORE_CLK_UPDATE_ENBL); mssnvlink_0_writel(tdev, MSSNVLINK_CLK_SLCG, val); } int t19x_nvlink_dev_interface_disable(struct nvlink_device *ndev) { struct tnvlink_dev *tdev = NULL; if (!ndev) { nvlink_err("Invalid nvlink_device struct pointer"); return -EINVAL; } tdev = (struct tnvlink_dev *)ndev->priv; /* Ensure there is no pending request in MSSNVLINK */ /* FIXME : If is_nvlink_idle() returns true, there's no guarantee * that NVLINK will remain idle. */ if (!is_nvlink_idle(tdev)) { nvlink_err("Can't shutdown nvlink, link still active."); return -EPERM; } disable_nvlink_aperture(); enable_nvlink_slcg(tdev); nvlink_dbg("nvlink dev interface disable successful"); return 0; } int t19x_nvlink_dev_car_disable(struct nvlink_device *ndev) { struct tnvlink_dev *tdev = NULL; int ret = 0; if (!ndev) { nvlink_err("Invalid nvlink_device struct pointer"); return -EINVAL; } tdev = (struct tnvlink_dev *)ndev->priv; /* Switch the TX clock from brick PLL to OSC */ ret = clk_set_parent(tdev->clk_nvlink_tx, tdev->clk_m); if (ret < 0) { nvlink_err("clk_nvlink_tx's clk_set_parent() call failed"); goto fail; } ret = reset_control_assert(tdev->rst_mssnvl); if (ret < 0) { nvlink_err("Reset assert failed for mssnvl"); goto fail; } if (tdev->refclk == NVLINK_REFCLK_150) clk_disable_unprepare(tdev->clk_pllnvhs); ret = reset_control_assert(tdev->rst_nvhs_uphy); if (ret < 0) { nvlink_err("Reset assert failed nvhs_uphy"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_pll0); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_pll0"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l7); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l7"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l6); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l6"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l5); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l5"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l4); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l4"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l3); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l3"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l2); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l2"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l1); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l1"); goto fail; } ret = reset_control_assert(tdev->rst_nvhs_uphy_l0); if (ret < 0) { nvlink_err("Reset assert failed for nvhs_uphy_l0"); goto fail; } clk_disable_unprepare(tdev->clk_nvlink_sys); ret = reset_control_assert(tdev->rst_nvhs_uphy_pm); if (ret < 0) { nvlink_err("Reset assert failed nvhs_uphy_pm"); goto fail; } clk_disable_unprepare(tdev->clk_nvhs_pll0_mgmt); fail: return ret; } static int tegra_nvlink_clk_rst_init(struct tnvlink_dev *tdev) { /* clocks */ tdev->clk_nvhs_pll0_mgmt = devm_clk_get(tdev->dev, "nvhs_pll0_mgmt"); if (IS_ERR(tdev->clk_nvhs_pll0_mgmt)) { nvlink_err("missing mgmt clock"); return PTR_ERR(tdev->clk_nvhs_pll0_mgmt); } tdev->clk_pllrefe_vcoout_gated = devm_clk_get(tdev->dev, "pllrefe_vcoout_gated"); if (IS_ERR(tdev->clk_pllrefe_vcoout_gated)) { nvlink_err("missing pllrefe clock"); return PTR_ERR(tdev->clk_pllrefe_vcoout_gated); } tdev->clk_nvlink_sys = devm_clk_get(tdev->dev, "nvlink_sys"); if (IS_ERR(tdev->clk_nvlink_sys)) { nvlink_err("missing sys clock"); return PTR_ERR(tdev->clk_nvlink_sys); } tdev->clk_pllnvhs = devm_clk_get(tdev->dev, "pllnvhs"); if (IS_ERR(tdev->clk_pllnvhs)) { nvlink_err("missing pllnvhs clock"); return PTR_ERR(tdev->clk_pllnvhs); } tdev->clk_m = devm_clk_get(tdev->dev, "clk_m"); if (IS_ERR(tdev->clk_m)) { nvlink_err("missing clk_m clock"); return PTR_ERR(tdev->clk_m); } tdev->clk_nvlink_pll_txclk = devm_clk_get(tdev->dev, "nvlink_pll_txclk"); if (IS_ERR(tdev->clk_nvlink_pll_txclk)) { nvlink_err("missing nvlink_pll_txclk clock"); return PTR_ERR(tdev->clk_nvlink_pll_txclk); } tdev->clk_nvlink_tx = devm_clk_get(tdev->dev, "nvlink_tx"); if (IS_ERR(tdev->clk_nvlink_tx)) { nvlink_err("missing nvlink_tx clock"); return PTR_ERR(tdev->clk_nvlink_tx); } /* Resets */ tdev->rst_mssnvl = devm_reset_control_get(tdev->dev, "mssnvl"); if (IS_ERR(tdev->rst_mssnvl)) { nvlink_err("missing rst_mssnvl reset"); return PTR_ERR(tdev->rst_mssnvl); } tdev->rst_nvhs_uphy_pm = devm_reset_control_get(tdev->dev, "nvhs_uphy_pm"); if (IS_ERR(tdev->rst_nvhs_uphy_pm)) { nvlink_err("missing rst_nvhs_uphy_pm reset"); return PTR_ERR(tdev->rst_nvhs_uphy_pm); } tdev->rst_nvhs_uphy = devm_reset_control_get(tdev->dev, "nvhs_uphy"); if (IS_ERR(tdev->rst_nvhs_uphy)) { nvlink_err("missing rst_nvhs_uphy reset"); return PTR_ERR(tdev->rst_nvhs_uphy); } tdev->rst_nvhs_uphy_pll0 = devm_reset_control_get(tdev->dev, "nvhs_uphy_pll0"); if (IS_ERR(tdev->rst_nvhs_uphy_pll0)) { nvlink_err("missing rst_nvhs_uphy_pll0 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_pll0); } tdev->rst_nvhs_uphy_l0 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l0"); if (IS_ERR(tdev->rst_nvhs_uphy_l0)) { nvlink_err("missing rst_nvhs_uphy_l0 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l0); } tdev->rst_nvhs_uphy_l1 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l1"); if (IS_ERR(tdev->rst_nvhs_uphy_l1)) { nvlink_err("missing rst_nvhs_uphy_l1 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l1); } tdev->rst_nvhs_uphy_l2 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l2"); if (IS_ERR(tdev->rst_nvhs_uphy_l2)) { nvlink_err("missing rst_nvhs_uphy_l2 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l2); } tdev->rst_nvhs_uphy_l3 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l3"); if (IS_ERR(tdev->rst_nvhs_uphy_l3)) { nvlink_err("missing rst_nvhs_uphy_l3 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l3); } tdev->rst_nvhs_uphy_l4 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l4"); if (IS_ERR(tdev->rst_nvhs_uphy_l4)) { nvlink_err("missing rst_nvhs_uphy_l4 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l4); } tdev->rst_nvhs_uphy_l5 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l5"); if (IS_ERR(tdev->rst_nvhs_uphy_l5)) { nvlink_err("missing rst_nvhs_uphy_l5 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l5); } tdev->rst_nvhs_uphy_l6 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l6"); if (IS_ERR(tdev->rst_nvhs_uphy_l6)) { nvlink_err("missing rst_nvhs_uphy_l6 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l6); } tdev->rst_nvhs_uphy_l7 = devm_reset_control_get(tdev->dev, "nvhs_uphy_l7"); if (IS_ERR(tdev->rst_nvhs_uphy_l7)) { nvlink_err("missing rst_nvhs_uphy_l7 reset"); return PTR_ERR(tdev->rst_nvhs_uphy_l7); } tdev->rst_nvlink = devm_reset_control_get(tdev->dev, "nvlink"); if (IS_ERR(tdev->rst_nvlink)) { nvlink_err("missing rst_nvlink reset"); return PTR_ERR(tdev->rst_nvlink); } return 0; } #if IS_ENABLED(CONFIG_PM_SLEEP) /* This function invokes core driver's nvlink shutdown api */ int t19x_nvlink_suspend(struct device *dev) { struct tnvlink_dev *tdev = dev_get_drvdata(dev); int ret = 0; if (!tdev) { nvlink_err("Invalid tnvlink_dev struct pointer"); ret = -EINVAL; goto fail; } ret = nvlink_shutdown(tdev->ndev); if (ret < 0) goto fail; nvlink_dbg("t19x nvlink suspend successful"); goto exit; fail: nvlink_err("t19x nvlink suspend failed!"); exit: return ret; } static const struct dev_pm_ops tegra_nvlink_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(t19x_nvlink_suspend, NULL) }; #endif static void tegra_nvlink_clk_rst_deinit(struct tnvlink_dev *tdev) { /* clocks */ if (tdev->clk_nvhs_pll0_mgmt) devm_clk_put(tdev->dev, tdev->clk_nvhs_pll0_mgmt); if (tdev->clk_pllrefe_vcoout_gated) devm_clk_put(tdev->dev, tdev->clk_pllrefe_vcoout_gated); if (tdev->clk_nvlink_sys) devm_clk_put(tdev->dev, tdev->clk_nvlink_sys); if (tdev->clk_pllnvhs) devm_clk_put(tdev->dev, tdev->clk_pllnvhs); if (tdev->clk_m) devm_clk_put(tdev->dev, tdev->clk_m); if (tdev->clk_nvlink_pll_txclk) devm_clk_put(tdev->dev, tdev->clk_nvlink_pll_txclk); if (tdev->clk_nvlink_tx) devm_clk_put(tdev->dev, tdev->clk_nvlink_tx); reset_control_assert(tdev->rst_nvhs_uphy_pm); reset_control_assert(tdev->rst_nvhs_uphy); reset_control_assert(tdev->rst_nvhs_uphy_pll0); reset_control_assert(tdev->rst_nvhs_uphy_l0); reset_control_assert(tdev->rst_nvhs_uphy_l1); reset_control_assert(tdev->rst_nvhs_uphy_l2); reset_control_assert(tdev->rst_nvhs_uphy_l3); reset_control_assert(tdev->rst_nvhs_uphy_l4); reset_control_assert(tdev->rst_nvhs_uphy_l5); reset_control_assert(tdev->rst_nvhs_uphy_l6); reset_control_assert(tdev->rst_nvhs_uphy_l7); } static int t19x_nvlink_check_fuse(void) { u32 fuse_val; int ret = 0; /* fuse clock is enabled by fuse driver before reading fuse */ ret = tegra_fuse_readl(FUSE_IP_DISABLE_0, &fuse_val); if (ret) { nvlink_err("nvlink ip fuse cannot be read"); } else { nvlink_dbg("fuse_ip_disable_0: 0x%x", fuse_val); } if (!ret) { /* fuse is read. check if ip is enabled */ if (fuse_val & FUSE_IP_DISABLE_0_NVLINK_MASK) { nvlink_err("nvlink ip is disabled"); ret = -EPERM; } } if (!ret) { if (tegra_fuse_readl(FUSE_UCODE_MINION_REV_0, &fuse_val)) { nvlink_err("ucode minion rev fuse cannot be read"); } else { nvlink_dbg("fuse_ucode_minion_rev_0: 0x%x", fuse_val & FUSE_UCODE_MINION_REV_0_MASK); } if (tegra_fuse_readl(FUSE_SECURE_MINION_DEBUG_DIS_0, &fuse_val)) { nvlink_err("minion debug dis fuse cannot be read"); } else { nvlink_dbg("fuse_secure_minion_debug_dis_0: 0x%x", fuse_val & FUSE_SECURE_MINION_DEBUG_DIS_0_MASK); } } return ret; } static int t19x_nvlink_endpt_probe(struct platform_device *pdev) { int ret = 0; struct tnvlink_dev *tdev; struct nvlink_device *ndev; struct device_node *np = pdev->dev.of_node; struct device_node *endpt_dt_node = NULL; struct device *dev = NULL; struct nvlink_device_pci_info *local_pci_info; struct nvlink_device_pci_info *remote_pci_info; struct tegra_prod *nvlink_prod; if (!np) { nvlink_err("Invalid device_node"); ret = -ENODEV; goto fail; } ret = t19x_nvlink_check_fuse(); /* proceed only if fuse is read and nvlink ip is not disabled */ if (ret) { ret = -ENODEV; goto fail; } ndev = kzalloc(sizeof(struct nvlink_device), GFP_KERNEL); if (!ndev) { nvlink_err("Couldn't allocate memory for nvlink device struct"); ret = -ENOMEM; goto err_alloc_ndev; } tdev = kzalloc(sizeof(struct tnvlink_dev), GFP_KERNEL); if (!tdev) { nvlink_err("Couldn't allocate memory for tegra nvlink device"); ret = -ENOMEM; goto err_alloc_tdev; } /* * By default the RM shim driver is disabled. The shim driver can be * enabled if userspace calls the ENABLE_SHIM_DRIVER IOCTL. */ tdev->rm_shim_enabled = false; /* Map NVLINK apertures listed in device tree node */ tdev->nvlw_tioctrl_base = of_io_request_and_map(np, 0, "NVLW_TIOCTRL aperture"); if (IS_ERR(tdev->nvlw_tioctrl_base)) { nvlink_err("Couldn't map the NVLW_TIOCTRL aperture"); ret = PTR_ERR(tdev->nvlw_tioctrl_base); goto err_mapping; } tdev->nvlw_nvlipt_base = of_io_request_and_map(np, 1, "NVLW_NVLIPT aperture"); if (IS_ERR(tdev->nvlw_nvlipt_base)) { nvlink_err("Couldn't map the NVLW_NVLIPT aperture"); ret = PTR_ERR(tdev->nvlw_nvlipt_base); goto err_mapping; } tdev->nvlw_minion_base = of_io_request_and_map(np, 2, "NVLW_MINION aperture"); if (IS_ERR(tdev->nvlw_minion_base)) { nvlink_err("Couldn't map the NVLW_MINION aperture"); ret = PTR_ERR(tdev->nvlw_minion_base); goto err_mapping; } tdev->tlink.nvlw_nvl_base = of_io_request_and_map(np, 3, "NVLW_NVL aperture"); if (IS_ERR(tdev->tlink.nvlw_nvl_base)) { nvlink_err("Couldn't map the NVLW_NVL aperture"); ret = PTR_ERR(tdev->tlink.nvlw_nvl_base); goto err_mapping; } tdev->nvlw_sync2x_base = of_io_request_and_map(np, 4, "NVLW_SYNC2X aperture"); if (IS_ERR(tdev->nvlw_sync2x_base)) { nvlink_err("Couldn't map the NVLW_SYNC2X aperture"); ret = PTR_ERR(tdev->nvlw_sync2x_base); goto err_mapping; } tdev->tlink.nvlw_nvltlc_base = of_io_request_and_map(np, 5, "NVLW_NVLTLC aperture"); if (IS_ERR(tdev->tlink.nvlw_nvltlc_base)) { nvlink_err("Couldn't map the NVLW_NVLTLC aperture"); ret = PTR_ERR(tdev->tlink.nvlw_nvltlc_base); goto err_mapping; } tdev->mssnvlink_0_base = of_io_request_and_map(np, 6, "MSSNVLINK_0 aperture"); if (IS_ERR(tdev->mssnvlink_0_base)) { nvlink_err("Couldn't map the MSSNVLINK_0 aperture"); ret = PTR_ERR(tdev->mssnvlink_0_base); goto err_mapping; } tdev->is_nea = DEFAULT_IS_NEA; tdev->is_tp_cntr_running = false; tdev->irq = platform_get_irq(pdev, 0); if (tdev->irq < 0) { nvlink_err("Couldn't get interrupt listed in device tree"); ret = -EINVAL; goto err_mapping; } tdev->dev = &pdev->dev; tdev->class.owner = THIS_MODULE; tdev->class.name = NVLINK_MODULE_NAME; /* Create device node */ ret = class_register(&tdev->class); if (ret) { nvlink_err("Failed to register class"); goto err_mapping; } ret = alloc_chrdev_region(&tdev->dev_t, 0, 1, dev_name(tdev->dev)); if (ret) { nvlink_err("Failed to allocate dev_t"); goto err_chrdev_region; } cdev_init(&tdev->cdev, &t19x_nvlink_endpt_ops); tdev->cdev.owner = THIS_MODULE; ret = cdev_add(&tdev->cdev, tdev->dev_t, 1); if (ret) { nvlink_err("Failed to add cdev"); goto err_cdev; } dev = device_create(&tdev->class, NULL, tdev->dev_t, NULL, NVLINK_MODULE_NAME); if (IS_ERR(dev)) { nvlink_err("Failed to create device"); ret = PTR_ERR(dev); goto err_device; } ret = tegra_nvlink_clk_rst_init(tdev); if (ret) goto err_clk_rst; nvlink_prod = devm_tegra_prod_get(tdev->dev); if (IS_ERR_OR_NULL(nvlink_prod)) { nvlink_err("Prod-setting not available"); nvlink_prod = NULL; } tdev->prod_list = nvlink_prod; tdev->refclk = NVLINK_REFCLK_156; ndev->speed = NVLINK_SPEED_20; ndev->link_bitrate = LINK_BITRATE_156MHZ_20GBPS; tdev->ndev = ndev; tdev->tlink.sl_params = entry_100us_sl_params; tdev->tlink.tdev = tdev; tdev->tlink.nlink = &(ndev->link); t19x_nvlink_endpt_debugfs_init(tdev); /* Fill in the nvlink_device struct */ /* Read NVLINK topology information in device tree */ endpt_dt_node = of_get_child_by_name(np, "endpoint"); if (endpt_dt_node == NULL) { nvlink_err("Topology information is missing from the" " device tree!"); ret = -ENODATA; goto err_topology; } ret = of_property_read_u32(endpt_dt_node, "local_dev_id", &ndev->device_id); if (ret < 0) { nvlink_dbg("Couldn't read the \"local_dev_id\" device tree" " property. Choosing default value" " (i.e. NVLINK_ENDPT_T19X)."); ndev->device_id = NVLINK_ENDPT_T19X; } ret = of_property_read_u32(endpt_dt_node, "local_link_id", &ndev->link.link_id); if (ret < 0) { nvlink_dbg("Couldn't read the \"local_link_id\" device tree" " property. Choosing default value (i.e. 0)."); ndev->link.link_id = 0; } ndev->is_master = of_property_read_bool(endpt_dt_node, "is_master"); ret = of_property_read_u32(endpt_dt_node, "remote_dev_id", &ndev->link.remote_dev_info.device_id); if (ret < 0) { nvlink_err("The \"remote_dev_id\" property is missing from the" " device tree!"); ret = -ENODATA; goto err_topology; } ret = of_property_read_u32(endpt_dt_node, "remote_link_id", &ndev->link.remote_dev_info.link_id); if (ret < 0) { nvlink_err("The \"remote_link_id\" property is missing from the" " device tree!"); ret = -ENODATA; goto err_topology; } nvlink_dbg("Device Tree Topology Information:"); nvlink_dbg(" - Local Device: Device ID = %d, Link ID = %d, Is master? = %s", ndev->device_id, ndev->link.link_id, ndev->is_master ? "True" : "False"); nvlink_dbg(" - Remote Device: Device ID = %d, Link ID = %d", ndev->link.remote_dev_info.device_id, ndev->link.remote_dev_info.link_id); /* * MODS/RM identifies devices using PCI device information. Tegra * however, doesn't have any valid PCI information because Tegra * is not a PCI device. Therefore we use the following bogus PCI * info for Tegra. We're hoping that this bogus PCI info doesn't * clash with the PCI info of an actual PCI device. */ local_pci_info = &ndev->pci_info; remote_pci_info = &ndev->link.remote_dev_info.pci_info; local_pci_info->domain = 0; local_pci_info->bus = ~0; local_pci_info->device = ~0; local_pci_info->function = ~0; local_pci_info->pci_device_id = ~0; if (ndev->device_id == ndev->link.remote_dev_info.device_id) { /* Tegra loopback topology */ /* Local and remote PCI info should be identical */ memcpy(remote_pci_info, local_pci_info, sizeof(struct nvlink_device_pci_info)); } else { /* Non-loopback topology */ /* Set all fields to 0xffff... */ memset(remote_pci_info, 0xff, sizeof(struct nvlink_device_pci_info)); } ndev->dev_ops.dev_early_init = t19x_nvlink_dev_early_init; ndev->dev_ops.dev_interface_init = t19x_nvlink_dev_interface_init; ndev->dev_ops.dev_reg_init = t19x_nvlink_dev_reg_init; ndev->dev_ops.dev_interface_disable = t19x_nvlink_dev_interface_disable; /* Point priv of ndev to the tegra nvlink endpoint device struct */ ndev->priv = (void *) tdev; /* Fill in the nvlink_link struct */ ndev->link.device_id = ndev->device_id; ndev->link.link_ops.get_link_mode = t19x_nvlink_get_link_mode; ndev->link.link_ops.set_link_mode = t19x_nvlink_set_link_mode; ndev->link.link_ops.get_sublink_mode = t19x_nvlink_get_sublink_mode; ndev->link.link_ops.set_sublink_mode = t19x_nvlink_set_sublink_mode; ndev->link.link_ops.get_link_state = t19x_nvlink_get_link_state; ndev->link.link_ops.get_tx_sublink_state = t19x_nvlink_get_tx_sublink_state; ndev->link.link_ops.get_rx_sublink_state = t19x_nvlink_get_rx_sublink_state; ndev->link.link_ops.link_early_init = t19x_nvlink_link_early_init; ndev->link.ndev = ndev; /* Point priv of ndev->link to the tegra nvlink endpoint link struct */ ndev->link.priv = (void *)&(tdev->tlink); ndev->link.is_sl_supported = false; platform_set_drvdata(pdev, tdev); /* Register device with the Tegra NVLINK core driver */ ret = nvlink_register_device(ndev); if (ret < 0) { goto err_ndev_register; } /* Register link with the Tegra NVLINK core driver */ ret = nvlink_register_link(&ndev->link); if (ret < 0) { goto err_nlink_register; } nvlink_dbg("Probe successful!"); goto success; err_nlink_register: nvlink_unregister_device(ndev); err_ndev_register: tegra_nvlink_clk_rst_deinit(tdev); err_topology: t19x_nvlink_endpt_debugfs_deinit(tdev); err_clk_rst: device_destroy(&tdev->class, tdev->dev_t); err_device: cdev_del(&tdev->cdev); err_cdev: unregister_chrdev_region(tdev->dev_t, 1); err_chrdev_region: class_unregister(&tdev->class); err_mapping: if (!IS_ERR(tdev->nvlw_tioctrl_base)) iounmap(tdev->nvlw_tioctrl_base); if (!IS_ERR(tdev->nvlw_nvlipt_base)) iounmap(tdev->nvlw_nvlipt_base); if (!IS_ERR(tdev->nvlw_minion_base)) iounmap(tdev->nvlw_minion_base); if (!IS_ERR(tdev->tlink.nvlw_nvl_base)) iounmap(tdev->tlink.nvlw_nvl_base); if (!IS_ERR(tdev->nvlw_sync2x_base)) iounmap(tdev->nvlw_sync2x_base); if (!IS_ERR(tdev->tlink.nvlw_nvltlc_base)) iounmap(tdev->tlink.nvlw_nvltlc_base); if (!IS_ERR(tdev->mssnvlink_0_base)) iounmap(tdev->mssnvlink_0_base); kfree(tdev); err_alloc_tdev: kfree(ndev); err_alloc_ndev: fail: nvlink_err("Probe failed!"); success: return ret; } static int t19x_nvlink_endpt_remove(struct platform_device *pdev) { struct nvlink_device *ndev = NULL; struct tnvlink_dev *tdev = NULL; tdev = platform_get_drvdata(pdev); ndev = tdev->ndev; if (!tdev->rm_shim_enabled) { nvlink_unregister_link(&ndev->link); nvlink_unregister_device(ndev); } t19x_nvlink_endpt_debugfs_deinit(tdev); tegra_nvlink_clk_rst_deinit(tdev); device_destroy(&tdev->class, tdev->dev_t); cdev_del(&tdev->cdev); unregister_chrdev_region(tdev->dev_t, 1); class_unregister(&tdev->class); iounmap(tdev->nvlw_tioctrl_base); iounmap(tdev->nvlw_nvlipt_base); iounmap(tdev->nvlw_minion_base); iounmap(tdev->tlink.nvlw_nvl_base); iounmap(tdev->nvlw_sync2x_base); iounmap(tdev->tlink.nvlw_nvltlc_base); iounmap(tdev->mssnvlink_0_base); kfree(tdev); kfree(ndev); return 0; } static void t19x_nvlink_endpt_shutdown(struct platform_device *pdev) { struct tnvlink_dev *tdev = platform_get_drvdata(pdev); int ret = 0; if (!tdev) { nvlink_err("Invalid tnvlink_dev struct pointer"); nvlink_err("t19x nvlink shutdown failed!"); return; } ret = nvlink_shutdown(tdev->ndev); if (ret < 0) { nvlink_err("t19x nvlink shutdown failed!"); return; } nvlink_dbg("t19x nvlink shutdown successful"); } static struct platform_driver t19x_nvlink_endpt_pdrv = { .probe = t19x_nvlink_endpt_probe, .remove = t19x_nvlink_endpt_remove, .driver = { .name = NVLINK_MODULE_NAME, #if IS_ENABLED(CONFIG_PM_SLEEP) .pm = &tegra_nvlink_pm_ops, #endif .of_match_table = of_match_ptr(t19x_nvlink_controller_of_match), }, .shutdown = t19x_nvlink_endpt_shutdown, }; static int __init t19x_nvlink_endpt_init(void) { int ret = 0; ret = platform_driver_register(&t19x_nvlink_endpt_pdrv); if (ret < 0) nvlink_err("Platform driver register failed"); return ret; } static void __exit t19x_nvlink_endpt_exit(void) { nvlink_dbg("Unloading the T19x NVLINK endpoint driver"); platform_driver_unregister(&t19x_nvlink_endpt_pdrv); } module_init(t19x_nvlink_endpt_init); module_exit(t19x_nvlink_endpt_exit); MODULE_ALIAS(NVLINK_MODULE_NAME); MODULE_DESCRIPTION("T19x NVLINK Endpoint Driver"); MODULE_LICENSE("GPL v2");