NED
/
tegrakernel
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
tegrakernel/kernel/nvidia/drivers/nvlink/t19x-nvlink-endpt-link.c

1157 lines
33 KiB
C
Raw Permalink Normal View History

initial commit tegra kernel 32.6.1
2022-02-16 09:13:02 -06:00
/*
* t19x-nvlink-endpt-link.c:
* This file contains link state transition and link trainig code for the Tegra
* NVLINK controller.
*
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <soc/tegra/fuse.h>
#include "t19x-nvlink-endpt.h"
#include "nvlink-hw.h"
#define SUBLINK_TIMEOUT_MS 200 /* msec */
#define R4TX_TIMEOUT_US 1000 /* usec */
const struct single_lane_params entry_100us_sl_params = {
.fb_ic_inc = 1,
.lp_ic_inc = 1,
.fb_ic_dec = 1,
.lp_ic_dec = 65535,
.enter_thresh = 161100,
.exit_thresh = 0,
.ic_limit = 161100,
};
u32 t19x_nvlink_get_link_state(struct nvlink_device *ndev)
{
struct tnvlink_dev *tdev = (struct tnvlink_dev *)ndev->priv;
return (nvlw_nvl_readl(tdev, NVL_LINK_STATE) &
NVL_LINK_STATE_STATE_MASK);
}
u32 t19x_nvlink_get_link_mode(struct nvlink_device *ndev)
{
u32 link_mode;
u32 link_state = t19x_nvlink_get_link_state(ndev);
switch (link_state) {
case NVL_LINK_STATE_STATE_INIT:
link_mode = NVLINK_LINK_OFF;
break;
case NVL_LINK_STATE_STATE_HWCFG:
link_mode = NVLINK_LINK_DETECT;
break;
case NVL_LINK_STATE_STATE_SWCFG:
link_mode = NVLINK_LINK_SAFE;
break;
case NVL_LINK_STATE_STATE_ACTIVE:
link_mode = NVLINK_LINK_HS;
break;
case NVL_LINK_STATE_STATE_FAULT:
link_mode = NVLINK_LINK_FAULT;
break;
case NVL_LINK_STATE_STATE_RCVY_AC:
link_mode = NVLINK_LINK_RCVY_AC;
break;
case NVL_LINK_STATE_STATE_RCVY_SW:
link_mode = NVLINK_LINK_RCVY_SW;
break;
case NVL_LINK_STATE_STATE_RCVY_RX:
link_mode = NVLINK_LINK_RCVY_RX;
break;
default:
nvlink_err("Invalid link state (link state = %u)", link_state);
link_mode = NVLINK_LINK_OFF;
break;
}
return link_mode;
}
u32 t19x_nvlink_get_sublink_mode(struct nvlink_device *ndev, bool is_rx_sublink)
{
u32 reg_val;
u8 state;
u32 sublink_mode;
struct tnvlink_dev *tdev = (struct tnvlink_dev *)ndev->priv;
if (!is_rx_sublink) {
reg_val = nvlw_nvl_readl(tdev, NVL_SL0_SLSM_STATUS_TX);
state = (reg_val & NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_MASK) >>
NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_SHIFT;
switch (state) {
case NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_EIGHTH:
sublink_mode = NVLINK_TX_SINGLE_LANE;
break;
case NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_HS:
sublink_mode = NVLINK_TX_HS;
break;
case NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_SAFE:
sublink_mode = NVLINK_TX_SAFE;
break;
case NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_OFF:
sublink_mode = NVLINK_TX_OFF;
break;
default:
nvlink_err("Invalid TX sublink state"
" (sublink state = %u)",
state);
sublink_mode = NVLINK_TX_OFF;
break;
}
} else {
reg_val = nvlw_nvl_readl(tdev, NVL_SL1_SLSM_STATUS_RX);
state = (reg_val & NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_MASK) >>
NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_SHIFT;
switch (state) {
case NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_EIGHTH:
sublink_mode = NVLINK_RX_SINGLE_LANE;
break;
case NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_HS:
sublink_mode = NVLINK_RX_HS;
break;
case NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_SAFE:
sublink_mode = NVLINK_RX_SAFE;
break;
case NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_OFF:
sublink_mode = NVLINK_RX_OFF;
break;
default:
nvlink_err("Invalid RX sublink state"
" (sublink state = %u)",
state);
sublink_mode = NVLINK_RX_OFF;
break;
}
}
return sublink_mode;
}
void t19x_nvlink_get_tx_sublink_state(struct nvlink_device *ndev,
u32 *tx_sublink_state)
{
struct tnvlink_dev *tdev = (struct tnvlink_dev *)ndev->priv;
u32 reg_val;
reg_val = nvlw_nvl_readl(tdev, NVL_SL0_SLSM_STATUS_TX);
*tx_sublink_state = (reg_val &
NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_MASK) >>
NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_SHIFT;
}
void t19x_nvlink_get_rx_sublink_state(struct nvlink_device *ndev,
u32 *rx_sublink_state)
{
struct tnvlink_dev *tdev = (struct tnvlink_dev *)ndev->priv;
u32 reg_val;
reg_val = nvlw_nvl_readl(tdev, NVL_SL1_SLSM_STATUS_RX);
*rx_sublink_state = (reg_val &
NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_MASK) >>
NVL_SL1_SLSM_STATUS_RX_PRIMARY_STATE_SHIFT;
}
/* From the point of view of the Tegra endpoint, is the link in SAFE mode? */
static inline bool is_link_in_safe(struct tnvlink_link *tlink)
{
struct nvlink_device *ndev = tlink->tdev->ndev;
enum link_mode link_mode = t19x_nvlink_get_link_mode(ndev);
enum tx_mode tx_mode = t19x_nvlink_get_sublink_mode(ndev, false);
enum rx_mode rx_mode = t19x_nvlink_get_sublink_mode(ndev, true);
return ((link_mode == NVLINK_LINK_SAFE) &&
(tx_mode == NVLINK_TX_SAFE) &&
(rx_mode == NVLINK_RX_SAFE));
}
/* From the point of view of the Tegra endpoint, is the link in HISPEED mode? */
static inline bool is_link_in_hs(struct tnvlink_link *tlink)
{
struct nvlink_device *ndev = tlink->tdev->ndev;
enum link_mode link_mode = t19x_nvlink_get_link_mode(ndev);
enum tx_mode tx_mode = t19x_nvlink_get_sublink_mode(ndev, false);
enum rx_mode rx_mode = t19x_nvlink_get_sublink_mode(ndev, true);
return ((link_mode == NVLINK_LINK_HS) &&
(tx_mode == NVLINK_TX_HS || tx_mode == NVLINK_TX_SINGLE_LANE) &&
(rx_mode == NVLINK_RX_HS || rx_mode == NVLINK_RX_SINGLE_LANE));
}
bool is_link_connected(struct tnvlink_link *tlink)
{
return (is_link_in_safe(tlink) || is_link_in_hs(tlink));
}
/* Configure and Start the PRBS generator */
static int t19x_nvlink_prbs_gen_en(struct tnvlink_dev *tdev)
{
u32 reg_val;
/* Not specified in doc but is required as per HW team */
nvlw_nvl_writel(tdev, NVL_SL1_RXSLSM_TIMEOUT_2, 0);
/*
* Minion will set these *_PBRS_* and *_SCRAM_* registers
* as a part of DLPL init.
*
* Note: A remote endpoint is expected to set exact same *_PBRS_*
* and *_SCRAM_* values. Otherwise, link training fails.
*/
reg_val = nvlw_nvl_readl(tdev, NVL_TXIOBIST_CONFIG);
reg_val |= BIT(NVL_TXIOBIST_CONFIG_DPG_PRBSSEEDLD);
nvlw_nvl_writel(tdev, NVL_TXIOBIST_CONFIG, reg_val);
reg_val = nvlw_nvl_readl(tdev, NVL_TXIOBIST_CONFIG);
reg_val &= ~BIT(NVL_TXIOBIST_CONFIG_DPG_PRBSSEEDLD);
nvlw_nvl_writel(tdev, NVL_TXIOBIST_CONFIG, reg_val);
return 0;
}
/* Put RX in calibration */
static int t19x_nvlink_rxcal_enable(struct tnvlink_dev *tdev)
{
/* TODO: Move the RX calibration code from init_nvhs_phy() to here */
return 0;
}
int t19x_nvlink_set_sublink_mode(struct nvlink_device *ndev, bool is_rx_sublink,
u32 mode)
{
struct tnvlink_dev *tdev = (struct tnvlink_dev *)ndev->priv;
u32 rx_sublink_state, tx_sublink_state;
u32 reg_val, timeout_us;
int status = 0;
timeout_us = SUBLINK_TIMEOUT_MS * 1000;
t19x_nvlink_get_tx_sublink_state(ndev, &tx_sublink_state);
t19x_nvlink_get_rx_sublink_state(ndev, &rx_sublink_state);
/* Check if SLSM is ready to accept a sublink change request */
do {
reg_val = nvlw_nvl_readl(tdev, NVL_SUBLINK_CHANGE);
if ((reg_val & NVL_SUBLINK_CHANGE_STATUS_MASK) ==
NVL_SUBLINK_CHANGE_STATUS_DONE)
break;
usleep_range(DEFAULT_LOOP_SLEEP_US, DEFAULT_LOOP_SLEEP_US * 2);
timeout_us = timeout_us - DEFAULT_LOOP_SLEEP_US;
} while (timeout_us > 0);
if (timeout_us <= 0) {
nvlink_err("SLSM not ready to accept a state change request");
return -EINVAL;
}
if (!is_rx_sublink) {
switch (mode) {
case NVLINK_TX_COMMON:
/* TODO */
nvlink_err("Putting the TX sublink in NVLINK_TX_COMMON"
" mode is currently not supported by the"
" driver");
status = -EOPNOTSUPP;
break;
case NVLINK_TX_COMMON_DISABLE:
nvlink_err("Putting the TX sublink in"
" NVLINK_TX_COMMON_DISABLE mode is currently"
" not supported by the driver");
status = -EOPNOTSUPP;
break;
case NVLINK_TX_DATA_READY:
/* TODO */
nvlink_err("Putting the TX sublink in"
" NVLINK_TX_DATA_READY mode is currently not"
" supported by the driver");
status = -EOPNOTSUPP;
break;
case NVLINK_TX_PRBS_EN:
status = t19x_nvlink_prbs_gen_en(tdev);
if (status) {
nvlink_err("Unable to start PRBS generator"
" for link");
return status;
}
break;
case NVLINK_TX_HS:
if (tx_sublink_state ==
NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_HS) {
nvlink_dbg("TX already in High Speed mode");
break;
} else if (tx_sublink_state ==
NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_OFF) {
nvlink_err("TX cannot be taken from OFF to"
" High Speed directly");
return -EPERM;
}
nvlink_dbg("Changing TX sublink state to High Speed");
reg_val = nvlw_nvl_readl(tdev, NVL_SUBLINK_CHANGE);
reg_val &= ~NVL_SUBLINK_CHANGE_NEWSTATE_F(~0);
reg_val |= NVL_SUBLINK_CHANGE_NEWSTATE_F(
NVL_SUBLINK_CHANGE_NEWSTATE_HS);
reg_val &= ~NVL_SUBLINK_CHANGE_SUBLINK_F(~0);
reg_val |= NVL_SUBLINK_CHANGE_SUBLINK_F(
NVL_SUBLINK_CHANGE_SUBLINK_TX);
reg_val &= ~NVL_SUBLINK_CHANGE_ACTION_F(~0);
reg_val |= NVL_SUBLINK_CHANGE_ACTION_F(
NVL_SUBLINK_CHANGE_ACTION_SLSM_CHANGE);
nvlw_nvl_writel(tdev, NVL_SUBLINK_CHANGE, reg_val);
timeout_us = SUBLINK_TIMEOUT_MS * 1000;
do {
reg_val = nvlw_nvl_readl(tdev,
NVL_SUBLINK_CHANGE);
if ((reg_val &
NVL_SUBLINK_CHANGE_STATUS_MASK) ==
NVL_SUBLINK_CHANGE_STATUS_DONE)
break;
else if ((reg_val &
NVL_SUBLINK_CHANGE_STATUS_MASK) ==
NVL_SUBLINK_CHANGE_STATUS_FAULT) {
nvlink_err("Fault while changing TX"
" sublink to High Speed");
return -EPROTO;
}
usleep_range(DEFAULT_LOOP_SLEEP_US,
DEFAULT_LOOP_SLEEP_US * 2);
timeout_us = timeout_us - DEFAULT_LOOP_SLEEP_US;
} while (timeout_us > 0);
if (timeout_us <= 0) {
nvlink_err("Timeout while waiting for TX"
" sublink to go to High Speed");
return -ETIMEDOUT;
}
break;
case NVLINK_TX_SAFE:
if (tx_sublink_state ==
NVL_SL0_SLSM_STATUS_TX_PRIMARY_STATE_SAFE) {
nvlink_dbg("TX already in Safe mode");
break;
}
nvlink_dbg("Changing TX sublink state to Safe mode");
reg_val = nvlw_nvl_readl(tdev, NVL_SUBLINK_CHANGE);
reg_val &= ~NVL_SUBLINK_CHANGE_NEWSTATE_F(~0);
reg_val |= NVL_SUBLINK_CHANGE_NEWSTATE_F(
NVL_SUBLINK_CHANGE_NEWSTATE_SAFE);
reg_val &= ~NVL_SUBLINK_CHANGE_SUBLINK_F(~0);
reg_val |= NVL_SUBLINK_CHANGE_SUBLINK_F(
NVL_SUBLINK_CHANGE_SUBLINK_TX);
reg_val &= ~NVL_SUBLINK_CHANGE_ACTION_F(~0);
reg_val |= NVL_SUBLINK_CHANGE_ACTION_F(
NVL_SUBLINK_CHANGE_ACTION_SLSM_CHANGE);
nvlw_nvl_writel(tdev, NVL_SUBLINK_CHANGE, reg_val);
timeout_us = SUBLINK_TIMEOUT_MS * 1000;
do {
reg_val = nvlw_nvl_readl(tdev,
NVL_SUBLINK_CHANGE);
if ((reg_val &
NVL_SUBLINK_CHANGE_STATUS_MASK) ==
NVL_SUBLINK_CHANGE_STATUS_DONE)
break;
else if ((reg_val &
NVL_SUBLINK_CHANGE_STATUS_MASK) ==
NVL_SUBLINK_CHANGE_STATUS_FAULT) {
nvlink_err("Fault while changing TX"
" sublink to SAFE MODE");
return -EPROTO;
}
usleep_range(DEFAULT_LOOP_SLEEP_US,
DEFAULT_LOOP_SLEEP_US * 2);
timeout_us = timeout_us - DEFAULT_LOOP_SLEEP_US;
} while (timeout_us > 0);
if (timeout_us <= 0) {
nvlink_err("Timeout while waiting for TX"
" sublink to go to SAFE MODE");
return -ETIMEDOUT;
}
break;
case NVLINK_TX_ENABLE_PM:
nvlink_dbg("Enabling Single-Lane (1/8th) mode for the"
" TX sublink");
reg_val = nvlw_nvltlc_readl(tdev,
NVLTLC_TX_PWRM_IC_SW_CTRL);
reg_val |=
BIT(NVLTLC_TX_PWRM_IC_SW_CTRL_SOFTWAREDESIRED);
reg_val &=
~BIT(NVLTLC_TX_PWRM_IC_SW_CTRL_HARDWAREDISABLE);
reg_val |=
BIT(NVLTLC_TX_PWRM_IC_SW_CTRL_COUNTSTART);
nvlw_nvltlc_writel(tdev,
NVLTLC_TX_PWRM_IC_SW_CTRL,
reg_val);
break;
case NVLINK_TX_DISABLE_PM:
nvlink_dbg("Disabling Single-Lane (1/8th) mode for the"
" TX sublink");
reg_val = nvlw_nvltlc_readl(tdev,
NVLTLC_TX_PWRM_IC_SW_CTRL);
reg_val &=
~BIT(NVLTLC_TX_PWRM_IC_SW_CTRL_SOFTWAREDESIRED);
reg_val |=
BIT(NVLTLC_TX_PWRM_IC_SW_CTRL_HARDWAREDISABLE);
reg_val &=
~BIT(NVLTLC_TX_PWRM_IC_SW_CTRL_COUNTSTART);
nvlw_nvltlc_writel(tdev,
NVLTLC_TX_PWRM_IC_SW_CTRL,
reg_val);
break;
default:
nvlink_err("Invalid TX sublink mode"
" (sublink mode = %u)",
mode);
status = -EINVAL;
break;
}
} else {
switch (mode) {
case NVLINK_RX_RXCAL:
status = t19x_nvlink_rxcal_enable(tdev);
if (status) {
nvlink_err("Unable to put RX"
" in RXCAL for link");
return status;
}
break;
case NVLINK_RX_ENABLE_PM:
nvlink_dbg("Enabling Single-Lane (1/8th) mode for the"
" RX sublink");
reg_val = nvlw_nvltlc_readl(tdev,
NVLTLC_RX_PWRM_IC_SW_CTRL);
reg_val |=
BIT(NVLTLC_RX_PWRM_IC_SW_CTRL_SOFTWAREDESIRED);
reg_val &=
~BIT(NVLTLC_RX_PWRM_IC_SW_CTRL_HARDWAREDISABLE);
reg_val |=
BIT(NVLTLC_RX_PWRM_IC_SW_CTRL_COUNTSTART);
nvlw_nvltlc_writel(tdev,
NVLTLC_RX_PWRM_IC_SW_CTRL,
reg_val);
break;
case NVLINK_RX_DISABLE_PM:
nvlink_dbg("Disabling Single-Lane (1/8th) mode for the"
" RX sublink");
reg_val = nvlw_nvltlc_readl(tdev,
NVLTLC_RX_PWRM_IC_SW_CTRL);
reg_val &=
~BIT(NVLTLC_RX_PWRM_IC_SW_CTRL_SOFTWAREDESIRED);
reg_val |=
BIT(NVLTLC_RX_PWRM_IC_SW_CTRL_HARDWAREDISABLE);
reg_val &=
~BIT(NVLTLC_RX_PWRM_IC_SW_CTRL_COUNTSTART);
nvlw_nvltlc_writel(tdev,
NVLTLC_RX_PWRM_IC_SW_CTRL,
reg_val);
break;
default:
nvlink_err("Invalid RX sublink mode"
" (sublink mode = %u)",
mode);
status = -EINVAL;
break;
}
}
return status;
}
int t19x_nvlink_set_link_mode(struct nvlink_device *ndev, u32 mode)
{
struct tnvlink_dev *tdev = (struct tnvlink_dev *)ndev->priv;
u32 link_state;
u32 reg_val;
int status = 0;
link_state = t19x_nvlink_get_link_state(ndev);
switch (mode) {
case NVLINK_LINK_OFF:
if (link_state == NVL_LINK_STATE_STATE_INIT) {
nvlink_dbg("Link is already in OFF mode");
break;
} else if (link_state == NVL_LINK_STATE_STATE_ACTIVE) {
nvlink_err("Link cannot be taken from ACTIVE state"
" to OFF mode");
return -EPERM;
}
/* Free the interrupt line on the device */
devm_free_irq(tdev->dev, tdev->irq, tdev);
status = t19x_nvlink_dev_car_disable(ndev);
if (status < 0) {
nvlink_err("set link mode to OFF failed");
}
/* WAR for Bug 2301575 */
status = tegra_fuse_clock_disable();
if (status < 0) {
nvlink_err("failed to disable fuse clocks");
} else {
nvlink_dbg("fuse clocks are turned OFF");
}
break;
case NVLINK_LINK_SAFE:
if (link_state == NVL_LINK_STATE_STATE_SWCFG) {
nvlink_dbg("Link is already in Safe mode");
break;
} else if (link_state == NVL_LINK_STATE_STATE_HWCFG) {
nvlink_dbg("Link already transitioning to"
" Safe mode");
break;
}
nvlink_dbg("Changing Link state to Safe for link");
if (link_state == NVL_LINK_STATE_STATE_INIT) {
reg_val = nvlw_nvl_readl(tdev, NVL_LINK_CHANGE);
reg_val &= ~NVL_LINK_CHANGE_NEWSTATE_F(~0);
reg_val |= NVL_LINK_CHANGE_NEWSTATE_F(
NVL_LINK_CHANGE_NEWSTATE_HWCFG);
reg_val &= ~NVL_LINK_CHANGE_OLDSTATE_MASK_F(~0);
reg_val |= NVL_LINK_CHANGE_OLDSTATE_MASK_F(
NVL_LINK_CHANGE_OLDSTATE_MASK_DONTCARE);
reg_val &= ~NVL_LINK_CHANGE_ACTION_F(~0);
reg_val |= NVL_LINK_CHANGE_ACTION_F(
NVL_LINK_CHANGE_ACTION_LTSSM_CHANGE);
nvlw_nvl_writel(tdev, NVL_LINK_CHANGE, reg_val);
} else if (link_state == NVL_LINK_STATE_STATE_ACTIVE) {
/* TODO :
* Disable PM first since we are moving out of
* ACTIVE state.
*/
reg_val = nvlw_nvl_readl(tdev, NVL_LINK_CHANGE);
reg_val &= ~NVL_LINK_CHANGE_NEWSTATE_F(~0);
reg_val |= NVL_LINK_CHANGE_NEWSTATE_F(
NVL_LINK_CHANGE_NEWSTATE_SWCFG);
reg_val &= ~NVL_LINK_CHANGE_OLDSTATE_MASK_F(~0);
reg_val |= NVL_LINK_CHANGE_OLDSTATE_MASK_F(
NVL_LINK_CHANGE_OLDSTATE_MASK_DONTCARE);
reg_val &= ~NVL_LINK_CHANGE_ACTION_F(~0);
reg_val |= NVL_LINK_CHANGE_ACTION_F(
NVL_LINK_CHANGE_ACTION_LTSSM_CHANGE);
nvlw_nvl_writel(tdev, NVL_LINK_CHANGE, reg_val);
}
break;
case NVLINK_LINK_HS:
if (link_state == NVL_LINK_STATE_STATE_ACTIVE) {
nvlink_dbg("Link is already in Active mode");
break;
} else if (link_state == NVL_LINK_STATE_STATE_INIT) {
nvlink_err("Link cannot be taken from INIT state"
" to Active mode");
return -EPERM;
}
nvlink_dbg("changing Link state to Active...");
reg_val = nvlw_nvl_readl(tdev, NVL_LINK_CHANGE);
reg_val &= ~NVL_LINK_CHANGE_NEWSTATE_F(~0);
reg_val |= NVL_LINK_CHANGE_NEWSTATE_F(
NVL_LINK_CHANGE_NEWSTATE_ACTIVE);
reg_val &= ~NVL_LINK_CHANGE_OLDSTATE_MASK_F(~0);
reg_val |= NVL_LINK_CHANGE_OLDSTATE_MASK_F(
NVL_LINK_CHANGE_OLDSTATE_MASK_DONTCARE);
reg_val &= ~NVL_LINK_CHANGE_ACTION_F(~0);
reg_val |= NVL_LINK_CHANGE_ACTION_F(
NVL_LINK_CHANGE_ACTION_LTSSM_CHANGE);
nvlw_nvl_writel(tdev, NVL_LINK_CHANGE, reg_val);
break;
case NVLINK_LINK_RCVY_AC:
if (link_state != NVL_LINK_STATE_STATE_ACTIVE) {
nvlink_err("Link is not in ACTIVE state. The link can"
" only go to RCVY_AC state from ACTIVE.");
status = -EPERM;
break;
}
nvlink_dbg("Changing link state to RCVY_AC");
reg_val = nvlw_nvl_readl(tdev, NVL_LINK_CHANGE);
reg_val &= ~NVL_LINK_CHANGE_NEWSTATE_F(~0);
reg_val |= NVL_LINK_CHANGE_NEWSTATE_F(
NVL_LINK_CHANGE_NEWSTATE_RCVY_AC);
reg_val &= ~NVL_LINK_CHANGE_OLDSTATE_MASK_F(~0);
reg_val |= NVL_LINK_CHANGE_OLDSTATE_MASK_F(
NVL_LINK_CHANGE_OLDSTATE_MASK_DONTCARE);
reg_val &= ~NVL_LINK_CHANGE_ACTION_F(~0);
reg_val |= NVL_LINK_CHANGE_ACTION_F(
NVL_LINK_CHANGE_ACTION_LTSSM_CHANGE);
nvlw_nvl_writel(tdev, NVL_LINK_CHANGE, reg_val);
break;
case NVLINK_LINK_ENABLE_PM:
if (link_state == NVL_LINK_STATE_STATE_ACTIVE) {
nvlink_dbg("Link is in Active state."
" Enabling Single-Lane (1/8th) mode.");
/* Enable Single-Lane mode for the TX sublink */
status = t19x_nvlink_set_sublink_mode(ndev,
false,
NVLINK_TX_ENABLE_PM);
if (status) {
nvlink_err("Failed to enable SL (1/8th) mode"
" for the TX sublink");
break;
}
/* Enable Single-Lane mode for the RX sublink */
status = t19x_nvlink_set_sublink_mode(ndev,
true,
NVLINK_RX_ENABLE_PM);
if (status) {
nvlink_err("Failed to enable SL (1/8th) mode"
" for the RX sublink");
break;
}
/*
* This is the final piece for enabling Single-Lane (SL)
* mode. We send the ENABLEPM command to the MINION to
* instruct MINION to enter/exit SL mode as per the
* programmed SL policy.
*/
status = minion_send_cmd(tdev,
MINION_NVLINK_DL_CMD_COMMAND_ENABLEPM,
0);
if (status < 0) {
nvlink_err("Error encountered while sending the"
" ENABLEPM command to the MINION");
nvlink_err("Failed to enable SL (1/8th) mode");
minion_dump_pc_trace(tdev);
minion_dump_registers(tdev);
break;
}
} else {
nvlink_err("Link is not in Active state."
" Single-Lane (1/8th) mode can only be enabled"
" from the Active state.");
status = -EPERM;
break;
}
break;
case NVLINK_LINK_DISABLE_PM:
if (link_state == NVL_LINK_STATE_STATE_ACTIVE) {
nvlink_dbg("Link is in Active state."
" Disabling Single-Lane (1/8th) mode.");
/* Disable Single-Lane mode for the TX sublink */
status = t19x_nvlink_set_sublink_mode(ndev,
false,
NVLINK_TX_DISABLE_PM);
if (status) {
nvlink_err("Failed to disable SL (1/8th) mode"
" for the TX sublink");
break;
}
/* Disable Single-Lane mode for the RX sublink */
status = t19x_nvlink_set_sublink_mode(ndev,
true,
NVLINK_RX_DISABLE_PM);
if (status) {
nvlink_err("Failed to disable SL (1/8th) mode"
" for the RX sublink");
break;
}
/* Disable Single-Lane mode in MINION */
status = minion_send_cmd(tdev,
MINION_NVLINK_DL_CMD_COMMAND_DISABLEPM,
0);
if (status < 0) {
nvlink_err("Error encountered while sending the"
" DISABLEPM command to the MINION");
nvlink_err("Failed to disable SL (1/8th) mode");
minion_dump_pc_trace(tdev);
minion_dump_registers(tdev);
break;
}
} else {
nvlink_err("Link is not in Active state."
" Single-Lane (1/8th) mode can only be disabled"
" from the Active state.");
status = -EPERM;
break;
}
break;
case NVLINK_LINK_DISABLE_ERR_DETECT:
/* Disable link interrupts */
nvlink_disable_link_interrupts(tdev);
break;
case NVLINK_LINK_LANE_DISABLE:
status = minion_send_cmd(tdev,
MINION_NVLINK_DL_CMD_COMMAND_LANEDISABLE, 0);
if (status < 0) {
nvlink_err("Error setting link mode to LANE DISABLE");
break;
}
break;
case NVLINK_LINK_LANE_SHUTDOWN:
status = minion_send_cmd(tdev,
MINION_NVLINK_DL_CMD_COMMAND_LANESHUTDOWN, 0);
if (status < 0) {
nvlink_err("Error setting link mode to LANE SHUTDOWN");
break;
}
break;
/* TODO: other "case"s need to be implemented here */
default:
nvlink_err("Invalid link mode specified (link mode = %u)",
mode);
status = -EINVAL;
break;
}
return status;
}
/* Initialize TLC settings which dictate Single-Lane (1/8th) mode policy */
void init_single_lane_params(struct tnvlink_dev *tdev)
{
u32 reg_val = 0;
struct single_lane_params *sl_params = &tdev->tlink.sl_params;
nvlink_dbg("Initializing Single-Lane parameters");
/* Idle counter increments */
reg_val = NVLTLC_TX_PWRM_IC_INC_FBINC_F(sl_params->fb_ic_inc) |
NVLTLC_TX_PWRM_IC_INC_LPINC_F(sl_params->lp_ic_inc);
nvlw_nvltlc_writel(tdev, NVLTLC_TX_PWRM_IC_INC, reg_val);
/* Idle counter decrements */
reg_val = NVLTLC_TX_PWRM_IC_DEC_FBDEC_F(sl_params->fb_ic_dec) |
NVLTLC_TX_PWRM_IC_DEC_LPDEC_F(sl_params->lp_ic_dec);
nvlw_nvltlc_writel(tdev, NVLTLC_TX_PWRM_IC_DEC, reg_val);
/* Entry threshold */
nvlw_nvltlc_writel(tdev,
NVLTLC_TX_PWRM_IC_LP_ENTER_THRESHOLD,
sl_params->enter_thresh);
/* Exit threshold */
nvlw_nvltlc_writel(tdev,
NVLTLC_TX_PWRM_IC_LP_EXIT_THRESHOLD,
sl_params->exit_thresh);
/* Idle counter saturation limit */
nvlw_nvltlc_writel(tdev, NVLTLC_TX_PWRM_IC_LIMIT, sl_params->ic_limit);
}
/* Enable ANO packets over link */
void nvlink_enable_AN0_packets(struct tnvlink_dev *tdev)
{
u32 reg_val = 0;
reg_val = nvlw_nvl_readl(tdev, NVL_LINK_CONFIG);
reg_val |= BIT(NVL_LINK_CONFIG_LINK_EN);
nvlw_nvl_writel(tdev, NVL_LINK_CONFIG, reg_val);
}
static int nvlink_retrain_link_from_off(struct tnvlink_dev *tdev)
{
/* We don't need this for now */
nvlink_err("Retraining the link from OFF mode is currently not"
" supported by the driver");
return -EOPNOTSUPP;
}
static int nvlink_retrain_link_from_safe(struct tnvlink_dev *tdev)
{
int ret;
struct nvlink_device *ndev = tdev->ndev;
ret = nvlink_transition_intranode_conn_hs_to_safe(ndev);
if (ret < 0) {
nvlink_err("Transiting intranode conn to safe failed");
return ret;
}
ret = nvlink_train_intranode_conn_safe_to_hs(ndev);
if (ret < 0) {
nvlink_err("Train intranode conn to HS failed");
return ret;
}
/* The link has successfully retrained */
tdev->tlink.error_recoveries++;
return 0;
}
/* Retrain a link from either safe mode or off */
int nvlink_retrain_link(struct tnvlink_dev *tdev, bool from_off)
{
struct nvlink_device *ndev = tdev->ndev;
int ret;
if (!ndev->is_master) {
/* TODO :
* Not needed for loopback but needed for other topologies.
*
* If this is a slave endpoint requesting the retrain,
* kick off a request to the master instead.
* There is no need to (and indeed, we must not) hold
* the master endpoint lock here.
*/
nvlink_err("Link retraining from the slave endpoint is"
" currently not supported");
return -EPERM;
}
if (from_off)
ret = nvlink_retrain_link_from_off(tdev);
else
ret = nvlink_retrain_link_from_safe(tdev);
return ret;
}
int t19x_nvlink_write_discovery_token(struct tnvlink_dev *tdev, u64 token)
{
int ret = 0;
u32 reg_val = 0;
u32 token_low_32 = (u32)(token & (u64)(0xffffffff));
u32 token_high_32 = (u32)((token >> (u64)(32)) & (u64)(0xffffffff));
/* Check if R4TX interface is ready to accept a new request */
reg_val = nvlw_nvl_readl(tdev, NVL_SL0_R4TX_COMMAND);
if (!(reg_val & BIT(NVL_SL0_R4TX_COMMAND_READY))) {
nvlink_err("Unable to write discovery token because R4TX"
" interface is not ready for a new request");
ret = -EBUSY;
goto fail;
}
/* Write the token in little endian format */
nvlw_nvl_writel(tdev, NVL_SL0_R4TX_WDATA0, token_low_32);
nvlw_nvl_writel(tdev, NVL_SL0_R4TX_WDATA1, token_high_32);
/* Issue the command to write the token */
reg_val &= ~NVL_SL0_R4TX_COMMAND_REQUEST_F(~0);
reg_val |= NVL_SL0_R4TX_COMMAND_REQUEST_F(
NVL_SL0_R4TX_COMMAND_REQUEST_WRITE);
reg_val |= BIT(NVL_SL0_R4TX_COMMAND_COMPLETE);
reg_val &= ~NVL_SL0_R4TX_COMMAND_WADDR_F(~0);
reg_val |= NVL_SL0_R4TX_COMMAND_WADDR_F(NVL_SL0_R4TX_COMMAND_WADDR_SC);
nvlw_nvl_writel(tdev, NVL_SL0_R4TX_COMMAND, reg_val);
/* Wait for the token write command to complete */
ret = wait_for_reg_cond_nvlink(tdev,
NVL_SL0_R4TX_COMMAND,
NVL_SL0_R4TX_COMMAND_COMPLETE,
true,
"NVL_SL0_R4TX_COMMAND_COMPLETE",
nvlw_nvl_readl,
&reg_val,
R4TX_TIMEOUT_US);
if (ret < 0)
goto fail;
nvlink_dbg("Successfully wrote discovery token. Token = 0x%llx.",
token);
goto exit;
fail:
nvlink_err("Failed to write discovery token!");
exit:
return ret;
}
int t19x_nvlink_read_discovery_token(struct tnvlink_dev *tdev, u64 *token)
{
int ret = 0;
u32 reg_val = 0;
u32 token_low_32 = 0;
u32 token_high_32 = 0;
/* Check if R4TX interface is ready to accept a new request */
reg_val = nvlw_nvl_readl(tdev, NVL_SL1_R4LOCAL_COMMAND);
if (!(reg_val & BIT(NVL_SL1_R4LOCAL_COMMAND_READY))) {
nvlink_err("Unable to read discovery token because R4TX"
" interface is not ready for a new request");
ret = -EBUSY;
goto fail;
}
/* Issue the command to read the token */
reg_val &= ~NVL_SL1_R4LOCAL_COMMAND_REQUEST_F(~0);
reg_val |= NVL_SL1_R4LOCAL_COMMAND_REQUEST_F(
NVL_SL1_R4LOCAL_COMMAND_REQUEST_READ);
reg_val |= BIT(NVL_SL1_R4LOCAL_COMMAND_COMPLETE);
reg_val &= ~NVL_SL1_R4LOCAL_COMMAND_RADDR_F(~0);
reg_val |= NVL_SL1_R4LOCAL_COMMAND_RADDR_F(
NVL_SL1_R4LOCAL_COMMAND_RADDR_SC);
nvlw_nvl_writel(tdev, NVL_SL1_R4LOCAL_COMMAND, reg_val);
/* Wait for the token read command to complete */
ret = wait_for_reg_cond_nvlink(tdev,
NVL_SL1_R4LOCAL_COMMAND,
NVL_SL1_R4LOCAL_COMMAND_COMPLETE,
true,
"NVL_SL1_R4LOCAL_COMMAND_COMPLETE",
nvlw_nvl_readl,
&reg_val,
R4TX_TIMEOUT_US);
if (ret < 0)
goto fail;
/* Read the token in little endian format */
token_low_32 = nvlw_nvl_readl(tdev, NVL_SL1_R4LOCAL_RDATA0);
token_high_32 = nvlw_nvl_readl(tdev, NVL_SL1_R4LOCAL_RDATA1);
*token = ((u64)(token_high_32) << (u64)(32)) |
((u64)(token_low_32) & (u64)(0xffffffff));
nvlink_dbg("Successfully read discovery token. Token = 0x%llx.",
*token);
goto exit;
fail:
nvlink_err("Failed to read discovery token!");
exit:
return ret;
}
/*
*-----------------------------------------------------------------------------*
* TLC THROUGHPUT COUNTERS
*-----------------------------------------------------------------------------*
*/
/* Reset the TLC throughput counters for both RX and TX */
int t19x_nvlink_reset_tp_counters(struct tnvlink_dev *tdev)
{
int ret = 0;
u32 data;
if(tdev == NULL) {
nvlink_err("Invalid tnvlink_dev pointer");
return -EINVAL;
}
/* Reset TX tp counters */
data = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR_CTRL);
data |= BIT(NVLTLC_TX_DEBUG_TP_CNTR_CTRL_RESETTX0);
data |= BIT(NVLTLC_TX_DEBUG_TP_CNTR_CTRL_RESETTX1);
nvlw_nvltlc_writel(tdev, NVLTLC_TX_DEBUG_TP_CNTR_CTRL, data);
/* Reset RX tp counters */
data = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR_CTRL);
data |= BIT(NVLTLC_RX_DEBUG_TP_CNTR_CTRL_RESETRX0);
data |= BIT(NVLTLC_RX_DEBUG_TP_CNTR_CTRL_RESETRX1);
nvlw_nvltlc_writel(tdev, NVLTLC_RX_DEBUG_TP_CNTR_CTRL, data);
return ret;
}
/* Start/Stop the TLC throughput counters for both RX and TX */
int t19x_nvlink_freeze_tp_counters(struct tnvlink_dev *tdev, bool freeze)
{
int ret = 0;
u32 data_tx, data_rx;
if(tdev == NULL) {
nvlink_err("Invalid tnvlink_dev pointer");
return -EINVAL;
}
/* TX tp counters */
data_tx = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR_CTRL);
/* RX tp counters */
data_rx = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR_CTRL);
if (!freeze) {
tdev->is_tp_cntr_running = true;
data_tx |= BIT(NVLTLC_TX_DEBUG_TP_CNTR_CTRL_ENTX0);
data_tx |= BIT(NVLTLC_TX_DEBUG_TP_CNTR_CTRL_ENTX1);
data_rx |= BIT(NVLTLC_RX_DEBUG_TP_CNTR_CTRL_ENRX0);
data_rx |= BIT(NVLTLC_RX_DEBUG_TP_CNTR_CTRL_ENRX1);
} else {
tdev->is_tp_cntr_running = false;
data_tx &= ~(BIT(NVLTLC_TX_DEBUG_TP_CNTR_CTRL_ENTX0));
data_tx &= ~(BIT(NVLTLC_TX_DEBUG_TP_CNTR_CTRL_ENTX1));
data_rx &= ~(BIT(NVLTLC_RX_DEBUG_TP_CNTR_CTRL_ENRX0));
data_rx &= ~(BIT(NVLTLC_RX_DEBUG_TP_CNTR_CTRL_ENRX1));
}
nvlw_nvltlc_writel(tdev, NVLTLC_TX_DEBUG_TP_CNTR_CTRL, data_tx);
nvlw_nvltlc_writel(tdev, NVLTLC_RX_DEBUG_TP_CNTR_CTRL, data_rx);
return ret;
}
/* Configure the TLC throughput counters 0 and 1 for both RX and TX
* Configuration for cntr0-
* 1. Setting CTRL0.UNIT of traffic the counter will counter to "PACKETS"
* 2. Setting CTRL0.VCSETFILTERMODE to count both VCSET0 and 1
* Configuration for cntr1-
* 1. Setting CTRL0.UNIT of traffic the counter will counter to "CYCLES"
* 2. Setting CTRL0.VCSETFILTERMODE to count both VCSET0 and 1
* 3. Setting CTRL0.FLITFILTER to count idle cycles
*/
int t19x_nvlink_config_tp_counters(struct tnvlink_dev *tdev)
{
int ret = 0;
u32 data;
if(tdev == NULL) {
nvlink_err("Invalid tnvlink_dev pointer");
return -EINVAL;
}
/* TX tp counter cntr0 */
data = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0);
data &= ~(NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0_UNIT_M() |
NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0_VCSETFILTERMODE_M());
data |= NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0_UNIT_F(
NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0_UNIT_PACKETS);
data |= NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0_VCSETFILTERMODE_F(
NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0_VCSETFILTERMODE_INIT);
nvlw_nvltlc_writel(tdev, NVLTLC_TX_DEBUG_TP_CNTR0_CTRL_0, data);
/* TX tp counter cntr1 */
data = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0);
data &= ~(NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_UNIT_M() |
NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_VCSETFILTERMODE_M() |
NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_FLITFILTER_M());
data |= NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_UNIT_F(
NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_UNIT_CYCLES);
data |= NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_VCSETFILTERMODE_F(
NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_VCSETFILTERMODE_INIT);
data |= NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_FLITFILTER_F(
NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0_FLITFILTER_IDLE);
nvlw_nvltlc_writel(tdev, NVLTLC_TX_DEBUG_TP_CNTR1_CTRL_0, data);
/* RX tp counter cntr0 */
data = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0);
data &= ~(NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0_UNIT_M() |
NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0_VCSETFILTERMODE_M());
data |= NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0_UNIT_F(
NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0_UNIT_PACKETS);
data |= NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0_VCSETFILTERMODE_F(
NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0_VCSETFILTERMODE_INIT);
nvlw_nvltlc_writel(tdev, NVLTLC_RX_DEBUG_TP_CNTR0_CTRL_0, data);
/* RX tp counter cntr1 */
data = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0);
data &= ~(NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_UNIT_M() |
NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_VCSETFILTERMODE_M() |
NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_FLITFILTER_M());
data |= NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_UNIT_F(
NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_UNIT_CYCLES);
data |= NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_VCSETFILTERMODE_F(
NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_VCSETFILTERMODE_INIT);
data |= NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_FLITFILTER_F(
NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0_FLITFILTER_IDLE);
nvlw_nvltlc_writel(tdev, NVLTLC_RX_DEBUG_TP_CNTR1_CTRL_0, data);
return ret;
};
/* Get the value of TLC throughput counters/controls for both RX and TX */
int t19x_nvlink_get_tp_counters(struct tnvlink_dev *tdev, u64 *tx0cnt,
u64 *tx1cnt, u64 *rx0cnt, u64 *rx1cnt)
{
int ret = 0;
u64 tx0cntlo, tx1cntlo, rx0cntlo, rx1cntlo;
u64 tx0cnthi, tx1cnthi, rx0cnthi, rx1cnthi;
if(tdev == NULL) {
nvlink_err("Invalid tnvlink_dev pointer");
return -EINVAL;
}
/* TX tp counters */
tx0cntlo = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR0_LO);
tx0cnthi = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR0_HI);
tx1cntlo = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR1_LO);
tx1cnthi = nvlw_nvltlc_readl(tdev, NVLTLC_TX_DEBUG_TP_CNTR1_HI);
/* RX tp counters */
rx0cntlo = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR0_LO);
rx0cnthi = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR0_HI);
rx1cntlo = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR1_LO);
rx1cnthi = nvlw_nvltlc_readl(tdev, NVLTLC_RX_DEBUG_TP_CNTR1_HI);
*tx0cnt = ((u64)tx0cnthi << 32) | tx0cntlo;
*tx1cnt = ((u64)tx1cnthi << 32) | tx1cntlo;
*rx0cnt = ((u64)rx0cnthi << 32) | rx0cntlo;
*rx1cnt = ((u64)rx1cnthi << 32) | rx1cntlo;
nvlink_dbg("tx packets : %llu\n", *tx0cnt);
nvlink_dbg("tx idle cycles : %llu\n", *tx1cnt);
nvlink_dbg("rx packets : %llu\n", *rx0cnt);
nvlink_dbg("rx idle cycles : %llu\n", *rx1cnt);
return ret;
}