/* * QSPI driver for NVIDIA's Tegra210 QUAD SPI Controller. * * Copyright (c) 2013-2021, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define QSPI_COMMAND1 0x000 #define QSPI_BIT_LENGTH(x) (((x) & 0x1f) << 0) #define QSPI_PACKED BIT(5) #define QSPI_INTERFACE_WIDTH(x) (((x) & 0x03) << 7) #define QSPI_INTERFACE_WIDTH_MASK (0x03 << 7) #define QSPI_SDR_DDR_SEL BIT(9) #define QSPI_TX_EN BIT(11) #define QSPI_RX_EN BIT(12) #define QSPI_CS_SW_VAL BIT(20) #define QSPI_CS_SW_HW BIT(21) #define QSPI_CONTROL_MODE_0 (0 << 28) #define QSPI_CONTROL_MODE_3 (3 << 28) #define QSPI_CONTROL_MODE_MASK (3 << 28) #define QSPI_M_S BIT(30) #define QSPI_PIO BIT(31) #define QSPI_COMMAND2 0x004 #define QSPI_RX_TAP_DELAY(x) (((x) & 0xFF) << 0) #define QSPI_TX_TAP_DELAY(x) (((x) & 0x1F) << 10) #define QSPI_RX_EXT_TAP_DELAY(x) (((x) & 0xFF) << 24) #define QSPI_SETUP_HOLD(setup, hold) \ (((setup) << 4) | ((hold) & 0x0F)) #define QSPI_CS_SETUP_HOLD(reg, cs, val) \ ((((val) & 0xFFu) << ((cs) * 8)) | \ ((reg) & ~(0xFFu << ((cs) * 8)))) #define QSPI_CS_TIMING1 0x008 #define QSPI_CS_TIMING2 0x00C #define QSPI_CS_TIMING3 0x198 #define CYCLES_BETWEEN_PACKETS_0(x) (((x) & 0x1F) << 0) #define CS_ACTIVE_BETWEEN_PACKETS_0 BIT(5) #define QSPI_SET_CYCLES_BETWEEN_PACKETS(reg, cs, val) \ (reg = (((val) & 0xF) << ((cs) * 8)) | \ ((reg) & ~(0xF << ((cs) * 8)))) #define QSPI_HALF_FULL_CYCLE_SAMPLE BIT(31) #define QSPI_TRANS_STATUS 0x010 #define QSPI_BLK_CNT(val) (((val) >> 0) & 0xFFFF) #define QSPI_RDY BIT(30) #define QSPI_FIFO_STATUS 0x014 #define QSPI_RX_FIFO_EMPTY BIT(0) #define QSPI_RX_FIFO_FULL BIT(1) #define QSPI_TX_FIFO_EMPTY BIT(2) #define QSPI_TX_FIFO_FULL BIT(3) #define QSPI_RX_FIFO_UNF BIT(4) #define QSPI_RX_FIFO_OVF BIT(5) #define QSPI_TX_FIFO_UNF BIT(6) #define QSPI_TX_FIFO_OVF BIT(7) #define QSPI_ERR BIT(8) #define QSPI_TX_FIFO_FLUSH BIT(14) #define QSPI_RX_FIFO_FLUSH BIT(15) #define QSPI_TX_FIFO_EMPTY_COUNT(val) (((val) >> 16) & 0x7F) #define QSPI_RX_FIFO_FULL_COUNT(val) (((val) >> 23) & 0x7F) #define QSPI_FIFO_ERROR (QSPI_RX_FIFO_UNF | \ QSPI_RX_FIFO_OVF | QSPI_TX_FIFO_UNF | QSPI_TX_FIFO_OVF) #define QSPI_FIFO_EMPTY (QSPI_RX_FIFO_EMPTY | \ QSPI_TX_FIFO_EMPTY) #define QSPI_TX_DATA 0x018 #define QSPI_MISC_REG 0x194 #define QSPI_NUM_DUMMY_CYCLE(x) (((x) & 0xFF) << 0) #define QSPI_GLOBAL_CONFIG 0x1a4 #define QSPI_RX_DATA 0x01C #define QSPI_DMA_CTL 0x020 #define QSPI_TX_TRIG(n) (((n) & 0x3) << 15) #define QSPI_TX_TRIG_1 QSPI_TX_TRIG(0) #define QSPI_TX_TRIG_4 QSPI_TX_TRIG(1) #define QSPI_TX_TRIG_8 QSPI_TX_TRIG(2) #define QSPI_TX_TRIG_16 QSPI_TX_TRIG(3) #define QSPI_RX_TRIG(n) (((n) & 0x3) << 19) #define QSPI_RX_TRIG_1 QSPI_RX_TRIG(0) #define QSPI_RX_TRIG_4 QSPI_RX_TRIG(1) #define QSPI_RX_TRIG_8 QSPI_RX_TRIG(2) #define QSPI_RX_TRIG_16 QSPI_RX_TRIG(3) #define QSPI_IE_TX BIT(28) #define QSPI_IE_RX BIT(29) #define QSPI_DMA BIT(31) #define QSPI_DMA_EN QSPI_DMA #define QSPI_DMA_BLK 0x024 #define QSPI_DMA_BLK_SET(x) (((x) & 0xFFFF) << 0) #define QSPI_TX_FIFO 0x108 #define QSPI_RX_FIFO 0x188 #define QSPI_INTR_MASK 0x18c #define QSPI_INTR_RX_FIFO_UNF_MASK BIT(25) #define QSPI_INTR_RX_FIFO_OVF_MASK BIT(26) #define QSPI_INTR_TX_FIFO_UNF_MASK BIT(27) #define QSPI_INTR_TX_FIFO_OVF_MASK BIT(28) #define QSPI_INTR_RDY_MASK BIT(29) #define QSPI_INTR_RX_TX_FIFO_ERR (QSPI_INTR_RX_FIFO_UNF_MASK | \ QSPI_INTR_RX_FIFO_OVF_MASK | \ QSPI_INTR_TX_FIFO_UNF_MASK | \ QSPI_INTR_TX_FIFO_OVF_MASK) #define QSPI_CMB_SEQ_CMD 0x19c #define QSPI_COMMAND_VALUE_SET(X) (((x) & 0xFF) << 0) #define QSPI_CMB_SEQ_CMD_CFG 0x1a0 #define QSPI_COMMAND_X1_X2_X4(x) (((x) & 0x3) << 13) #define QSPI_COMMAND_X1_X2_X4_MASK (0x03 << 13) #define QSPI_COMMAND_SDR_DDR BIT(12) #define QSPI_COMMAND_SIZE_SET(x) (((x) & 0xFF) << 0) #define QSPI_GLOBAL_CONFIG 0x1a4 #define QSPI_CMB_SEQ_EN BIT(0) #define QSPI_CMB_SEQ_ADDR 0x1a8 #define QSPI_ADDRESS_VALUE_SET(X) (((x) & 0xFFFF) << 0) #define QSPI_CMB_SEQ_ADDR_CFG 0x1ac #define QSPI_ADDRESS_X1_X2_X4(x) (((x) & 0x3) << 13) #define QSPI_ADDRESS_X1_X2_X4_MASK (0x03 << 13) #define QSPI_ADDRESS_SDR_DDR BIT(12) #define QSPI_ADDRESS_SIZE_SET(x) (((x) & 0xFF) << 0) #define DATA_DIR_TX BIT(0) #define DATA_DIR_RX BIT(1) #define QSPI_DMA_TIMEOUT (msecs_to_jiffies(10000)) #define DEFAULT_SPI_DMA_BUF_LEN (64 * 1024) #define TX_FIFO_EMPTY_COUNT_MAX SPI_TX_FIFO_EMPTY_COUNT(0x40) #define RX_FIFO_FULL_COUNT_ZERO SPI_RX_FIFO_FULL_COUNT(0) #define MAX_PROD_NAME 15 /* * NOTE: Actual chip has only one CS. Below is WAR to enable * spidev and mtd layer register same time. */ #define MAX_CHIP_SELECT 2 #define QSPI_FIFO_DEPTH 64 #define QSPI_FIFO_FLUSH_MAX_DELAY 2000 #define CMD_TRANSFER 0 #define ADDR_TRANSFER 1 #define DATA_TRANSFER 2 static int rx_tap_delay = -1; module_param(rx_tap_delay, int, 0444); static int tx_tap_delay = -1; module_param(tx_tap_delay, int, 0444); struct tegra_qspi_data { struct device *dev; struct spi_master *master; /* Lock fro register access */ spinlock_t lock; struct clk *clk; struct clk *sdr_ddr_clk; struct reset_control *rstc; void __iomem *base; phys_addr_t phys; unsigned irq; bool enable_dma_support; bool clock_always_on; bool is_ddr_mode; u8 bus_clk_div; u32 qspi_max_frequency; u32 cur_speed; struct spi_device *cur_qspi; unsigned cur_pos; unsigned cur_len; unsigned words_per_32bit; unsigned bytes_per_word; unsigned curr_dma_words; unsigned cur_direction; unsigned cur_rx_pos; unsigned cur_tx_pos; unsigned dma_buf_size; unsigned max_buf_size; bool is_curr_dma_xfer; bool is_hw_based_cs; bool dcycle_non_cmbseq_mode; struct completion rx_dma_complete; struct completion tx_dma_complete; u32 tx_status; u32 rx_status; u32 status_reg; bool is_packed; unsigned long packed_size; u32 command1_reg; u32 dma_control_reg; u32 def_command1_reg; u32 def_command2_reg; u32 qspi_cs_timing; struct completion xfer_completion; struct spi_transfer *curr_xfer; struct dma_chan *rx_dma_chan; u32 *rx_dma_buf; dma_addr_t rx_dma_phys; struct dma_async_tx_descriptor *rx_dma_desc; struct dma_chan *tx_dma_chan; u32 *tx_dma_buf; dma_addr_t tx_dma_phys; struct dma_async_tx_descriptor *tx_dma_desc; struct tegra_prod *prod_list; int qspi_enable_cmbseq_mode; #ifdef QSPI_BRINGUP_BUILD int qspi_force_unpacked_mode; int qspi_enable_prod_override; int qspi_force_pio_mode; int qspi_force_dma_mode; bool qspi_force_bus_speed; #endif }; static int tegra_qspi_runtime_suspend(struct device *dev); static int tegra_qspi_runtime_resume(struct device *dev); static int tegra_qspi_clk_enable(struct tegra_qspi_data *tqspi); static void tegra_qspi_clk_disable(struct tegra_qspi_data *tqspi); static struct tegra_qspi_device_controller_data *tegra_qspi_get_cdata_dt(struct spi_device *spi); static void set_best_clk_source(struct tegra_qspi_data *tqspi, unsigned long rate); static unsigned long tegra_qspi_readl(struct tegra_qspi_data *tqspi, unsigned long reg) { return readl(tqspi->base + reg); } static void tegra_qspi_writel(struct tegra_qspi_data *tqspi, unsigned long val, unsigned long reg) { writel(val, tqspi->base + reg); /* Read back register to make sure that register writes completed */ if (reg != QSPI_TX_FIFO) readl(tqspi->base + QSPI_COMMAND1); } #ifdef QSPI_BRINGUP_BUILD static ssize_t force_unpacked_mode_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); if (tqspi && count) { tqspi->qspi_force_unpacked_mode = ((buf[0] - '0') > 0); return count; } return -ENODEV; } static ssize_t force_unpacked_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); return sprintf(buf, "%d", tqspi->qspi_force_unpacked_mode); } static DEVICE_ATTR(qspi_force_unpacked_mode, 0644, force_unpacked_mode_show, force_unpacked_mode_set); static ssize_t force_cmbseq_mode_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); if (tqspi && count) { tqspi->qspi_enable_cmbseq_mode = ((buf[0] - '0') > 0); return count; } return -ENODEV; } static ssize_t force_cmbseq_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); return sprintf(buf, "%d", tqspi->qspi_enable_cmbseq_mode); return -ENODEV; } static DEVICE_ATTR(qspi_enable_cmbseq_mode, 0644, force_cmbseq_mode_show, force_cmbseq_mode_set); static ssize_t enable_prod_override_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); if (tqspi && count) { tqspi->qspi_enable_prod_override = ((buf[0] - '0') > 0); return count; } return -ENODEV; } static ssize_t enable_prod_override_show(struct device *dev, struct device_attribute *attr, char *buf) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); return sprintf(buf, "%d\n", tqspi->qspi_enable_prod_override); return -ENODEV; } static DEVICE_ATTR(qspi_enable_prod_override, 0644, enable_prod_override_show, enable_prod_override_set); static ssize_t enable_clk_always_on_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); if (tqspi && count) { tqspi->clock_always_on = ((buf[0] - '0') > 0); if (tqspi->clock_always_on) tegra_qspi_clk_enable(tqspi); else tegra_qspi_clk_disable(tqspi); return count; } return -ENODEV; } static ssize_t enable_clk_always_on_show(struct device *dev, struct device_attribute *attr, char *buf) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); return sprintf(buf, "%d\n", tqspi->clock_always_on); return -ENODEV; } static DEVICE_ATTR(qspi_enable_clk_always_on, 0644, enable_clk_always_on_show, enable_clk_always_on_set); static ssize_t bus_speed_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; int ret = 0; long speed; u32 actual_speed; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); if (tqspi && count) { ret = kstrtol(buf, 10, &speed); if (ret != 0) return -EINVAL; if (speed == 0) { tqspi->qspi_force_bus_speed = false; return count; } if (speed > tqspi->qspi_max_frequency/tqspi->bus_clk_div) return -EINVAL; if (tqspi->cur_speed == speed) return count; tqspi->cur_speed = speed; set_best_clk_source(tqspi, speed); ret = clk_set_rate(tqspi->clk, speed*tqspi->bus_clk_div); if (ret < 0) { dev_err(tqspi->dev, "Failed to set QSPI clock freq: %d\n", ret); return -EINVAL; } actual_speed = clk_get_rate(tqspi->clk)/tqspi->bus_clk_div; if (actual_speed <= 0) return -EINVAL; ret = clk_set_rate(tqspi->sdr_ddr_clk, actual_speed); if (ret < 0) { dev_err(tqspi->dev, "Failed to set QSPI OUT clock freq: %d\n", ret); return -EINVAL; } tqspi->qspi_force_bus_speed = true; return count; } return -ENODEV; } static ssize_t bus_speed_show(struct device *dev, struct device_attribute *attr, char *buf) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; u32 actual_qspi_speed; u32 actual_qspi_out_speed; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); actual_qspi_speed = clk_get_rate(tqspi->clk); actual_qspi_out_speed = clk_get_rate(tqspi->sdr_ddr_clk); return sprintf(buf, "qspi:%d, qspi_out:%d\n", actual_qspi_speed, actual_qspi_out_speed); return -ENODEV; } static DEVICE_ATTR(qspi_bus_speed, 0644, bus_speed_show, bus_speed_set); static ssize_t force_pio_mode_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); if (tqspi && count) { tqspi->qspi_force_pio_mode = ((buf[0] - '0') > 0); return count; } return -ENODEV; } static ssize_t force_pio_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); return sprintf(buf, "%d\n", tqspi->qspi_force_pio_mode); return -ENODEV; } static DEVICE_ATTR(qspi_force_pio_mode, 0644, force_pio_mode_show, force_pio_mode_set); static ssize_t force_dma_mode_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); if (tqspi && count) { tqspi->qspi_force_dma_mode = ((buf[0] - '0') > 0); return count; } return -ENODEV; } static ssize_t force_dma_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi; if (!master) return -ENODEV; tqspi = spi_master_get_devdata(master); return sprintf(buf, "%d\n", tqspi->qspi_force_dma_mode); return -ENODEV; } static DEVICE_ATTR(qspi_force_dma_mode, 0644, force_dma_mode_show, force_dma_mode_set); static struct attribute *tegra_qspi_attrs[] = { &dev_attr_qspi_force_unpacked_mode.attr, &dev_attr_qspi_enable_cmbseq_mode.attr, &dev_attr_qspi_enable_prod_override.attr, &dev_attr_qspi_enable_clk_always_on.attr, &dev_attr_qspi_bus_speed.attr, &dev_attr_qspi_force_pio_mode.attr, &dev_attr_qspi_force_dma_mode.attr, NULL, }; ATTRIBUTE_GROUPS(tegra_qspi); #endif #ifdef QSPI_DUMP_REGISTERS static void tegra_qspi_dump_regs(const char *heading, struct tegra_qspi_data *tqspi) { u32 command1_reg; u32 fifo_status_reg, misc_reg, gl_config_reg; u32 dma_ctrl_reg, dma_blk_reg, intr_mask_reg; u32 trans_status_reg; u32 cmd_config; u32 addr_config; u32 addr_value; u32 cmd_value; cmd_value = tegra_qspi_readl(tqspi, QSPI_CMB_SEQ_CMD); addr_value = tegra_qspi_readl(tqspi, QSPI_CMB_SEQ_ADDR); cmd_config = tegra_qspi_readl(tqspi, QSPI_CMB_SEQ_CMD_CFG); addr_config = tegra_qspi_readl(tqspi, QSPI_CMB_SEQ_ADDR_CFG); command1_reg = tegra_qspi_readl(tqspi, QSPI_COMMAND1); fifo_status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); dma_ctrl_reg = tegra_qspi_readl(tqspi, QSPI_DMA_CTL); trans_status_reg = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS); dma_blk_reg = tegra_qspi_readl(tqspi, QSPI_DMA_BLK); intr_mask_reg = tegra_qspi_readl(tqspi, QSPI_INTR_MASK); misc_reg = tegra_qspi_readl(tqspi, QSPI_MISC_REG); gl_config_reg = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG); if (heading) dev_info(tqspi->dev, "%s\n", heading); dev_err(tqspi->dev, "CMD_0: \t\t\t0x%08lx\n", command1_reg); dev_err(tqspi->dev, "FIFO_STS: \t\t\t0x%08x\n", fifo_status_reg); dev_err(tqspi->dev, "DMA_CTL: \t\t\t0x%08lx\n", dma_ctrl_reg); dev_err(tqspi->dev, "TRANS_STS: \t\t\t0x%08lx\n", trans_status_reg); dev_err(tqspi->dev, "GLOBAL_CONFIG: \t\t\t0x%08lx\n", gl_config_reg); dev_err(tqspi->dev, "DMA_BLK: \t\t\t0x%08lx\n", dma_blk_reg); dev_err(tqspi->dev, "INTR: \t\t\t0x%08lx\n", intr_mask_reg); dev_err(tqspi->dev, "MISC-REG: \t\t\t0x%08lx\n", misc_reg); dev_err(tqspi->dev, "CMD_VAl: \t\t\t0x%08lx\n", cmd_value); dev_err(tqspi->dev, "ADR_VAL: \t\t\t0x%08lx\n", addr_value); dev_err(tqspi->dev, "CMD_CFG: \t\t\t0x%08lx\n", cmd_config); dev_err(tqspi->dev, "ADR_CFG: \t\t\t0x%08lx\n", addr_config); #else static void tegra_qspi_dump_regs(const char *heading, struct tegra_qspi_data *tqspi) {} #endif static void tegra_qspi_clear_status(struct tegra_qspi_data *tqspi) { unsigned long val; /* Write 1 to clear status register */ val = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS); tegra_qspi_writel(tqspi, val, QSPI_TRANS_STATUS); val = tegra_qspi_readl(tqspi, QSPI_INTR_MASK); if (!(val & QSPI_INTR_RDY_MASK)) { val |= (QSPI_INTR_RDY_MASK | QSPI_INTR_RX_TX_FIFO_ERR); tegra_qspi_writel(tqspi, val, QSPI_INTR_MASK); } /* Clear fifo status error if any */ val = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); if (val & QSPI_ERR) tegra_qspi_writel(tqspi, QSPI_ERR | QSPI_FIFO_ERROR, QSPI_FIFO_STATUS); } static int check_and_clear_fifo(struct tegra_qspi_data *tqspi) { unsigned long status; int cnt = QSPI_FIFO_FLUSH_MAX_DELAY; /* Make sure that Rx and Tx fifo are empty */ status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); if ((status & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY) return 0; /* flush the fifo */ status |= (QSPI_RX_FIFO_FLUSH | QSPI_TX_FIFO_FLUSH); tegra_qspi_writel(tqspi, status, QSPI_FIFO_STATUS); do { status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); if ((status & QSPI_FIFO_EMPTY) == QSPI_FIFO_EMPTY) return 0; udelay(1); } while (cnt--); dev_err(tqspi->dev, "Failed to flush Rx/Tx fifo(status 0x%08lx)\n", status); return -EIO; } static unsigned tegra_qspi_calculate_curr_xfer_param( struct spi_device *spi, struct tegra_qspi_data *tqspi, struct spi_transfer *t) { unsigned remain_len = t->len - tqspi->cur_pos; unsigned max_word; unsigned bits_per_word; unsigned max_len; unsigned total_fifo_words; bits_per_word = t->bits_per_word ? t->bits_per_word : spi->bits_per_word; tqspi->bytes_per_word = (bits_per_word - 1) / 8 + 1; #ifdef QSPI_BRINGUP_BUILD if (!tqspi->qspi_force_unpacked_mode && (bits_per_word % 8 == 0) && (t->len > 3)) { tqspi->is_packed = true; tqspi->words_per_32bit = 32 / bits_per_word; } else { tqspi->is_packed = false; tqspi->words_per_32bit = 1; } #else if ((bits_per_word == 8 || bits_per_word == 16) && (t->len > 3)) { tqspi->is_packed = true; tqspi->words_per_32bit = 32 / bits_per_word; } else { tqspi->is_packed = false; tqspi->words_per_32bit = 1; } #endif if (tqspi->is_packed) { max_len = min(remain_len, tqspi->max_buf_size); tqspi->curr_dma_words = max_len / tqspi->bytes_per_word; total_fifo_words = (max_len + 3) / 4; #ifdef QSPI_BRINGUP_BUILD if (tqspi->qspi_force_pio_mode) if (tqspi->curr_dma_words > QSPI_FIFO_DEPTH) { tqspi->curr_dma_words = QSPI_FIFO_DEPTH; total_fifo_words = QSPI_FIFO_DEPTH; } #endif } else { max_word = (remain_len - 1) / tqspi->bytes_per_word + 1; max_word = min(max_word, tqspi->max_buf_size / 4); tqspi->curr_dma_words = max_word; total_fifo_words = max_word; #ifdef QSPI_BRINGUP_BUILD if (tqspi->qspi_force_pio_mode) if (tqspi->curr_dma_words > QSPI_FIFO_DEPTH) { tqspi->curr_dma_words = QSPI_FIFO_DEPTH; total_fifo_words = QSPI_FIFO_DEPTH; } #endif } return total_fifo_words; } static unsigned tegra_qspi_fill_tx_fifo_from_client_txbuf( struct tegra_qspi_data *tqspi, struct spi_transfer *t) { unsigned nbytes; unsigned tx_empty_count; unsigned long fifo_status; unsigned max_n_32bit; unsigned i, count; unsigned long x; unsigned int written_words; unsigned fifo_words_left; u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos; fifo_status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); tx_empty_count = QSPI_TX_FIFO_EMPTY_COUNT(fifo_status); if (tqspi->is_packed) { fifo_words_left = tx_empty_count * tqspi->words_per_32bit; written_words = min(fifo_words_left, tqspi->curr_dma_words); nbytes = written_words * tqspi->bytes_per_word; max_n_32bit = DIV_ROUND_UP(nbytes, 4); for (count = 0; count < max_n_32bit; count++) { x = 0; for (i = 0; (i < 4) && nbytes; i++, nbytes--) x |= (unsigned long) (((unsigned)(*tx_buf++)) << (i * 8)); tegra_qspi_writel(tqspi, x, QSPI_TX_FIFO); } } else { max_n_32bit = min(tqspi->curr_dma_words, tx_empty_count); written_words = max_n_32bit; nbytes = written_words * tqspi->bytes_per_word; for (count = 0; count < max_n_32bit; count++) { x = 0; for (i = 0; nbytes && (i < tqspi->bytes_per_word); i++, nbytes--) x |= (unsigned long) (((unsigned)(*tx_buf++)) << (i * 8)); tegra_qspi_writel(tqspi, x, QSPI_TX_FIFO); } } tqspi->cur_tx_pos += written_words * tqspi->bytes_per_word; return written_words; } static unsigned int tegra_qspi_read_rx_fifo_to_client_rxbuf( struct tegra_qspi_data *tqspi, struct spi_transfer *t) { unsigned rx_full_count; unsigned long fifo_status; unsigned i, count; unsigned long x; unsigned int read_words = 0; unsigned len; u8 *rx_buf = (u8 *)t->rx_buf + tqspi->cur_rx_pos; fifo_status = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); rx_full_count = QSPI_RX_FIFO_FULL_COUNT(fifo_status); if (tqspi->is_packed) { len = tqspi->curr_dma_words * tqspi->bytes_per_word; for (count = 0; count < rx_full_count; count++) { x = tegra_qspi_readl(tqspi, QSPI_RX_FIFO); for (i = 0; len && (i < 4); i++, len--) *rx_buf++ = (x >> i * 8) & 0xFF; } tqspi->cur_rx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word; read_words += tqspi->curr_dma_words; } else { unsigned int bits_per_word; bits_per_word = t->bits_per_word ? t->bits_per_word : tqspi->cur_qspi->bits_per_word; for (count = 0; count < rx_full_count; count++) { x = tegra_qspi_readl(tqspi, QSPI_RX_FIFO); for (i = 0; (i < tqspi->bytes_per_word); i++) *rx_buf++ = (x >> (i * 8)) & 0xFF; } tqspi->cur_rx_pos += rx_full_count * tqspi->bytes_per_word; read_words += rx_full_count; } return read_words; } static void tegra_qspi_copy_client_txbuf_to_qspi_txbuf( struct tegra_qspi_data *tqspi, struct spi_transfer *t) { unsigned len; /* Make the dma buffer to read by cpu */ dma_sync_single_for_cpu(tqspi->dev, tqspi->tx_dma_phys, tqspi->dma_buf_size, DMA_TO_DEVICE); if (tqspi->is_packed) { len = tqspi->curr_dma_words * tqspi->bytes_per_word; memcpy(tqspi->tx_dma_buf, t->tx_buf + tqspi->cur_pos, len); } else { unsigned int i; unsigned int count; u8 *tx_buf = (u8 *)t->tx_buf + tqspi->cur_tx_pos; unsigned consume = tqspi->curr_dma_words * tqspi->bytes_per_word; for (count = 0; count < tqspi->curr_dma_words; count++) { u32 x = 0; for (i = 0; consume && (i < tqspi->bytes_per_word); i++, consume--) x |= ((*tx_buf++) << i * 8); tqspi->tx_dma_buf[count] = x; } } tqspi->cur_tx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word; /* Make the dma buffer to read by dma */ dma_sync_single_for_device(tqspi->dev, tqspi->tx_dma_phys, tqspi->dma_buf_size, DMA_TO_DEVICE); } static void tegra_qspi_copy_qspi_rxbuf_to_client_rxbuf( struct tegra_qspi_data *tqspi, struct spi_transfer *t) { unsigned len; /* Make the dma buffer to read by cpu */ dma_sync_single_for_cpu(tqspi->dev, tqspi->rx_dma_phys, tqspi->dma_buf_size, DMA_FROM_DEVICE); if (tqspi->is_packed) { len = tqspi->curr_dma_words * tqspi->bytes_per_word; memcpy(t->rx_buf + tqspi->cur_rx_pos, tqspi->rx_dma_buf, len); } else { unsigned int i; unsigned int count; unsigned char *rx_buf = t->rx_buf + tqspi->cur_rx_pos; unsigned int x; unsigned int rx_mask, bits_per_word; bits_per_word = t->bits_per_word ? t->bits_per_word : tqspi->cur_qspi->bits_per_word; rx_mask = (1ULL << bits_per_word) - 1; for (count = 0; count < tqspi->curr_dma_words; count++) { x = tqspi->rx_dma_buf[count]; x &= rx_mask; for (i = 0; (i < tqspi->bytes_per_word); i++) *rx_buf++ = (x >> (i * 8)) & 0xFF; } } tqspi->cur_rx_pos += tqspi->curr_dma_words * tqspi->bytes_per_word; /* Make the dma buffer to read by dma */ dma_sync_single_for_device(tqspi->dev, tqspi->rx_dma_phys, tqspi->dma_buf_size, DMA_FROM_DEVICE); } static void tegra_qspi_dma_complete(void *args) { struct completion *dma_complete = args; complete(dma_complete); } static int tegra_qspi_start_tx_dma(struct tegra_qspi_data *tqspi, int len) { reinit_completion(&tqspi->tx_dma_complete); tqspi->tx_dma_desc = dmaengine_prep_slave_single(tqspi->tx_dma_chan, tqspi->tx_dma_phys, len, DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!tqspi->tx_dma_desc) { dev_err(tqspi->dev, "Failed to get Tx DMA Desc\n"); return -EIO; } tqspi->tx_dma_desc->callback = tegra_qspi_dma_complete; tqspi->tx_dma_desc->callback_param = &tqspi->tx_dma_complete; dmaengine_submit(tqspi->tx_dma_desc); dma_async_issue_pending(tqspi->tx_dma_chan); return 0; } static int tegra_qspi_start_rx_dma(struct tegra_qspi_data *tqspi, int len) { reinit_completion(&tqspi->rx_dma_complete); tqspi->rx_dma_desc = dmaengine_prep_slave_single(tqspi->rx_dma_chan, tqspi->rx_dma_phys, len, DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); if (!tqspi->rx_dma_desc) { dev_err(tqspi->dev, "Failed to get Rx Dma Desc\n"); return -EIO; } tqspi->rx_dma_desc->callback = tegra_qspi_dma_complete; tqspi->rx_dma_desc->callback_param = &tqspi->rx_dma_complete; dmaengine_submit(tqspi->rx_dma_desc); dma_async_issue_pending(tqspi->rx_dma_chan); return 0; } static int tegra_qspi_start_dma_based_transfer( struct tegra_qspi_data *tqspi, struct spi_transfer *t) { unsigned long val, command1; unsigned int len, intr_mask; int ret = 0, maxburst; struct dma_slave_config dma_sconfig; /* Make sure that Rx and Tx fifo are empty */ ret = check_and_clear_fifo(tqspi); if (ret != 0) return ret; /* TX_EN/RX_EN should not be set here */ command1 = tqspi->command1_reg; tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1); val = QSPI_DMA_BLK_SET(tqspi->curr_dma_words - 1); tegra_qspi_writel(tqspi, val, QSPI_DMA_BLK); if (tqspi->is_packed) len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4; else len = tqspi->curr_dma_words * 4; val = 0; /* Set attention level based on length of transfer */ if (len & 0xF) { val |= QSPI_TX_TRIG_1 | QSPI_RX_TRIG_1; maxburst = 1; } else if (((len) >> 4) & 0x1) { val |= QSPI_TX_TRIG_4 | QSPI_RX_TRIG_4; maxburst = 4; } else { val |= QSPI_TX_TRIG_8 | QSPI_RX_TRIG_8; maxburst = 8; } if ((tqspi->cur_direction & DATA_DIR_TX) || (tqspi->cur_direction & DATA_DIR_RX)) { intr_mask = tegra_qspi_readl(tqspi, QSPI_INTR_MASK); intr_mask &= ~(QSPI_INTR_RDY_MASK | QSPI_INTR_RX_TX_FIFO_ERR); tegra_qspi_writel(tqspi, intr_mask, QSPI_INTR_MASK); } tegra_qspi_writel(tqspi, val, QSPI_DMA_CTL); tqspi->dma_control_reg = val; if (tqspi->cur_direction & DATA_DIR_TX) { command1 |= QSPI_TX_EN; dma_sconfig.dst_addr = tqspi->phys + QSPI_TX_FIFO; dma_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; dma_sconfig.dst_maxburst = maxburst; dmaengine_slave_config(tqspi->tx_dma_chan, &dma_sconfig); tegra_qspi_copy_client_txbuf_to_qspi_txbuf(tqspi, t); ret = tegra_qspi_start_tx_dma(tqspi, len); if (ret < 0) { dev_err(tqspi->dev, "Failed to start Tx DMA: %d\n", ret); return ret; } } if (tqspi->cur_direction & DATA_DIR_RX) { command1 |= QSPI_RX_EN; /* Make the dma buffer to read by dma */ dma_sync_single_for_device(tqspi->dev, tqspi->rx_dma_phys, tqspi->dma_buf_size, DMA_FROM_DEVICE); dma_sconfig.src_addr = tqspi->phys + QSPI_RX_FIFO; dma_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; dma_sconfig.src_maxburst = maxburst; dmaengine_slave_config(tqspi->rx_dma_chan, &dma_sconfig); ret = tegra_qspi_start_rx_dma(tqspi, len); if (ret < 0) { dev_err(tqspi->dev, "Failed to start Rx DMA: %d\n", ret); if (tqspi->cur_direction & DATA_DIR_TX) dmaengine_terminate_all(tqspi->tx_dma_chan); return ret; } } tqspi->is_curr_dma_xfer = true; tqspi->dma_control_reg = val; val |= QSPI_DMA_EN; /* TX_EN/RX_EN need to set after DMA_BLK to avoid spurious interrupt */ tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1); tegra_qspi_dump_regs("DMA Transfer started", tqspi); tegra_qspi_writel(tqspi, val, QSPI_DMA_CTL); return ret; } static int tegra_qspi_start_cpu_based_transfer( struct tegra_qspi_data *tqspi, struct spi_transfer *t) { unsigned long val; unsigned int cur_words, intr_mask; int ret = 0; /* Make sure Tx/Rx fifo is empty */ ret = check_and_clear_fifo(tqspi); if (ret != 0) return ret; /* TX_EN/RX_EN should not be set here */ tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1); if (tqspi->cur_direction & DATA_DIR_TX) cur_words = tegra_qspi_fill_tx_fifo_from_client_txbuf(tqspi, t); else cur_words = tqspi->curr_dma_words; val = QSPI_DMA_BLK_SET(cur_words - 1); tegra_qspi_writel(tqspi, val, QSPI_DMA_BLK); val = 0; if ((tqspi->cur_direction & DATA_DIR_TX) || (tqspi->cur_direction & DATA_DIR_RX)) { intr_mask = tegra_qspi_readl(tqspi, QSPI_INTR_MASK); intr_mask &= ~(QSPI_INTR_RDY_MASK | QSPI_INTR_RX_TX_FIFO_ERR); tegra_qspi_writel(tqspi, intr_mask, QSPI_INTR_MASK); } tqspi->is_curr_dma_xfer = false; val = tqspi->command1_reg; /* TX_EN/RX_EN need to set after DMA_BLK to avoid spurious interrupt */ if (tqspi->cur_direction & DATA_DIR_RX) val |= QSPI_RX_EN; if (tqspi->cur_direction & DATA_DIR_TX) val |= QSPI_TX_EN; tegra_qspi_writel(tqspi, val, QSPI_COMMAND1); tegra_qspi_dump_regs("CPU Transfer started", tqspi); val |= QSPI_PIO; tegra_qspi_writel(tqspi, val, QSPI_COMMAND1); return 0; } static int tegra_qspi_init_dma_param(struct tegra_qspi_data *tqspi, bool dma_to_memory) { struct dma_chan *dma_chan; u32 *dma_buf; dma_addr_t dma_phys; int ret; struct dma_slave_config dma_sconfig; dma_chan = dma_request_slave_channel_reason(tqspi->dev, dma_to_memory ? "rx" : "tx"); if (IS_ERR(dma_chan)) { ret = PTR_ERR(dma_chan); dev_err(tqspi->dev, "Failed to get DMA channel, will retryi: %d\n", ret); return ret; } dma_buf = dma_alloc_coherent(tqspi->dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL); if (!dma_buf) { dev_err(tqspi->dev, "Failed to allocate coherant DMA buffer\n"); dma_release_channel(dma_chan); return -ENOMEM; } if (dma_to_memory) { dma_sconfig.src_addr = tqspi->phys + QSPI_RX_FIFO; dma_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; dma_sconfig.src_maxburst = 0; } else { dma_sconfig.dst_addr = tqspi->phys + QSPI_TX_FIFO; dma_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; dma_sconfig.dst_maxburst = 0; } ret = dmaengine_slave_config(dma_chan, &dma_sconfig); if (ret) goto scrub; if (dma_to_memory) { tqspi->rx_dma_chan = dma_chan; tqspi->rx_dma_buf = dma_buf; tqspi->rx_dma_phys = dma_phys; } else { tqspi->tx_dma_chan = dma_chan; tqspi->tx_dma_buf = dma_buf; tqspi->tx_dma_phys = dma_phys; } return 0; scrub: dma_free_coherent(tqspi->dev, tqspi->dma_buf_size, dma_buf, dma_phys); dma_release_channel(dma_chan); return ret; } static void tegra_qspi_deinit_dma_param(struct tegra_qspi_data *tqspi, bool dma_to_memory) { u32 *dma_buf; dma_addr_t dma_phys; struct dma_chan *dma_chan; if (dma_to_memory) { dma_buf = tqspi->rx_dma_buf; dma_chan = tqspi->rx_dma_chan; dma_phys = tqspi->rx_dma_phys; tqspi->rx_dma_chan = NULL; tqspi->rx_dma_buf = NULL; } else { dma_buf = tqspi->tx_dma_buf; dma_chan = tqspi->tx_dma_chan; dma_phys = tqspi->tx_dma_phys; tqspi->tx_dma_buf = NULL; tqspi->tx_dma_chan = NULL; } if (!dma_chan) return; dma_free_coherent(tqspi->dev, tqspi->dma_buf_size, dma_buf, dma_phys); dma_release_channel(dma_chan); } static int tegra_qspi_validate_request(struct spi_device *spi, struct tegra_qspi_data *tqspi, struct spi_transfer *t, bool is_ddr) { int req_mode; req_mode = spi->mode & 0x3; if ((req_mode == SPI_MODE_1) || (req_mode == SPI_MODE_2)) { dev_err(tqspi->dev, "QSPI does not support mode %d\n", req_mode); return -EINVAL; } if ((req_mode == SPI_MODE_3) && is_ddr) { dev_err(tqspi->dev, "DDR is not supported in mode 3\n"); return -EINVAL; } if ((t->bits_per_word != 8) && (t->bits_per_word != 16) && (t->bits_per_word != 32)) { dev_err(tqspi->dev, "QSPI does not support bpw = %d\n", t->bits_per_word); return -EINVAL; } if (((t->bits_per_word == 16) && (t->len & 0x1)) || ((t->bits_per_word == 32) && (t->len & 0x3))) { dev_err(tqspi->dev, "QSPI: length %d and bits-per-word %d must align\n", t->len, t->bits_per_word); return -EINVAL; } return 0; } static bool tegra_qspi_tap(struct tegra_qspi_data *tqspi, char *name, int tap, unsigned int mask, unsigned int shift, u32 *cmd2) { if (tap < 0) return false; if (tap & ~mask) dev_warn(tqspi->dev, "%s (%d) should be within %d..%d range\n", name, tap, 0, mask); tap &= mask; *cmd2 &= ~(mask << shift); *cmd2 |= (tap << shift); return true; } static void tegra_qspi_tap_delays(struct tegra_qspi_data *tqspi, int tx_tap_delay, int rx_tap_delay) { u32 cmd2 = tegra_qspi_readl(tqspi, QSPI_COMMAND2); bool changed = false; if (tegra_qspi_tap(tqspi, "tx_tap_delay", tx_tap_delay, 0x1F, 10, &cmd2)) changed = true; if (tegra_qspi_tap(tqspi, "rx_tap_delay", rx_tap_delay, 0xFF, 0, &cmd2)) changed = true; if (changed) { dev_info(tqspi->dev, "rx tap = %d, tx tap = %d, COMMAND2 = %08x\n", rx_tap_delay, tx_tap_delay, cmd2); tegra_qspi_writel(tqspi, cmd2, QSPI_COMMAND2); } } static void tegra_qspi_set_gr_registers(struct tegra_qspi_data *tqspi) { char prod_name[MAX_PROD_NAME]; int clk_mhz; int err; #ifdef QSPI_BRINGUP_BUILD if (tqspi->qspi_enable_prod_override) return; #endif if (!tqspi->prod_list) goto regs_por; /* If available, initialise the config registers * for QSPI with the values mentioned in prod list. */ err = tegra_prod_set_by_name(&tqspi->base, "prod", tqspi->prod_list); if (err < 0) dev_info_once(tqspi->dev, "Prod config not found for QSPI: %d\n", err); clk_mhz = tqspi->cur_speed / 1000000; if (tqspi->is_ddr_mode) sprintf(prod_name, "prod_c_ddr"); else sprintf(prod_name, "prod_c_sdr"); err = tegra_prod_set_by_name(&tqspi->base, prod_name, tqspi->prod_list); if (!err) dev_info_once(tqspi->dev, "Failed to apply prod name[%s] for qspi\n", prod_name); return; regs_por: /* If NOT defined in prod list or error in applying prod settings, * then initialise golden registers with POR values. */ tegra_qspi_writel(tqspi, 0, QSPI_COMMAND2); tegra_qspi_writel(tqspi, 0, QSPI_CS_TIMING1); tegra_qspi_writel(tqspi, CS_ACTIVE_BETWEEN_PACKETS_0, QSPI_CS_TIMING2); tegra_qspi_writel(tqspi, 0, QSPI_CS_TIMING3); } static int tegra_qspi_start_transfer_one(struct spi_device *spi, struct spi_transfer *t, bool is_first_of_msg, bool is_single_xfer) { struct tegra_qspi_data *tqspi = spi_master_get_devdata(spi->master); #ifndef QSPI_BRINGUP_BUILD struct tegra_qspi_device_controller_data *cdata = spi->controller_data; #endif u32 speed = 0; u32 actual_speed = 0; u8 bits_per_word; unsigned total_fifo_words; int ret; unsigned long command1; int req_mode; u8 bus_width = X1; bool is_ddr = false; u8 bus_clk_div = tqspi->bus_clk_div; bits_per_word = t->bits_per_word; tqspi->cur_qspi = spi; tqspi->cur_pos = 0; tqspi->cur_rx_pos = 0; tqspi->cur_tx_pos = 0; tqspi->curr_xfer = t; tqspi->tx_status = 0; tqspi->rx_status = 0; total_fifo_words = tegra_qspi_calculate_curr_xfer_param(spi, tqspi, t); #ifndef QSPI_BRINGUP_BUILD if (cdata) { if ((t->len - tqspi->cur_pos) > cdata->x1_len_limit) speed = cdata->x4_bus_speed; else { is_ddr = false; speed = cdata->x1_bus_speed; } bus_clk_div = cdata->bus_clk_div; } else { dev_err(tqspi->dev, "Controller Data is not available\n"); return -EINVAL; } #else if (tqspi->qspi_force_bus_speed) speed = tqspi->cur_speed; else speed = t->speed_hz; #endif if (bus_clk_div < 1 || bus_clk_div > 2) bus_clk_div = tqspi->bus_clk_div; /* * NOTE: * 1.Bus width can be x4 even for command/addr for QPI commands. * So caller requested bus width should be considered. * 2. is_ddr is not applicable for write. Write is always in SDR mode. */ is_ddr = get_sdr_ddr(t->delay_usecs); bus_width = get_bus_width(t->delay_usecs); ret = tegra_qspi_validate_request(spi, tqspi, t, is_ddr); if (ret) return ret; if (!speed || speed > tqspi->qspi_max_frequency/bus_clk_div) speed = tqspi->qspi_max_frequency/bus_clk_div; if (speed != tqspi->cur_speed || bus_clk_div != tqspi->bus_clk_div) { ret = clk_set_rate(tqspi->clk, speed*bus_clk_div); if (ret < 0) { dev_err(tqspi->dev, "Failed to set QSPI clock freq: %d\n", ret); return -EINVAL; } actual_speed = clk_get_rate(tqspi->clk)/bus_clk_div; ret = clk_set_rate(tqspi->sdr_ddr_clk, actual_speed); if (ret < 0) { dev_err(tqspi->dev, "Failed to set QSPI clock freq: %d\n", ret); return -EINVAL; } tqspi->cur_speed = speed; tqspi->bus_clk_div = bus_clk_div; } #if 0 if (is_ddr != tqspi->is_ddr_mode) { actual_speed = clk_get_rate(tqspi->clk); if (is_ddr) ret = clk_set_rate(tqspi->sdr_ddr_clk, (actual_speed >> 1)); else ret = clk_set_rate(tqspi->sdr_ddr_clk, actual_speed); if (ret < 0) { dev_err(tqspi->dev, "Failed to set QSPI-out clock freq: %d\n", ret); return -EINVAL; } tqspi->is_ddr_mode = is_ddr; } #endif if (is_first_of_msg) { tegra_qspi_clear_status(tqspi); command1 = tqspi->def_command1_reg; command1 |= QSPI_BIT_LENGTH(bits_per_word - 1); command1 &= ~QSPI_CONTROL_MODE_MASK; req_mode = spi->mode & 0x3; if (req_mode == SPI_MODE_0) { command1 |= QSPI_CONTROL_MODE_0; } else if (req_mode == SPI_MODE_3) { command1 |= QSPI_CONTROL_MODE_3; } else { dev_err(tqspi->dev, "QSPI does not support mode %d\n", req_mode); return -EINVAL; } /* Programming mode first suggested by HW - Bug 200082074 */ tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1); /* Toggle CS to active state now */ if (spi->mode & SPI_CS_HIGH) command1 |= QSPI_CS_SW_VAL; else command1 &= ~QSPI_CS_SW_VAL; tegra_qspi_writel(tqspi, command1, QSPI_COMMAND1); } else { command1 = tqspi->command1_reg; command1 &= ~QSPI_BIT_LENGTH(~0); command1 |= QSPI_BIT_LENGTH(bits_per_word - 1); } command1 &= ~QSPI_SDR_DDR_SEL; if (is_ddr) command1 |= QSPI_SDR_DDR_SEL; command1 &= ~QSPI_INTERFACE_WIDTH_MASK; command1 |= QSPI_INTERFACE_WIDTH(bus_width); command1 &= ~QSPI_PACKED; if (tqspi->is_packed) command1 |= QSPI_PACKED; command1 &= ~(QSPI_TX_EN | QSPI_RX_EN); tqspi->cur_direction = 0; if (t->rx_buf) tqspi->cur_direction |= DATA_DIR_RX; if (t->tx_buf) tqspi->cur_direction |= DATA_DIR_TX; tqspi->command1_reg = command1; #ifdef QSPI_BRINGUP_BUILD if (tqspi->qspi_force_dma_mode) { ret = tegra_qspi_start_dma_based_transfer(tqspi, t); return ret; } if (tqspi->qspi_force_pio_mode) { ret = tegra_qspi_start_cpu_based_transfer(tqspi, t); return ret; } #endif if (total_fifo_words > QSPI_FIFO_DEPTH) ret = tegra_qspi_start_dma_based_transfer(tqspi, t); else ret = tegra_qspi_start_cpu_based_transfer(tqspi, t); return ret; } static void tegra_qspi_clean(struct spi_device *spi) { kfree(spi->controller_data); spi->controller_data = NULL; } static int tegra_qspi_setup(struct spi_device *spi) { struct tegra_qspi_data *tqspi = spi_master_get_devdata(spi->master); struct tegra_qspi_device_controller_data *cdata = spi->controller_data; unsigned long val; unsigned long flags; int ret; dev_dbg(&spi->dev, "setup %d bpw, %scpol, %scpha, %dHz\n", spi->bits_per_word, spi->mode & SPI_CPOL ? "" : "~", spi->mode & SPI_CPHA ? "" : "~", spi->max_speed_hz); if (spi->chip_select >= MAX_CHIP_SELECT) { dev_err(tqspi->dev, "QSPI Chip select %d is not supported\n", spi->chip_select); return -EINVAL; } if (!cdata) { cdata = tegra_qspi_get_cdata_dt(spi); spi->controller_data = cdata; } /* Set speed to the spi max fequency if qspi device has not set */ spi->max_speed_hz = spi->max_speed_hz ? : tqspi->qspi_max_frequency; ret = pm_runtime_get_sync(tqspi->dev); if (ret < 0) { dev_err(tqspi->dev, "Failed to get runtime PM: %d\n", ret); return ret; } spin_lock_irqsave(&tqspi->lock, flags); /* keep default cs state to inactive */ val = tqspi->def_command1_reg; if (spi->mode & SPI_CS_HIGH) val &= ~QSPI_CS_SW_VAL; else val |= QSPI_CS_SW_VAL; tqspi->def_command1_reg = val; tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1); spin_unlock_irqrestore(&tqspi->lock, flags); pm_runtime_mark_last_busy(tqspi->dev); pm_runtime_put_autosuspend(tqspi->dev); return 0; } static int tegra_qspi_cs_low(struct spi_device *spi, bool state) { struct tegra_qspi_data *tqspi = spi_master_get_devdata(spi->master); int ret; unsigned long val; unsigned long flags; if (spi->chip_select >= MAX_CHIP_SELECT) { dev_err(tqspi->dev, "QSPI Chip select %d is not supported\n", spi->chip_select); return -EINVAL; } ret = pm_runtime_get_sync(tqspi->dev); if (ret < 0) { dev_err(tqspi->dev, "Failed to get runtime PM: %d\n", ret); return ret; } spin_lock_irqsave(&tqspi->lock, flags); val = tegra_qspi_readl(tqspi, QSPI_COMMAND1); if (state) val &= ~QSPI_CS_SW_VAL; else val |= QSPI_CS_SW_VAL; tegra_qspi_writel(tqspi, val, QSPI_COMMAND1); spin_unlock_irqrestore(&tqspi->lock, flags); pm_runtime_mark_last_busy(tqspi->dev); pm_runtime_put_autosuspend(tqspi->dev); return 0; } static int tegra_qspi_combined_sequence_transfer(struct tegra_qspi_data *tqspi, struct spi_message *msg) { bool is_first_msg = true, is_ddr = false; int single_xfer; struct spi_transfer *xfer; struct spi_device *spi = msg->spi; u8 transfer_phase = 0, bus_width = X1; int ret; u32 qspi_setting = 0; u32 address_value = 0; u32 cmd_config = 0, addr_config = 0; u8 cmd_value = 0, len = 0, val = 0; /* Enable Combined sequence mode */ val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG); val |= QSPI_CMB_SEQ_EN; tegra_qspi_writel(tqspi, val, QSPI_GLOBAL_CONFIG); single_xfer = list_is_singular(&msg->transfers); /* Process individual transfer list */ list_for_each_entry(xfer, &msg->transfers, transfer_list) { if (transfer_phase == CMD_TRANSFER) { /* Extract Command configuration and value */ qspi_setting = xfer->delay_usecs; is_ddr = get_sdr_ddr(qspi_setting); bus_width = get_bus_width(qspi_setting); if (is_ddr) cmd_config |= QSPI_COMMAND_SDR_DDR; else cmd_config &= ~QSPI_COMMAND_SDR_DDR; cmd_config |= QSPI_COMMAND_X1_X2_X4(bus_width); cmd_config |= QSPI_COMMAND_SIZE_SET( (xfer->len * 8) - 1); cmd_value = *((const u8 *)(xfer->tx_buf)); } else if (transfer_phase == ADDR_TRANSFER) { /* Extract Address configuration and value */ qspi_setting = xfer->delay_usecs; is_ddr = get_sdr_ddr(qspi_setting); bus_width = get_bus_width(qspi_setting); len = xfer->len; if (is_ddr) addr_config |= QSPI_ADDRESS_SDR_DDR; else addr_config &= ~QSPI_ADDRESS_SDR_DDR; addr_config |= QSPI_ADDRESS_X1_X2_X4(bus_width); addr_config |= QSPI_ADDRESS_SIZE_SET( (xfer->len * 8) - 1); address_value = *((const u32 *)(xfer->tx_buf)); } else { /* Program Command, Address value in register */ tegra_qspi_writel(tqspi, cmd_value, QSPI_CMB_SEQ_CMD); tegra_qspi_writel(tqspi, address_value, QSPI_CMB_SEQ_ADDR); /* Program Command and Address config in register */ tegra_qspi_writel(tqspi, cmd_config, QSPI_CMB_SEQ_CMD_CFG); tegra_qspi_writel(tqspi, addr_config, QSPI_CMB_SEQ_ADDR_CFG); reinit_completion(&tqspi->xfer_completion); /* Start Data transfer */ ret = tegra_qspi_start_transfer_one(spi, xfer, is_first_msg, single_xfer); if (ret < 0) { dev_err(tqspi->dev, "Failed to start transfer-one: %d\n", ret); return ret; } is_first_msg = false; ret = wait_for_completion_timeout (&tqspi->xfer_completion, QSPI_DMA_TIMEOUT); if (WARN_ON(ret == 0)) { dev_err(tqspi->dev, "QSPI Transfer failed with timeout: %d\n", ret); if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_TX)) dmaengine_terminate_all( tqspi->tx_dma_chan); if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_RX)) dmaengine_terminate_all( tqspi->rx_dma_chan); /* Reset controller if timeout happens */ reset_control_reset(tqspi->rstc); ret = -EIO; return ret; } if (tqspi->tx_status || tqspi->rx_status) { dev_err(tqspi->dev, "QSPI Transfer failed\n"); tqspi->tx_status = 0; tqspi->rx_status = 0; ret = -EIO; return ret; } } msg->actual_length += xfer->len; transfer_phase++; } return 0; } static int tegra_qspi_non_combined_sequence_transfer (struct tegra_qspi_data *tqspi, struct spi_message *msg) { bool is_first_msg = true; int single_xfer; struct spi_transfer *xfer; struct spi_device *spi = msg->spi; int ret; u8 val = 0; val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG); val &= ~QSPI_CMB_SEQ_EN; tegra_qspi_writel(tqspi, val, QSPI_GLOBAL_CONFIG); single_xfer = list_is_singular(&msg->transfers); list_for_each_entry(xfer, &msg->transfers, transfer_list) { reinit_completion(&tqspi->xfer_completion); ret = tegra_qspi_start_transfer_one(spi, xfer, is_first_msg, single_xfer); if (ret < 0) { dev_err(tqspi->dev, "Failed to start transfer-one: %d\n", ret); return ret; } is_first_msg = false; ret = wait_for_completion_timeout(&tqspi->xfer_completion, QSPI_DMA_TIMEOUT); if (WARN_ON(ret == 0)) { dev_err(tqspi->dev, "QSPI Transfer failed with timeout\n"); if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_TX)) dmaengine_terminate_all(tqspi->tx_dma_chan); if (tqspi->is_curr_dma_xfer && (tqspi->cur_direction & DATA_DIR_RX)) dmaengine_terminate_all(tqspi->rx_dma_chan); /* Reset controller in case of timeout happens */ reset_control_reset(tqspi->rstc); ret = -EIO; return ret; } if (tqspi->tx_status || tqspi->rx_status) { dev_err(tqspi->dev, "QSPI Transfer failed\n"); tqspi->tx_status = 0; tqspi->rx_status = 0; ret = -EIO; return ret; } msg->actual_length += xfer->len; } return 0; } static int tegra_qspi_transfer_one_message(struct spi_master *master, struct spi_message *msg) { struct tegra_qspi_data *tqspi = spi_master_get_devdata(master); struct spi_transfer *xfer; u8 count_number_of_transfers = 0; int ret; msg->status = 0; msg->actual_length = 0; ret = pm_runtime_get_sync(tqspi->dev); if (ret < 0) { dev_err(tqspi->dev, "Failed to get runtime PM: %d\n", ret); msg->status = ret; spi_finalize_current_message(master); return ret; } list_for_each_entry(xfer, &msg->transfers, transfer_list) { count_number_of_transfers++; } /* * Do Combined sequence mode related processing if it is enabled in DT * Support for Combined Sequence mode is available from T18X onwards * Combined sequence mode is applicable for transfers * Non combined mode transfer is used in other transfers */ if (tqspi->qspi_enable_cmbseq_mode && count_number_of_transfers == 3) { tqspi->dcycle_non_cmbseq_mode = false; ret = tegra_qspi_combined_sequence_transfer(tqspi, msg); if (ret < 0) { dev_err(tqspi->dev, "QSPI combined sequence transfer failed: %d\n", ret); goto exit; } } else { tqspi->dcycle_non_cmbseq_mode = true; ret = tegra_qspi_non_combined_sequence_transfer(tqspi, msg); if (ret < 0) { dev_err(tqspi->dev, "QSPI non-combined sequence transfer failed: %d\n", ret); goto exit; } } ret = 0; exit: tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1); pm_runtime_mark_last_busy(tqspi->dev); pm_runtime_put_autosuspend(tqspi->dev); msg->status = ret; spi_finalize_current_message(master); return ret; } #ifdef QSPI_BRINGUP_BUILD static void handle_combined_sequence(struct tegra_qspi_data *tqspi) { u8 val = 0; /* clear combined sequence enable */ val = tegra_qspi_readl(tqspi, QSPI_GLOBAL_CONFIG); val &= ~QSPI_CMB_SEQ_EN; tegra_qspi_writel(tqspi, val, QSPI_GLOBAL_CONFIG); } #endif static irqreturn_t handle_cpu_based_xfer(struct tegra_qspi_data *tqspi) { struct spi_transfer *t = tqspi->curr_xfer; unsigned long flags; spin_lock_irqsave(&tqspi->lock, flags); if (tqspi->tx_status || tqspi->rx_status) { dev_err(tqspi->dev, "CpuXfer ERROR, status 0x%08x\n", tqspi->status_reg); dev_err(tqspi->dev, "CpuXfer command1:dmacontro->0x%08x:0x%08x\n", tqspi->command1_reg, tqspi->dma_control_reg); reset_control_reset(tqspi->rstc); complete(&tqspi->xfer_completion); tqspi->tx_status = 0; tqspi->rx_status = 0; goto exit; } if (tqspi->cur_direction & DATA_DIR_RX) tegra_qspi_read_rx_fifo_to_client_rxbuf(tqspi, t); if (tqspi->cur_direction & DATA_DIR_TX) tqspi->cur_pos = tqspi->cur_tx_pos; else tqspi->cur_pos = tqspi->cur_rx_pos; if (tqspi->cur_pos >= t->len) { complete(&tqspi->xfer_completion); goto exit; } tegra_qspi_calculate_curr_xfer_param(tqspi->cur_qspi, tqspi, t); tegra_qspi_start_cpu_based_transfer(tqspi, t); exit: spin_unlock_irqrestore(&tqspi->lock, flags); return IRQ_HANDLED; } static irqreturn_t handle_dma_based_xfer(struct tegra_qspi_data *tqspi) { struct spi_transfer *t = tqspi->curr_xfer; long wait_status; int err = 0; unsigned total_fifo_words; unsigned long flags; /* Abort dmas if any error */ if (tqspi->cur_direction & DATA_DIR_TX) { if (tqspi->tx_status) { dmaengine_terminate_all(tqspi->tx_dma_chan); err += 1; } else { wait_status = wait_for_completion_interruptible_timeout( &tqspi->tx_dma_complete, QSPI_DMA_TIMEOUT); if (wait_status <= 0) { dmaengine_terminate_all(tqspi->tx_dma_chan); dev_err(tqspi->dev, "TxDma Xfer failed\n"); err += 1; } } } if (tqspi->cur_direction & DATA_DIR_RX) { if (tqspi->rx_status) { dmaengine_terminate_all(tqspi->rx_dma_chan); err += 2; } else { wait_status = wait_for_completion_interruptible_timeout( &tqspi->rx_dma_complete, QSPI_DMA_TIMEOUT); if (wait_status <= 0) { dmaengine_terminate_all(tqspi->rx_dma_chan); dev_err(tqspi->dev, "RxDma Xfer failed\n"); err += 2; } } } spin_lock_irqsave(&tqspi->lock, flags); if (err) { dev_err(tqspi->dev, "DmaXfer ERROR, status 0x%08x\n", tqspi->status_reg); dev_err(tqspi->dev, "DmaXfer command1:dmacontro->0x%08x:0x%08x\n", tqspi->command1_reg, tqspi->dma_control_reg); reset_control_reset(tqspi->rstc); complete(&tqspi->xfer_completion); tqspi->rx_status = 0; tqspi->tx_status = 0; spin_unlock_irqrestore(&tqspi->lock, flags); return IRQ_HANDLED; } if (tqspi->cur_direction & DATA_DIR_RX) tegra_qspi_copy_qspi_rxbuf_to_client_rxbuf(tqspi, t); if (tqspi->cur_direction & DATA_DIR_TX) tqspi->cur_pos = tqspi->cur_tx_pos; else tqspi->cur_pos = tqspi->cur_rx_pos; if (tqspi->cur_pos >= t->len) { complete(&tqspi->xfer_completion); goto exit; } #ifdef QSPI_BRINGUP_BUILD if (tqspi->qspi_force_dma_mode) { err = tegra_qspi_start_dma_based_transfer(tqspi, t); goto exit; } #endif /* Continue transfer in current message */ total_fifo_words = tegra_qspi_calculate_curr_xfer_param(tqspi->cur_qspi, tqspi, t); if (total_fifo_words > QSPI_FIFO_DEPTH) err = tegra_qspi_start_dma_based_transfer(tqspi, t); else err = tegra_qspi_start_cpu_based_transfer(tqspi, t); exit: spin_unlock_irqrestore(&tqspi->lock, flags); return IRQ_HANDLED; } static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data) { struct tegra_qspi_data *tqspi = context_data; #ifdef QSPI_BRINGUP_BUILD if (tqspi->qspi_enable_cmbseq_mode) handle_combined_sequence(tqspi); #endif if (!tqspi->is_curr_dma_xfer) return handle_cpu_based_xfer(tqspi); return handle_dma_based_xfer(tqspi); } static irqreturn_t tegra_qspi_isr(int irq, void *context_data) { struct tegra_qspi_data *tqspi = context_data; tegra_qspi_dump_regs("From QSPI ISR", tqspi); tqspi->status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS); if (tqspi->cur_direction & DATA_DIR_TX) tqspi->tx_status = tqspi->status_reg & (QSPI_TX_FIFO_UNF | QSPI_TX_FIFO_OVF); if (tqspi->cur_direction & DATA_DIR_RX) tqspi->rx_status = tqspi->status_reg & (QSPI_RX_FIFO_OVF | QSPI_RX_FIFO_UNF); if (!(tqspi->cur_direction & DATA_DIR_TX) && !(tqspi->cur_direction & DATA_DIR_RX)) dev_err(tqspi->dev, "QSPI get spurious interrupt, Status = 0x%08x\n", tqspi->status_reg); tegra_qspi_clear_status(tqspi); return IRQ_WAKE_THREAD; } static int tegra_qspi_clk_enable(struct tegra_qspi_data *tqspi) { int ret; ret = clk_prepare_enable(tqspi->clk); if (ret < 0) { dev_err(tqspi->dev, "Failed to enable QSPI clock: %d\n", ret); return ret; } ret = clk_prepare_enable(tqspi->sdr_ddr_clk); if (ret < 0) { dev_err(tqspi->dev, "Failed to enable QSPI-OUT clk: %d\n", ret); clk_disable_unprepare(tqspi->clk); return ret; } return ret; } static void tegra_qspi_clk_disable(struct tegra_qspi_data *tqspi) { clk_disable_unprepare(tqspi->sdr_ddr_clk); clk_disable_unprepare(tqspi->clk); } static void set_best_clk_source(struct tegra_qspi_data *tqspi, unsigned long rate) { long new_rate; unsigned long err_rate; unsigned int fin_err = rate; int ret; struct clk *pclk, *fpclk = NULL; const char *pclk_name, *fpclk_name = NULL; struct device_node *node = tqspi->dev->of_node; struct property *prop; if (!of_property_count_strings(node, "nvidia,clk-parents")) return; of_property_for_each_string(node, "nvidia,clk-parents", prop, pclk_name) { pclk = clk_get(tqspi->dev, pclk_name); if (IS_ERR(pclk)) continue; ret = clk_set_parent(tqspi->clk, pclk); if (ret < 0) { dev_warn(tqspi->dev, "Failed to set parent clk: %d\n", ret); continue; } new_rate = clk_round_rate(tqspi->clk, rate); if (new_rate < 0) continue; err_rate = abs(new_rate - rate); if (err_rate < fin_err) { fpclk = pclk; fin_err = err_rate; fpclk_name = pclk_name; } } if (fpclk) { dev_dbg(tqspi->dev, "Setting clk_src %s\n", fpclk_name); clk_set_parent(tqspi->clk, fpclk); } } static struct tegra_qspi_device_controller_data *tegra_qspi_get_cdata_dt( struct spi_device *spi) { struct tegra_qspi_device_controller_data *cdata = NULL; struct device_node *np = spi->dev.of_node, *data_np = NULL; u32 pval; if (!np) { dev_dbg(&spi->dev, "Device must have DT node handle\n"); return NULL; } data_np = of_get_child_by_name(np, "controller-data"); if (!data_np) { dev_dbg(&spi->dev, "child node 'controller-data' not found\n"); return NULL; } cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); if (!cdata) return NULL; if (!of_property_read_u32(data_np, "nvidia,x1-len-limit", &pval)) cdata->x1_len_limit = pval; if (!of_property_read_u32(data_np, "nvidia,x1-bus-speed", &pval)) cdata->x1_bus_speed = pval; if (!of_property_read_u32(data_np, "nvidia,x1-dymmy-cycle", &pval)) cdata->x1_dymmy_cycle = pval; if (!of_property_read_u32(data_np, "nvidia,x4-bus-speed", &pval)) cdata->x4_bus_speed = pval; if (!of_property_read_u32(data_np, "nvidia,x4-dymmy-cycle", &pval)) cdata->x4_dymmy_cycle = pval; if (!of_property_read_u32(data_np, "nvidia,x4-is-ddr", &pval)) cdata->x4_is_ddr = pval; if (!of_property_read_u32(data_np, "nvidia,ifddr-div2-sdr", &pval)) cdata->ifddr_div2_sdr = pval; if (!of_property_read_u32(data_np, "nvidia,ctrl-bus-clk-ratio", &pval)) cdata->bus_clk_div = (u8)pval; cdata->is_combined_seq_mode_en = of_property_read_bool(data_np, "nvidia,combined-seq-mode-en"); /* Bus speed mentioned in device tree should be what is applied * on interface. Earlier version used to apply half of the bus * speed defined in device tree. To maintain backward compatibility * with old device tree, applied bus speed is half of that defined in * device tree if 'nvidia,ifddr-div2-sdr' is defined */ if (cdata->ifddr_div2_sdr && cdata->x1_bus_speed) cdata->x1_bus_speed /= 2; if (cdata->ifddr_div2_sdr && cdata->x4_bus_speed) cdata->x4_bus_speed /= 2; return cdata; } static void tegra_qspi_parse_dt(struct device *dev, struct tegra_qspi_data *tqspi) { struct device_node *np = dev->of_node; u32 pval; int ret; tqspi->enable_dma_support = of_property_read_bool(np, "dma-names"); ret = of_property_read_u32(np, "spi-max-frequency", &pval); if (!ret) tqspi->qspi_max_frequency = pval; else tqspi->qspi_max_frequency = 136000000; /* 136MHz */ tqspi->clock_always_on = of_property_read_bool(np, "nvidia,clock-always-on"); } static int tegra_qspi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct spi_master *master; struct tegra_qspi_data *tqspi; struct resource *r; int ret, qspi_irq; u32 as_delay; u32 actual_speed = 0; master = spi_alloc_master(&pdev->dev, sizeof(*tqspi)); if (!master) { dev_err(&pdev->dev, "SPI master allocation failed\n"); return -ENOMEM; } master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH; master->setup = tegra_qspi_setup; master->cleanup = tegra_qspi_clean; master->transfer_one_message = tegra_qspi_transfer_one_message; master->num_chipselect = MAX_CHIP_SELECT; master->bus_num = -1; master->spi_cs_low = tegra_qspi_cs_low; dev_set_drvdata(&pdev->dev, master); tqspi = spi_master_get_devdata(master); tqspi->master = master; tegra_qspi_parse_dt(&pdev->dev, tqspi); tqspi->dev = &pdev->dev; tqspi->prod_list = devm_tegra_prod_get(&pdev->dev); if (IS_ERR(tqspi->prod_list)) { dev_info(&pdev->dev, "Prod settings list not found\n"); tqspi->prod_list = NULL; } spin_lock_init(&tqspi->lock); r = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!r) { dev_err(&pdev->dev, "Failed to get IO memory\n"); ret = -ENODEV; goto exit_free_master; } tqspi->phys = r->start; tqspi->base = devm_ioremap_resource(&pdev->dev, r); if (IS_ERR(tqspi->base)) { ret = PTR_ERR(tqspi->base); dev_err(dev, "Failed to request memregion/iomap address: %d\n", ret); goto exit_free_master; } tqspi->rstc = devm_reset_control_get(&pdev->dev, "qspi"); if (IS_ERR(tqspi->rstc)) { ret = PTR_ERR(tqspi->rstc); dev_err(&pdev->dev, "Failed to get reset control: %d\n", ret); goto exit_free_master; } reset_control_reset(tqspi->rstc); qspi_irq = platform_get_irq(pdev, 0); tqspi->irq = qspi_irq; ret = devm_request_threaded_irq(dev, tqspi->irq, tegra_qspi_isr, tegra_qspi_isr_thread, IRQF_ONESHOT, dev_name(&pdev->dev), tqspi); if (ret < 0) { dev_err(dev, "Failed to register interrupt: %d\n", tqspi->irq); goto exit_free_master; } tqspi->clk = devm_clk_get(&pdev->dev, "qspi"); if (IS_ERR(tqspi->clk)) { ret = PTR_ERR(tqspi->clk); dev_err(&pdev->dev, "Failed to get QSPI clock: %d\n", ret); goto exit_free_master; } tqspi->sdr_ddr_clk = devm_clk_get(&pdev->dev, "qspi_out"); if (IS_ERR(tqspi->sdr_ddr_clk)) { ret = PTR_ERR(tqspi->sdr_ddr_clk); dev_err(&pdev->dev, "Failed to get QSPI-OUT: %d\n", ret); goto exit_free_master; } /* Set default mode to SDR */ tqspi->is_ddr_mode = false; tqspi->max_buf_size = QSPI_FIFO_DEPTH << 2; tqspi->dma_buf_size = DEFAULT_SPI_DMA_BUF_LEN; if (tqspi->enable_dma_support) { ret = tegra_qspi_init_dma_param(tqspi, true); if (ret < 0) { dev_err(&pdev->dev, "Failed to initialise RxDma: %d\n", ret); goto exit_free_master; } ret = tegra_qspi_init_dma_param(tqspi, false); if (ret < 0) { dev_err(&pdev->dev, "Failed to initialise TxDma: %d\n", ret); goto exit_rx_dma_free; } tqspi->max_buf_size = tqspi->dma_buf_size; init_completion(&tqspi->tx_dma_complete); init_completion(&tqspi->rx_dma_complete); } init_completion(&tqspi->xfer_completion); if (tqspi->clock_always_on) { ret = tegra_qspi_clk_enable(tqspi); if (ret < 0) goto exit_deinit_dma; } ret = of_property_read_u32(dev->of_node, "qspi-autosuspend-delay", &as_delay); if (ret) as_delay = 3000; /* defalut autosuspend delay */ pm_runtime_set_autosuspend_delay(&pdev->dev, as_delay); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_enable(&pdev->dev); if (!pm_runtime_enabled(&pdev->dev)) { ret = tegra_qspi_runtime_resume(&pdev->dev); if (ret) goto exit_pm_disable; } ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) { dev_err(tqspi->dev, "Failed to get runtime PM: %d\n", ret); goto exit_pm_disable; } set_best_clk_source(tqspi, tqspi->qspi_max_frequency); ret = clk_set_rate(tqspi->clk, tqspi->qspi_max_frequency); if (ret) { dev_err(dev, "Failed to set qspi clk freq %d\n", ret); goto exit_pm_disable; } tqspi->cur_speed = tqspi->qspi_max_frequency; actual_speed = clk_get_rate(tqspi->clk); if (actual_speed > 0) { ret = clk_set_rate(tqspi->sdr_ddr_clk, actual_speed >> 1); if (ret) { dev_err(dev, "Failed to set qspi_out clk freq %d\n", ret); goto exit_pm_disable; } tqspi->bus_clk_div = 2; } tqspi->def_command1_reg = QSPI_M_S | QSPI_CS_SW_HW | QSPI_CS_SW_VAL; tegra_qspi_writel(tqspi, tqspi->def_command1_reg, QSPI_COMMAND1); tqspi->def_command2_reg = tegra_qspi_readl(tqspi, QSPI_COMMAND2); tegra_qspi_set_gr_registers(tqspi); tegra_qspi_tap_delays(tqspi, tx_tap_delay, rx_tap_delay); pm_runtime_mark_last_busy(&pdev->dev); pm_runtime_put_autosuspend(&pdev->dev); master->dev.of_node = pdev->dev.of_node; ret = devm_spi_register_master(&pdev->dev, master); if (ret < 0) { dev_err(dev, "Failed to register spi master: %d\n", ret); goto exit_pm_disable; } #ifdef QSPI_BRINGUP_BUILD ret = sysfs_create_group(&dev->kobj, tegra_qspi_groups[0]); if (ret) goto exit_pm_disable; tqspi->qspi_force_bus_speed = false; #endif return ret; exit_pm_disable: pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) tegra_qspi_runtime_suspend(&pdev->dev); if (tqspi->clock_always_on) tegra_qspi_clk_disable(tqspi); exit_deinit_dma: tegra_qspi_deinit_dma_param(tqspi, false); exit_rx_dma_free: tegra_qspi_deinit_dma_param(tqspi, true); exit_free_master: spi_master_put(master); return ret; } static int tegra_qspi_remove(struct platform_device *pdev) { struct spi_master *master = dev_get_drvdata(&pdev->dev); struct tegra_qspi_data *tqspi = spi_master_get_devdata(master); #ifdef QSPI_BRINGUP_BUILD sysfs_remove_group(&pdev->dev.kobj, tegra_qspi_groups[0]); #endif if (tqspi->tx_dma_chan) tegra_qspi_deinit_dma_param(tqspi, false); if (tqspi->rx_dma_chan) tegra_qspi_deinit_dma_param(tqspi, true); pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) tegra_qspi_runtime_suspend(&pdev->dev); if (tqspi->clock_always_on) tegra_qspi_clk_disable(tqspi); return 0; } #ifdef CONFIG_PM_SLEEP static int tegra_qspi_suspend(struct device *dev) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi = spi_master_get_devdata(master); int ret; ret = spi_master_suspend(master); if (tqspi->clock_always_on) tegra_qspi_clk_disable(tqspi); return ret; } static int tegra_qspi_resume(struct device *dev) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi = spi_master_get_devdata(master); int ret; if (tqspi->clock_always_on) { ret = tegra_qspi_clk_enable(tqspi); if (ret < 0) return ret; } ret = pm_runtime_get_sync(dev); if (ret < 0) { dev_err(tqspi->dev, "Failed to get runtime PM: %d\n", ret); return ret; } tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1); tegra_qspi_set_gr_registers(tqspi); tegra_qspi_tap_delays(tqspi, tx_tap_delay, rx_tap_delay); pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); return spi_master_resume(master); } #endif static int tegra_qspi_runtime_suspend(struct device *dev) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi = spi_master_get_devdata(master); /* Flush all write which are in PPSB queue by reading back */ tegra_qspi_readl(tqspi, QSPI_COMMAND1); tegra_qspi_clk_disable(tqspi); return 0; } static int tegra_qspi_runtime_resume(struct device *dev) { struct spi_master *master = dev_get_drvdata(dev); struct tegra_qspi_data *tqspi = spi_master_get_devdata(master); return tegra_qspi_clk_enable(tqspi); } static const struct dev_pm_ops tegra_qspi_pm_ops = { SET_RUNTIME_PM_OPS(tegra_qspi_runtime_suspend, tegra_qspi_runtime_resume, NULL) SET_SYSTEM_SLEEP_PM_OPS(tegra_qspi_suspend, tegra_qspi_resume) }; static const struct of_device_id tegra_qspi_of_match[] = { { .compatible = "nvidia,tegra186-qspi", }, { .compatible = "nvidia,tegra210-qspi", }, {} }; MODULE_DEVICE_TABLE(of, tegra_qspi_of_match); static struct platform_driver tegra_qspi_driver = { .driver = { .name = "tegra-qspi", .pm = &tegra_qspi_pm_ops, .of_match_table = tegra_qspi_of_match, }, .probe = tegra_qspi_probe, .remove = tegra_qspi_remove, }; module_platform_driver(tegra_qspi_driver); MODULE_DESCRIPTION("NVIDIA Tegra186 QSPI Controller Driver"); MODULE_AUTHOR("Laxman Dewangan "); MODULE_AUTHOR("Amlan Kundu "); MODULE_LICENSE("GPL v2");