From 7c649c3ce50c3fbbc5e6d91ef03b2bbcb7cd1a04 Mon Sep 17 00:00:00 2001 From: Teoh Choon Zone Date: Tue, 30 Sep 2025 14:34:34 +0800 Subject: [PATCH] mmc: Add Efinix eMMC driver --- drivers/mmc/host/Kconfig | 12 + drivers/mmc/host/Makefile | 2 + drivers/mmc/host/efx_emmc.h | 304 +++++++++ drivers/mmc/host/efx_emmc_core.c | 975 +++++++++++++++++++++++++++ drivers/mmc/host/efx_emmc_dma.c | 209 ++++++ drivers/mmc/host/efx_emmc_platform.c | 370 ++++++++++ drivers/mmc/host/efx_emmc_tuning.c | 459 +++++++++++++ 7 files changed, 2331 insertions(+) create mode 100644 drivers/mmc/host/efx_emmc.h create mode 100644 drivers/mmc/host/efx_emmc_core.c create mode 100644 drivers/mmc/host/efx_emmc_dma.c create mode 100644 drivers/mmc/host/efx_emmc_platform.c create mode 100644 drivers/mmc/host/efx_emmc_tuning.c diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 9321cf6dc..17aa83d4d 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -1109,3 +1109,15 @@ config MMC_SDHCI_EFX config MMC_SDHCI_EXTERNAL_DMA bool + +config MMC_EFX_EMMC + tristate "Efinix eMMC host controller support" + depends on OF + help + This selects support for the Efinix eMMC Host Controller. + The controller supports eMMC 5.1 specification with HS200 and HS400 modes. + It includes hardware reset support and is designed for embedded applications. + + If you have an Efinix platform with an eMMC device, say Y here. + + If unsure, say N. diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile index c6a7ed2ab..ef488f128 100644 --- a/drivers/mmc/host/Makefile +++ b/drivers/mmc/host/Makefile @@ -106,6 +106,8 @@ obj-$(CONFIG_MMC_SDHCI_SPRD) += sdhci-sprd.o obj-$(CONFIG_MMC_CQHCI) += cqhci.o obj-$(CONFIG_MMC_HSQ) += mmc_hsq.o obj-$(CONFIG_MMC_SDHCI_EFX) += sdhci-efx.o +obj-$(CONFIG_MMC_EFX_EMMC) += efx-emmc.o +efx-emmc-y += efx_emmc_core.o efx_emmc_platform.o efx_emmc_dma.o efx_emmc_tuning.o ifeq ($(CONFIG_CB710_DEBUG),y) CFLAGS-cb710-mmc += -DDEBUG diff --git a/drivers/mmc/host/efx_emmc.h b/drivers/mmc/host/efx_emmc.h new file mode 100644 index 000000000..e7d484639 --- /dev/null +++ b/drivers/mmc/host/efx_emmc.h @@ -0,0 +1,304 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Efinix eMMC Host Controller Driver Header with DMA Support + * + * Copyright (C) 2025 Efinix, Inc. + * Author: Teoh Choon Zone + */ + + +#ifndef __EFX_EMMC_H__ +#define __EFX_EMMC_H__ + +#include +#include +#include +#include +#include +#include + +#ifndef min3 +#define min3(x, y, z) min(min(x, y), z) +#endif + +#define EFX_ADMA_DESC_VALID BIT(0) +#define EFX_ADMA_DESC_END BIT(1) +#define EFX_ADMA_DESC_INT BIT(2) +#define EFX_ADMA_DESC_NOP (0 << 4) +#define EFX_ADMA_DESC_TRAN (2 << 4) +#define EFX_ADMA_DESC_LINK (3 << 4) + +#define EFX_ADMA_MAX_LEN 65536 +#define EFX_ADMA_DESC_ALIGN 8 +#define EFX_ADMA_TABLE_SZ (512 * 8) + +#define EFX_DMA_BOUNDARY_4K 0 +#define EFX_DMA_BOUNDARY_8K 1 +#define EFX_DMA_BOUNDARY_16K 2 +#define EFX_DMA_BOUNDARY_32K 3 +#define EFX_DMA_BOUNDARY_64K 4 +#define EFX_DMA_BOUNDARY_128K 5 +#define EFX_DMA_BOUNDARY_256K 6 +#define EFX_DMA_BOUNDARY_512K 7 + +#define EFX_EMMC_VERSION 0x000 +#define EFX_EMMC_BASE_REG0 0x004 +#define EFX_EMMC_BASE_STATUS_REG0 0x008 +#define EFX_EMMC_BASE_REG1 0x00C +#define EFX_EMMC_ARG2 0x100 +#define EFX_EMMC_BLOCK_SIZE 0x104 +#define EFX_EMMC_ARG1 0x108 +#define EFX_EMMC_TRANSFER_MODE 0x10C +#define EFX_EMMC_RESPONSE0 0x110 +#define EFX_EMMC_RESPONSE1 0x114 +#define EFX_EMMC_RESPONSE2 0x118 +#define EFX_EMMC_RESPONSE3 0x11C +#define EFX_EMMC_BUFFER_DATA_PORT 0x120 +#define EFX_EMMC_PRESENT_STATE 0x124 +#define EFX_EMMC_HOST_CONTROL 0x128 +#define EFX_EMMC_INT_STATUS 0x130 +#define EFX_EMMC_INT_STATUS_EN 0x134 +#define EFX_EMMC_INT_SIGNAL_EN 0x138 +#define EFX_EMMC_HOST_CAPABILITIES 0x140 +#define EFX_EMMC_ADMA_SYS_ADDR_LOW 0x158 +#define EFX_EMMC_ADMA_SYS_ADDR_HIGH 0x15C + +#define EFX_SYS_DATE_REG 0x000 +#define EFX_SYS_TEST_REG 0x004 +#define EFX_SYS_RESET_REG 0x008 + +#define EFX_EMMC_BASE_REG0_CLK_EN BIT(16) +#define EFX_EMMC_BASE_REG0_CLK_DIV_MASK 0xFFFF + +#define EFX_EMMC_BASE_STATUS_DAT_BUSY BIT(1) +#define EFX_EMMC_BASE_STATUS_CMD_BUSY BIT(0) + +#define EFX_EMMC_BASE_REG1_SAMPLE_CNT_SHIFT 16 +#define EFX_EMMC_BASE_REG1_SAMPLE_CNT_MASK (0xFFFF << 16) +#define EFX_EMMC_BASE_REG1_PHASE_SHIFT 6 +#define EFX_EMMC_BASE_REG1_PHASE_MASK (0x7 << 6) +#define EFX_EMMC_BASE_REG1_PHASE_PULSE BIT(0) + +#define EFX_EMMC_BLOCK_COUNT_SHIFT 16 +#define EFX_EMMC_BLOCK_COUNT_MASK (0xFFFF << 16) +#define EFX_EMMC_BLOCK_SIZE_MASK 0xFFF +#define EFX_EMMC_DMA_BOUNDARY_SHIFT 12 +#define EFX_EMMC_DMA_BOUNDARY_MASK (0x7 << 12) + +#define EFX_EMMC_CMD_INDEX_SHIFT 24 +#define EFX_EMMC_CMD_INDEX_MASK (0x3F << 24) +#define EFX_EMMC_DATA_PRESENT BIT(21) +#define EFX_EMMC_CMD_INDEX_CHECK_EN BIT(20) +#define EFX_EMMC_CMD_CRC_CHECK_EN BIT(19) +#define EFX_EMMC_RESP_TYPE_SHIFT 16 +#define EFX_EMMC_RESP_TYPE_MASK (0x3 << 16) +#define EFX_EMMC_RESP_TYPE_NONE 0 +#define EFX_EMMC_RESP_TYPE_136 1 +#define EFX_EMMC_RESP_TYPE_48 2 +#define EFX_EMMC_RESP_TYPE_48_BUSY 3 +#define EFX_EMMC_MULTI_BLOCK_SEL BIT(5) +#define EFX_EMMC_DATA_XFER_DIR BIT(4) +#define EFX_EMMC_AUTO_CMD_EN_SHIFT 2 +#define EFX_EMMC_AUTO_CMD_EN_MASK (0x3 << 2) +#define EFX_EMMC_BLOCK_COUNT_EN BIT(1) +#define EFX_EMMC_DMA_EN BIT(0) + +#define EFX_EMMC_BUFFER_READ_EN BIT(11) +#define EFX_EMMC_BUFFER_WRITE_EN BIT(10) +#define EFX_EMMC_READ_XFER_ACTIVE BIT(9) +#define EFX_EMMC_WRITE_XFER_ACTIVE BIT(8) +#define EFX_EMMC_DAT_LINE_ACTIVE BIT(2) +#define EFX_EMMC_CMD_INHIBIT_DAT BIT(1) +#define EFX_EMMC_CMD_INHIBIT_CMD BIT(0) + +#define EFX_EMMC_DATA_SAMPLING_MODE BIT(3) +#define EFX_EMMC_DATA_WIDTH_SHIFT 1 +#define EFX_EMMC_DATA_WIDTH_MASK (0x3 << 1) +#define EFX_EMMC_DATA_WIDTH_1BIT 0 +#define EFX_EMMC_DATA_WIDTH_4BIT 1 +#define EFX_EMMC_DATA_WIDTH_8BIT 2 + +#define EFX_EMMC_INT_ADMA_ERROR BIT(25) +#define EFX_EMMC_INT_DATA_TIMEOUT_ERR BIT(22) +#define EFX_EMMC_INT_DATA_CRC_ERR BIT(21) +#define EFX_EMMC_INT_DATA_END_BIT_ERR BIT(20) +#define EFX_EMMC_INT_CMD_INDEX_ERR BIT(19) +#define EFX_EMMC_INT_CMD_END_BIT_ERR BIT(18) +#define EFX_EMMC_INT_CMD_CRC_ERR BIT(17) +#define EFX_EMMC_INT_CMD_TIMEOUT_ERR BIT(16) +#define EFX_EMMC_INT_BUFFER_READ_RDY BIT(5) +#define EFX_EMMC_INT_BUFFER_WRITE_RDY BIT(4) +#define EFX_EMMC_INT_BLOCK_GAP_EVENT BIT(2) +#define EFX_EMMC_INT_XFER_COMPLETE BIT(1) +#define EFX_EMMC_INT_CMD_COMPLETE BIT(0) + +#define EFX_EMMC_INT_ERROR_MASK (EFX_EMMC_INT_ADMA_ERROR | \ + EFX_EMMC_INT_DATA_TIMEOUT_ERR | \ + EFX_EMMC_INT_DATA_CRC_ERR | \ + EFX_EMMC_INT_DATA_END_BIT_ERR | \ + EFX_EMMC_INT_CMD_INDEX_ERR | \ + EFX_EMMC_INT_CMD_END_BIT_ERR | \ + EFX_EMMC_INT_CMD_CRC_ERR | \ + EFX_EMMC_INT_CMD_TIMEOUT_ERR) + +#define EFX_EMMC_INT_ALL_MASK (EFX_EMMC_INT_ERROR_MASK | \ + EFX_EMMC_INT_BUFFER_READ_RDY | \ + EFX_EMMC_INT_BUFFER_WRITE_RDY | \ + EFX_EMMC_INT_BLOCK_GAP_EVENT | \ + EFX_EMMC_INT_XFER_COMPLETE | \ + EFX_EMMC_INT_CMD_COMPLETE) + +#define EFX_SYS_RESET_EMMC_DEV BIT(1) +#define EFX_SYS_RESET_EMMC_IP BIT(0) + +#define EFX_EMMC_BASE_CLK_FREQ_MHZ 200 +#define EFX_EMMC_MAX_BLOCK_LENGTH 512 +#define EFX_EMMC_TIMEOUT_CLK_FREQ 200000000 + +#define EFX_EMMC_MIN_FREQ 400000 +#define EFX_EMMC_MAX_FREQ 200000000 +#define EFX_EMMC_PIO_TIMEOUT_MS 1000 + +#define EFX_EMMC_CLOCK_STABILIZE_DELAY 1000 +#define EFX_EMMC_RESET_PULSE_WIDTH 1 +#define EFX_EMMC_POST_RESET_DELAY 200 +#define EFX_EMMC_CMD_RETRY_COUNT 3 + +#define EFX_EMMC_MAX_PLL_SHIFT 8 +#define EFX_EMMC_PLL_SETTLING_TIME 50 +#define EFX_EMMC_TUNING_TIMEOUT_MS 50 +#define EFX_EMMC_TUNING_POLL_INTERVAL 200 +#define EFX_EMMC_TUNING_BLOCK_SIZE_4BIT 64 +#define EFX_EMMC_TUNING_BLOCK_SIZE_8BIT 128 +#define EFX_EMMC_MIN_TIMING_MARGIN 1 + +struct efx_adma_desc { + u16 attr; + u16 len; + u32 addr; +} __packed __aligned(8); + +struct efx_emmc_host { + struct mmc_host *mmc; + void __iomem *ioaddr; + void __iomem *sys_ioaddr; + struct clk *clk; + int irq; + + struct mmc_request *mrq; + struct mmc_request *mrq_done; + struct mmc_command *cmd; + struct mmc_data *data; + + u32 base_clk; + u32 current_clk; + + int bytes_to_transfer; + int blocks_done; + unsigned int sg_offset; + + struct efx_adma_desc *adma_desc; + dma_addr_t adma_desc_dma; + size_t adma_desc_sz; + + void *bounce_buffer; + dma_addr_t bounce_dma; + unsigned int bounce_buffer_size; + unsigned int bounce_used; + bool bounce_active; + + bool use_dma; + bool dma_64bit; + + u32 clk_div; + u32 hs200_sample_count; + u32 hs200_pll_shift; + u32 hs200_margin; + u32 hs400_sample_count; + u32 hs400_pll_shift; + u32 hs400_margin; + u32 hs400_default_sample; + u32 hs400_default_pll; + bool tuning_done; + bool tuning_in_progress; + unsigned int prev_timing; + bool hs400_retune_pending; + struct delayed_work hs400_retune_work; + + unsigned long tuned_timing_modes; + + spinlock_t lock; + + struct platform_device *pdev; +}; + +#define efx_emmc_dbg_irq(host, fmt, ...) \ + dev_dbg(&(host)->pdev->dev, fmt, ##__VA_ARGS__) +#define efx_emmc_dbg_pio(host, fmt, ...) \ + dev_dbg(&(host)->pdev->dev, fmt, ##__VA_ARGS__) +#define efx_emmc_dbg_cmd(host, fmt, ...) \ + dev_dbg(&(host)->pdev->dev, fmt, ##__VA_ARGS__) + +int efx_emmc_probe(struct platform_device *pdev); +int efx_emmc_remove(struct platform_device *pdev); +int efx_emmc_init_hw(struct efx_emmc_host *host); +void efx_emmc_reset_hw(struct efx_emmc_host *host); +void efx_emmc_hs400_retune_work(struct work_struct *work); + +void efx_emmc_request(struct mmc_host *mmc, struct mmc_request *mrq); +void efx_emmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios); +int efx_emmc_get_cd(struct mmc_host *mmc); +int efx_emmc_card_busy_wrapper(struct mmc_host *mmc); +int efx_emmc_get_ro(struct mmc_host *mmc); +irqreturn_t efx_emmc_irq(int irq, void *dev_id); + +void efx_emmc_send_command(struct efx_emmc_host *host, struct mmc_command *cmd); +void efx_emmc_finish_request(struct efx_emmc_host *host, + struct mmc_request *mrq); +void efx_emmc_finish_command(struct efx_emmc_host *host); +void efx_emmc_finish_data(struct efx_emmc_host *host); +void efx_emmc_transfer_pio(struct efx_emmc_host *host); +void efx_emmc_set_clock(struct efx_emmc_host *host, unsigned int clock); +void efx_emmc_set_bus_width(struct efx_emmc_host *host, int width); +void efx_emmc_set_timing(struct efx_emmc_host *host, unsigned int timing); +bool efx_emmc_card_busy(struct efx_emmc_host *host); + +int efx_emmc_adma_table_pre(struct efx_emmc_host *host, + struct mmc_data *data); +void efx_emmc_adma_table_post(struct efx_emmc_host *host, + struct mmc_data *data); +void efx_emmc_prepare_dma(struct efx_emmc_host *host, struct mmc_data *data); +void efx_emmc_cleanup_dma(struct efx_emmc_host *host, struct mmc_data *data); +void efx_emmc_set_adma_addr(struct efx_emmc_host *host, dma_addr_t addr); + +int efx_emmc_execute_tuning(struct mmc_host *mmc, u32 opcode); +int efx_emmc_execute_tuning_command(struct efx_emmc_host *host, + int bus_width); +void efx_emmc_set_timing_config(struct efx_emmc_host *host, + u32 sample_count, u32 pll_shift); +int efx_emmc_find_optimal_timing(struct efx_emmc_host *host, + u8 result_map[][EFX_EMMC_MAX_PLL_SHIFT], + u32 max_sample_count); + +static inline u32 efx_emmc_readl(struct efx_emmc_host *host, u32 reg) +{ + return readl(host->ioaddr + reg); +} + +static inline void efx_emmc_writel(struct efx_emmc_host *host, u32 val, u32 reg) +{ + writel(val, host->ioaddr + reg); +} + +static inline u32 efx_emmc_sys_readl(struct efx_emmc_host *host, u32 reg) +{ + return readl(host->sys_ioaddr + reg); +} + +static inline void efx_emmc_sys_writel(struct efx_emmc_host *host, u32 val, + u32 reg) +{ + writel(val, host->sys_ioaddr + reg); +} + +#endif diff --git a/drivers/mmc/host/efx_emmc_core.c b/drivers/mmc/host/efx_emmc_core.c new file mode 100644 index 000000000..4c4a975a3 --- /dev/null +++ b/drivers/mmc/host/efx_emmc_core.c @@ -0,0 +1,975 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Efinix eMMC Host Controller Core Operations + * + * Copyright (C) 2025 Efinix, Inc. + * Author: Teoh Choon Zone + */ + + +#include +#include +#include +#include +#include +#include + +#include "efx_emmc.h" + +static void efx_emmc_complete_pending_request(struct efx_emmc_host *host) +{ + struct mmc_request *mrq; + + mrq = READ_ONCE(host->mrq_done); + if (!mrq) + return; + + WRITE_ONCE(host->mrq_done, NULL); + mmc_request_done(host->mmc, mrq); +} + +bool efx_emmc_card_busy(struct efx_emmc_host *host) +{ + return !!(efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE) & + EFX_EMMC_DAT_LINE_ACTIVE); +} + +static bool efx_emmc_need_stop_command(struct mmc_data *data, + struct mmc_request *mrq) +{ + if (data->blocks <= 1) + return false; + + if (!data->stop) + return false; + + if (mrq->sbc) + return false; + + return true; +} + +void efx_emmc_finish_request(struct efx_emmc_host *host, + struct mmc_request *mrq) +{ + + if (host->data) { + efx_emmc_cleanup_dma(host, host->data); + } + + host->mrq = NULL; + host->cmd = NULL; + host->data = NULL; + host->bytes_to_transfer = 0; + host->blocks_done = 0; + host->sg_offset = 0; + host->use_dma = false; + + if (mrq) + WRITE_ONCE(host->mrq_done, mrq); +} + +static void efx_emmc_transfer_pio_read(struct efx_emmc_host *host) +{ + struct mmc_data *data; + struct scatterlist *sg; + u32 *buf; + int words_in_fifo, words_transferred, i; + unsigned int sg_offset, remaining_in_sg, words_to_transfer; + u32 present_state; + + data = host->data; + if (!data) { + return; + } + + sg = data->sg; + if (!sg) { + dev_err(&host->pdev->dev, "No scatter-gather list for read\n"); + return; + } + + words_in_fifo = data->blksz / sizeof(u32); + sg_offset = host->sg_offset; + words_transferred = 0; + + while (words_transferred < words_in_fifo && + host->bytes_to_transfer > 0) { + while (sg && sg_offset >= sg->length) { + sg_offset -= sg->length; + sg = sg_next(sg); + } + + if (!sg) { + dev_err(&host->pdev->dev, + "No more scatter-gather entries\n"); + break; + } + + buf = (u32 *)(sg_virt(sg) + sg_offset); + remaining_in_sg = sg->length - sg_offset; + words_to_transfer = min3(remaining_in_sg / sizeof(u32), + (unsigned int)(words_in_fifo - + words_transferred), + (unsigned int)(host->bytes_to_transfer / + sizeof(u32))); + + if (words_to_transfer == 0) { + break; + } + + for (i = 0; i < words_to_transfer; i++) { + buf[i] = efx_emmc_readl(host, EFX_EMMC_BUFFER_DATA_PORT); + } + + words_transferred += words_to_transfer; + sg_offset += words_to_transfer * 4; + host->bytes_to_transfer -= words_to_transfer * 4; + + efx_emmc_dbg_pio(host, + "PIO read: %d words, %d bytes remaining\n", + words_to_transfer, host->bytes_to_transfer); + } + + host->sg_offset += words_transferred * 4; + host->blocks_done = (data->blksz * data->blocks - + host->bytes_to_transfer) / data->blksz; + + efx_emmc_dbg_pio(host, + "PIO read completed: %d words total, " + "%d blocks done, %d bytes remaining\n", + words_transferred, host->blocks_done, + host->bytes_to_transfer); + + present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE); + efx_emmc_dbg_pio(host, "Present state after PIO read: 0x%08x\n", + present_state); +} + +static void efx_emmc_transfer_pio_write(struct efx_emmc_host *host) +{ + struct mmc_data *data; + struct scatterlist *sg; + u32 *buf; + int words_in_fifo, words_transferred, i; + unsigned int sg_offset, remaining_in_sg, words_to_transfer; + u32 present_state; + + data = host->data; + if (!data) { + return; + } + + sg = data->sg; + if (!sg) { + dev_err(&host->pdev->dev, "No scatter-gather list for write\n"); + return; + } + + words_in_fifo = data->blksz / sizeof(u32); + sg_offset = host->sg_offset; + words_transferred = 0; + + while (words_transferred < words_in_fifo && + host->bytes_to_transfer > 0) { + while (sg && sg_offset >= sg->length) { + sg_offset -= sg->length; + sg = sg_next(sg); + } + + if (!sg) { + dev_err(&host->pdev->dev, + "No more scatter-gather entries\n"); + break; + } + + buf = (u32 *)(sg_virt(sg) + sg_offset); + remaining_in_sg = sg->length - sg_offset; + words_to_transfer = min3(remaining_in_sg / sizeof(u32), + (unsigned int)(words_in_fifo - + words_transferred), + (unsigned int)(host->bytes_to_transfer / + sizeof(u32))); + + if (words_to_transfer == 0) { + break; + } + + for (i = 0; i < words_to_transfer; i++) { + efx_emmc_writel(host, buf[i], EFX_EMMC_BUFFER_DATA_PORT); + } + + words_transferred += words_to_transfer; + sg_offset += words_to_transfer * 4; + host->bytes_to_transfer -= words_to_transfer * 4; + + efx_emmc_dbg_pio(host, + "PIO write: %d words, %d bytes remaining\n", + words_to_transfer, host->bytes_to_transfer); + } + + host->sg_offset += words_transferred * 4; + host->blocks_done = (data->blksz * data->blocks - + host->bytes_to_transfer) / data->blksz; + + efx_emmc_dbg_pio(host, + "PIO write completed: %d words total, %d blocks done, " + "%d bytes remaining\\n", + words_transferred, host->blocks_done, + host->bytes_to_transfer); + + present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE); + efx_emmc_dbg_pio(host, "Present state after PIO write: 0x%08x\n", + present_state); +} + +void efx_emmc_transfer_pio(struct efx_emmc_host *host) +{ + struct mmc_data *data; + u32 present_state; + + data = host->data; + if (!data) { + return; + } + + present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE); + + if (data->flags & MMC_DATA_READ) { + if (present_state & EFX_EMMC_BUFFER_READ_EN) { + efx_emmc_transfer_pio_read(host); + } + } else { + if (present_state & EFX_EMMC_BUFFER_WRITE_EN) { + efx_emmc_transfer_pio_write(host); + } + } +} + +void efx_emmc_finish_data(struct efx_emmc_host *host) +{ + struct mmc_data *data; + u32 present_state; + int retry_count; + + data = host->data; + if (!data) { + return; + } + + if (data->error == 0) { + data->bytes_xfered = data->blksz * data->blocks; + efx_emmc_dbg_cmd(host, + "Data transfer completed: %d bytes (DMA: %s)\n", + data->bytes_xfered, + host->use_dma ? "enabled" : "disabled"); + } else { + data->bytes_xfered = 0; + dev_err(&host->pdev->dev, + "Data transfer failed with error %d\n", + data->error); + } + + efx_emmc_cleanup_dma(host, data); + + host->data = NULL; + host->bytes_to_transfer = 0; + host->blocks_done = 0; + host->sg_offset = 0; + + if (efx_emmc_need_stop_command(data, host->mrq) && !data->error) { + retry_count = 0; + while (retry_count < 100) { + present_state = efx_emmc_readl(host, + EFX_EMMC_PRESENT_STATE); + if (!(present_state & (EFX_EMMC_DAT_LINE_ACTIVE | + EFX_EMMC_READ_XFER_ACTIVE | + EFX_EMMC_WRITE_XFER_ACTIVE))) { + break; + } + udelay(10); + retry_count++; + } + + if (retry_count >= 100) { + dev_warn(&host->pdev->dev, + "Data lines still active before CMD12, proceeding anyway\n"); + } + + udelay(100); + + efx_emmc_dbg_cmd(host, + "Sending CMD12 (STOP) for %d-block transfer\n", + data->blocks); + host->cmd = data->stop; + efx_emmc_send_command(host, data->stop); + } else { + efx_emmc_dbg_cmd(host, "Finishing request\n"); + if (host->mrq) { + efx_emmc_finish_request(host, host->mrq); + } + } +} + +void efx_emmc_finish_command(struct efx_emmc_host *host) +{ + struct mmc_command *cmd; + u32 resp[4]; + u32 present_state; + + cmd = host->cmd; + if (!cmd) { + return; + } + + if (cmd->flags & MMC_RSP_PRESENT) { + if (cmd->flags & MMC_RSP_136) { + resp[0] = efx_emmc_readl(host, EFX_EMMC_RESPONSE0); + resp[1] = efx_emmc_readl(host, EFX_EMMC_RESPONSE1); + resp[2] = efx_emmc_readl(host, EFX_EMMC_RESPONSE2); + resp[3] = efx_emmc_readl(host, EFX_EMMC_RESPONSE3) & 0xFFFFFF; + + cmd->resp[0] = resp[3] << 8 | resp[2] >> 24; + cmd->resp[1] = resp[2] << 8 | resp[1] >> 24; + cmd->resp[2] = resp[1] << 8 | resp[0] >> 24; + cmd->resp[3] = resp[0] << 8; + + efx_emmc_dbg_cmd(host, + "CMD%d 136-bit response: %08x %08x %08x %08x\n", + cmd->opcode, cmd->resp[0], cmd->resp[1], + cmd->resp[2], cmd->resp[3]); + } else { + cmd->resp[0] = efx_emmc_readl(host, EFX_EMMC_RESPONSE0); + efx_emmc_dbg_cmd(host, "CMD%d response: 0x%08x\n", + cmd->opcode, cmd->resp[0]); + } + } + + if (cmd->opcode == 6 && cmd->error == 0) { + u32 arg = cmd->arg; + u8 index = (arg >> 16) & 0xFF; + u8 value = (arg >> 8) & 0xFF; + + if (index == 185) { + switch (value) { + case 0: + case 1: + dev_dbg(&host->pdev->dev, + "CMD6 mode switch to legacy/HS completed\n"); + efx_emmc_set_timing_config(host, 0, 0); + break; + case 2: + dev_dbg(&host->pdev->dev, + "CMD6 mode switch to DDR52 completed\n"); + efx_emmc_set_timing_config(host, 2, 2); + break; + case 3: + dev_dbg(&host->pdev->dev, + "CMD6 mode switch to HS200 completed\n"); + break; + default: + efx_emmc_dbg_cmd(host, + "CMD6 mode switch to timing value %u\n", + value); + break; + } + + efx_emmc_writel(host, 0xFFFFFFFF, EFX_EMMC_INT_STATUS); + + } + } + + if (cmd->opcode == 23 && host->mrq && host->mrq->cmd) { + efx_emmc_dbg_cmd(host, + "CMD23 complete, sending main command CMD%d\n", + host->mrq->cmd->opcode); + host->cmd = host->mrq->cmd; + efx_emmc_send_command(host, host->mrq->cmd); + return; + } + + host->cmd = NULL; + + if (!host->data) { + efx_emmc_dbg_cmd(host, "Command complete, finishing request\n"); + if (host->mrq) { + efx_emmc_finish_request(host, host->mrq); + } + } else { + efx_emmc_dbg_cmd(host, + "Command complete, data transfer continues (DMA: %s)\n", + host->use_dma ? "enabled" : "disabled"); + + if (!host->use_dma) { + bool buffer_ready_read, buffer_ready_write; + + present_state = efx_emmc_readl(host, + EFX_EMMC_PRESENT_STATE); + efx_emmc_dbg_cmd(host, + "Present state after command: 0x%08x\n", + present_state); + + buffer_ready_read = !!(present_state & EFX_EMMC_BUFFER_READ_EN); + buffer_ready_write = !!(present_state & EFX_EMMC_BUFFER_WRITE_EN); + + if (host->data->flags & MMC_DATA_READ) { + if (buffer_ready_read) { + efx_emmc_dbg_pio(host, + "Buffer immediately ready for read, starting PIO\n"); + efx_emmc_transfer_pio(host); + } + } else { + if (buffer_ready_write) { + efx_emmc_dbg_pio(host, + "Buffer immediately ready for write, starting PIO\n"); + efx_emmc_transfer_pio(host); + } + } + } + } +} + +void efx_emmc_send_command(struct efx_emmc_host *host, struct mmc_command *cmd) +{ + u32 command, present_state; + unsigned long timeout; + struct mmc_data *data; + u16 cmd_timeout; + + data = cmd->data; + host->cmd = cmd; + + efx_emmc_dbg_cmd(host, "Sending CMD%d, arg=0x%08x%s\n", + cmd->opcode, cmd->arg, (cmd->opcode == 12) ? " (STOP)" : ""); + + if (cmd->opcode == 6) { + u32 arg = cmd->arg; + u8 index = (arg >> 16) & 0xFF; + u8 value = (arg >> 8) & 0xFF; + + if (index == 185) { + dev_dbg(&host->pdev->dev, + "CMD6 mode switch detected (index=%u, value=%u)\n", + index, value); + efx_emmc_set_timing_config(host, 0, 0); + msleep(10); + + efx_emmc_writel(host, 0xFFFFFFFF, EFX_EMMC_INT_STATUS); + } + } + + switch (cmd->opcode) { + case 6: + cmd_timeout = 2000; + break; + case 12: + cmd_timeout = 1000; + break; + case 23: + cmd_timeout = 500; + break; + default: + cmd_timeout = 500; + break; + } + timeout = jiffies + msecs_to_jiffies(cmd_timeout); + while (time_before(jiffies, timeout)) { + present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE); + if (!(present_state & EFX_EMMC_CMD_INHIBIT_CMD)) { + if (!data || !(present_state & EFX_EMMC_CMD_INHIBIT_DAT)) { + break; + } + } + cpu_relax(); + } + + if (time_after_eq(jiffies, timeout)) { + dev_err(&host->pdev->dev, + "Command line timeout, present_state=0x%08x\n", + efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE)); + cmd->error = -ETIMEDOUT; + if (host->mrq) { + efx_emmc_finish_request(host, host->mrq); + } + return; + } + + efx_emmc_writel(host, cmd->arg, EFX_EMMC_ARG1); + + command = (cmd->opcode << EFX_EMMC_CMD_INDEX_SHIFT) & + EFX_EMMC_CMD_INDEX_MASK; + + if (cmd->flags & MMC_RSP_PRESENT) { + if (cmd->flags & MMC_RSP_136) { + command |= (EFX_EMMC_RESP_TYPE_136 << EFX_EMMC_RESP_TYPE_SHIFT); + } else if (cmd->flags & MMC_RSP_BUSY) { + command |= (EFX_EMMC_RESP_TYPE_48_BUSY << EFX_EMMC_RESP_TYPE_SHIFT); + } else { + command |= (EFX_EMMC_RESP_TYPE_48 << EFX_EMMC_RESP_TYPE_SHIFT); + } + + if (cmd->flags & MMC_RSP_CRC) { + command |= EFX_EMMC_CMD_CRC_CHECK_EN; + } + + if (cmd->flags & MMC_RSP_OPCODE) { + command |= EFX_EMMC_CMD_INDEX_CHECK_EN; + } + } + + if (data) { + command |= EFX_EMMC_DATA_PRESENT; + + host->use_dma = (data->blksz * data->blocks >= 512); + + if (host->use_dma) { + efx_emmc_prepare_dma(host, data); + } + + efx_emmc_writel(host, (data->blocks << EFX_EMMC_BLOCK_COUNT_SHIFT) | + (data->blksz & EFX_EMMC_BLOCK_SIZE_MASK), + EFX_EMMC_BLOCK_SIZE); + + if (data->blocks > 1) { + command |= EFX_EMMC_MULTI_BLOCK_SEL; + command |= EFX_EMMC_BLOCK_COUNT_EN; + if (host->mrq && host->mrq->sbc) { + efx_emmc_dbg_cmd(host, + "Multi-block transfer: CMD23 used, no CMD12 needed\n"); + } else { + efx_emmc_dbg_cmd(host, + "Multi-block transfer: manual CMD12 will be used\n"); + } + } + + if (data->flags & MMC_DATA_READ) { + command |= EFX_EMMC_DATA_XFER_DIR; + } + + if (host->use_dma) { + command |= EFX_EMMC_DMA_EN; + efx_emmc_dbg_cmd(host, "DMA enabled for data transfer\n"); + } else { + host->bytes_to_transfer = data->blksz * data->blocks; + host->blocks_done = 0; + host->sg_offset = 0; + efx_emmc_dbg_cmd(host, "Using PIO for data transfer\n"); + } + + host->data = data; + + efx_emmc_dbg_cmd(host, "Data transfer: %d blocks of %d bytes, %s, %s\n", + data->blocks, data->blksz, + (data->flags & MMC_DATA_READ) ? "read" : "write", + host->use_dma ? "DMA" : "PIO"); + } + + efx_emmc_dbg_cmd(host, "Command register: 0x%08x\n", command); + + efx_emmc_writel(host, command, EFX_EMMC_TRANSFER_MODE); +} + +void efx_emmc_set_clock(struct efx_emmc_host *host, unsigned int clock) +{ + u32 div, reg; + unsigned long timeout; + + if (clock == 0) { + reg = efx_emmc_readl(host, EFX_EMMC_BASE_REG0); + reg &= ~EFX_EMMC_BASE_REG0_CLK_EN; + efx_emmc_writel(host, reg, EFX_EMMC_BASE_REG0); + host->current_clk = 0; + return; + } + + if (clock > EFX_EMMC_MAX_FREQ) { + clock = EFX_EMMC_MAX_FREQ; + } + + if (clock >= host->base_clk) { + div = 1; + } else { + div = (host->base_clk + clock - 1) / clock; + if (div > 1 && (div & 1)) { + div += 1; + } + } + + reg = efx_emmc_readl(host, EFX_EMMC_BASE_REG0); + reg &= ~EFX_EMMC_BASE_REG0_CLK_EN; + efx_emmc_writel(host, reg, EFX_EMMC_BASE_REG0); + + reg = (reg & ~EFX_EMMC_BASE_REG0_CLK_DIV_MASK) | + (div & EFX_EMMC_BASE_REG0_CLK_DIV_MASK); + efx_emmc_writel(host, reg, EFX_EMMC_BASE_REG0); + + reg |= EFX_EMMC_BASE_REG0_CLK_EN; + efx_emmc_writel(host, reg, EFX_EMMC_BASE_REG0); + + if (clock <= 400000) { + usleep_range(500, 1000); + } else if (clock <= 25000000) { + usleep_range(100, 200); + } else { + usleep_range(50, 100); + } + + timeout = jiffies + msecs_to_jiffies(50); + while (time_before(jiffies, timeout)) { + if (!(efx_emmc_readl(host, EFX_EMMC_BASE_STATUS_REG0) & + (EFX_EMMC_BASE_STATUS_CMD_BUSY | + EFX_EMMC_BASE_STATUS_DAT_BUSY))) { + break; + } + cpu_relax(); + } + + host->current_clk = host->base_clk / div; + host->clk_div = div; + dev_dbg(&host->pdev->dev, "Set clock to %u Hz (div=%u, actual=%u)\n", + clock, div, host->current_clk); +} + +void efx_emmc_set_bus_width(struct efx_emmc_host *host, int width) +{ + u32 reg; + + reg = efx_emmc_readl(host, EFX_EMMC_HOST_CONTROL); + reg &= ~EFX_EMMC_DATA_WIDTH_MASK; + + switch (width) { + case MMC_BUS_WIDTH_1: + reg |= (EFX_EMMC_DATA_WIDTH_1BIT << EFX_EMMC_DATA_WIDTH_SHIFT); + break; + case MMC_BUS_WIDTH_4: + reg |= (EFX_EMMC_DATA_WIDTH_4BIT << EFX_EMMC_DATA_WIDTH_SHIFT); + break; + case MMC_BUS_WIDTH_8: + reg |= (EFX_EMMC_DATA_WIDTH_8BIT << EFX_EMMC_DATA_WIDTH_SHIFT); + break; + default: + dev_warn(&host->pdev->dev, "Unsupported bus width: %d\n", width); + return; + } + + efx_emmc_writel(host, reg, EFX_EMMC_HOST_CONTROL); + dev_dbg(&host->pdev->dev, "Set bus width to %d bits\n", width); +} + +void efx_emmc_set_timing(struct efx_emmc_host *host, unsigned int timing) +{ + u32 reg; + bool needs_tuning = false; + + reg = efx_emmc_readl(host, EFX_EMMC_HOST_CONTROL); + + switch (timing) { + case MMC_TIMING_LEGACY: + case MMC_TIMING_MMC_HS: + reg &= ~EFX_EMMC_DATA_SAMPLING_MODE; + break; + case MMC_TIMING_MMC_DDR52: + reg |= EFX_EMMC_DATA_SAMPLING_MODE; + host->tuning_done = true; + set_bit(timing, &host->tuned_timing_modes); + break; + case MMC_TIMING_MMC_HS200: + reg &= ~EFX_EMMC_DATA_SAMPLING_MODE; + if (!test_bit(timing, &host->tuned_timing_modes)) { + needs_tuning = true; + host->tuning_done = false; + dev_dbg(&host->pdev->dev, "HS200 mode - tuning required\n"); + } else { + efx_emmc_set_timing_config(host, host->hs200_sample_count, + host->hs200_pll_shift); + dev_dbg(&host->pdev->dev, + "HS200 mode - already tuned, restored timing (sample=%u, pll=%u)\n", + host->hs200_sample_count, host->hs200_pll_shift); + } + break; + case MMC_TIMING_MMC_HS400: + reg |= EFX_EMMC_DATA_SAMPLING_MODE; + + if (!test_bit(timing, &host->tuned_timing_modes)) { + u32 initial_sample = host->hs400_default_sample; + u32 initial_pll = host->hs400_default_pll; + + efx_emmc_set_timing_config(host, initial_sample, initial_pll); + needs_tuning = true; + host->tuning_done = false; + host->hs400_retune_pending = true; + + dev_info(&host->pdev->dev, + "HS400 mode - using default sample=%u pll=%u, clock=%u Hz\n", + initial_sample, initial_pll, host->mmc->ios.clock); + dev_info(&host->pdev->dev, + "HS400 tuning scheduled (clock=%u Hz)\n", + host->mmc->ios.clock); + schedule_delayed_work(&host->hs400_retune_work, + msecs_to_jiffies(10)); + } else { + efx_emmc_set_timing_config(host, host->hs400_sample_count, + host->hs400_pll_shift); + dev_info(&host->pdev->dev, + "HS400 mode - already tuned, restored timing (sample=%u, pll=%u)\n", + host->hs400_sample_count, host->hs400_pll_shift); + } + break; + default: + dev_warn(&host->pdev->dev, "Unsupported timing: %d\n", timing); + return; + } + + efx_emmc_writel(host, reg, EFX_EMMC_HOST_CONTROL); + dev_dbg(&host->pdev->dev, "Set timing mode: %s (%d) - %s\n", + timing == MMC_TIMING_LEGACY ? "Legacy" : + timing == MMC_TIMING_MMC_HS ? "High Speed" : + timing == MMC_TIMING_MMC_DDR52 ? "DDR52" : + timing == MMC_TIMING_MMC_HS200 ? "HS200" : + timing == MMC_TIMING_MMC_HS400 ? "HS400" : "Unknown", timing, + needs_tuning ? "NEEDS TUNING" : "NO TUNING NEEDED"); + + host->prev_timing = timing; +} + +irqreturn_t efx_emmc_irq(int irq, void *dev_id) +{ + struct efx_emmc_host *host; + u32 intstat, present_state; + irqreturn_t result; + + host = dev_id; + result = IRQ_NONE; + + spin_lock(&host->lock); + + intstat = efx_emmc_readl(host, EFX_EMMC_INT_STATUS); + if (!intstat) { + goto out; + } + + present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE); + efx_emmc_dbg_irq(host, "IRQ: status=0x%08x, present=0x%08x\n", + intstat, present_state); + + efx_emmc_writel(host, intstat, EFX_EMMC_INT_STATUS); + result = IRQ_HANDLED; + + if (intstat & EFX_EMMC_INT_ERROR_MASK) { + if (intstat & EFX_EMMC_INT_ADMA_ERROR) { + dev_err(&host->pdev->dev, "ADMA error detected\n"); + if (host->data) { + host->data->error = -EIO; + } + } + + if (host->cmd && (host->cmd->opcode == 52 || host->cmd->opcode == 8 || + host->cmd->opcode == 5 || host->cmd->opcode == 55)) { + efx_emmc_dbg_irq(host, + "Expected timeout for CMD%d during card detection\n", + host->cmd->opcode); + } else if (host->tuning_in_progress) { + efx_emmc_dbg_irq(host, "Tuning error (expected): 0x%08x\n", + (unsigned int)(intstat & EFX_EMMC_INT_ERROR_MASK)); + } else { + dev_err(&host->pdev->dev, "Error interrupt: 0x%08x\n", + (unsigned int)(intstat & EFX_EMMC_INT_ERROR_MASK)); + } + + if (host->cmd) { + if (intstat & EFX_EMMC_INT_CMD_TIMEOUT_ERR) { + host->cmd->error = -ETIMEDOUT; + if (host->cmd->opcode == 12) { + dev_warn(&host->pdev->dev, + "CMD12 timeout - data may have completed normally\n"); + host->cmd = NULL; + if (host->mrq) { + efx_emmc_finish_request(host, host->mrq); + } + goto out; + } else if (host->cmd->opcode == 6) { + dev_err(&host->pdev->dev, + "CMD6 (mode switch) timeout - arg=0x%08x\n", + host->cmd->arg); + } else if (!(host->cmd->opcode == 52 || + host->cmd->opcode == 8 || + host->cmd->opcode == 5 || + host->cmd->opcode == 55)) { + dev_err(&host->pdev->dev, "CMD%d timeout\n", + host->cmd->opcode); + } + } else if (intstat & (EFX_EMMC_INT_CMD_CRC_ERR | + EFX_EMMC_INT_CMD_END_BIT_ERR | + EFX_EMMC_INT_CMD_INDEX_ERR)) { + host->cmd->error = -EILSEQ; + if (host->cmd->opcode == 6) { + dev_err(&host->pdev->dev, + "CMD6 (mode switch) CRC/protocol error - arg=0x%08x\n", + host->cmd->arg); + } else { + dev_err(&host->pdev->dev, + "CMD%d CRC/protocol error\n", + host->cmd->opcode); + } + } + } + + if (host->data) { + if (intstat & EFX_EMMC_INT_DATA_TIMEOUT_ERR) { + host->data->error = -ETIMEDOUT; + dev_err(&host->pdev->dev, "Data timeout error\n"); + } + if (intstat & EFX_EMMC_INT_DATA_CRC_ERR) { + host->data->error = -EILSEQ; + dev_err(&host->pdev->dev, "Data CRC error\n"); + } + if (intstat & EFX_EMMC_INT_DATA_END_BIT_ERR) { + host->data->error = -EILSEQ; + dev_err(&host->pdev->dev, "Data end bit error\n"); + } + } + + if (!(host->cmd && host->cmd->opcode == 12 && + (intstat & EFX_EMMC_INT_CMD_TIMEOUT_ERR))) { + if (host->mrq) { + efx_emmc_finish_request(host, host->mrq); + } + } + goto out; + } + + if (intstat & EFX_EMMC_INT_CMD_COMPLETE) { + efx_emmc_dbg_irq(host, "Command complete\n"); + efx_emmc_finish_command(host); + } + + if (likely(!host->use_dma) && + (intstat & (EFX_EMMC_INT_BUFFER_READ_RDY | + EFX_EMMC_INT_BUFFER_WRITE_RDY))) { + efx_emmc_dbg_irq(host, "Buffer ready for %s\n", + (intstat & EFX_EMMC_INT_BUFFER_READ_RDY) ? "read" : "write"); + efx_emmc_transfer_pio(host); + + if (host->data && host->bytes_to_transfer > 0) { + efx_emmc_dbg_pio(host, + "Waiting for more data: %d bytes remaining\n", + host->bytes_to_transfer); + } else if (host->data && host->bytes_to_transfer == 0) { + efx_emmc_dbg_pio(host, + "All data transferred via PIO, completing transfer\n"); + efx_emmc_finish_data(host); + } + } + + if (intstat & EFX_EMMC_INT_XFER_COMPLETE) { + efx_emmc_dbg_irq(host, "Transfer complete interrupt (DMA: %s)\n", + host->use_dma ? "enabled" : "disabled"); + efx_emmc_finish_data(host); + } + + if (intstat & ~(EFX_EMMC_INT_CMD_COMPLETE | EFX_EMMC_INT_BUFFER_READ_RDY | + EFX_EMMC_INT_BUFFER_WRITE_RDY | + EFX_EMMC_INT_XFER_COMPLETE | EFX_EMMC_INT_ERROR_MASK)) { + efx_emmc_dbg_irq(host, "Unhandled interrupt bits: 0x%08x\n", + (unsigned int)(intstat & + ~(EFX_EMMC_INT_CMD_COMPLETE | + EFX_EMMC_INT_BUFFER_READ_RDY | + EFX_EMMC_INT_BUFFER_WRITE_RDY | + EFX_EMMC_INT_XFER_COMPLETE | + EFX_EMMC_INT_ERROR_MASK))); + } + +out: + spin_unlock(&host->lock); + efx_emmc_complete_pending_request(host); + return result; +} + +void efx_emmc_request(struct mmc_host *mmc, struct mmc_request *mrq) +{ + struct efx_emmc_host *host; + unsigned long flags; + + host = mmc_priv(mmc); + + spin_lock_irqsave(&host->lock, flags); + + if (!host->clk) { + dev_err(&host->pdev->dev, "No clock available\n"); + mrq->cmd->error = -ENODEV; + spin_unlock_irqrestore(&host->lock, flags); + mmc_request_done(mmc, mrq); + return; + } + + efx_emmc_dbg_cmd(host, "New request: CMD%d\n", mrq->cmd->opcode); + + host->mrq = mrq; + + if (mrq->sbc) { + efx_emmc_dbg_cmd(host, + "Sending CMD23 (SET_BLOCK_COUNT) first, blocks=%u\n", + mrq->sbc->arg); + host->cmd = mrq->sbc; + efx_emmc_send_command(host, mrq->sbc); + } else { + efx_emmc_send_command(host, mrq->cmd); + } + + spin_unlock_irqrestore(&host->lock, flags); + efx_emmc_complete_pending_request(host); +} + +void efx_emmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) +{ + struct efx_emmc_host *host; + unsigned long flags; + bool need_host_control_update = false; + + host = mmc_priv(mmc); + + spin_lock_irqsave(&host->lock, flags); + + if (ios->clock != host->current_clk) { + efx_emmc_set_clock(host, ios->clock); + } + + if (ios->bus_width != MMC_BUS_WIDTH_1) { + efx_emmc_set_bus_width(host, ios->bus_width); + need_host_control_update = true; + } + + if (ios->timing != host->prev_timing) { + dev_dbg(&host->pdev->dev, "Timing mode: %u -> %u, Clock: %u Hz\n", + host->prev_timing, ios->timing, ios->clock); + } + + efx_emmc_set_timing(host, ios->timing); + + spin_unlock_irqrestore(&host->lock, flags); +} + +int efx_emmc_get_cd(struct mmc_host *mmc) +{ + return 1; +} + +int efx_emmc_card_busy_wrapper(struct mmc_host *mmc) +{ + struct efx_emmc_host *host; + unsigned long flags; + bool busy; + + host = mmc_priv(mmc); + + spin_lock_irqsave(&host->lock, flags); + busy = efx_emmc_card_busy(host); + spin_unlock_irqrestore(&host->lock, flags); + + return busy; +} + +int efx_emmc_get_ro(struct mmc_host *mmc) +{ + return 0; +} diff --git a/drivers/mmc/host/efx_emmc_dma.c b/drivers/mmc/host/efx_emmc_dma.c new file mode 100644 index 000000000..a50634996 --- /dev/null +++ b/drivers/mmc/host/efx_emmc_dma.c @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Efinix eMMC Host Controller DMA Support + * + * Copyright (C) 2025 Efinix, Inc. + * Author: Teoh Choon Zone + */ + + +#include +#include +#include + +#include "efx_emmc.h" + +void efx_emmc_set_adma_addr(struct efx_emmc_host *host, dma_addr_t addr) +{ + efx_emmc_writel(host, (u32)addr, EFX_EMMC_ADMA_SYS_ADDR_LOW); + + efx_emmc_writel(host, 0, EFX_EMMC_ADMA_SYS_ADDR_HIGH); +} + +static void efx_emmc_adma_mark_end(struct efx_adma_desc *desc) +{ + desc->attr |= EFX_ADMA_DESC_END; +} + +static void efx_emmc_adma_set_desc(struct efx_adma_desc *desc, u32 addr, + u16 len, u16 attr) +{ + desc->attr = attr; + desc->len = len; + desc->addr = addr; +} + +int efx_emmc_adma_table_pre(struct efx_emmc_host *host, struct mmc_data *data) +{ + struct efx_adma_desc *desc; + struct scatterlist *sg; + dma_addr_t addr, align_addr; + u32 len, offset, align_len; + int i, desc_count = 0; + bool use_bounce = false; + + for_each_sg(data->sg, sg, data->sg_len, i) { + addr = sg_dma_address(sg); + len = sg_dma_len(sg); + + if ((addr & 0x7) || (len & 0x7)) { + use_bounce = true; + break; + } + } + + if (use_bounce) { + if (!host->bounce_buffer) { + dev_err(&host->pdev->dev, "Bounce buffer not available\n"); + return -ENOMEM; + } + + if (data->blksz * data->blocks > host->bounce_buffer_size) { + dev_err(&host->pdev->dev, "Transfer too large for bounce buffer\n"); + return -EINVAL; + } + + if (data->flags & MMC_DATA_WRITE) { + struct scatterlist *sg; + char *bounce_pos; + int i; + + bounce_pos = host->bounce_buffer; + + for_each_sg(data->sg, sg, data->sg_len, i) { + memcpy(bounce_pos, sg_virt(sg), sg->length); + bounce_pos += sg->length; + } + } + + desc = host->adma_desc; + efx_emmc_adma_set_desc(desc, host->bounce_dma, + data->blksz * data->blocks, + EFX_ADMA_DESC_VALID | EFX_ADMA_DESC_TRAN); + efx_emmc_adma_mark_end(desc); + desc_count = 1; + host->bounce_active = true; + host->bounce_used = data->blksz * data->blocks; + } else { + desc = host->adma_desc; + host->bounce_active = false; + host->bounce_used = 0; + + for_each_sg(data->sg, sg, data->sg_len, i) { + addr = sg_dma_address(sg); + len = sg_dma_len(sg); + offset = 0; + + while (len > 0) { + align_addr = addr + offset; + align_len = min(len, (u32)EFX_ADMA_MAX_LEN); + + if (desc_count >= + (EFX_ADMA_TABLE_SZ / sizeof(struct efx_adma_desc))) { + dev_err(&host->pdev->dev, "Too many ADMA descriptors\n"); + return -EINVAL; + } + + efx_emmc_adma_set_desc(&desc[desc_count], align_addr, align_len, + EFX_ADMA_DESC_VALID | EFX_ADMA_DESC_TRAN); + + offset += align_len; + len -= align_len; + desc_count++; + } + } + + if (desc_count > 0) { + efx_emmc_adma_mark_end(&desc[desc_count - 1]); + } + } + + if (desc_count == 0) { + dev_err(&host->pdev->dev, "No ADMA descriptors created\n"); + return -EINVAL; + } + + dma_sync_single_for_device(&host->pdev->dev, host->adma_desc_dma, + host->adma_desc_sz, DMA_TO_DEVICE); + + return 0; +} + +void efx_emmc_adma_table_post(struct efx_emmc_host *host, struct mmc_data *data) +{ + dma_sync_single_for_cpu(&host->pdev->dev, host->adma_desc_dma, + host->adma_desc_sz, DMA_FROM_DEVICE); + + if (host->bounce_buffer && (data->flags & MMC_DATA_READ)) { + struct scatterlist *sg; + char *bounce_pos; + int i; + unsigned int remaining; + + if (!host->bounce_active) + goto out; + + remaining = host->bounce_used; + + bounce_pos = host->bounce_buffer; + + for_each_sg(data->sg, sg, data->sg_len, i) { + unsigned int len = min(sg->length, remaining); + + if (!len) + break; + + memcpy(sg_virt(sg), bounce_pos, len); + bounce_pos += len; + remaining -= len; + } + } +out: + host->bounce_active = false; + host->bounce_used = 0; +} + +void efx_emmc_prepare_dma(struct efx_emmc_host *host, struct mmc_data *data) +{ + int ret; + + if (!host->use_dma || !data) { + return; + } + + ret = dma_map_sg(&host->pdev->dev, data->sg, data->sg_len, + (data->flags & MMC_DATA_READ) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (ret == 0) { + dev_err(&host->pdev->dev, "Failed to map DMA scatter-gather list\n"); + host->use_dma = false; + return; + } + + data->sg_len = ret; + + ret = efx_emmc_adma_table_pre(host, data); + if (ret) { + dev_err(&host->pdev->dev, "Failed to setup ADMA table: %d\n", ret); + dma_unmap_sg(&host->pdev->dev, data->sg, data->sg_len, + (data->flags & MMC_DATA_READ) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + host->use_dma = false; + return; + } + + efx_emmc_set_adma_addr(host, host->adma_desc_dma); +} + +void efx_emmc_cleanup_dma(struct efx_emmc_host *host, struct mmc_data *data) +{ + if (!host->use_dma || !data) { + return; + } + + efx_emmc_adma_table_post(host, data); + + dma_unmap_sg(&host->pdev->dev, data->sg, data->sg_len, + (data->flags & MMC_DATA_READ) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); +} diff --git a/drivers/mmc/host/efx_emmc_platform.c b/drivers/mmc/host/efx_emmc_platform.c new file mode 100644 index 000000000..dcfc5222d --- /dev/null +++ b/drivers/mmc/host/efx_emmc_platform.c @@ -0,0 +1,370 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Efinix eMMC Host Controller Platform Driver + * + * Copyright (C) 2025 Efinix, Inc. + * Author: Teoh Choon Zone + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "efx_emmc.h" + +static const struct mmc_host_ops efx_emmc_ops = { + .request = efx_emmc_request, + .set_ios = efx_emmc_set_ios, + .get_cd = efx_emmc_get_cd, + .get_ro = efx_emmc_get_ro, + .card_busy = efx_emmc_card_busy_wrapper, + .execute_tuning = efx_emmc_execute_tuning, +}; + +void efx_emmc_hs400_retune_work(struct work_struct *work) +{ + struct efx_emmc_host *host = container_of(work, struct efx_emmc_host, + hs400_retune_work.work); + + dev_info(&host->pdev->dev, "HS400 delayed retune worker started\n"); + + if (host->hs400_retune_pending && + host->mmc && + host->mmc->ios.timing == MMC_TIMING_MMC_HS400) { + + dev_info(&host->pdev->dev, + "HS400 conditions met (clock=%u Hz), executing tuning\n", + host->mmc->ios.clock); + + efx_emmc_execute_tuning(host->mmc, MMC_SEND_TUNING_BLOCK_HS200); + + } else { + dev_warn(&host->pdev->dev, + "HS400 conditions not met: pending=%s, timing=%u, clock=%u\n", + host->hs400_retune_pending ? "true" : "false", + host->mmc ? host->mmc->ios.timing : 0, + host->mmc ? host->mmc->ios.clock : 0); + } + + dev_info(&host->pdev->dev, "HS400 delayed retune worker completed\n"); +} + +void efx_emmc_reset_hw(struct efx_emmc_host *host) +{ + u32 reg; + + reg = efx_emmc_sys_readl(host, EFX_SYS_RESET_REG); + reg |= EFX_SYS_RESET_EMMC_IP; + efx_emmc_sys_writel(host, reg, EFX_SYS_RESET_REG); + udelay(EFX_EMMC_RESET_PULSE_WIDTH); + + reg &= ~EFX_SYS_RESET_EMMC_IP; + efx_emmc_sys_writel(host, reg, EFX_SYS_RESET_REG); + udelay(EFX_EMMC_RESET_PULSE_WIDTH); + + reg |= EFX_SYS_RESET_EMMC_DEV; + efx_emmc_sys_writel(host, reg, EFX_SYS_RESET_REG); + udelay(EFX_EMMC_RESET_PULSE_WIDTH); + + reg &= ~EFX_SYS_RESET_EMMC_DEV; + efx_emmc_sys_writel(host, reg, EFX_SYS_RESET_REG); + + udelay(EFX_EMMC_POST_RESET_DELAY); +} + +int efx_emmc_init_hw(struct efx_emmc_host *host) +{ + u32 caps, reg; + + efx_emmc_reset_hw(host); + + caps = efx_emmc_readl(host, EFX_EMMC_HOST_CAPABILITIES); + host->base_clk = (caps & 0x3FF) * 1000000; + + if (host->base_clk == 0) { + host->base_clk = EFX_EMMC_BASE_CLK_FREQ_MHZ * 1000000; + } + + dev_info(&host->pdev->dev, "Base clock: %u Hz, Capabilities: 0x%08x\n", + host->base_clk, caps); + + efx_emmc_writel(host, 0, EFX_EMMC_INT_SIGNAL_EN); + efx_emmc_writel(host, 0, EFX_EMMC_INT_STATUS_EN); + + efx_emmc_writel(host, EFX_EMMC_INT_ALL_MASK, EFX_EMMC_INT_STATUS); + + reg = efx_emmc_readl(host, EFX_EMMC_HOST_CONTROL); + reg &= ~EFX_EMMC_DATA_WIDTH_MASK; + reg |= (EFX_EMMC_DATA_WIDTH_1BIT << EFX_EMMC_DATA_WIDTH_SHIFT); + efx_emmc_writel(host, reg, EFX_EMMC_HOST_CONTROL); + + efx_emmc_set_clock(host, EFX_EMMC_MIN_FREQ); + + msleep(10); + + efx_emmc_writel(host, EFX_EMMC_INT_ALL_MASK, EFX_EMMC_INT_STATUS_EN); + efx_emmc_writel(host, EFX_EMMC_INT_ALL_MASK, EFX_EMMC_INT_SIGNAL_EN); + + dev_info(&host->pdev->dev, "Hardware initialized successfully\n"); + + return 0; +} + +int efx_emmc_probe(struct platform_device *pdev) +{ + struct mmc_host *mmc; + struct efx_emmc_host *host; + struct resource *res; + int ret; + u32 version, present_state; + u32 hs400_defaults[2]; + + mmc = mmc_alloc_host(sizeof(struct efx_emmc_host), &pdev->dev); + if (!mmc) { + return -ENOMEM; + } + + host = mmc_priv(mmc); + host->mmc = mmc; + host->pdev = pdev; + + spin_lock_init(&host->lock); + + host->tuning_done = false; + host->tuning_in_progress = false; + host->hs200_sample_count = 0; + host->hs200_pll_shift = 0; + host->hs200_margin = 0; + host->hs400_sample_count = 0; + host->hs400_pll_shift = 0; + host->hs400_margin = 0; + host->prev_timing = MMC_TIMING_LEGACY; + host->hs400_retune_pending = false; + host->tuned_timing_modes = 0; + + INIT_DELAYED_WORK(&host->hs400_retune_work, efx_emmc_hs400_retune_work); + + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + host->ioaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(host->ioaddr)) { + ret = PTR_ERR(host->ioaddr); + goto err_free_host; + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + host->sys_ioaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(host->sys_ioaddr)) { + ret = PTR_ERR(host->sys_ioaddr); + goto err_free_host; + } + + host->hs400_default_sample = 0; + host->hs400_default_pll = 6; + + if (!of_property_read_u32_array(pdev->dev.of_node, + "hs400-default-timing", + hs400_defaults, 2)) { + host->hs400_default_sample = hs400_defaults[0]; + host->hs400_default_pll = hs400_defaults[1]; + dev_info(&pdev->dev, "HS400 default timing from DT: sample=%u pll=%u\n", + host->hs400_default_sample, host->hs400_default_pll); + } + + host->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(host->clk)) { + ret = PTR_ERR(host->clk); + dev_err(&pdev->dev, "Failed to get clock: %d\n", ret); + goto err_free_host; + } + + ret = clk_prepare_enable(host->clk); + if (ret) { + dev_err(&pdev->dev, "Failed to enable clock: %d\n", ret); + goto err_free_host; + } + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(&pdev->dev, "Failed to set DMA mask\n"); + goto err_clk_disable; + } + host->dma_64bit = false; + + host->adma_desc_sz = EFX_ADMA_TABLE_SZ; + host->adma_desc = dma_alloc_coherent(&pdev->dev, host->adma_desc_sz, + &host->adma_desc_dma, GFP_KERNEL); + if (!host->adma_desc) { + dev_err(&pdev->dev, "Failed to allocate ADMA descriptor table\n"); + ret = -ENOMEM; + goto err_clk_disable; + } + + host->bounce_buffer_size = 512 * 1024; + host->bounce_buffer = dma_alloc_coherent(&pdev->dev, + host->bounce_buffer_size, + &host->bounce_dma, GFP_KERNEL); + if (!host->bounce_buffer) { + dev_warn(&pdev->dev, + "Failed to allocate bounce buffer, using software alignment\n"); + host->bounce_buffer_size = 0; + } + host->bounce_used = 0; + host->bounce_active = false; + + host->irq = platform_get_irq(pdev, 0); + if (host->irq < 0) { + ret = host->irq; + goto err_free_dma; + } + + ret = devm_request_irq(&pdev->dev, host->irq, efx_emmc_irq, + IRQF_SHARED, mmc_hostname(mmc), host); + if (ret) { + dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret); + goto err_free_dma; + } + + ret = efx_emmc_init_hw(host); + if (ret) { + goto err_free_dma; + } + + version = efx_emmc_readl(host, EFX_EMMC_VERSION); + present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE); + dev_info(&pdev->dev, "Version: 0x%08x, Present state: 0x%08x\n", + version, present_state); + + ret = mmc_of_parse(mmc); + if (ret) { + dev_err(&pdev->dev, "Failed to parse DT: %d\n", ret); + goto err_free_dma; + } + + mmc->ops = &efx_emmc_ops; + + if (!mmc->f_min) + mmc->f_min = EFX_EMMC_MIN_FREQ; + if (!mmc->f_max) + mmc->f_max = EFX_EMMC_MAX_FREQ; + + if (!(mmc->caps & MMC_CAP_CMD23)) + mmc->caps |= MMC_CAP_CMD23; + + if (!mmc->ocr_avail) + mmc->ocr_avail = MMC_VDD_165_195 | MMC_VDD_27_28 | MMC_VDD_28_29 | + MMC_VDD_29_30 | MMC_VDD_30_31 | MMC_VDD_31_32 | + MMC_VDD_32_33 | MMC_VDD_33_34 | MMC_VDD_34_35 | + MMC_VDD_35_36; + + mmc->max_seg_size = 65536; + mmc->max_segs = 128; + mmc->max_req_size = mmc->max_seg_size * mmc->max_segs; + mmc->max_blk_size = EFX_EMMC_MAX_BLOCK_LENGTH; + mmc->max_blk_count = 65535; + + platform_set_drvdata(pdev, mmc); + + ret = mmc_add_host(mmc); + if (ret) { + dev_err(&pdev->dev, "Failed to add MMC host: %d\n", ret); + goto err_free_dma; + } + + mmc_detect_change(mmc, msecs_to_jiffies(500)); + + dev_info(&pdev->dev, "Efinix eMMC Host Controller registered (DMA: %s)\n", + host->adma_desc ? "enabled" : "disabled"); + dev_info(&pdev->dev, "MMC caps: 0x%08x, OCR: 0x%08x\n", + mmc->caps, mmc->ocr_avail); + dev_info(&pdev->dev, "Clock range: %u - %u Hz\n", mmc->f_min, mmc->f_max); + dev_info(&pdev->dev, "Max block size: %u, Max segments: %u\n", + mmc->max_blk_size, mmc->max_segs); + dev_info(&pdev->dev, + "ADMA desc table: %zu bytes, Bounce buffer: %u bytes\n", + host->adma_desc_sz, host->bounce_buffer_size); + + return 0; + +err_free_dma: + if (host->bounce_buffer) { + dma_free_coherent(&pdev->dev, host->bounce_buffer_size, + host->bounce_buffer, host->bounce_dma); + } + if (host->adma_desc) { + dma_free_coherent(&pdev->dev, host->adma_desc_sz, + host->adma_desc, host->adma_desc_dma); + } +err_clk_disable: + clk_disable_unprepare(host->clk); +err_free_host: + mmc_free_host(mmc); + return ret; +} + +int efx_emmc_remove(struct platform_device *pdev) +{ + struct mmc_host *mmc; + struct efx_emmc_host *host; + + mmc = platform_get_drvdata(pdev); + host = mmc_priv(mmc); + + mmc_remove_host(mmc); + + cancel_delayed_work_sync(&host->hs400_retune_work); + + efx_emmc_writel(host, 0, EFX_EMMC_INT_SIGNAL_EN); + efx_emmc_writel(host, 0, EFX_EMMC_INT_STATUS_EN); + + efx_emmc_reset_hw(host); + + if (host->bounce_buffer) { + dma_free_coherent(&pdev->dev, host->bounce_buffer_size, + host->bounce_buffer, host->bounce_dma); + } + if (host->adma_desc) { + dma_free_coherent(&pdev->dev, host->adma_desc_sz, + host->adma_desc, host->adma_desc_dma); + } + + clk_disable_unprepare(host->clk); + mmc_free_host(mmc); + + return 0; +} + +static const struct of_device_id efx_emmc_of_match[] = { + { .compatible = "efinix,emmc-host-controller", }, + { } +}; +MODULE_DEVICE_TABLE(of, efx_emmc_of_match); + +static struct platform_driver efx_emmc_driver = { + .probe = efx_emmc_probe, + .remove = efx_emmc_remove, + .driver = { + .name = "efx-emmc", + .of_match_table = efx_emmc_of_match, + }, +}; + +module_platform_driver(efx_emmc_driver); + +MODULE_DESCRIPTION("Efinix eMMC Host Controller Driver with DMA Support"); +MODULE_AUTHOR("Teoh Choon Zone "); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION("1.0"); diff --git a/drivers/mmc/host/efx_emmc_tuning.c b/drivers/mmc/host/efx_emmc_tuning.c new file mode 100644 index 000000000..f38a77091 --- /dev/null +++ b/drivers/mmc/host/efx_emmc_tuning.c @@ -0,0 +1,459 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Efinix eMMC Host Controller Tuning Support + * + * Copyright (C) 2025 Efinix, Inc. + * Author: Teoh Choon Zone + */ + + +#include +#include +#include +#include + +#include "efx_emmc.h" + +static const u32 tuning_block_pattern_8b_mode[] = { + 0xff00ffff, 0x0000ffff, 0xccccffff, 0xcccc33cc, + 0xcc3333cc, 0xffffcccc, 0xffffeeff, 0xffeeeeff, + 0xffddffff, 0xddddffff, 0xbbffffff, 0xbbffffff, + 0xffffffbb, 0xffffff77, 0x77ff7777, 0xffeeddbb, + 0x00ffffff, 0x00ffffff, 0xccffff00, 0xcc33cccc, + 0x3333cccc, 0xffcccccc, 0xffeeffff, 0xeeeeffff, + 0xddffffff, 0xddffffff, 0xffffffdd, 0xffffffbb, + 0xffffbbbb, 0xffff77ff, 0xff7777ff, 0xeeddbb77 +}; + +static const u32 tuning_block_pattern_4b_mode[] = { + 0x00ff0fff, 0xccc3ccff, 0xffcc3cc3, 0xeffefffe, + 0xddffdfff, 0xfbfffbff, 0xff7fffbf, 0xefbdf777, + 0xf0fff0ff, 0x3cccfc0f, 0xcfcc33cc, 0xeeffefff, + 0xfdfffdff, 0xffbfffdf, 0xfff7ffbb, 0xde7b7ff7 +}; + +void efx_emmc_set_timing_config(struct efx_emmc_host *host, + u32 sample_count, u32 pll_shift) +{ + u32 config_value; + + config_value = (sample_count << 16) | (pll_shift << 6); + + efx_emmc_writel(host, config_value | 0x0, EFX_EMMC_BASE_REG1); + efx_emmc_writel(host, config_value | 0x1, EFX_EMMC_BASE_REG1); + efx_emmc_writel(host, config_value | 0x0, EFX_EMMC_BASE_REG1); + + udelay(100); +} + +int efx_emmc_execute_tuning_command(struct efx_emmc_host *host, int bus_width) +{ + u32 block_size, command_config, word_count; + const u32 *reference_pattern; + u32 received_data; + int i, mismatches = 0; + unsigned long timeout; + u32 tuning_present_state; + + if (bus_width == 8) { + block_size = EFX_EMMC_TUNING_BLOCK_SIZE_8BIT; + word_count = 32; + reference_pattern = tuning_block_pattern_8b_mode; + } else { + block_size = EFX_EMMC_TUNING_BLOCK_SIZE_4BIT; + word_count = 16; + reference_pattern = tuning_block_pattern_4b_mode; + } + + efx_emmc_writel(host, (1 << 16) | block_size, EFX_EMMC_BLOCK_SIZE); + efx_emmc_writel(host, 0x0, EFX_EMMC_ARG1); + + efx_emmc_writel(host, 0xFFFFFFFF, EFX_EMMC_INT_STATUS); + + command_config = 0x153A0010; + efx_emmc_writel(host, command_config, EFX_EMMC_TRANSFER_MODE); + + timeout = jiffies + msecs_to_jiffies(5); + + do { + tuning_present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE); + if (tuning_present_state & EFX_EMMC_BUFFER_READ_EN) { + break; + } + + if (time_after(jiffies, timeout)) { + dev_dbg(&host->pdev->dev, "Tuning command timeout after 5ms\n"); + return 0; + } + + cpu_relax(); + } while (1); + + for (i = 0; i < word_count; i++) { + received_data = efx_emmc_readl(host, EFX_EMMC_BUFFER_DATA_PORT); + + if (received_data != reference_pattern[i]) { + mismatches++; + dev_dbg(&host->pdev->dev, + "Tuning data mismatch at word %d: got 0x%08x, expected 0x%08x\n", + i, received_data, reference_pattern[i]); + } + } + + if (mismatches <= 2) { + return 1; + } else { + dev_dbg(&host->pdev->dev, "Too many mismatches: %d\n", mismatches); + return 0; + } +} + +static int efx_emmc_find_longest_consecutive_ones(u8 *row, int length) +{ + int max_len, current_len, i; + + max_len = 0; + current_len = 0; + + for (i = 0; i < length; i++) { + if (row[i] == 1) { + current_len++; + if (current_len > max_len) { + max_len = current_len; + } + } else { + current_len = 0; + } + } + + return max_len; +} + +static int efx_emmc_find_center_of_consecutive_ones(u8 *row, int length) +{ + int max_len, current_len, max_start, current_start, center, i; + + max_len = 0; + current_len = 0; + max_start = 0; + current_start = 0; + + for (i = 0; i < length; i++) { + if (row[i] == 1) { + if (current_len == 0) { + current_start = i; + } + current_len++; + if (current_len > max_len) { + max_len = current_len; + max_start = current_start; + } + } else { + current_len = 0; + } + } + + center = max_start + (max_len / 2); + return center; +} + +int efx_emmc_find_optimal_timing(struct efx_emmc_host *host, + u8 result_map[][EFX_EMMC_MAX_PLL_SHIFT], + u32 max_sample_count) +{ + int max_consecutive_length, row_length, optimal_sample_count, + optimal_pll_shift; + int *optimal_rows; + int optimal_row_count, center_row, i; + char optimal_row_str[32]; + + optimal_rows = kmalloc(max_sample_count * sizeof(int), GFP_KERNEL); + if (!optimal_rows) { + return -ENOMEM; + } + + dev_dbg(&host->pdev->dev, "Analyzing timing results\n"); + max_consecutive_length = 0; + for (i = 0; i < max_sample_count; i++) { + row_length = efx_emmc_find_longest_consecutive_ones(result_map[i], + EFX_EMMC_MAX_PLL_SHIFT); + dev_dbg(&host->pdev->dev, "Sample[%u]: consecutive_length=%d\n", + i, row_length); + if (row_length > max_consecutive_length) { + dev_dbg(&host->pdev->dev, + "New best: Sample[%u] length=%d\n", + i, row_length); + max_consecutive_length = row_length; + } + } + dev_dbg(&host->pdev->dev, "Best consecutive length: %d\n", + max_consecutive_length); + + + if (max_consecutive_length < EFX_EMMC_MIN_TIMING_MARGIN) { + dev_warn(&host->pdev->dev, + "Insufficient timing margin: %d (minimum %d)\n", + max_consecutive_length, EFX_EMMC_MIN_TIMING_MARGIN); + } + + optimal_row_count = 0; + for (i = 0; i < max_sample_count; i++) { + if (efx_emmc_find_longest_consecutive_ones(result_map[i], + EFX_EMMC_MAX_PLL_SHIFT) == + max_consecutive_length) { + optimal_rows[optimal_row_count++] = i; + } + } + + if (optimal_row_count == 0) { + kfree(optimal_rows); + return -ENODEV; + } + + center_row = optimal_row_count / 2; + optimal_sample_count = optimal_rows[center_row]; + + dev_dbg(&host->pdev->dev, + "Selected sample_count=%d from %d optimal rows\n", + optimal_sample_count, optimal_row_count); + + optimal_row_str[0] = '\0'; + for (i = 0; i < EFX_EMMC_MAX_PLL_SHIFT; i++) { + sprintf(optimal_row_str + strlen(optimal_row_str), "%d", + result_map[optimal_sample_count][i]); + } + dev_dbg(&host->pdev->dev, + "Analyzing row[%d]: [%s] for center calculation\n", + optimal_sample_count, optimal_row_str); + + optimal_pll_shift = + efx_emmc_find_center_of_consecutive_ones(result_map[optimal_sample_count], + EFX_EMMC_MAX_PLL_SHIFT); + + dev_dbg(&host->pdev->dev, + "Center PLL calculation result: pll_shift=%d\n", + optimal_pll_shift); + + if (host->mmc->ios.timing == MMC_TIMING_MMC_HS200) { + host->hs200_sample_count = optimal_sample_count; + host->hs200_pll_shift = optimal_pll_shift; + host->hs200_margin = max_consecutive_length; + } else if (host->mmc->ios.timing == MMC_TIMING_MMC_HS400) { + host->hs400_sample_count = optimal_sample_count; + host->hs400_pll_shift = optimal_pll_shift; + host->hs400_margin = max_consecutive_length; + } + + dev_dbg(&host->pdev->dev, + "Optimal timing found: sample_count=%u, pll_shift=%u, margin=%d\n", + optimal_sample_count, optimal_pll_shift, max_consecutive_length); + + kfree(optimal_rows); + return 0; +} + +int efx_emmc_execute_tuning(struct mmc_host *mmc, u32 opcode) +{ + struct efx_emmc_host *host; + u32 max_sample_count, sample_count, pll_shift; + int bus_width, success, ret; + unsigned long flags, timeout; + u8 (*tuning_result_map)[EFX_EMMC_MAX_PLL_SHIFT]; + + host = mmc_priv(mmc); + + host->tuning_in_progress = true; + + dev_info(&host->pdev->dev, "execute_tuning called: timing=%u, clock=%u Hz\n", + mmc->ios.timing, mmc->ios.clock); + + if (!host->hs400_retune_pending && + test_bit(mmc->ios.timing, &host->tuned_timing_modes) && + host->tuning_done && + (mmc->ios.timing == MMC_TIMING_MMC_HS200 || + mmc->ios.timing == MMC_TIMING_MMC_HS400)) { + u32 sample = (mmc->ios.timing == MMC_TIMING_MMC_HS200) ? + host->hs200_sample_count : host->hs400_sample_count; + u32 pll = (mmc->ios.timing == MMC_TIMING_MMC_HS200) ? + host->hs200_pll_shift : host->hs400_pll_shift; + + efx_emmc_set_timing_config(host, sample, pll); + + dev_info(&host->pdev->dev, + "Tuning skipped: already successfully tuned for timing=%u (sample=%u, pll=%u)\n", + mmc->ios.timing, sample, pll); + return 0; + } + + if (host->hs400_retune_pending && + mmc->ios.timing == MMC_TIMING_MMC_HS400) { + dev_info(&host->pdev->dev, + "HS400 forced retuning (clock=%u Hz)\n", + mmc->ios.clock); + host->hs400_retune_pending = false; + } + + if (opcode != MMC_SEND_TUNING_BLOCK && + opcode != MMC_SEND_TUNING_BLOCK_HS200) { + dev_err(&host->pdev->dev, "Unsupported tuning opcode: %u\n", opcode); + return -EINVAL; + } + + if (mmc->ios.timing == MMC_TIMING_MMC_HS400) { + dev_info(&host->pdev->dev, + "Tuning in HS400 mode (DDR has different timing than HS200 SDR)\n"); + } + + bus_width = (mmc->ios.bus_width == MMC_BUS_WIDTH_8) ? 8 : 4; + max_sample_count = host->clk_div ? host->clk_div : 1; + + dev_dbg(&host->pdev->dev, + "Starting tuning algorithm (bus_width=%d, max_sample=%u)\n", + bus_width, max_sample_count); + + tuning_result_map = kmalloc(max_sample_count * + sizeof(u8[EFX_EMMC_MAX_PLL_SHIFT]), + GFP_KERNEL); + if (!tuning_result_map) { + dev_err(&host->pdev->dev, "Failed to allocate tuning result map\n"); + return -ENOMEM; + } + + memset(tuning_result_map, 0, + max_sample_count * sizeof(u8[EFX_EMMC_MAX_PLL_SHIFT])); + + dev_dbg(&host->pdev->dev, "Using clk_div=%u for sample count\n", + max_sample_count); + + timeout = jiffies + msecs_to_jiffies(5000); + + dev_dbg(&host->pdev->dev, "Testing %d samples × %d PLL positions\n", + max_sample_count, EFX_EMMC_MAX_PLL_SHIFT); + for (sample_count = 0; sample_count < max_sample_count; sample_count++) { + int consecutive_passes = 0; + + for (pll_shift = 0; pll_shift < EFX_EMMC_MAX_PLL_SHIFT; pll_shift++) { + + efx_emmc_set_timing_config(host, sample_count, pll_shift); + + success = efx_emmc_execute_tuning_command(host, bus_width); + + if (!success) { + tuning_result_map[sample_count][pll_shift] = 0; + consecutive_passes = 0; + } else { + tuning_result_map[sample_count][pll_shift] = 1; + consecutive_passes++; + } + + dev_dbg(&host->pdev->dev, + "Tuning [%u][%u]: %s (consecutive: %d)\n", + sample_count, pll_shift, success ? "PASS" : "FAIL", + consecutive_passes); + + if (time_after(jiffies, timeout)) { + dev_warn(&host->pdev->dev, "Tuning timeout after 5 seconds\n"); + goto find_optimal; + } + } + } + +find_optimal: + + if (!test_bit(mmc->ios.timing, &host->tuned_timing_modes)) { + dev_info(&host->pdev->dev, "Tuning result map:\n"); + for (sample_count = 0; sample_count < max_sample_count; + sample_count++) { + char row_str[32] = ""; + for (pll_shift = 0; pll_shift < EFX_EMMC_MAX_PLL_SHIFT; + pll_shift++) { + sprintf(row_str + strlen(row_str), "%d", + tuning_result_map[sample_count][pll_shift]); + } + dev_info(&host->pdev->dev, "Sample[%u]: [%s]\n", + sample_count, row_str); + } + } + + ret = efx_emmc_find_optimal_timing(host, tuning_result_map, + max_sample_count); + if (ret == 0) { + u32 sample, pll, margin; + if (mmc->ios.timing == MMC_TIMING_MMC_HS200) { + sample = host->hs200_sample_count; + pll = host->hs200_pll_shift; + margin = host->hs200_margin; + } else if (mmc->ios.timing == MMC_TIMING_MMC_HS400) { + sample = host->hs400_sample_count; + pll = host->hs400_pll_shift; + margin = host->hs400_margin; + } else { + sample = 0; + pll = 4; + margin = 0; + } + + efx_emmc_set_timing_config(host, sample, pll); + + if (!test_bit(mmc->ios.timing, &host->tuned_timing_modes)) { + dev_info(&host->pdev->dev, + "Tuning completed: sample=%u, pll=%u, margin=%u\n", + sample, pll, margin); + } else { + dev_info(&host->pdev->dev, + "Tuning reconfirmed: sample=%u, pll=%u\n", + sample, pll); + } + + host->tuning_done = true; + set_bit(mmc->ios.timing, &host->tuned_timing_modes); + } else { + dev_warn(&host->pdev->dev, + "Tuning failed: %d, using fallback configuration\n", + ret); + if (max_sample_count == 1) { + efx_emmc_set_timing_config(host, 0, 2); + if (mmc->ios.timing == MMC_TIMING_MMC_HS200) { + host->hs200_sample_count = 0; + host->hs200_pll_shift = 2; + } else if (mmc->ios.timing == MMC_TIMING_MMC_HS400) { + host->hs400_sample_count = 0; + host->hs400_pll_shift = 2; + } + } else { + efx_emmc_set_timing_config(host, 1, 1); + if (mmc->ios.timing == MMC_TIMING_MMC_HS200) { + host->hs200_sample_count = 1; + host->hs200_pll_shift = 1; + } else if (mmc->ios.timing == MMC_TIMING_MMC_HS400) { + host->hs400_sample_count = 1; + host->hs400_pll_shift = 1; + } + } + host->tuning_done = true; + + if (mmc->ios.timing != MMC_TIMING_MMC_HS400) { + set_bit(mmc->ios.timing, &host->tuned_timing_modes); + } else { + dev_warn(&host->pdev->dev, + "HS400 tuning fallback - will retry on next access\n"); + } + ret = 0; + } + + spin_lock_irqsave(&host->lock, flags); + host->prev_timing = mmc->ios.timing; + spin_unlock_irqrestore(&host->lock, flags); + + kfree(tuning_result_map); + + host->tuning_in_progress = false; + + if (ret == 0) { + dev_info(&host->pdev->dev, "Tuning completed successfully\n"); + } else { + dev_err(&host->pdev->dev, "Tuning failed: %d\n", ret); + } + + return ret; +}