460 lines
13 KiB
C
460 lines
13 KiB
C
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||
/*
|
||
* Efinix eMMC Host Controller Tuning Support
|
||
*
|
||
* Copyright (C) 2025 Efinix, Inc.
|
||
* Author: Teoh Choon Zone <czteoh@efinixinc.com>
|
||
*/
|
||
|
||
|
||
#include <linux/delay.h>
|
||
#include <linux/slab.h>
|
||
#include <linux/jiffies.h>
|
||
#include <linux/mmc/mmc.h>
|
||
|
||
#include "efx_emmc.h"
|
||
|
||
static const u32 tuning_block_pattern_8b_mode[] = {
|
||
0xff00ffff, 0x0000ffff, 0xccccffff, 0xcccc33cc,
|
||
0xcc3333cc, 0xffffcccc, 0xffffeeff, 0xffeeeeff,
|
||
0xffddffff, 0xddddffff, 0xbbffffff, 0xbbffffff,
|
||
0xffffffbb, 0xffffff77, 0x77ff7777, 0xffeeddbb,
|
||
0x00ffffff, 0x00ffffff, 0xccffff00, 0xcc33cccc,
|
||
0x3333cccc, 0xffcccccc, 0xffeeffff, 0xeeeeffff,
|
||
0xddffffff, 0xddffffff, 0xffffffdd, 0xffffffbb,
|
||
0xffffbbbb, 0xffff77ff, 0xff7777ff, 0xeeddbb77
|
||
};
|
||
|
||
static const u32 tuning_block_pattern_4b_mode[] = {
|
||
0x00ff0fff, 0xccc3ccff, 0xffcc3cc3, 0xeffefffe,
|
||
0xddffdfff, 0xfbfffbff, 0xff7fffbf, 0xefbdf777,
|
||
0xf0fff0ff, 0x3cccfc0f, 0xcfcc33cc, 0xeeffefff,
|
||
0xfdfffdff, 0xffbfffdf, 0xfff7ffbb, 0xde7b7ff7
|
||
};
|
||
|
||
void efx_emmc_set_timing_config(struct efx_emmc_host *host,
|
||
u32 sample_count, u32 pll_shift)
|
||
{
|
||
u32 config_value;
|
||
|
||
config_value = (sample_count << 16) | (pll_shift << 6);
|
||
|
||
efx_emmc_writel(host, config_value | 0x0, EFX_EMMC_BASE_REG1);
|
||
efx_emmc_writel(host, config_value | 0x1, EFX_EMMC_BASE_REG1);
|
||
efx_emmc_writel(host, config_value | 0x0, EFX_EMMC_BASE_REG1);
|
||
|
||
udelay(100);
|
||
}
|
||
|
||
int efx_emmc_execute_tuning_command(struct efx_emmc_host *host, int bus_width)
|
||
{
|
||
u32 block_size, command_config, word_count;
|
||
const u32 *reference_pattern;
|
||
u32 received_data;
|
||
int i, mismatches = 0;
|
||
unsigned long timeout;
|
||
u32 tuning_present_state;
|
||
|
||
if (bus_width == 8) {
|
||
block_size = EFX_EMMC_TUNING_BLOCK_SIZE_8BIT;
|
||
word_count = 32;
|
||
reference_pattern = tuning_block_pattern_8b_mode;
|
||
} else {
|
||
block_size = EFX_EMMC_TUNING_BLOCK_SIZE_4BIT;
|
||
word_count = 16;
|
||
reference_pattern = tuning_block_pattern_4b_mode;
|
||
}
|
||
|
||
efx_emmc_writel(host, (1 << 16) | block_size, EFX_EMMC_BLOCK_SIZE);
|
||
efx_emmc_writel(host, 0x0, EFX_EMMC_ARG1);
|
||
|
||
efx_emmc_writel(host, 0xFFFFFFFF, EFX_EMMC_INT_STATUS);
|
||
|
||
command_config = 0x153A0010;
|
||
efx_emmc_writel(host, command_config, EFX_EMMC_TRANSFER_MODE);
|
||
|
||
timeout = jiffies + msecs_to_jiffies(5);
|
||
|
||
do {
|
||
tuning_present_state = efx_emmc_readl(host, EFX_EMMC_PRESENT_STATE);
|
||
if (tuning_present_state & EFX_EMMC_BUFFER_READ_EN) {
|
||
break;
|
||
}
|
||
|
||
if (time_after(jiffies, timeout)) {
|
||
dev_dbg(&host->pdev->dev, "Tuning command timeout after 5ms\n");
|
||
return 0;
|
||
}
|
||
|
||
cpu_relax();
|
||
} while (1);
|
||
|
||
for (i = 0; i < word_count; i++) {
|
||
received_data = efx_emmc_readl(host, EFX_EMMC_BUFFER_DATA_PORT);
|
||
|
||
if (received_data != reference_pattern[i]) {
|
||
mismatches++;
|
||
dev_dbg(&host->pdev->dev,
|
||
"Tuning data mismatch at word %d: got 0x%08x, expected 0x%08x\n",
|
||
i, received_data, reference_pattern[i]);
|
||
}
|
||
}
|
||
|
||
if (mismatches <= 2) {
|
||
return 1;
|
||
} else {
|
||
dev_dbg(&host->pdev->dev, "Too many mismatches: %d\n", mismatches);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
static int efx_emmc_find_longest_consecutive_ones(u8 *row, int length)
|
||
{
|
||
int max_len, current_len, i;
|
||
|
||
max_len = 0;
|
||
current_len = 0;
|
||
|
||
for (i = 0; i < length; i++) {
|
||
if (row[i] == 1) {
|
||
current_len++;
|
||
if (current_len > max_len) {
|
||
max_len = current_len;
|
||
}
|
||
} else {
|
||
current_len = 0;
|
||
}
|
||
}
|
||
|
||
return max_len;
|
||
}
|
||
|
||
static int efx_emmc_find_center_of_consecutive_ones(u8 *row, int length)
|
||
{
|
||
int max_len, current_len, max_start, current_start, center, i;
|
||
|
||
max_len = 0;
|
||
current_len = 0;
|
||
max_start = 0;
|
||
current_start = 0;
|
||
|
||
for (i = 0; i < length; i++) {
|
||
if (row[i] == 1) {
|
||
if (current_len == 0) {
|
||
current_start = i;
|
||
}
|
||
current_len++;
|
||
if (current_len > max_len) {
|
||
max_len = current_len;
|
||
max_start = current_start;
|
||
}
|
||
} else {
|
||
current_len = 0;
|
||
}
|
||
}
|
||
|
||
center = max_start + (max_len / 2);
|
||
return center;
|
||
}
|
||
|
||
int efx_emmc_find_optimal_timing(struct efx_emmc_host *host,
|
||
u8 result_map[][EFX_EMMC_MAX_PLL_SHIFT],
|
||
u32 max_sample_count)
|
||
{
|
||
int max_consecutive_length, row_length, optimal_sample_count,
|
||
optimal_pll_shift;
|
||
int *optimal_rows;
|
||
int optimal_row_count, center_row, i;
|
||
char optimal_row_str[32];
|
||
|
||
optimal_rows = kmalloc(max_sample_count * sizeof(int), GFP_KERNEL);
|
||
if (!optimal_rows) {
|
||
return -ENOMEM;
|
||
}
|
||
|
||
dev_dbg(&host->pdev->dev, "Analyzing timing results\n");
|
||
max_consecutive_length = 0;
|
||
for (i = 0; i < max_sample_count; i++) {
|
||
row_length = efx_emmc_find_longest_consecutive_ones(result_map[i],
|
||
EFX_EMMC_MAX_PLL_SHIFT);
|
||
dev_dbg(&host->pdev->dev, "Sample[%u]: consecutive_length=%d\n",
|
||
i, row_length);
|
||
if (row_length > max_consecutive_length) {
|
||
dev_dbg(&host->pdev->dev,
|
||
"New best: Sample[%u] length=%d\n",
|
||
i, row_length);
|
||
max_consecutive_length = row_length;
|
||
}
|
||
}
|
||
dev_dbg(&host->pdev->dev, "Best consecutive length: %d\n",
|
||
max_consecutive_length);
|
||
|
||
|
||
if (max_consecutive_length < EFX_EMMC_MIN_TIMING_MARGIN) {
|
||
dev_warn(&host->pdev->dev,
|
||
"Insufficient timing margin: %d (minimum %d)\n",
|
||
max_consecutive_length, EFX_EMMC_MIN_TIMING_MARGIN);
|
||
}
|
||
|
||
optimal_row_count = 0;
|
||
for (i = 0; i < max_sample_count; i++) {
|
||
if (efx_emmc_find_longest_consecutive_ones(result_map[i],
|
||
EFX_EMMC_MAX_PLL_SHIFT) ==
|
||
max_consecutive_length) {
|
||
optimal_rows[optimal_row_count++] = i;
|
||
}
|
||
}
|
||
|
||
if (optimal_row_count == 0) {
|
||
kfree(optimal_rows);
|
||
return -ENODEV;
|
||
}
|
||
|
||
center_row = optimal_row_count / 2;
|
||
optimal_sample_count = optimal_rows[center_row];
|
||
|
||
dev_dbg(&host->pdev->dev,
|
||
"Selected sample_count=%d from %d optimal rows\n",
|
||
optimal_sample_count, optimal_row_count);
|
||
|
||
optimal_row_str[0] = '\0';
|
||
for (i = 0; i < EFX_EMMC_MAX_PLL_SHIFT; i++) {
|
||
sprintf(optimal_row_str + strlen(optimal_row_str), "%d",
|
||
result_map[optimal_sample_count][i]);
|
||
}
|
||
dev_dbg(&host->pdev->dev,
|
||
"Analyzing row[%d]: [%s] for center calculation\n",
|
||
optimal_sample_count, optimal_row_str);
|
||
|
||
optimal_pll_shift =
|
||
efx_emmc_find_center_of_consecutive_ones(result_map[optimal_sample_count],
|
||
EFX_EMMC_MAX_PLL_SHIFT);
|
||
|
||
dev_dbg(&host->pdev->dev,
|
||
"Center PLL calculation result: pll_shift=%d\n",
|
||
optimal_pll_shift);
|
||
|
||
if (host->mmc->ios.timing == MMC_TIMING_MMC_HS200) {
|
||
host->hs200_sample_count = optimal_sample_count;
|
||
host->hs200_pll_shift = optimal_pll_shift;
|
||
host->hs200_margin = max_consecutive_length;
|
||
} else if (host->mmc->ios.timing == MMC_TIMING_MMC_HS400) {
|
||
host->hs400_sample_count = optimal_sample_count;
|
||
host->hs400_pll_shift = optimal_pll_shift;
|
||
host->hs400_margin = max_consecutive_length;
|
||
}
|
||
|
||
dev_dbg(&host->pdev->dev,
|
||
"Optimal timing found: sample_count=%u, pll_shift=%u, margin=%d\n",
|
||
optimal_sample_count, optimal_pll_shift, max_consecutive_length);
|
||
|
||
kfree(optimal_rows);
|
||
return 0;
|
||
}
|
||
|
||
int efx_emmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
|
||
{
|
||
struct efx_emmc_host *host;
|
||
u32 max_sample_count, sample_count, pll_shift;
|
||
int bus_width, success, ret;
|
||
unsigned long flags, timeout;
|
||
u8 (*tuning_result_map)[EFX_EMMC_MAX_PLL_SHIFT];
|
||
|
||
host = mmc_priv(mmc);
|
||
|
||
host->tuning_in_progress = true;
|
||
|
||
dev_info(&host->pdev->dev, "execute_tuning called: timing=%u, clock=%u Hz\n",
|
||
mmc->ios.timing, mmc->ios.clock);
|
||
|
||
if (!host->hs400_retune_pending &&
|
||
test_bit(mmc->ios.timing, &host->tuned_timing_modes) &&
|
||
host->tuning_done &&
|
||
(mmc->ios.timing == MMC_TIMING_MMC_HS200 ||
|
||
mmc->ios.timing == MMC_TIMING_MMC_HS400)) {
|
||
u32 sample = (mmc->ios.timing == MMC_TIMING_MMC_HS200) ?
|
||
host->hs200_sample_count : host->hs400_sample_count;
|
||
u32 pll = (mmc->ios.timing == MMC_TIMING_MMC_HS200) ?
|
||
host->hs200_pll_shift : host->hs400_pll_shift;
|
||
|
||
efx_emmc_set_timing_config(host, sample, pll);
|
||
|
||
dev_info(&host->pdev->dev,
|
||
"Tuning skipped: already successfully tuned for timing=%u (sample=%u, pll=%u)\n",
|
||
mmc->ios.timing, sample, pll);
|
||
return 0;
|
||
}
|
||
|
||
if (host->hs400_retune_pending &&
|
||
mmc->ios.timing == MMC_TIMING_MMC_HS400) {
|
||
dev_info(&host->pdev->dev,
|
||
"HS400 forced retuning (clock=%u Hz)\n",
|
||
mmc->ios.clock);
|
||
host->hs400_retune_pending = false;
|
||
}
|
||
|
||
if (opcode != MMC_SEND_TUNING_BLOCK &&
|
||
opcode != MMC_SEND_TUNING_BLOCK_HS200) {
|
||
dev_err(&host->pdev->dev, "Unsupported tuning opcode: %u\n", opcode);
|
||
return -EINVAL;
|
||
}
|
||
|
||
if (mmc->ios.timing == MMC_TIMING_MMC_HS400) {
|
||
dev_info(&host->pdev->dev,
|
||
"Tuning in HS400 mode (DDR has different timing than HS200 SDR)\n");
|
||
}
|
||
|
||
bus_width = (mmc->ios.bus_width == MMC_BUS_WIDTH_8) ? 8 : 4;
|
||
max_sample_count = host->clk_div ? host->clk_div : 1;
|
||
|
||
dev_dbg(&host->pdev->dev,
|
||
"Starting tuning algorithm (bus_width=%d, max_sample=%u)\n",
|
||
bus_width, max_sample_count);
|
||
|
||
tuning_result_map = kmalloc(max_sample_count *
|
||
sizeof(u8[EFX_EMMC_MAX_PLL_SHIFT]),
|
||
GFP_KERNEL);
|
||
if (!tuning_result_map) {
|
||
dev_err(&host->pdev->dev, "Failed to allocate tuning result map\n");
|
||
return -ENOMEM;
|
||
}
|
||
|
||
memset(tuning_result_map, 0,
|
||
max_sample_count * sizeof(u8[EFX_EMMC_MAX_PLL_SHIFT]));
|
||
|
||
dev_dbg(&host->pdev->dev, "Using clk_div=%u for sample count\n",
|
||
max_sample_count);
|
||
|
||
timeout = jiffies + msecs_to_jiffies(5000);
|
||
|
||
dev_dbg(&host->pdev->dev, "Testing %d samples × %d PLL positions\n",
|
||
max_sample_count, EFX_EMMC_MAX_PLL_SHIFT);
|
||
for (sample_count = 0; sample_count < max_sample_count; sample_count++) {
|
||
int consecutive_passes = 0;
|
||
|
||
for (pll_shift = 0; pll_shift < EFX_EMMC_MAX_PLL_SHIFT; pll_shift++) {
|
||
|
||
efx_emmc_set_timing_config(host, sample_count, pll_shift);
|
||
|
||
success = efx_emmc_execute_tuning_command(host, bus_width);
|
||
|
||
if (!success) {
|
||
tuning_result_map[sample_count][pll_shift] = 0;
|
||
consecutive_passes = 0;
|
||
} else {
|
||
tuning_result_map[sample_count][pll_shift] = 1;
|
||
consecutive_passes++;
|
||
}
|
||
|
||
dev_dbg(&host->pdev->dev,
|
||
"Tuning [%u][%u]: %s (consecutive: %d)\n",
|
||
sample_count, pll_shift, success ? "PASS" : "FAIL",
|
||
consecutive_passes);
|
||
|
||
if (time_after(jiffies, timeout)) {
|
||
dev_warn(&host->pdev->dev, "Tuning timeout after 5 seconds\n");
|
||
goto find_optimal;
|
||
}
|
||
}
|
||
}
|
||
|
||
find_optimal:
|
||
|
||
if (!test_bit(mmc->ios.timing, &host->tuned_timing_modes)) {
|
||
dev_info(&host->pdev->dev, "Tuning result map:\n");
|
||
for (sample_count = 0; sample_count < max_sample_count;
|
||
sample_count++) {
|
||
char row_str[32] = "";
|
||
for (pll_shift = 0; pll_shift < EFX_EMMC_MAX_PLL_SHIFT;
|
||
pll_shift++) {
|
||
sprintf(row_str + strlen(row_str), "%d",
|
||
tuning_result_map[sample_count][pll_shift]);
|
||
}
|
||
dev_info(&host->pdev->dev, "Sample[%u]: [%s]\n",
|
||
sample_count, row_str);
|
||
}
|
||
}
|
||
|
||
ret = efx_emmc_find_optimal_timing(host, tuning_result_map,
|
||
max_sample_count);
|
||
if (ret == 0) {
|
||
u32 sample, pll, margin;
|
||
if (mmc->ios.timing == MMC_TIMING_MMC_HS200) {
|
||
sample = host->hs200_sample_count;
|
||
pll = host->hs200_pll_shift;
|
||
margin = host->hs200_margin;
|
||
} else if (mmc->ios.timing == MMC_TIMING_MMC_HS400) {
|
||
sample = host->hs400_sample_count;
|
||
pll = host->hs400_pll_shift;
|
||
margin = host->hs400_margin;
|
||
} else {
|
||
sample = 0;
|
||
pll = 4;
|
||
margin = 0;
|
||
}
|
||
|
||
efx_emmc_set_timing_config(host, sample, pll);
|
||
|
||
if (!test_bit(mmc->ios.timing, &host->tuned_timing_modes)) {
|
||
dev_info(&host->pdev->dev,
|
||
"Tuning completed: sample=%u, pll=%u, margin=%u\n",
|
||
sample, pll, margin);
|
||
} else {
|
||
dev_info(&host->pdev->dev,
|
||
"Tuning reconfirmed: sample=%u, pll=%u\n",
|
||
sample, pll);
|
||
}
|
||
|
||
host->tuning_done = true;
|
||
set_bit(mmc->ios.timing, &host->tuned_timing_modes);
|
||
} else {
|
||
dev_warn(&host->pdev->dev,
|
||
"Tuning failed: %d, using fallback configuration\n",
|
||
ret);
|
||
if (max_sample_count == 1) {
|
||
efx_emmc_set_timing_config(host, 0, 2);
|
||
if (mmc->ios.timing == MMC_TIMING_MMC_HS200) {
|
||
host->hs200_sample_count = 0;
|
||
host->hs200_pll_shift = 2;
|
||
} else if (mmc->ios.timing == MMC_TIMING_MMC_HS400) {
|
||
host->hs400_sample_count = 0;
|
||
host->hs400_pll_shift = 2;
|
||
}
|
||
} else {
|
||
efx_emmc_set_timing_config(host, 1, 1);
|
||
if (mmc->ios.timing == MMC_TIMING_MMC_HS200) {
|
||
host->hs200_sample_count = 1;
|
||
host->hs200_pll_shift = 1;
|
||
} else if (mmc->ios.timing == MMC_TIMING_MMC_HS400) {
|
||
host->hs400_sample_count = 1;
|
||
host->hs400_pll_shift = 1;
|
||
}
|
||
}
|
||
host->tuning_done = true;
|
||
|
||
if (mmc->ios.timing != MMC_TIMING_MMC_HS400) {
|
||
set_bit(mmc->ios.timing, &host->tuned_timing_modes);
|
||
} else {
|
||
dev_warn(&host->pdev->dev,
|
||
"HS400 tuning fallback - will retry on next access\n");
|
||
}
|
||
ret = 0;
|
||
}
|
||
|
||
spin_lock_irqsave(&host->lock, flags);
|
||
host->prev_timing = mmc->ios.timing;
|
||
spin_unlock_irqrestore(&host->lock, flags);
|
||
|
||
kfree(tuning_result_map);
|
||
|
||
host->tuning_in_progress = false;
|
||
|
||
if (ret == 0) {
|
||
dev_info(&host->pdev->dev, "Tuning completed successfully\n");
|
||
} else {
|
||
dev_err(&host->pdev->dev, "Tuning failed: %d\n", ret);
|
||
}
|
||
|
||
return ret;
|
||
}
|