2018-12-11 21:54:06 +08:00
|
|
|
/*
|
2019-01-24 14:11:10 +08:00
|
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
|
|
|
*
|
|
|
|
|
* Copyright (c) 2019 Western Digital Corporation or its affiliates.
|
2018-12-11 21:54:06 +08:00
|
|
|
*
|
|
|
|
|
* Authors:
|
|
|
|
|
* Anup Patel <anup.patel@wdc.com>
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <sbi/riscv_asm.h>
|
|
|
|
|
#include <sbi/riscv_atomic.h>
|
2020-08-15 22:11:11 +08:00
|
|
|
#include <sbi/riscv_barrier.h>
|
2020-03-02 19:20:01 +08:00
|
|
|
#include <sbi/riscv_locks.h>
|
2018-12-11 21:54:06 +08:00
|
|
|
#include <sbi/sbi_console.h>
|
2023-03-29 16:57:27 +08:00
|
|
|
#include <sbi/sbi_cppc.h>
|
2020-09-18 19:37:49 +08:00
|
|
|
#include <sbi/sbi_domain.h>
|
2018-12-11 21:54:06 +08:00
|
|
|
#include <sbi/sbi_ecall.h>
|
|
|
|
|
#include <sbi/sbi_hart.h>
|
2020-03-04 17:00:49 +08:00
|
|
|
#include <sbi/sbi_hartmask.h>
|
2020-02-12 09:32:37 +08:00
|
|
|
#include <sbi/sbi_hsm.h>
|
2018-12-11 21:54:06 +08:00
|
|
|
#include <sbi/sbi_ipi.h>
|
2022-02-19 01:48:41 +08:00
|
|
|
#include <sbi/sbi_irqchip.h>
|
2018-12-11 21:54:06 +08:00
|
|
|
#include <sbi/sbi_platform.h>
|
2021-07-11 00:18:11 +08:00
|
|
|
#include <sbi/sbi_pmu.h>
|
2018-12-11 21:54:06 +08:00
|
|
|
#include <sbi/sbi_system.h>
|
2020-05-10 07:47:31 +08:00
|
|
|
#include <sbi/sbi_string.h>
|
2018-12-11 21:54:06 +08:00
|
|
|
#include <sbi/sbi_timer.h>
|
2020-01-15 16:01:12 +08:00
|
|
|
#include <sbi/sbi_tlb.h>
|
2019-01-25 13:49:22 +08:00
|
|
|
#include <sbi/sbi_version.h>
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2019-04-11 08:41:52 +08:00
|
|
|
#define BANNER \
|
|
|
|
|
" ____ _____ ____ _____\n" \
|
|
|
|
|
" / __ \\ / ____| _ \\_ _|\n" \
|
|
|
|
|
" | | | |_ __ ___ _ __ | (___ | |_) || |\n" \
|
2019-01-18 12:18:25 +08:00
|
|
|
" | | | | '_ \\ / _ \\ '_ \\ \\___ \\| _ < | |\n" \
|
2019-04-11 08:41:52 +08:00
|
|
|
" | |__| | |_) | __/ | | |____) | |_) || |_\n" \
|
|
|
|
|
" \\____/| .__/ \\___|_| |_|_____/|____/_____|\n" \
|
|
|
|
|
" | |\n" \
|
2019-01-18 12:18:25 +08:00
|
|
|
" |_|\n\n"
|
2018-12-21 13:58:23 +08:00
|
|
|
|
2020-11-04 17:46:20 +08:00
|
|
|
static void sbi_boot_print_banner(struct sbi_scratch *scratch)
|
2019-03-27 22:39:08 +08:00
|
|
|
{
|
2020-11-04 17:46:20 +08:00
|
|
|
if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
|
|
|
|
|
return;
|
2019-03-27 22:39:08 +08:00
|
|
|
|
2019-08-17 22:11:09 +08:00
|
|
|
#ifdef OPENSBI_VERSION_GIT
|
2019-11-12 08:40:34 +08:00
|
|
|
sbi_printf("\nOpenSBI %s\n", OPENSBI_VERSION_GIT);
|
2019-08-17 22:11:09 +08:00
|
|
|
#else
|
2019-11-12 08:40:34 +08:00
|
|
|
sbi_printf("\nOpenSBI v%d.%d\n", OPENSBI_VERSION_MAJOR,
|
|
|
|
|
OPENSBI_VERSION_MINOR);
|
2019-08-17 22:11:09 +08:00
|
|
|
#endif
|
2019-03-27 22:39:08 +08:00
|
|
|
|
2021-09-30 18:11:20 +08:00
|
|
|
#ifdef OPENSBI_BUILD_TIME_STAMP
|
|
|
|
|
sbi_printf("Build time: %s\n", OPENSBI_BUILD_TIME_STAMP);
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef OPENSBI_BUILD_COMPILER_VERSION
|
|
|
|
|
sbi_printf("Build compiler: %s\n", OPENSBI_BUILD_COMPILER_VERSION);
|
|
|
|
|
#endif
|
|
|
|
|
|
2019-03-27 22:39:08 +08:00
|
|
|
sbi_printf(BANNER);
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
2019-03-27 22:39:08 +08:00
|
|
|
|
2020-11-04 17:46:20 +08:00
|
|
|
static void sbi_boot_print_general(struct sbi_scratch *scratch)
|
|
|
|
|
{
|
|
|
|
|
char str[128];
|
2022-08-24 17:59:27 +08:00
|
|
|
const struct sbi_pmu_device *pdev;
|
2021-04-22 17:19:44 +08:00
|
|
|
const struct sbi_hsm_device *hdev;
|
|
|
|
|
const struct sbi_ipi_device *idev;
|
|
|
|
|
const struct sbi_timer_device *tdev;
|
|
|
|
|
const struct sbi_console_device *cdev;
|
|
|
|
|
const struct sbi_system_reset_device *srdev;
|
2023-02-27 18:31:02 +08:00
|
|
|
const struct sbi_system_suspend_device *susp_dev;
|
2023-03-29 16:57:27 +08:00
|
|
|
const struct sbi_cppc_device *cppc_dev;
|
2020-11-04 17:46:20 +08:00
|
|
|
const struct sbi_platform *plat = sbi_platform_ptr(scratch);
|
|
|
|
|
|
|
|
|
|
if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
|
|
|
|
|
return;
|
2019-11-26 18:36:29 +08:00
|
|
|
|
2019-03-27 22:39:08 +08:00
|
|
|
/* Platform details */
|
2020-10-27 05:03:34 +08:00
|
|
|
sbi_printf("Platform Name : %s\n",
|
|
|
|
|
sbi_platform_name(plat));
|
2020-05-10 14:06:14 +08:00
|
|
|
sbi_platform_get_features_str(plat, str, sizeof(str));
|
2020-10-27 05:03:34 +08:00
|
|
|
sbi_printf("Platform Features : %s\n", str);
|
|
|
|
|
sbi_printf("Platform HART Count : %u\n",
|
2020-05-10 14:19:59 +08:00
|
|
|
sbi_platform_hart_count(plat));
|
2021-04-22 17:19:44 +08:00
|
|
|
idev = sbi_ipi_get_device();
|
|
|
|
|
sbi_printf("Platform IPI Device : %s\n",
|
|
|
|
|
(idev) ? idev->name : "---");
|
|
|
|
|
tdev = sbi_timer_get_device();
|
2021-09-15 11:21:42 +08:00
|
|
|
sbi_printf("Platform Timer Device : %s @ %luHz\n",
|
|
|
|
|
(tdev) ? tdev->name : "---",
|
|
|
|
|
(tdev) ? tdev->timer_freq : 0);
|
2021-04-22 17:19:44 +08:00
|
|
|
cdev = sbi_console_get_device();
|
|
|
|
|
sbi_printf("Platform Console Device : %s\n",
|
|
|
|
|
(cdev) ? cdev->name : "---");
|
|
|
|
|
hdev = sbi_hsm_get_device();
|
|
|
|
|
sbi_printf("Platform HSM Device : %s\n",
|
|
|
|
|
(hdev) ? hdev->name : "---");
|
2022-08-24 17:59:27 +08:00
|
|
|
pdev = sbi_pmu_get_device();
|
|
|
|
|
sbi_printf("Platform PMU Device : %s\n",
|
|
|
|
|
(pdev) ? pdev->name : "---");
|
2021-10-01 16:31:16 +08:00
|
|
|
srdev = sbi_system_reset_get_device(SBI_SRST_RESET_TYPE_COLD_REBOOT, 0);
|
|
|
|
|
sbi_printf("Platform Reboot Device : %s\n",
|
|
|
|
|
(srdev) ? srdev->name : "---");
|
|
|
|
|
srdev = sbi_system_reset_get_device(SBI_SRST_RESET_TYPE_SHUTDOWN, 0);
|
|
|
|
|
sbi_printf("Platform Shutdown Device : %s\n",
|
2021-04-22 17:19:44 +08:00
|
|
|
(srdev) ? srdev->name : "---");
|
2023-02-27 18:31:02 +08:00
|
|
|
susp_dev = sbi_system_suspend_get_device();
|
|
|
|
|
sbi_printf("Platform Suspend Device : %s\n",
|
|
|
|
|
(susp_dev) ? susp_dev->name : "---");
|
2023-03-29 16:57:27 +08:00
|
|
|
cppc_dev = sbi_cppc_get_device();
|
|
|
|
|
sbi_printf("Platform CPPC Device : %s\n",
|
|
|
|
|
(cppc_dev) ? cppc_dev->name : "---");
|
2020-05-10 07:47:31 +08:00
|
|
|
|
2019-03-27 22:39:08 +08:00
|
|
|
/* Firmware details */
|
2020-10-27 05:03:34 +08:00
|
|
|
sbi_printf("Firmware Base : 0x%lx\n", scratch->fw_start);
|
|
|
|
|
sbi_printf("Firmware Size : %d KB\n",
|
2019-03-27 22:39:08 +08:00
|
|
|
(u32)(scratch->fw_size / 1024));
|
2023-01-19 23:18:25 +08:00
|
|
|
sbi_printf("Firmware RW Offset : 0x%lx\n", scratch->fw_rw_offset);
|
2020-05-10 14:06:14 +08:00
|
|
|
|
2020-09-24 00:19:01 +08:00
|
|
|
/* SBI details */
|
2020-10-27 05:03:34 +08:00
|
|
|
sbi_printf("Runtime SBI Version : %d.%d\n",
|
2019-03-27 22:39:08 +08:00
|
|
|
sbi_ecall_version_major(), sbi_ecall_version_minor());
|
|
|
|
|
sbi_printf("\n");
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void sbi_boot_print_domains(struct sbi_scratch *scratch)
|
|
|
|
|
{
|
|
|
|
|
if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
|
|
|
|
|
return;
|
2019-03-27 22:39:08 +08:00
|
|
|
|
2020-09-24 00:19:01 +08:00
|
|
|
/* Domain details */
|
2020-10-27 05:03:34 +08:00
|
|
|
sbi_domain_dump_all(" ");
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void sbi_boot_print_hart(struct sbi_scratch *scratch, u32 hartid)
|
|
|
|
|
{
|
|
|
|
|
int xlen;
|
|
|
|
|
char str[128];
|
|
|
|
|
const struct sbi_domain *dom = sbi_domain_thishart_ptr();
|
|
|
|
|
|
|
|
|
|
if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
/* Determine MISA XLEN and MISA string */
|
|
|
|
|
xlen = misa_xlen();
|
|
|
|
|
if (xlen < 1) {
|
|
|
|
|
sbi_printf("Error %d getting MISA XLEN\n", xlen);
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
}
|
2020-09-24 00:19:01 +08:00
|
|
|
|
|
|
|
|
/* Boot HART details */
|
2020-10-27 05:03:34 +08:00
|
|
|
sbi_printf("Boot HART ID : %u\n", hartid);
|
|
|
|
|
sbi_printf("Boot HART Domain : %s\n", dom->name);
|
2022-04-28 20:00:02 +08:00
|
|
|
sbi_hart_get_priv_version_str(scratch, str, sizeof(str));
|
|
|
|
|
sbi_printf("Boot HART Priv Version : %s\n", str);
|
2020-09-24 00:19:01 +08:00
|
|
|
misa_string(xlen, str, sizeof(str));
|
2022-04-28 20:41:02 +08:00
|
|
|
sbi_printf("Boot HART Base ISA : %s\n", str);
|
2022-04-28 23:59:22 +08:00
|
|
|
sbi_hart_get_extensions_str(scratch, str, sizeof(str));
|
|
|
|
|
sbi_printf("Boot HART ISA Extensions : %s\n", str);
|
2020-10-27 05:03:34 +08:00
|
|
|
sbi_printf("Boot HART PMP Count : %d\n",
|
|
|
|
|
sbi_hart_pmp_count(scratch));
|
|
|
|
|
sbi_printf("Boot HART PMP Granularity : %lu\n",
|
|
|
|
|
sbi_hart_pmp_granularity(scratch));
|
|
|
|
|
sbi_printf("Boot HART PMP Address Bits: %d\n",
|
|
|
|
|
sbi_hart_pmp_addrbits(scratch));
|
|
|
|
|
sbi_printf("Boot HART MHPM Count : %d\n",
|
|
|
|
|
sbi_hart_mhpm_count(scratch));
|
|
|
|
|
sbi_hart_delegation_dump(scratch, "Boot HART ", " ");
|
2019-03-27 22:39:08 +08:00
|
|
|
}
|
|
|
|
|
|
2020-03-02 19:20:01 +08:00
|
|
|
static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
|
2020-03-04 20:00:21 +08:00
|
|
|
static struct sbi_hartmask coldboot_wait_hmask = { 0 };
|
2020-03-02 19:20:01 +08:00
|
|
|
|
2020-08-15 22:11:11 +08:00
|
|
|
static unsigned long coldboot_done;
|
|
|
|
|
|
2020-03-02 19:20:01 +08:00
|
|
|
static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
|
|
|
|
|
{
|
2020-03-21 06:10:36 +08:00
|
|
|
unsigned long saved_mie, cmip;
|
2020-03-02 19:20:01 +08:00
|
|
|
|
|
|
|
|
/* Save MIE CSR */
|
|
|
|
|
saved_mie = csr_read(CSR_MIE);
|
|
|
|
|
|
2021-06-29 15:59:52 +08:00
|
|
|
/* Set MSIE and MEIE bits to receive IPI */
|
|
|
|
|
csr_set(CSR_MIE, MIP_MSIP | MIP_MEIP);
|
2020-03-02 19:20:01 +08:00
|
|
|
|
|
|
|
|
/* Acquire coldboot lock */
|
|
|
|
|
spin_lock(&coldboot_lock);
|
|
|
|
|
|
|
|
|
|
/* Mark current HART as waiting */
|
2020-03-04 20:00:21 +08:00
|
|
|
sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
|
2020-03-02 19:20:01 +08:00
|
|
|
|
2020-08-15 22:11:11 +08:00
|
|
|
/* Release coldboot lock */
|
|
|
|
|
spin_unlock(&coldboot_lock);
|
|
|
|
|
|
2020-03-02 19:20:01 +08:00
|
|
|
/* Wait for coldboot to finish using WFI */
|
2020-08-15 22:11:11 +08:00
|
|
|
while (!__smp_load_acquire(&coldboot_done)) {
|
2020-03-21 06:10:36 +08:00
|
|
|
do {
|
|
|
|
|
wfi();
|
|
|
|
|
cmip = csr_read(CSR_MIP);
|
2021-06-29 15:59:52 +08:00
|
|
|
} while (!(cmip & (MIP_MSIP | MIP_MEIP)));
|
2020-03-02 19:20:01 +08:00
|
|
|
};
|
|
|
|
|
|
2020-08-15 22:11:11 +08:00
|
|
|
/* Acquire coldboot lock */
|
|
|
|
|
spin_lock(&coldboot_lock);
|
|
|
|
|
|
2020-03-02 19:20:01 +08:00
|
|
|
/* Unmark current HART as waiting */
|
2020-03-04 20:00:21 +08:00
|
|
|
sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
|
2020-03-02 19:20:01 +08:00
|
|
|
|
|
|
|
|
/* Release coldboot lock */
|
|
|
|
|
spin_unlock(&coldboot_lock);
|
|
|
|
|
|
|
|
|
|
/* Restore MIE CSR */
|
|
|
|
|
csr_write(CSR_MIE, saved_mie);
|
|
|
|
|
|
2021-02-19 12:24:05 +08:00
|
|
|
/*
|
|
|
|
|
* The wait for coldboot is common for both warm startup and
|
|
|
|
|
* warm resume path so clearing IPI here would result in losing
|
|
|
|
|
* an IPI in warm resume path.
|
|
|
|
|
*
|
|
|
|
|
* Also, the sbi_platform_ipi_init() called from sbi_ipi_init()
|
|
|
|
|
* will automatically clear IPI for current HART.
|
|
|
|
|
*/
|
2020-03-02 19:20:01 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
|
|
|
|
|
{
|
2020-08-15 22:11:11 +08:00
|
|
|
/* Mark coldboot done */
|
|
|
|
|
__smp_store_release(&coldboot_done, 1);
|
|
|
|
|
|
2020-03-02 19:20:01 +08:00
|
|
|
/* Acquire coldboot lock */
|
|
|
|
|
spin_lock(&coldboot_lock);
|
|
|
|
|
|
|
|
|
|
/* Send an IPI to all HARTs waiting for coldboot */
|
2021-06-22 03:37:54 +08:00
|
|
|
for (u32 i = 0; i <= sbi_scratch_last_hartid(); i++) {
|
2020-03-04 20:00:21 +08:00
|
|
|
if ((i != hartid) &&
|
|
|
|
|
sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
|
2021-04-22 12:57:15 +08:00
|
|
|
sbi_ipi_raw_send(i);
|
2020-03-02 19:20:01 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Release coldboot lock */
|
|
|
|
|
spin_unlock(&coldboot_lock);
|
|
|
|
|
}
|
|
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
static unsigned long entry_count_offset;
|
2020-01-03 17:43:33 +08:00
|
|
|
static unsigned long init_count_offset;
|
|
|
|
|
|
2019-01-14 16:28:29 +08:00
|
|
|
static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
|
2018-12-11 21:54:06 +08:00
|
|
|
{
|
|
|
|
|
int rc;
|
2023-03-20 21:22:02 +08:00
|
|
|
unsigned long *count;
|
2019-03-06 15:29:34 +08:00
|
|
|
const struct sbi_platform *plat = sbi_platform_ptr(scratch);
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2020-03-14 12:27:45 +08:00
|
|
|
/* Note: This has to be first thing in coldboot init sequence */
|
|
|
|
|
rc = sbi_scratch_init(scratch);
|
|
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2020-09-18 19:37:49 +08:00
|
|
|
/* Note: This has to be second thing in coldboot init sequence */
|
|
|
|
|
rc = sbi_domain_init(scratch, hartid);
|
|
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
entry_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__);
|
|
|
|
|
if (!entry_count_offset)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2021-05-30 03:54:29 +08:00
|
|
|
init_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__);
|
2020-01-03 17:43:33 +08:00
|
|
|
if (!init_count_offset)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
count = sbi_scratch_offset_ptr(scratch, entry_count_offset);
|
|
|
|
|
(*count)++;
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_hsm_init(scratch, hartid, true);
|
2020-02-12 09:32:37 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_platform_early_init(plat, true);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_hart_init(scratch, true);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
|
|
|
|
rc = sbi_console_init(scratch);
|
|
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_pmu_init(scratch, true);
|
2021-07-11 00:18:11 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2020-11-04 17:46:20 +08:00
|
|
|
sbi_boot_print_banner(scratch);
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_irqchip_init(scratch, true);
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
2022-02-19 01:48:41 +08:00
|
|
|
sbi_printf("%s: irqchip init failed (error %d)\n",
|
2020-11-04 17:46:20 +08:00
|
|
|
__func__, rc);
|
2018-12-11 21:54:06 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_ipi_init(scratch, true);
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
|
|
|
|
sbi_printf("%s: ipi init failed (error %d)\n", __func__, rc);
|
2018-12-11 21:54:06 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_tlb_init(scratch, true);
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
|
|
|
|
sbi_printf("%s: tlb init failed (error %d)\n", __func__, rc);
|
2020-01-15 16:01:12 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
2020-01-15 16:01:12 +08:00
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_timer_init(scratch, true);
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
|
|
|
|
sbi_printf("%s: timer init failed (error %d)\n", __func__, rc);
|
2018-12-11 21:54:06 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2020-01-16 14:08:49 +08:00
|
|
|
rc = sbi_ecall_init();
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
|
|
|
|
sbi_printf("%s: ecall init failed (error %d)\n", __func__, rc);
|
2020-01-16 14:08:49 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
|
|
|
|
|
2020-09-18 19:37:49 +08:00
|
|
|
/*
|
|
|
|
|
* Note: Finalize domains after HSM initialization so that we
|
|
|
|
|
* can startup non-root domains.
|
|
|
|
|
* Note: Finalize domains before HART PMP configuration so
|
|
|
|
|
* that we use correct domain for configuring PMP.
|
|
|
|
|
*/
|
|
|
|
|
rc = sbi_domain_finalize(scratch, hartid);
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
|
|
|
|
sbi_printf("%s: domain finalize failed (error %d)\n",
|
|
|
|
|
__func__, rc);
|
2020-09-18 19:37:49 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
|
|
|
|
|
2020-09-20 12:39:32 +08:00
|
|
|
rc = sbi_hart_pmp_configure(scratch);
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
|
|
|
|
sbi_printf("%s: PMP configure failed (error %d)\n",
|
|
|
|
|
__func__, rc);
|
2020-09-20 12:39:32 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
2020-09-20 12:39:32 +08:00
|
|
|
|
2020-09-18 19:37:49 +08:00
|
|
|
/*
|
|
|
|
|
* Note: Platform final initialization should be last so that
|
|
|
|
|
* it sees correct domain assignment and PMP configuration.
|
|
|
|
|
*/
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_platform_final_init(plat, true);
|
2020-11-04 17:46:20 +08:00
|
|
|
if (rc) {
|
|
|
|
|
sbi_printf("%s: platform final init failed (error %d)\n",
|
|
|
|
|
__func__, rc);
|
2018-12-11 21:54:06 +08:00
|
|
|
sbi_hart_hang();
|
2020-11-04 17:46:20 +08:00
|
|
|
}
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2021-11-06 17:54:49 +08:00
|
|
|
sbi_boot_print_general(scratch);
|
|
|
|
|
|
|
|
|
|
sbi_boot_print_domains(scratch);
|
|
|
|
|
|
2020-11-04 17:46:20 +08:00
|
|
|
sbi_boot_print_hart(scratch, hartid);
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2020-03-02 19:20:01 +08:00
|
|
|
wake_coldboot_harts(scratch, hartid);
|
2020-01-03 11:04:51 +08:00
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
count = sbi_scratch_offset_ptr(scratch, init_count_offset);
|
|
|
|
|
(*count)++;
|
2020-01-03 17:43:33 +08:00
|
|
|
|
2023-03-06 05:22:45 +08:00
|
|
|
sbi_hsm_hart_start_finish(scratch, hartid);
|
2018-12-11 21:54:06 +08:00
|
|
|
}
|
|
|
|
|
|
2023-03-06 05:22:45 +08:00
|
|
|
static void __noreturn init_warm_startup(struct sbi_scratch *scratch,
|
|
|
|
|
u32 hartid)
|
2018-12-11 21:54:06 +08:00
|
|
|
{
|
|
|
|
|
int rc;
|
2023-03-20 21:22:02 +08:00
|
|
|
unsigned long *count;
|
2019-03-06 15:29:34 +08:00
|
|
|
const struct sbi_platform *plat = sbi_platform_ptr(scratch);
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
if (!entry_count_offset || !init_count_offset)
|
2020-01-03 17:43:33 +08:00
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
count = sbi_scratch_offset_ptr(scratch, entry_count_offset);
|
|
|
|
|
(*count)++;
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_hsm_init(scratch, hartid, false);
|
2020-02-12 09:32:37 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_platform_early_init(plat, false);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_hart_init(scratch, false);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_pmu_init(scratch, false);
|
2021-07-11 00:18:11 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_irqchip_init(scratch, false);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_ipi_init(scratch, false);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_tlb_init(scratch, false);
|
2020-01-15 16:01:12 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_timer_init(scratch, false);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2020-09-20 12:39:32 +08:00
|
|
|
rc = sbi_hart_pmp_configure(scratch);
|
|
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2022-12-21 19:38:06 +08:00
|
|
|
rc = sbi_platform_final_init(plat, false);
|
2018-12-11 21:54:06 +08:00
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
count = sbi_scratch_offset_ptr(scratch, init_count_offset);
|
|
|
|
|
(*count)++;
|
2020-01-03 17:43:33 +08:00
|
|
|
|
2023-03-06 05:22:45 +08:00
|
|
|
sbi_hsm_hart_start_finish(scratch, hartid);
|
2021-02-06 17:18:56 +08:00
|
|
|
}
|
|
|
|
|
|
2023-03-06 05:22:45 +08:00
|
|
|
static void __noreturn init_warm_resume(struct sbi_scratch *scratch,
|
|
|
|
|
u32 hartid)
|
2021-02-06 17:18:56 +08:00
|
|
|
{
|
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
|
|
sbi_hsm_hart_resume_start(scratch);
|
|
|
|
|
|
|
|
|
|
rc = sbi_hart_reinit(scratch);
|
|
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
|
|
|
|
rc = sbi_hart_pmp_configure(scratch);
|
|
|
|
|
if (rc)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2023-03-06 05:22:45 +08:00
|
|
|
sbi_hsm_hart_resume_finish(scratch, hartid);
|
2021-02-06 17:18:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
|
|
|
|
|
{
|
|
|
|
|
int hstate;
|
|
|
|
|
|
|
|
|
|
wait_for_coldboot(scratch, hartid);
|
|
|
|
|
|
|
|
|
|
hstate = sbi_hsm_hart_get_state(sbi_domain_thishart_ptr(), hartid);
|
|
|
|
|
if (hstate < 0)
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
lib: sbi: Clear IPIs before init_warm_startup in non-boot harts
Since commit 50d4fde1c5a4 ("lib: Remove redundant sbi_platform_ipi_clear()
calls"), the IPI sent from the boot hart in wake_coldboot_harts() is not
cleared in the secondary harts until they reach sbi_ipi_init(). However,
sbi_hsm_init() and sbi_hsm_hart_wait() are called earlier, so a secondary
hart might enter sbi_hsm_hart_wait() with an already pending IPI.
sbi_hsm_hart_wait() makes sure the hart leaves the loop only when it is
actually ready, so a pending unrelated IPI should not cause safety issues.
However, it might be inefficient on certain hardware, because it prevents
"wfi" from stalling the hart even if the hardware supports this, making the
hart needlessly spin in a "busy-wait" loop.
This behaviour can be observed, for example, in a QEMU VM (QEMU 7.2.0) with
"-machine virt" running a Linux guest. Inserting delays in
sbi_hsm_hart_start() allows reproducing the issue more reliably.
The comment in wait_for_coldboot() suggests that the initial IPI is needed
in the warm resume path, so let us clear it before init_warm_startup()
only.
To do this, sbi_ipi_raw_clear() was created similar to sbi_ipi_raw_send().
Signed-off-by: Evgenii Shatokhin <e.shatokhin@yadro.com>
Reviewed-by: Anup Patel <anup@brainfault.org>
2023-03-06 05:22:47 +08:00
|
|
|
if (hstate == SBI_HSM_STATE_SUSPENDED) {
|
2023-03-06 05:22:45 +08:00
|
|
|
init_warm_resume(scratch, hartid);
|
lib: sbi: Clear IPIs before init_warm_startup in non-boot harts
Since commit 50d4fde1c5a4 ("lib: Remove redundant sbi_platform_ipi_clear()
calls"), the IPI sent from the boot hart in wake_coldboot_harts() is not
cleared in the secondary harts until they reach sbi_ipi_init(). However,
sbi_hsm_init() and sbi_hsm_hart_wait() are called earlier, so a secondary
hart might enter sbi_hsm_hart_wait() with an already pending IPI.
sbi_hsm_hart_wait() makes sure the hart leaves the loop only when it is
actually ready, so a pending unrelated IPI should not cause safety issues.
However, it might be inefficient on certain hardware, because it prevents
"wfi" from stalling the hart even if the hardware supports this, making the
hart needlessly spin in a "busy-wait" loop.
This behaviour can be observed, for example, in a QEMU VM (QEMU 7.2.0) with
"-machine virt" running a Linux guest. Inserting delays in
sbi_hsm_hart_start() allows reproducing the issue more reliably.
The comment in wait_for_coldboot() suggests that the initial IPI is needed
in the warm resume path, so let us clear it before init_warm_startup()
only.
To do this, sbi_ipi_raw_clear() was created similar to sbi_ipi_raw_send().
Signed-off-by: Evgenii Shatokhin <e.shatokhin@yadro.com>
Reviewed-by: Anup Patel <anup@brainfault.org>
2023-03-06 05:22:47 +08:00
|
|
|
} else {
|
|
|
|
|
sbi_ipi_raw_clear(hartid);
|
2021-02-06 17:18:56 +08:00
|
|
|
init_warm_startup(scratch, hartid);
|
lib: sbi: Clear IPIs before init_warm_startup in non-boot harts
Since commit 50d4fde1c5a4 ("lib: Remove redundant sbi_platform_ipi_clear()
calls"), the IPI sent from the boot hart in wake_coldboot_harts() is not
cleared in the secondary harts until they reach sbi_ipi_init(). However,
sbi_hsm_init() and sbi_hsm_hart_wait() are called earlier, so a secondary
hart might enter sbi_hsm_hart_wait() with an already pending IPI.
sbi_hsm_hart_wait() makes sure the hart leaves the loop only when it is
actually ready, so a pending unrelated IPI should not cause safety issues.
However, it might be inefficient on certain hardware, because it prevents
"wfi" from stalling the hart even if the hardware supports this, making the
hart needlessly spin in a "busy-wait" loop.
This behaviour can be observed, for example, in a QEMU VM (QEMU 7.2.0) with
"-machine virt" running a Linux guest. Inserting delays in
sbi_hsm_hart_start() allows reproducing the issue more reliably.
The comment in wait_for_coldboot() suggests that the initial IPI is needed
in the warm resume path, so let us clear it before init_warm_startup()
only.
To do this, sbi_ipi_raw_clear() was created similar to sbi_ipi_raw_send().
Signed-off-by: Evgenii Shatokhin <e.shatokhin@yadro.com>
Reviewed-by: Anup Patel <anup@brainfault.org>
2023-03-06 05:22:47 +08:00
|
|
|
}
|
2018-12-11 21:54:06 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
|
|
|
|
|
|
2019-01-14 16:28:29 +08:00
|
|
|
/**
|
|
|
|
|
* Initialize OpenSBI library for current HART and jump to next
|
|
|
|
|
* booting stage.
|
|
|
|
|
*
|
|
|
|
|
* The function expects following:
|
|
|
|
|
* 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
|
|
|
|
|
* 2. Stack pointer (SP) is setup for current HART
|
2019-01-14 17:30:15 +08:00
|
|
|
* 3. Interrupts are disabled in MSTATUS CSR
|
2019-01-22 13:36:31 +08:00
|
|
|
* 4. All interrupts are disabled in MIE CSR
|
2019-01-14 16:28:29 +08:00
|
|
|
*
|
|
|
|
|
* @param scratch pointer to sbi_scratch of current HART
|
|
|
|
|
*/
|
|
|
|
|
void __noreturn sbi_init(struct sbi_scratch *scratch)
|
2018-12-11 21:54:06 +08:00
|
|
|
{
|
2022-12-21 19:38:06 +08:00
|
|
|
bool next_mode_supported = false;
|
|
|
|
|
bool coldboot = false;
|
2020-03-12 12:51:25 +08:00
|
|
|
u32 hartid = current_hartid();
|
2019-03-06 15:29:34 +08:00
|
|
|
const struct sbi_platform *plat = sbi_platform_ptr(scratch);
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2020-03-04 17:00:49 +08:00
|
|
|
if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
|
2020-03-15 14:09:25 +08:00
|
|
|
sbi_platform_hart_invalid(plat, hartid))
|
2018-12-22 03:29:28 +08:00
|
|
|
sbi_hart_hang();
|
2019-04-11 08:41:52 +08:00
|
|
|
|
2020-09-09 11:49:09 +08:00
|
|
|
switch (scratch->next_mode) {
|
|
|
|
|
case PRV_M:
|
2022-12-21 19:38:06 +08:00
|
|
|
next_mode_supported = true;
|
2020-09-09 11:49:09 +08:00
|
|
|
break;
|
|
|
|
|
case PRV_S:
|
|
|
|
|
if (misa_extension('S'))
|
2022-12-21 19:38:06 +08:00
|
|
|
next_mode_supported = true;
|
2020-09-09 11:49:09 +08:00
|
|
|
break;
|
|
|
|
|
case PRV_U:
|
|
|
|
|
if (misa_extension('U'))
|
2022-12-21 19:38:06 +08:00
|
|
|
next_mode_supported = true;
|
2020-09-09 11:49:09 +08:00
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Only the HART supporting privilege mode specified in the
|
|
|
|
|
* scratch->next_mode should be allowed to become the coldboot
|
|
|
|
|
* HART because the coldboot HART will be directly jumping to
|
|
|
|
|
* the next booting stage.
|
|
|
|
|
*
|
|
|
|
|
* We use a lottery mechanism to select coldboot HART among
|
|
|
|
|
* HARTs which satisfy above condition.
|
|
|
|
|
*/
|
|
|
|
|
|
2022-12-29 10:56:15 +08:00
|
|
|
if (sbi_platform_cold_boot_allowed(plat, hartid)) {
|
|
|
|
|
if (next_mode_supported &&
|
|
|
|
|
atomic_xchg(&coldboot_lottery, 1) == 0)
|
|
|
|
|
coldboot = true;
|
|
|
|
|
}
|
2018-12-11 21:54:06 +08:00
|
|
|
|
2021-06-29 15:44:04 +08:00
|
|
|
/*
|
|
|
|
|
* Do platform specific nascent (very early) initialization so
|
|
|
|
|
* that platform can initialize platform specific per-HART CSRs
|
|
|
|
|
* or per-HART devices.
|
|
|
|
|
*/
|
|
|
|
|
if (sbi_platform_nascent_init(plat))
|
|
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2018-12-11 21:54:06 +08:00
|
|
|
if (coldboot)
|
|
|
|
|
init_coldboot(scratch, hartid);
|
|
|
|
|
else
|
|
|
|
|
init_warmboot(scratch, hartid);
|
|
|
|
|
}
|
2020-01-03 11:19:23 +08:00
|
|
|
|
2023-03-20 21:22:02 +08:00
|
|
|
unsigned long sbi_entry_count(u32 hartid)
|
|
|
|
|
{
|
|
|
|
|
struct sbi_scratch *scratch;
|
|
|
|
|
unsigned long *entry_count;
|
|
|
|
|
|
|
|
|
|
if (!entry_count_offset)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
scratch = sbi_hartid_to_scratch(hartid);
|
|
|
|
|
if (!scratch)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
entry_count = sbi_scratch_offset_ptr(scratch, entry_count_offset);
|
|
|
|
|
|
|
|
|
|
return *entry_count;
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-03 17:43:33 +08:00
|
|
|
unsigned long sbi_init_count(u32 hartid)
|
|
|
|
|
{
|
|
|
|
|
struct sbi_scratch *scratch;
|
|
|
|
|
unsigned long *init_count;
|
|
|
|
|
|
2020-03-14 22:13:26 +08:00
|
|
|
if (!init_count_offset)
|
2020-01-03 17:43:33 +08:00
|
|
|
return 0;
|
|
|
|
|
|
2020-03-14 16:50:22 +08:00
|
|
|
scratch = sbi_hartid_to_scratch(hartid);
|
2020-03-14 21:25:53 +08:00
|
|
|
if (!scratch)
|
|
|
|
|
return 0;
|
|
|
|
|
|
2020-01-03 17:43:33 +08:00
|
|
|
init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
|
|
|
|
|
|
|
|
|
|
return *init_count;
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-03 11:19:23 +08:00
|
|
|
/**
|
|
|
|
|
* Exit OpenSBI library for current HART and stop HART
|
|
|
|
|
*
|
|
|
|
|
* The function expects following:
|
|
|
|
|
* 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
|
|
|
|
|
* 2. Stack pointer (SP) is setup for current HART
|
|
|
|
|
*
|
|
|
|
|
* @param scratch pointer to sbi_scratch of current HART
|
|
|
|
|
*/
|
|
|
|
|
void __noreturn sbi_exit(struct sbi_scratch *scratch)
|
|
|
|
|
{
|
2020-03-12 12:51:25 +08:00
|
|
|
u32 hartid = current_hartid();
|
2020-01-03 11:19:23 +08:00
|
|
|
const struct sbi_platform *plat = sbi_platform_ptr(scratch);
|
|
|
|
|
|
2020-03-15 14:09:25 +08:00
|
|
|
if (sbi_platform_hart_invalid(plat, hartid))
|
2020-01-03 11:19:23 +08:00
|
|
|
sbi_hart_hang();
|
|
|
|
|
|
2020-01-03 11:40:41 +08:00
|
|
|
sbi_platform_early_exit(plat);
|
|
|
|
|
|
2021-07-11 00:18:11 +08:00
|
|
|
sbi_pmu_exit(scratch);
|
|
|
|
|
|
2020-01-03 11:48:42 +08:00
|
|
|
sbi_timer_exit(scratch);
|
|
|
|
|
|
2020-01-03 12:09:10 +08:00
|
|
|
sbi_ipi_exit(scratch);
|
|
|
|
|
|
2022-02-19 01:48:41 +08:00
|
|
|
sbi_irqchip_exit(scratch);
|
2020-01-03 12:21:58 +08:00
|
|
|
|
2020-01-03 11:40:41 +08:00
|
|
|
sbi_platform_final_exit(plat);
|
|
|
|
|
|
2020-02-12 09:32:37 +08:00
|
|
|
sbi_hsm_exit(scratch);
|
2020-01-03 11:19:23 +08:00
|
|
|
}
|