Commit 68c88ef1 authored by Vincenzo Frascino's avatar Vincenzo Frascino
Browse files

arm64: mte: Enable TCO in functions that can read beyond buffer limits



load_unaligned_zeropad() and __get/put_kernel_nofault() functions can
read passed some buffer limits which may include some MTE granule with a
different tag.

When MTE async mode is enable, the load operation crosses the boundaries
and the next granule has a different tag the PE sets the TFSR_EL1.TF1 bit
as if an asynchronous tag fault is happened.

Enable Tag Check Override (TCO) in these functions  before the load and
disable it afterwards to prevent this to happen.

Note: The same condition can be hit in MTE sync mode but we deal with it
through the exception handling.
In the current implementation, mte_async_mode flag is set only at boot
time but in future kasan might acquire some runtime features that
that change the mode dynamically, hence we disable it when sync mode is
selected for future proof.

Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Reported-by: default avatarBranislav Rankov <Branislav.Rankov@arm.com>
Tested-by: default avatarBranislav Rankov <Branislav.Rankov@arm.com>
Signed-off-by: Vincenzo Frascino's avatarVincenzo Frascino <vincenzo.frascino@arm.com>
parent db5a080b
......@@ -188,6 +188,26 @@ static inline void __uaccess_enable_tco(void)
ARM64_MTE, CONFIG_KASAN_HW_TAGS));
}
/* Whether the MTE asynchronous mode is enabled. */
DECLARE_STATIC_KEY_FALSE(mte_async_mode);
/*
* These functions disable tag checking only if in MTE async mode
* since the sync mode generates exceptions synchronously and the
* nofault or load_unaligned_zeropad can handle them.
*/
static inline void __uaccess_disable_tco_async(void)
{
if (static_branch_unlikely(&mte_async_mode))
__uaccess_disable_tco();
}
static inline void __uaccess_enable_tco_async(void)
{
if (static_branch_unlikely(&mte_async_mode))
__uaccess_enable_tco();
}
static inline void uaccess_disable_privileged(void)
{
__uaccess_disable_tco();
......@@ -307,8 +327,10 @@ do { \
do { \
int __gkn_err = 0; \
\
__uaccess_enable_tco_async(); \
__raw_get_mem("ldr", *((type *)(dst)), \
(__force type *)(src), __gkn_err); \
__uaccess_disable_tco_async(); \
if (unlikely(__gkn_err)) \
goto err_label; \
} while (0)
......@@ -380,8 +402,10 @@ do { \
do { \
int __pkn_err = 0; \
\
__uaccess_enable_tco_async(); \
__raw_put_mem("str", *((type *)(src)), \
(__force type *)(dst), __pkn_err); \
__uaccess_disable_tco_async(); \
if (unlikely(__pkn_err)) \
goto err_label; \
} while(0)
......
......@@ -55,6 +55,8 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
{
unsigned long ret, offset;
__uaccess_enable_tco_async();
/* Load word from unaligned pointer addr */
asm(
"1: ldr %0, %3\n"
......@@ -76,6 +78,8 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
: "=&r" (ret), "=&r" (offset)
: "r" (addr), "Q" (*(unsigned long *)addr));
__uaccess_disable_tco_async();
return ret;
}
......
......@@ -26,6 +26,10 @@ u64 gcr_kernel_excl __ro_after_init;
static bool report_fault_once = true;
/* Whether the MTE asynchronous mode is enabled. */
DEFINE_STATIC_KEY_FALSE(mte_async_mode);
EXPORT_SYMBOL_GPL(mte_async_mode);
static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap)
{
pte_t old_pte = READ_ONCE(*ptep);
......@@ -119,12 +123,24 @@ static inline void __mte_enable_kernel(const char *mode, unsigned long tcf)
void mte_enable_kernel_sync(void)
{
__mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC);
/*
* This function is called on each active smp core at boot
* time, hence we do not need to take cpu_hotplug_lock again.
*/
static_branch_disable_cpuslocked(&mte_async_mode);
}
EXPORT_SYMBOL_GPL(mte_enable_kernel_sync);
void mte_enable_kernel_async(void)
{
__mte_enable_kernel("asynchronous", SCTLR_ELx_TCF_ASYNC);
/*
* This function is called on each active smp core at boot
* time, hence we do not need to take cpu_hotplug_lock again.
*/
static_branch_enable_cpuslocked(&mte_async_mode);
}
EXPORT_SYMBOL_GPL(mte_enable_kernel_async);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment