diff options
Diffstat (limited to 'mm/kasan')
| -rw-r--r-- | mm/kasan/Makefile | 2 | ||||
| -rw-r--r-- | mm/kasan/common.c | 4 | ||||
| -rw-r--r-- | mm/kasan/hw_tags.c | 211 | ||||
| -rw-r--r-- | mm/kasan/kasan.h | 56 | ||||
| -rw-r--r-- | mm/kasan/report.c | 326 | ||||
| -rw-r--r-- | mm/kasan/report_generic.c | 34 | ||||
| -rw-r--r-- | mm/kasan/report_hw_tags.c | 1 | ||||
| -rw-r--r-- | mm/kasan/report_sw_tags.c | 16 | ||||
| -rw-r--r-- | mm/kasan/report_tags.c | 2 | ||||
| -rw-r--r-- | mm/kasan/shadow.c | 64 |
10 files changed, 480 insertions, 236 deletions
diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile index adcd9acaef61..1f84df9c302e 100644 --- a/mm/kasan/Makefile +++ b/mm/kasan/Makefile @@ -35,7 +35,7 @@ CFLAGS_shadow.o := $(CC_FLAGS_KASAN_RUNTIME) CFLAGS_hw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) CFLAGS_sw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) -obj-$(CONFIG_KASAN) := common.o report.o +obj-y := common.o report.o obj-$(CONFIG_KASAN_GENERIC) += init.o generic.o report_generic.o shadow.o quarantine.o obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o tags.o report_tags.o obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o tags.o report_tags.o diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 92196562687b..d9079ec11f31 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -387,7 +387,7 @@ static inline bool ____kasan_kfree_large(void *ptr, unsigned long ip) } /* - * The object will be poisoned by kasan_free_pages() or + * The object will be poisoned by kasan_poison_pages() or * kasan_slab_free_mempool(). */ @@ -538,7 +538,7 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, return NULL; /* - * The object has already been unpoisoned by kasan_alloc_pages() for + * The object has already been unpoisoned by kasan_unpoison_pages() for * alloc_pages() or by kasan_krealloc() for krealloc(). */ diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index 7355cb534e4f..07a76c46daa5 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -32,6 +32,12 @@ enum kasan_arg_mode { KASAN_ARG_MODE_ASYMM, }; +enum kasan_arg_vmalloc { + KASAN_ARG_VMALLOC_DEFAULT, + KASAN_ARG_VMALLOC_OFF, + KASAN_ARG_VMALLOC_ON, +}; + enum kasan_arg_stacktrace { KASAN_ARG_STACKTRACE_DEFAULT, KASAN_ARG_STACKTRACE_OFF, @@ -40,18 +46,28 @@ enum kasan_arg_stacktrace { static enum kasan_arg kasan_arg __ro_after_init; static enum kasan_arg_mode kasan_arg_mode __ro_after_init; -static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init; +static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata; +static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata; -/* Whether KASAN is enabled at all. */ +/* + * Whether KASAN is enabled at all. + * The value remains false until KASAN is initialized by kasan_init_hw_tags(). + */ DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); EXPORT_SYMBOL(kasan_flag_enabled); -/* Whether the selected mode is synchronous/asynchronous/asymmetric.*/ +/* + * Whether the selected mode is synchronous, asynchronous, or asymmetric. + * Defaults to KASAN_MODE_SYNC. + */ enum kasan_mode kasan_mode __ro_after_init; EXPORT_SYMBOL_GPL(kasan_mode); +/* Whether to enable vmalloc tagging. */ +DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc); + /* Whether to collect alloc/free stack traces. */ -DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace); +DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace); /* kasan=off/on */ static int __init early_kasan_flag(char *arg) @@ -89,6 +105,23 @@ static int __init early_kasan_mode(char *arg) } early_param("kasan.mode", early_kasan_mode); +/* kasan.vmalloc=off/on */ +static int __init early_kasan_flag_vmalloc(char *arg) +{ + if (!arg) + return -EINVAL; + + if (!strcmp(arg, "off")) + kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF; + else if (!strcmp(arg, "on")) + kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON; + else + return -EINVAL; + + return 0; +} +early_param("kasan.vmalloc", early_kasan_flag_vmalloc); + /* kasan.stacktrace=off/on */ static int __init early_kasan_flag_stacktrace(char *arg) { @@ -116,7 +149,10 @@ static inline const char *kasan_mode_info(void) return "sync"; } -/* kasan_init_hw_tags_cpu() is called for each CPU. */ +/* + * kasan_init_hw_tags_cpu() is called for each CPU. + * Not marked as __init as a CPU can be hot-plugged after boot. + */ void kasan_init_hw_tags_cpu(void) { /* @@ -124,7 +160,11 @@ void kasan_init_hw_tags_cpu(void) * as this function is only called for MTE-capable hardware. */ - /* If KASAN is disabled via command line, don't initialize it. */ + /* + * If KASAN is disabled via command line, don't initialize it. + * When this function is called, kasan_flag_enabled is not yet + * set by kasan_init_hw_tags(). Thus, check kasan_arg instead. + */ if (kasan_arg == KASAN_ARG_OFF) return; @@ -132,12 +172,7 @@ void kasan_init_hw_tags_cpu(void) * Enable async or asymm modes only when explicitly requested * through the command line. */ - if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) - hw_enable_tagging_async(); - else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM) - hw_enable_tagging_asymm(); - else - hw_enable_tagging_sync(); + kasan_enable_tagging(); } /* kasan_init_hw_tags() is called once on boot CPU. */ @@ -151,86 +186,168 @@ void __init kasan_init_hw_tags(void) if (kasan_arg == KASAN_ARG_OFF) return; - /* Enable KASAN. */ - static_branch_enable(&kasan_flag_enabled); - switch (kasan_arg_mode) { case KASAN_ARG_MODE_DEFAULT: - /* - * Default to sync mode. - */ - fallthrough; + /* Default is specified by kasan_mode definition. */ + break; case KASAN_ARG_MODE_SYNC: - /* Sync mode enabled. */ kasan_mode = KASAN_MODE_SYNC; break; case KASAN_ARG_MODE_ASYNC: - /* Async mode enabled. */ kasan_mode = KASAN_MODE_ASYNC; break; case KASAN_ARG_MODE_ASYMM: - /* Asymm mode enabled. */ kasan_mode = KASAN_MODE_ASYMM; break; } + switch (kasan_arg_vmalloc) { + case KASAN_ARG_VMALLOC_DEFAULT: + /* Default is specified by kasan_flag_vmalloc definition. */ + break; + case KASAN_ARG_VMALLOC_OFF: + static_branch_disable(&kasan_flag_vmalloc); + break; + case KASAN_ARG_VMALLOC_ON: + static_branch_enable(&kasan_flag_vmalloc); + break; + } + switch (kasan_arg_stacktrace) { case KASAN_ARG_STACKTRACE_DEFAULT: - /* Default to enabling stack trace collection. */ - static_branch_enable(&kasan_flag_stacktrace); + /* Default is specified by kasan_flag_stacktrace definition. */ break; case KASAN_ARG_STACKTRACE_OFF: - /* Do nothing, kasan_flag_stacktrace keeps its default value. */ + static_branch_disable(&kasan_flag_stacktrace); break; case KASAN_ARG_STACKTRACE_ON: static_branch_enable(&kasan_flag_stacktrace); break; } - pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, stacktrace=%s)\n", + /* KASAN is now initialized, enable it. */ + static_branch_enable(&kasan_flag_enabled); + + pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n", kasan_mode_info(), + kasan_vmalloc_enabled() ? "on" : "off", kasan_stack_collection_enabled() ? "on" : "off"); } -void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags) +#ifdef CONFIG_KASAN_VMALLOC + +static void unpoison_vmalloc_pages(const void *addr, u8 tag) { + struct vm_struct *area; + int i; + /* - * This condition should match the one in post_alloc_hook() in - * page_alloc.c. + * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations + * (see the comment in __kasan_unpoison_vmalloc), all of the pages + * should belong to a single area. */ - bool init = !want_init_on_free() && want_init_on_alloc(flags); - - if (flags & __GFP_SKIP_KASAN_POISON) - SetPageSkipKASanPoison(page); + area = find_vm_area((void *)addr); + if (WARN_ON(!area)) + return; - if (flags & __GFP_ZEROTAGS) { - int i; + for (i = 0; i < area->nr_pages; i++) { + struct page *page = area->pages[i]; - for (i = 0; i != 1 << order; ++i) - tag_clear_highpage(page + i); - } else { - kasan_unpoison_pages(page, order, init); + page_kasan_tag_set(page, tag); } } -void kasan_free_pages(struct page *page, unsigned int order) +void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, + kasan_vmalloc_flags_t flags) { + u8 tag; + unsigned long redzone_start, redzone_size; + + if (!kasan_vmalloc_enabled()) + return (void *)start; + + if (!is_vmalloc_or_module_addr(start)) + return (void *)start; + + /* + * Skip unpoisoning and assigning a pointer tag for non-VM_ALLOC + * mappings as: + * + * 1. Unlike the software KASAN modes, hardware tag-based KASAN only + * supports tagging physical memory. Therefore, it can only tag a + * single mapping of normal physical pages. + * 2. Hardware tag-based KASAN can only tag memory mapped with special + * mapping protection bits, see arch_vmalloc_pgprot_modify(). + * As non-VM_ALLOC mappings can be mapped outside of vmalloc code, + * providing these bits would require tracking all non-VM_ALLOC + * mappers. + * + * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags + * the first virtual mapping, which is created by vmalloc(). + * Tagging the page_alloc memory backing that vmalloc() allocation is + * skipped, see ___GFP_SKIP_KASAN_UNPOISON. + * + * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual. + */ + if (!(flags & KASAN_VMALLOC_VM_ALLOC)) + return (void *)start; + + /* + * Don't tag executable memory. + * The kernel doesn't tolerate having the PC register tagged. + */ + if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) + return (void *)start; + + tag = kasan_random_tag(); + start = set_tag(start, tag); + + /* Unpoison and initialize memory up to size. */ + kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT); + + /* + * Explicitly poison and initialize the in-page vmalloc() redzone. + * Unlike software KASAN modes, hardware tag-based KASAN doesn't + * unpoison memory when populating shadow for vmalloc() space. + */ + redzone_start = round_up((unsigned long)start + size, + KASAN_GRANULE_SIZE); + redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start; + kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID, + flags & KASAN_VMALLOC_INIT); + /* - * This condition should match the one in free_pages_prepare() in - * page_alloc.c. + * Set per-page tag flags to allow accessing physical memory for the + * vmalloc() mapping through page_address(vmalloc_to_page()). */ - bool init = want_init_on_free(); + unpoison_vmalloc_pages(start, tag); - kasan_poison_pages(page, order, init); + return (void *)start; +} + +void __kasan_poison_vmalloc(const void *start, unsigned long size) +{ + /* + * No tagging here. + * The physical pages backing the vmalloc() allocation are poisoned + * through the usual page_alloc paths. + */ } +#endif + #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) -void kasan_enable_tagging_sync(void) +void kasan_enable_tagging(void) { - hw_enable_tagging_sync(); + if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) + hw_enable_tagging_async(); + else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM) + hw_enable_tagging_asymm(); + else + hw_enable_tagging_sync(); } -EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync); +EXPORT_SYMBOL_GPL(kasan_enable_tagging); void kasan_force_async_fault(void) { diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index c17fa8d26ffe..d79b83d673b1 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -12,7 +12,8 @@ #include <linux/static_key.h> #include "../slab.h" -DECLARE_STATIC_KEY_FALSE(kasan_flag_stacktrace); +DECLARE_STATIC_KEY_TRUE(kasan_flag_vmalloc); +DECLARE_STATIC_KEY_TRUE(kasan_flag_stacktrace); enum kasan_mode { KASAN_MODE_SYNC, @@ -22,6 +23,11 @@ enum kasan_mode { extern enum kasan_mode kasan_mode __ro_after_init; +static inline bool kasan_vmalloc_enabled(void) +{ + return static_branch_likely(&kasan_flag_vmalloc); +} + static inline bool kasan_stack_collection_enabled(void) { return static_branch_unlikely(&kasan_flag_stacktrace); @@ -71,17 +77,19 @@ static inline bool kasan_sync_fault_possible(void) #define KASAN_PAGE_REDZONE 0xFE /* redzone for kmalloc_large allocations */ #define KASAN_KMALLOC_REDZONE 0xFC /* redzone inside slub object */ #define KASAN_KMALLOC_FREE 0xFB /* object was freed (kmem_cache_free/kfree) */ -#define KASAN_KMALLOC_FREETRACK 0xFA /* object was freed and has free track set */ +#define KASAN_VMALLOC_INVALID 0xF8 /* unallocated space in vmapped page */ #else #define KASAN_FREE_PAGE KASAN_TAG_INVALID #define KASAN_PAGE_REDZONE KASAN_TAG_INVALID #define KASAN_KMALLOC_REDZONE KASAN_TAG_INVALID #define KASAN_KMALLOC_FREE KASAN_TAG_INVALID -#define KASAN_KMALLOC_FREETRACK KASAN_TAG_INVALID +#define KASAN_VMALLOC_INVALID KASAN_TAG_INVALID /* only for SW_TAGS */ #endif +#ifdef CONFIG_KASAN_GENERIC + +#define KASAN_KMALLOC_FREETRACK 0xFA /* object was freed and has free track set */ #define KASAN_GLOBAL_REDZONE 0xF9 /* redzone for global variable */ -#define KASAN_VMALLOC_INVALID 0xF8 /* unallocated space in vmapped page */ /* * Stack redzone shadow values @@ -110,6 +118,8 @@ static inline bool kasan_sync_fault_possible(void) #define KASAN_ABI_VERSION 1 #endif +#endif /* CONFIG_KASAN_GENERIC */ + /* Metadata layout customization. */ #define META_BYTES_PER_BLOCK 1 #define META_BLOCKS_PER_ROW 16 @@ -117,9 +127,15 @@ static inline bool kasan_sync_fault_possible(void) #define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE) #define META_ROWS_AROUND_ADDR 2 -struct kasan_access_info { - const void *access_addr; - const void *first_bad_addr; +enum kasan_report_type { + KASAN_REPORT_ACCESS, + KASAN_REPORT_INVALID_FREE, +}; + +struct kasan_report_info { + enum kasan_report_type type; + void *access_addr; + void *first_bad_addr; size_t access_size; bool is_write; unsigned long ip; @@ -204,6 +220,14 @@ struct kasan_free_meta { #endif }; +#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) +/* Used in KUnit-compatible KASAN tests. */ +struct kunit_kasan_status { + bool report_found; + bool sync_fault; +}; +#endif + struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, const void *object); #ifdef CONFIG_KASAN_GENERIC @@ -221,7 +245,8 @@ static inline const void *kasan_shadow_to_mem(const void *shadow_addr) static inline bool addr_has_metadata(const void *addr) { - return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START)); + return (kasan_reset_tag(addr) >= + kasan_shadow_to_mem((void *)KASAN_SHADOW_START)); } /** @@ -251,10 +276,10 @@ static inline void kasan_print_tags(u8 addr_tag, const void *addr) { } #endif void *kasan_find_first_bad_addr(void *addr, size_t size); -const char *kasan_get_bug_type(struct kasan_access_info *info); +const char *kasan_get_bug_type(struct kasan_report_info *info); void kasan_metadata_fetch_row(char *buffer, void *row); -#if defined(CONFIG_KASAN_GENERIC) && defined(CONFIG_KASAN_STACK) +#if defined(CONFIG_KASAN_STACK) void kasan_print_address_stack_frame(const void *addr); #else static inline void kasan_print_address_stack_frame(const void *addr) { } @@ -340,12 +365,12 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag) #if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) -void kasan_enable_tagging_sync(void); +void kasan_enable_tagging(void); void kasan_force_async_fault(void); #else /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */ -static inline void kasan_enable_tagging_sync(void) { } +static inline void kasan_enable_tagging(void) { } static inline void kasan_force_async_fault(void) { } #endif /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */ @@ -467,6 +492,13 @@ static inline bool kasan_arch_is_ready(void) { return true; } #error kasan_arch_is_ready only works in KASAN generic outline mode! #endif +#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) || IS_ENABLED(CONFIG_KASAN_MODULE_TEST) + +bool kasan_save_enable_multi_shot(void); +void kasan_restore_multi_shot(bool enabled); + +#endif + /* * Exported functions for interfaces called from assembly or from generated * code. Declarations here to avoid warning about missing declarations. diff --git a/mm/kasan/report.c b/mm/kasan/report.c index f14146563d41..199d77cce21a 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -13,6 +13,7 @@ #include <linux/ftrace.h> #include <linux/init.h> #include <linux/kernel.h> +#include <linux/lockdep.h> #include <linux/mm.h> #include <linux/printk.h> #include <linux/sched.h> @@ -64,6 +65,40 @@ static int __init early_kasan_fault(char *arg) } early_param("kasan.fault", early_kasan_fault); +static int __init kasan_set_multi_shot(char *str) +{ + set_bit(KASAN_BIT_MULTI_SHOT, &kasan_flags); + return 1; +} +__setup("kasan_multi_shot", kasan_set_multi_shot); + +/* + * Used to suppress reports within kasan_disable/enable_current() critical + * sections, which are used for marking accesses to slab metadata. + */ +static bool report_suppressed(void) +{ +#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) + if (current->kasan_depth) + return true; +#endif + return false; +} + +/* + * Used to avoid reporting more than one KASAN bug unless kasan_multi_shot + * is enabled. Note that KASAN tests effectively enable kasan_multi_shot + * for their duration. + */ +static bool report_enabled(void) +{ + if (test_bit(KASAN_BIT_MULTI_SHOT, &kasan_flags)) + return true; + return !test_and_set_bit(KASAN_BIT_REPORTED, &kasan_flags); +} + +#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) || IS_ENABLED(CONFIG_KASAN_MODULE_TEST) + bool kasan_save_enable_multi_shot(void) { return test_and_set_bit(KASAN_BIT_MULTI_SHOT, &kasan_flags); @@ -77,53 +112,87 @@ void kasan_restore_multi_shot(bool enabled) } EXPORT_SYMBOL_GPL(kasan_restore_multi_shot); -static int __init kasan_set_multi_shot(char *str) -{ - set_bit(KASAN_BIT_MULTI_SHOT, &kasan_flags); - return 1; -} -__setup("kasan_multi_shot", kasan_set_multi_shot); +#endif -static void print_error_description(struct kasan_access_info *info) +#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) +static void update_kunit_status(bool sync) { - pr_err("BUG: KASAN: %s in %pS\n", - kasan_get_bug_type(info), (void *)info->ip); - if (info->access_size) - pr_err("%s of size %zu at addr %px by task %s/%d\n", - info->is_write ? "Write" : "Read", info->access_size, - info->access_addr, current->comm, task_pid_nr(current)); - else - pr_err("%s at addr %px by task %s/%d\n", - info->is_write ? "Write" : "Read", - info->access_addr, current->comm, task_pid_nr(current)); + struct kunit *test; + struct kunit_resource *resource; + struct kunit_kasan_status *status; + + test = current->kunit_test; + if (!test) + return; + + resource = kunit_find_named_resource(test, "kasan_status"); + if (!resource) { + kunit_set_failure(test); + return; + } + + status = (struct kunit_kasan_status *)resource->data; + WRITE_ONCE(status->report_found, true); + WRITE_ONCE(status->sync_fault, sync); + + kunit_put_resource(resource); } +#else +static void update_kunit_status(bool sync) { } +#endif static DEFINE_SPINLOCK(report_lock); -static void start_report(unsigned long *flags) +static void start_report(unsigned long *flags, bool sync) { - /* - * Make sure we don't end up in loop. - */ + /* Respect the /proc/sys/kernel/traceoff_on_warning interface. */ + disable_trace_on_warning(); + /* Update status of the currently running KASAN test. */ + update_kunit_status(sync); + /* Do not allow LOCKDEP mangling KASAN reports. */ + lockdep_off(); + /* Make sure we don't end up in loop. */ kasan_disable_current(); spin_lock_irqsave(&report_lock, *flags); pr_err("==================================================================\n"); } -static void end_report(unsigned long *flags, unsigned long addr) +static void end_report(unsigned long *flags, void *addr) { - if (!kasan_async_fault_possible()) - trace_error_report_end(ERROR_DETECTOR_KASAN, addr); + if (addr) + trace_error_report_end(ERROR_DETECTOR_KASAN, + (unsigned long)addr); pr_err("==================================================================\n"); - add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); spin_unlock_irqrestore(&report_lock, *flags); if (panic_on_warn && !test_bit(KASAN_BIT_MULTI_SHOT, &kasan_flags)) panic("panic_on_warn set ...\n"); if (kasan_arg_fault == KASAN_ARG_FAULT_PANIC) panic("kasan.fault=panic set ...\n"); + add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); + lockdep_on(); kasan_enable_current(); } +static void print_error_description(struct kasan_report_info *info) +{ + if (info->type == KASAN_REPORT_INVALID_FREE) { + pr_err("BUG: KASAN: double-free or invalid-free in %pS\n", + (void *)info->ip); + return; + } + + pr_err("BUG: KASAN: %s in %pS\n", + kasan_get_bug_type(info), (void *)info->ip); + if (info->access_size) + pr_err("%s of size %zu at addr %px by task %s/%d\n", + info->is_write ? "Write" : "Read", info->access_size, + info->access_addr, current->comm, task_pid_nr(current)); + else + pr_err("%s at addr %px by task %s/%d\n", + info->is_write ? "Write" : "Read", + info->access_addr, current->comm, task_pid_nr(current)); +} + static void print_track(struct kasan_track *track, const char *prefix) { pr_err("%s by task %u:\n", prefix, track->pid); @@ -162,9 +231,6 @@ static void describe_object_addr(struct kmem_cache *cache, void *object, " which belongs to the cache %s of size %d\n", object, cache->name, cache->object_size); - if (!addr) - return; - if (access_addr < object_addr) { rel_type = "to the left"; rel_bytes = object_addr - access_addr; @@ -253,19 +319,43 @@ static void print_address_description(void *addr, u8 tag) void *object = nearest_obj(cache, slab, addr); describe_object(cache, object, addr, tag); + pr_err("\n"); } if (kernel_or_module_addr(addr) && !init_task_stack_addr(addr)) { pr_err("The buggy address belongs to the variable:\n"); pr_err(" %pS\n", addr); + pr_err("\n"); + } + + if (object_is_on_stack(addr)) { + /* + * Currently, KASAN supports printing frame information only + * for accesses to the task's own stack. + */ + kasan_print_address_stack_frame(addr); + pr_err("\n"); + } + + if (is_vmalloc_addr(addr)) { + struct vm_struct *va = find_vm_area(addr); + + if (va) { + pr_err("The buggy address belongs to the virtual mapping at\n" + " [%px, %px) created by:\n" + " %pS\n", + va->addr, va->addr + va->size, va->caller); + pr_err("\n"); + + page = vmalloc_to_page(page); + } } if (page) { - pr_err("The buggy address belongs to the page:\n"); + pr_err("The buggy address belongs to the physical page:\n"); dump_page(page, "kasan: bad access detected"); + pr_err("\n"); } - - kasan_print_address_stack_frame(addr); } static bool meta_row_is_guilty(const void *row, const void *addr) @@ -324,138 +414,110 @@ static void print_memory_metadata(const void *addr) } } -static bool report_enabled(void) -{ -#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) - if (current->kasan_depth) - return false; -#endif - if (test_bit(KASAN_BIT_MULTI_SHOT, &kasan_flags)) - return true; - return !test_and_set_bit(KASAN_BIT_REPORTED, &kasan_flags); -} - -#if IS_ENABLED(CONFIG_KUNIT) -static void kasan_update_kunit_status(struct kunit *cur_test) +static void print_report(struct kasan_report_info *info) { - struct kunit_resource *resource; - struct kunit_kasan_expectation *kasan_data; + void *tagged_addr = info->access_addr; + void *untagged_addr = kasan_reset_tag(tagged_addr); + u8 tag = get_tag(tagged_addr); - resource = kunit_find_named_resource(cur_test, "kasan_data"); + print_error_description(info); + if (addr_has_metadata(untagged_addr)) + kasan_print_tags(tag, info->first_bad_addr); + pr_err("\n"); - if (!resource) { - kunit_set_failure(cur_test); - return; + if (addr_has_metadata(untagged_addr)) { + print_address_description(untagged_addr, tag); + print_memory_metadata(info->first_bad_addr); + } else { + dump_stack_lvl(KERN_ERR); } - - kasan_data = (struct kunit_kasan_expectation *)resource->data; - WRITE_ONCE(kasan_data->report_found, true); - kunit_put_resource(resource); } -#endif /* IS_ENABLED(CONFIG_KUNIT) */ -void kasan_report_invalid_free(void *object, unsigned long ip) +void kasan_report_invalid_free(void *ptr, unsigned long ip) { unsigned long flags; - u8 tag = get_tag(object); + struct kasan_report_info info; - object = kasan_reset_tag(object); - -#if IS_ENABLED(CONFIG_KUNIT) - if (current->kunit_test) - kasan_update_kunit_status(current->kunit_test); -#endif /* IS_ENABLED(CONFIG_KUNIT) */ + /* + * Do not check report_suppressed(), as an invalid-free cannot be + * caused by accessing slab metadata and thus should not be + * suppressed by kasan_disable/enable_current() critical sections. + */ + if (unlikely(!report_enabled())) + return; - start_report(&flags); - pr_err("BUG: KASAN: double-free or invalid-free in %pS\n", (void *)ip); - kasan_print_tags(tag, object); - pr_err("\n"); - print_address_description(object, tag); - pr_err("\n"); - print_memory_metadata(object); - end_report(&flags, (unsigned long)object); -} + start_report(&flags, true); -#ifdef CONFIG_KASAN_HW_TAGS -void kasan_report_async(void) -{ - unsigned long flags; + info.type = KASAN_REPORT_INVALID_FREE; + info.access_addr = ptr; + info.first_bad_addr = kasan_reset_tag(ptr); + info.access_size = 0; + info.is_write = false; + info.ip = ip; -#if IS_ENABLED(CONFIG_KUNIT) - if (current->kunit_test) - kasan_update_kunit_status(current->kunit_test); -#endif /* IS_ENABLED(CONFIG_KUNIT) */ + print_report(&info); - start_report(&flags); - pr_err("BUG: KASAN: invalid-access\n"); - pr_err("Asynchronous mode enabled: no access details available\n"); - pr_err("\n"); - dump_stack_lvl(KERN_ERR); - end_report(&flags, 0); + end_report(&flags, ptr); } -#endif /* CONFIG_KASAN_HW_TAGS */ -static void __kasan_report(unsigned long addr, size_t size, bool is_write, - unsigned long ip) +/* + * kasan_report() is the only reporting function that uses + * user_access_save/restore(): kasan_report_invalid_free() cannot be called + * from a UACCESS region, and kasan_report_async() is not used on x86. + */ +bool kasan_report(unsigned long addr, size_t size, bool is_write, + unsigned long ip) { - struct kasan_access_info info; - void *tagged_addr; - void *untagged_addr; - unsigned long flags; - -#if IS_ENABLED(CONFIG_KUNIT) - if (current->kunit_test) - kasan_update_kunit_status(current->kunit_test); -#endif /* IS_ENABLED(CONFIG_KUNIT) */ - - disable_trace_on_warning(); + bool ret = true; + void *ptr = (void *)addr; + unsigned long ua_flags = user_access_save(); + unsigned long irq_flags; + struct kasan_report_info info; + + if (unlikely(report_suppressed()) || unlikely(!report_enabled())) { + ret = false; + goto out; + } - tagged_addr = (void *)addr; - untagged_addr = kasan_reset_tag(tagged_addr); + start_report(&irq_flags, true); - info.access_addr = tagged_addr; - if (addr_has_metadata(untagged_addr)) - info.first_bad_addr = - kasan_find_first_bad_addr(tagged_addr, size); - else - info.first_bad_addr = untagged_addr; + info.type = KASAN_REPORT_ACCESS; + info.access_addr = ptr; + info.first_bad_addr = kasan_find_first_bad_addr(ptr, size); info.access_size = size; info.is_write = is_write; info.ip = ip; - start_report(&flags); + print_report(&info); - print_error_description(&info); - if (addr_has_metadata(untagged_addr)) - kasan_print_tags(get_tag(tagged_addr), info.first_bad_addr); - pr_err("\n"); + end_report(&irq_flags, ptr); - if (addr_has_metadata(untagged_addr)) { - print_address_description(untagged_addr, get_tag(tagged_addr)); - pr_err("\n"); - print_memory_metadata(info.first_bad_addr); - } else { - dump_stack_lvl(KERN_ERR); - } +out: + user_access_restore(ua_flags); - end_report(&flags, addr); + return ret; } -bool kasan_report(unsigned long addr, size_t size, bool is_write, - unsigned long ip) +#ifdef CONFIG_KASAN_HW_TAGS +void kasan_report_async(void) { - unsigned long flags = user_access_save(); - bool ret = false; - - if (likely(report_enabled())) { - __kasan_report(addr, size, is_write, ip); - ret = true; - } + unsigned long flags; - user_access_restore(flags); + /* + * Do not check report_suppressed(), as kasan_disable/enable_current() + * critical sections do not affect Hardware Tag-Based KASAN. + */ + if (unlikely(!report_enabled())) + return; - return ret; + start_report(&flags, false); + pr_err("BUG: KASAN: invalid-access\n"); + pr_err("Asynchronous fault: no details available\n"); + pr_err("\n"); + dump_stack_lvl(KERN_ERR); + end_report(&flags, NULL); } +#endif /* CONFIG_KASAN_HW_TAGS */ #ifdef CONFIG_KASAN_INLINE /* diff --git a/mm/kasan/report_generic.c b/mm/kasan/report_generic.c index 139615ef326b..efc5e79a103f 100644 --- a/mm/kasan/report_generic.c +++ b/mm/kasan/report_generic.c @@ -34,12 +34,16 @@ void *kasan_find_first_bad_addr(void *addr, size_t size) { void *p = addr; + if (!addr_has_metadata(p)) + return p; + while (p < addr + size && !(*(u8 *)kasan_mem_to_shadow(p))) p += KASAN_GRANULE_SIZE; + return p; } -static const char *get_shadow_bug_type(struct kasan_access_info *info) +static const char *get_shadow_bug_type(struct kasan_report_info *info) { const char *bug_type = "unknown-crash"; u8 *shadow_addr; @@ -91,7 +95,7 @@ static const char *get_shadow_bug_type(struct kasan_access_info *info) return bug_type; } -static const char *get_wild_bug_type(struct kasan_access_info *info) +static const char *get_wild_bug_type(struct kasan_report_info *info) { const char *bug_type = "unknown-crash"; @@ -105,7 +109,7 @@ static const char *get_wild_bug_type(struct kasan_access_info *info) return bug_type; } -const char *kasan_get_bug_type(struct kasan_access_info *info) +const char *kasan_get_bug_type(struct kasan_report_info *info) { /* * If access_size is a negative number, then it has reason to be @@ -180,7 +184,7 @@ static void print_decoded_frame_descr(const char *frame_descr) return; pr_err("\n"); - pr_err("this frame has %lu %s:\n", num_objects, + pr_err("This frame has %lu %s:\n", num_objects, num_objects == 1 ? "object" : "objects"); while (num_objects--) { @@ -211,6 +215,7 @@ static void print_decoded_frame_descr(const char *frame_descr) } } +/* Returns true only if the address is on the current task's stack. */ static bool __must_check get_address_stack_frame_info(const void *addr, unsigned long *offset, const char **frame_descr, @@ -224,13 +229,6 @@ static bool __must_check get_address_stack_frame_info(const void *addr, BUILD_BUG_ON(IS_ENABLED(CONFIG_STACK_GROWSUP)); - /* - * NOTE: We currently only support printing frame information for - * accesses to the task's own stack. - */ - if (!object_is_on_stack(addr)) - return false; - aligned_addr = round_down((unsigned long)addr, sizeof(long)); mem_ptr = round_down(aligned_addr, KASAN_GRANULE_SIZE); shadow_ptr = kasan_mem_to_shadow((void *)aligned_addr); @@ -269,17 +267,17 @@ void kasan_print_address_stack_frame(const void *addr) const char *frame_descr; const void *frame_pc; + if (WARN_ON(!object_is_on_stack(addr))) + return; + + pr_err("The buggy address belongs to stack of task %s/%d\n", + current->comm, task_pid_nr(current)); + if (!get_address_stack_frame_info(addr, &offset, &frame_descr, &frame_pc)) return; - /* - * get_address_stack_frame_info only returns true if the given addr is - * on the current task's stack. - */ - pr_err("\n"); - pr_err("addr %px is located in stack of task %s/%d at offset %lu in frame:\n", - addr, current->comm, task_pid_nr(current), offset); + pr_err(" and is located at offset %lu in frame:\n", offset); pr_err(" %pS\n", frame_pc); if (!frame_descr) diff --git a/mm/kasan/report_hw_tags.c b/mm/kasan/report_hw_tags.c index 5dbbbb930e7a..f3d3be614e4b 100644 --- a/mm/kasan/report_hw_tags.c +++ b/mm/kasan/report_hw_tags.c @@ -17,6 +17,7 @@ void *kasan_find_first_bad_addr(void *addr, size_t size) { + /* Return the same value regardless of whether addr_has_metadata(). */ return kasan_reset_tag(addr); } diff --git a/mm/kasan/report_sw_tags.c b/mm/kasan/report_sw_tags.c index d2298c357834..7a26397297ed 100644 --- a/mm/kasan/report_sw_tags.c +++ b/mm/kasan/report_sw_tags.c @@ -16,6 +16,7 @@ #include <linux/mm.h> #include <linux/printk.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/slab.h> #include <linux/stackdepot.h> #include <linux/stacktrace.h> @@ -35,8 +36,12 @@ void *kasan_find_first_bad_addr(void *addr, size_t size) void *p = kasan_reset_tag(addr); void *end = p + size; + if (!addr_has_metadata(p)) + return p; + while (p < end && tag == *(u8 *)kasan_mem_to_shadow(p)) p += KASAN_GRANULE_SIZE; + return p; } @@ -51,3 +56,14 @@ void kasan_print_tags(u8 addr_tag, const void *addr) pr_err("Pointer tag: [%02x], memory tag: [%02x]\n", addr_tag, *shadow); } + +#ifdef CONFIG_KASAN_STACK +void kasan_print_address_stack_frame(const void *addr) +{ + if (WARN_ON(!object_is_on_stack(addr))) + return; + + pr_err("The buggy address belongs to stack of task %s/%d\n", + current->comm, task_pid_nr(current)); +} +#endif diff --git a/mm/kasan/report_tags.c b/mm/kasan/report_tags.c index 1b41de88c53e..e25d2166e813 100644 --- a/mm/kasan/report_tags.c +++ b/mm/kasan/report_tags.c @@ -7,7 +7,7 @@ #include "kasan.h" #include "../slab.h" -const char *kasan_get_bug_type(struct kasan_access_info *info) +const char *kasan_get_bug_type(struct kasan_report_info *info) { #ifdef CONFIG_KASAN_TAGS_IDENTIFY struct kasan_alloc_meta *alloc_meta; diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 94136f84b449..a4f07de21771 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -345,27 +345,6 @@ int kasan_populate_vmalloc(unsigned long addr, unsigned long size) return 0; } -/* - * Poison the shadow for a vmalloc region. Called as part of the - * freeing process at the time the region is freed. - */ -void kasan_poison_vmalloc(const void *start, unsigned long size) -{ - if (!is_vmalloc_or_module_addr(start)) - return; - - size = round_up(size, KASAN_GRANULE_SIZE); - kasan_poison(start, size, KASAN_VMALLOC_INVALID, false); -} - -void kasan_unpoison_vmalloc(const void *start, unsigned long size) -{ - if (!is_vmalloc_or_module_addr(start)) - return; - - kasan_unpoison(start, size, false); -} - static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr, void *unused) { @@ -496,9 +475,48 @@ void kasan_release_vmalloc(unsigned long start, unsigned long end, } } +void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, + kasan_vmalloc_flags_t flags) +{ + /* + * Software KASAN modes unpoison both VM_ALLOC and non-VM_ALLOC + * mappings, so the KASAN_VMALLOC_VM_ALLOC flag is ignored. + * Software KASAN modes can't optimize zeroing memory by combining it + * with setting memory tags, so the KASAN_VMALLOC_INIT flag is ignored. + */ + + if (!is_vmalloc_or_module_addr(start)) + return (void *)start; + + /* + * Don't tag executable memory with the tag-based mode. + * The kernel doesn't tolerate having the PC register tagged. + */ + if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) && + !(flags & KASAN_VMALLOC_PROT_NORMAL)) + return (void *)start; + + start = set_tag(start, kasan_random_tag()); + kasan_unpoison(start, size, false); + return (void *)start; +} + +/* + * Poison the shadow for a vmalloc region. Called as part of the + * freeing process at the time the region is freed. + */ +void __kasan_poison_vmalloc(const void *start, unsigned long size) +{ + if (!is_vmalloc_or_module_addr(start)) + return; + + size = round_up(size, KASAN_GRANULE_SIZE); + kasan_poison(start, size, KASAN_VMALLOC_INVALID, false); +} + #else /* CONFIG_KASAN_VMALLOC */ -int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask) +int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) { void *ret; size_t scaled_size; @@ -534,7 +552,7 @@ int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask) return -ENOMEM; } -void kasan_free_shadow(const struct vm_struct *vm) +void kasan_free_module_shadow(const struct vm_struct *vm) { if (vm->flags & VM_KASAN) vfree(kasan_mem_to_shadow(vm->addr)); |