arm64: mm: always map fixmap at page granularity

For largely historical reasons, when using 4K pages we map the FDT using
2MiB block mappings. The boot protocol mandates that the FDT is placed
in memory such that the kernel can safely use such mappings, and we
reserve 4MiB of VA space within the fixmap to handle arbitrary
mis-alignment of the FDT.

We reserve the memory containing the FDT using memblock_reserve(), which
reserves the memory at page granularity. Any pages surrounding the FDT
which fall within the same 2MiB block will be mapped via the FDT
mapping, and so will always have a Normal Write Back Cacheable alias.
This is benign today, but could become a problem in future if we need to
map any memory with differing attributes.

We can avoid such problems and simplify the fixmap code overall by
allocating tables for the entire fixmap VA range such that all fixmap
addresses are mapped at the PTE level. Our VA layout means that the
fixmap will always fall within a single PMD table, and we only need to
allocate multiple PTE tables. With a static_assert() that the entire
fixmap falls within a single PMD table we can also remove runtime
warnings regarding the fixmap VA layout.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>

2 files changed, 80 insertions, 90 deletions

diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 52154a2b496ac..a6a92eb86d07c 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -36,17 +36,13 @@ enum fixed_addresses {
 	FIX_HOLE,
 
 	/*
-	 * Reserve a virtual window for the FDT that is 2 MB larger than the
-	 * maximum supported size, and put it at the top of the fixmap region.
-	 * The additional space ensures that any FDT that does not exceed
-	 * MAX_FDT_SIZE can be mapped regardless of whether it crosses any
-	 * 2 MB alignment boundaries.
-	 *
-	 * Keep this at the top so it remains 2 MB aligned.
+	 * Reserve a virtual window for the FDT that is a page bigger than the
+	 * maximum supported size. The additional space ensures that any FDT
+	 * that does not exceed MAX_FDT_SIZE can be mapped regardless of
+	 * whether it crosses any page boundary.
 	 */
-#define FIX_FDT_SIZE		(MAX_FDT_SIZE + SZ_2M)
 	FIX_FDT_END,
-	FIX_FDT = FIX_FDT_END + FIX_FDT_SIZE / PAGE_SIZE - 1,
+	FIX_FDT = FIX_FDT_END + MAX_FDT_SIZE / PAGE_SIZE,
 
 	FIX_EARLYCON_MEM_BASE,
 	FIX_TEXT_POKE0,
diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index 54e50552bfe3d..b886ff501c029 100644
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -16,34 +16,85 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 
-static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
+#define NR_BM_PTE_TABLES	SPAN_PMD_ENTRIES(FIXADDR_START, FIXADDR_TOP)
+#define NR_BM_PMD_TABLES	SPAN_PUD_ENTRIES(FIXADDR_START, FIXADDR_TOP)
+
+static_assert(NR_BM_PMD_TABLES == 1);
+
+#define __BM_TABLE_IDX(addr, shift) \
+	(((addr) >> (shift)) - (FIXADDR_START >> (shift)))
+
+#define BM_PTE_IDX(addr)	__BM_TABLE_IDX(addr, PMD_SHIFT)
+
+static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss;
 static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
 static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
 
-static inline pud_t *fixmap_pud(unsigned long addr)
+/*
+ * While using init_pg_dir, all tables are allocated from the kernel image, and
+ * are accesible from the kimg mapping, but the linear map is not mapped.
+ *
+ * While using swapper_pg_dir, intermediate levels of kernel page tables are
+ * allocated from arbitrary memory mapped by the linear map, and are not
+ * accessible from the kimg mapping.
+ *
+ * In both cases the bm_pte tables are used, which we can access directly via
+ * the kimg mapping.
+ */
+static inline pte_t *fixmap_pte(unsigned long addr)
 {
-	pgd_t *pgdp = pgd_offset_k(addr);
-	p4d_t *p4dp = p4d_offset(pgdp, addr);
-	p4d_t p4d = READ_ONCE(*p4dp);
+	return &bm_pte[BM_PTE_IDX(addr)][pte_index(addr)];
+}
 
-	BUG_ON(p4d_none(p4d) || p4d_bad(p4d));
+void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr)
+{
+	pmd_t pmd = READ_ONCE(*pmdp);
+	pte_t *ptep;
 
-	return pud_offset_kimg(p4dp, addr);
+	if (pmd_none(pmd)) {
+		ptep = bm_pte[BM_PTE_IDX(addr)];
+		__pmd_populate(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE);
+	}
 }
 
-static inline pmd_t *fixmap_pmd(unsigned long addr)
+void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr,
+				  unsigned long end)
 {
-	pud_t *pudp = fixmap_pud(addr);
+	unsigned long next;
 	pud_t pud = READ_ONCE(*pudp);
+	pmd_t *pmdp;
 
-	BUG_ON(pud_none(pud) || pud_bad(pud));
+	if (pud_none(pud))
+		__pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE);
 
-	return pmd_offset_kimg(pudp, addr);
+	pmdp = pmd_offset_kimg(pudp, addr);
+	do {
+		next = pmd_addr_end(addr, end);
+		early_fixmap_init_pte(pmdp, addr);
+	} while (pmdp++, addr = next, addr != end);
 }
 
-static inline pte_t *fixmap_pte(unsigned long addr)
+void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr,
+				  unsigned long end)
 {
-	return &bm_pte[pte_index(addr)];
+	p4d_t p4d = READ_ONCE(*p4dp);
+	pud_t *pudp;
+
+	if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) &&
+		   p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) {
+		/*
+		 * We only end up here if the kernel mapping and the fixmap
+		 * share the top level pgd entry, which should only happen on
+		 * 16k/4 levels configurations.
+		 */
+		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
+	}
+
+	if (p4d_none(p4d))
+		__p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
+
+	pudp = pud_offset_kimg(p4dp, addr);
+	early_fixmap_init_pmd(pudp, addr, end);
 }
 
 /*
@@ -54,55 +105,12 @@ static inline pte_t *fixmap_pte(unsigned long addr)
  */
 void __init early_fixmap_init(void)
 {
-	pgd_t *pgdp;
-	p4d_t *p4dp, p4d;
-	pud_t *pudp;
-	pmd_t *pmdp;
 	unsigned long addr = FIXADDR_START;
+	unsigned long end = FIXADDR_TOP;
+	pgd_t *pgdp = pgd_offset_k(addr);
+	p4d_t *p4dp = p4d_offset(pgdp, addr);
 
-	pgdp = pgd_offset_k(addr);
-	p4dp = p4d_offset(pgdp, addr);
-	p4d = READ_ONCE(*p4dp);
-	if (CONFIG_PGTABLE_LEVELS > 3 &&
-	    !(p4d_none(p4d) || p4d_page_paddr(p4d) == __pa_symbol(bm_pud))) {
-		/*
-		 * We only end up here if the kernel mapping and the fixmap
-		 * share the top level pgd entry, which should only happen on
-		 * 16k/4 levels configurations.
-		 */
-		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
-		pudp = pud_offset_kimg(p4dp, addr);
-	} else {
-		if (p4d_none(p4d))
-			__p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
-		pudp = fixmap_pud(addr);
-	}
-	if (pud_none(READ_ONCE(*pudp)))
-		__pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE);
-	pmdp = fixmap_pmd(addr);
-	__pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
-
-	/*
-	 * The boot-ioremap range spans multiple pmds, for which
-	 * we are not prepared:
-	 */
-	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
-		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
-
-	if ((pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
-	     || pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
-		WARN_ON(1);
-		pr_warn("pmdp %p != %p, %p\n",
-			pmdp, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
-			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
-		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
-			fix_to_virt(FIX_BTMAP_BEGIN));
-		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
-			fix_to_virt(FIX_BTMAP_END));
-
-		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
-		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
-	}
+	early_fixmap_init_pud(p4dp, addr, end);
 }
 
 /*
@@ -130,6 +138,7 @@ void __set_fixmap(enum fixed_addresses idx,
 void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
 {
 	const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
+	phys_addr_t dt_phys_base;
 	int offset;
 	void *dt_virt;
 
@@ -144,27 +153,12 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
 	if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
 		return NULL;
 
-	/*
-	 * Make sure that the FDT region can be mapped without the need to
-	 * allocate additional translation table pages, so that it is safe
-	 * to call create_mapping_noalloc() this early.
-	 *
-	 * On 64k pages, the FDT will be mapped using PTEs, so we need to
-	 * be in the same PMD as the rest of the fixmap.
-	 * On 4k pages, we'll use section mappings for the FDT so we only
-	 * have to be in the same PUD.
-	 */
-	BUILD_BUG_ON(dt_virt_base % SZ_2M);
-
-	BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
-		     __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
-
-	offset = dt_phys % SWAPPER_BLOCK_SIZE;
+	dt_phys_base = round_down(dt_phys, PAGE_SIZE);
+	offset = dt_phys % PAGE_SIZE;
 	dt_virt = (void *)dt_virt_base + offset;
 
 	/* map the first chunk so we can read the size from the header */
-	create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE),
-			dt_virt_base, SWAPPER_BLOCK_SIZE, prot);
+	create_mapping_noalloc(dt_phys_base, dt_virt_base, PAGE_SIZE, prot);
 
 	if (fdt_magic(dt_virt) != FDT_MAGIC)
 		return NULL;
@@ -173,9 +167,9 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot)
 	if (*size > MAX_FDT_SIZE)
 		return NULL;
 
-	if (offset + *size > SWAPPER_BLOCK_SIZE)
-		create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
-			       round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot);
+	if (offset + *size > PAGE_SIZE)
+		create_mapping_noalloc(dt_phys_base, dt_virt_base,
+				       offset + *size, prot);
 
 	return dt_virt;
 }