core: make OP-TEE relocatable without S-EL2 SPMC

This dissociates CFG_CORE_PHYS_RELOCATABLE from CFG_CORE_SEL2_SPMC,
so that OP-TEE can be physically relocated without an SPMC at EL2.

One of the possible use-cases of enabling CFG_CORE_PHYS_RELOCATABLE
without CFG_CORE_SEL2_SPMC is physical KASLR. In particular, with this
configuration enabled, earlier stage bootloader can load OP-TEE at
an arbitrary 4K-aligned physical address.

Unlike the case where OP-TEE is loaded at some offset by an SPMC at
EL2 (e.g., 16KB by Hafnium) and there are some metadata to keep
(e.g, FF-A manifest) below the physical offset, we can use the memory
region below the physical offset as TA RAM when OP-TEE is loaded at
some random location by earlier stage bootloader, but there's no
SPMC at EL2.

To that end, we introduce another memory pool `tee_mm_sec_ddr_extra`
that accompanies with existing `tee_mm_sec_ddr` to provide secure-
DDR-backed allocation, to support such spliti TA RAM scenarios.

Also note that memory statistic infrastructure for TA RAM will be
inaccurate as it doesn't take account into split TA RAM, i.e.,
memory blocks allocated from `tee_sec_mm_ddr_extra`, as of now.

Signed-off-by: Seonghyun Park <seonghp@amazon.com>

core/arch/arm/arm.mk

@@ -99,6 +99,10 @@ ifeq ($(CFG_CORE_LARGE_PHYS_ADDR),y)
 $(call force,CFG_WITH_LPAE,y)
 endif
 
+# Enable or not support in OP-TEE to relocate itself to allow it to run from a
+# physical address that differs from the link address
+CFG_CORE_PHYS_RELOCATABLE ?= n
+
 # SPMC configuration "S-EL1 SPMC" where SPM Core is implemented at S-EL1,
 # that is, OP-TEE.
 ifeq ($(CFG_CORE_SEL1_SPMC),y)
@@ -137,11 +141,6 @@ endif
 ifeq ($(CFG_CORE_PHYS_RELOCATABLE)-$(CFG_WITH_PAGER),y-y)
 $(error CFG_CORE_PHYS_RELOCATABLE and CFG_WITH_PAGER are not compatible)
 endif
-ifeq ($(CFG_CORE_PHYS_RELOCATABLE),y)
-ifneq ($(CFG_CORE_SEL2_SPMC),y)
-$(error CFG_CORE_PHYS_RELOCATABLE depends on CFG_CORE_SEL2_SPMC)
-endif
-endif
 
 ifeq ($(CFG_CORE_FFA)-$(CFG_WITH_PAGER),y-y)
 $(error CFG_CORE_FFA and CFG_WITH_PAGER are not compatible)

core/arch/arm/kernel/boot.c

@@ -1691,6 +1691,9 @@ void __weak boot_save_args(unsigned long a0, unsigned long a1,
 			boot_arg_nsec_entry = a4;
 #endif
 		}
+		if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE))
+			core_mmu_set_secure_memory(TRUSTED_DRAM_BASE,
+						   TRUSTED_DRAM_SIZE);
 	}
 }
 

core/arch/arm/kernel/secure_partition.c

@@ -518,7 +518,7 @@ static TEE_Result load_binary_sp(struct ts_session *s,
 	bin_page_count = bin_size_rounded / SMALL_PAGE_SIZE;
 
 	/* Allocate memory */
-	mm = tee_mm_alloc(&tee_mm_sec_ddr, bin_size_rounded);
+	mm = tee_mm_alloc_sec_mem(bin_size_rounded);
 	if (!mm) {
 		res = TEE_ERROR_OUT_OF_MEMORY;
 		goto err;
@@ -901,7 +901,7 @@ static TEE_Result handle_fdt_load_relative_mem_regions(struct sp_ctx *ctx,
 			struct mobj *m = NULL;
 			unsigned int idx = 0;
 
-			mm = tee_mm_alloc(&tee_mm_sec_ddr, size);
+			mm = tee_mm_alloc_sec_mem(size);
 			if (!mm)
 				return TEE_ERROR_OUT_OF_MEMORY;
 
@@ -1225,7 +1225,7 @@ static TEE_Result handle_fdt_mem_regions(struct sp_ctx *ctx, void *fdt)
 
 		if (alloc_needed) {
 			/* Base address is missing, we have to allocate */
-			mm = tee_mm_alloc(&tee_mm_sec_ddr, size);
+			mm = tee_mm_alloc_sec_mem(size);
 			if (!mm)
 				return TEE_ERROR_OUT_OF_MEMORY;
 

core/include/mm/tee_mm.h

@@ -41,6 +41,9 @@ typedef struct _tee_mm_pool_t tee_mm_pool_t;
 /* Physical Secure DDR pool */
 extern tee_mm_pool_t tee_mm_sec_ddr;
 
+/* Extra physical Secure DDR pool */
+extern tee_mm_pool_t tee_mm_sec_ddr_extra;
+
 /* Virtual eSRAM pool */
 extern tee_mm_pool_t tee_mm_vcore;
 
@@ -84,6 +87,9 @@ void tee_mm_final(tee_mm_pool_t *pool);
  */
 tee_mm_entry_t *tee_mm_alloc(tee_mm_pool_t *pool, size_t size);
 
+/* Allocate memory from secure memory */
+tee_mm_entry_t *tee_mm_alloc_sec_mem(size_t size);
+
 /* Allocate supplied memory range if it's free */
 tee_mm_entry_t *tee_mm_alloc2(tee_mm_pool_t *pool, paddr_t base, size_t size);
 

core/kernel/ree_fs_ta.c

@@ -719,7 +719,7 @@ static TEE_Result buf_ta_open(const TEE_UUID *uuid,
 	if (res)
 		goto err;
 
-	handle->mm = tee_mm_alloc(&tee_mm_sec_ddr, handle->ta_size);
+	handle->mm = tee_mm_alloc_sec_mem(handle->ta_size);
 	if (!handle->mm) {
 		res = TEE_ERROR_OUT_OF_MEMORY;
 		goto err;

core/mm/core_mmu.c

@@ -153,9 +153,23 @@ void core_mmu_get_ta_range(paddr_t *base, size_t *size)
 
 		assert(secure_only[0].size >
 		       load_offs + TEE_RAM_VA_SIZE + TEE_SDP_TEST_MEM_SIZE);
-		b = secure_only[0].paddr + load_offs + TEE_RAM_VA_SIZE;
-		s = secure_only[0].size - load_offs - TEE_RAM_VA_SIZE -
-		    TEE_SDP_TEST_MEM_SIZE;
+		if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) &&
+		    !IS_ENABLED(CFG_CORE_SEL2_SPMC)) {
+			/*
+			 * When CFG_CORE_PHYS_RELOCATABLE is enabled but
+			 * CFG_CORE_SEL2_SPMC is not, we first register the
+			 * whole secure DRAM as TA RAM so that we can use
+			 * the memory below the physical load offset as
+			 * TA RAM. Overlapped region with TEE RAM will be
+			 * invalidated by `core_mmu_init_ta_ram()`.
+			 */
+			b = secure_only[0].paddr;
+			s = secure_only[0].size - TEE_SDP_TEST_MEM_SIZE;
+		} else {
+			b = secure_only[0].paddr + load_offs + TEE_RAM_VA_SIZE;
+			s = secure_only[0].size - load_offs - TEE_RAM_VA_SIZE -
+			    TEE_SDP_TEST_MEM_SIZE;
+		}
 	} else {
 		assert(secure_only[1].size > TEE_SDP_TEST_MEM_SIZE);
 		b = secure_only[1].paddr;
@@ -1005,17 +1019,23 @@ static size_t collect_mem_ranges(struct tee_mmap_region *memory_map,
 				 size_t num_elems)
 {
 	const struct core_mmu_phys_mem *mem = NULL;
-	vaddr_t ram_start = secure_only[0].paddr;
+	vaddr_t tee_ram_start = 0;
 	size_t last = 0;
 
+	if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) &&
+	    !IS_ENABLED(CFG_CORE_SEL2_SPMC))
+		tee_ram_start = core_mmu_tee_load_pa;
+	else
+		tee_ram_start = secure_only[0].paddr;
+
 
 #define ADD_PHYS_MEM(_type, _addr, _size) \
 		add_phys_mem(memory_map, num_elems, #_addr, (_type), \
 			     (_addr), (_size),  &last)
 
 	if (IS_ENABLED(CFG_CORE_RWDATA_NOEXEC)) {
-		ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, ram_start,
-			     VCORE_UNPG_RX_PA - ram_start);
+		ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, tee_ram_start,
+			     VCORE_UNPG_RX_PA - tee_ram_start);
 		ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RX, VCORE_UNPG_RX_PA,
 			     VCORE_UNPG_RX_SZ);
 		ADD_PHYS_MEM(MEM_AREA_TEE_RAM_RO, VCORE_UNPG_RO_PA,
@@ -1053,7 +1073,31 @@ static size_t collect_mem_ranges(struct tee_mmap_region *memory_map,
 		size_t ta_size = 0;
 
 		core_mmu_get_ta_range(&ta_base, &ta_size);
-		ADD_PHYS_MEM(MEM_AREA_TA_RAM, ta_base, ta_size);
+
+		/*
+		 * When CFG_CORE_PHYS_RELOCATABLE, TEE can be loaded in the
+		 * middle of the secure DRAM, rather than the secure DRAM base.
+		 * As a result, there may be up to two remaining memory regions
+		 * (i.e., one in the front and one in the back), which can be
+		 * used for TA RAM.
+		 *
+		 * When the TEE is loaded at the secure DRAM base or at the top,
+		 * then either one of the below `ADD_PHYS_MEM()` can result in
+		 * an empty memory region.
+		 */
+		if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) &&
+		    !IS_ENABLED(CFG_CORE_SEL2_SPMC)) {
+			ADD_PHYS_MEM(MEM_AREA_TA_RAM, ta_base,
+				     core_mmu_tee_load_pa - ta_base);
+
+			ADD_PHYS_MEM(MEM_AREA_TA_RAM,
+				     core_mmu_tee_load_pa + TEE_RAM_VA_SIZE,
+				     ta_size -
+				     (core_mmu_tee_load_pa - ta_base) -
+				     TEE_RAM_VA_SIZE);
+		} else {
+			ADD_PHYS_MEM(MEM_AREA_TA_RAM, ta_base, ta_size);
+		}
 	}
 
 	if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS) &&
@@ -1350,13 +1394,19 @@ static unsigned long init_mem_map(struct tee_mmap_region *memory_map,
 	 */
 	vaddr_t id_map_start = (vaddr_t)__identity_map_init_start;
 	vaddr_t id_map_end = (vaddr_t)__identity_map_init_end;
-	vaddr_t start_addr = secure_only[0].paddr;
+	vaddr_t tee_start_addr = 0;
 	unsigned long offs = 0;
 	size_t last = 0;
 
 	last = collect_mem_ranges(memory_map, num_elems);
 	assign_mem_granularity(memory_map);
 
+	if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) &&
+	    !IS_ENABLED(CFG_CORE_SEL2_SPMC))
+		tee_start_addr = core_mmu_tee_load_pa;
+	else
+		tee_start_addr = secure_only[0].paddr;
+
 	/*
 	 * To ease mapping and lower use of xlat tables, sort mapping
 	 * description moving small-page regions after the pgdir regions.
@@ -1368,7 +1418,7 @@ static unsigned long init_mem_map(struct tee_mmap_region *memory_map,
 		add_pager_vaspace(memory_map, num_elems, &last);
 
 	if (IS_ENABLED(CFG_CORE_ASLR) && seed) {
-		vaddr_t base_addr = start_addr + seed;
+		vaddr_t base_addr = tee_start_addr + seed;
 		const unsigned int va_width = core_mmu_get_va_width();
 		const vaddr_t va_mask = GENMASK_64(va_width - 1,
 						   SMALL_PAGE_SHIFT);
@@ -1382,7 +1432,7 @@ static unsigned long init_mem_map(struct tee_mmap_region *memory_map,
 			if (assign_mem_va(ba, memory_map) &&
 			    mem_map_add_id_map(memory_map, num_elems, &last,
 					       id_map_start, id_map_end)) {
-				offs = ba - start_addr;
+				offs = ba - tee_start_addr;
 				DMSG("Mapping core at %#"PRIxVA" offs %#lx",
 				     ba, offs);
 				goto out;
@@ -1393,7 +1443,7 @@ static unsigned long init_mem_map(struct tee_mmap_region *memory_map,
 		EMSG("Failed to map core with seed %#lx", seed);
 	}
 
-	if (!assign_mem_va(start_addr, memory_map))
+	if (!assign_mem_va(tee_start_addr, memory_map))
 		panic();
 
 out:
@@ -2552,6 +2602,27 @@ static TEE_Result teecore_init_pub_ram(void)
 early_init(teecore_init_pub_ram);
 #endif /*CFG_CORE_RESERVED_SHM*/
 
+static bool get_extra_ta_ram(vaddr_t *s, vaddr_t *e)
+{
+	struct tee_mmap_region *map = NULL;
+	struct tee_mmap_region *lower_ta_ram = NULL;
+
+	for (map = get_memory_map(); !core_mmap_is_end_of_table(map); map++) {
+		if (map->type == MEM_AREA_TA_RAM) {
+			if (!lower_ta_ram) {
+				lower_ta_ram = map;
+				continue;
+			} else {
+				*s = map->va;
+				*e = map->va + map->size;
+				break;
+			}
+		}
+	}
+
+	return lower_ta_ram != map && map->type == MEM_AREA_TA_RAM;
+}
+
 void core_mmu_init_ta_ram(void)
 {
 	vaddr_t s = 0;
@@ -2586,4 +2657,29 @@ void core_mmu_init_ta_ram(void)
 	tee_mm_final(&tee_mm_sec_ddr);
 	tee_mm_init(&tee_mm_sec_ddr, ps, size, CORE_MMU_USER_CODE_SHIFT,
 		    TEE_MM_POOL_NO_FLAGS);
+
+	if (IS_ENABLED(CFG_CORE_PHYS_RELOCATABLE) &&
+	    !IS_ENABLED(CFG_CORE_SEL2_SPMC)) {
+		if (!tee_mm_is_empty(&tee_mm_sec_ddr_extra))
+			panic("Extra TA RAM pool is not empty");
+
+		if (!get_extra_ta_ram(&s, &e))
+			return;
+
+		ps = virt_to_phys((void *)s);
+		size = e - s;
+
+		if (!ps || (ps & CORE_MMU_USER_CODE_MASK) ||
+		    !size || (size & CORE_MMU_USER_CODE_MASK))
+			panic("invalid Extra TA RAM");
+
+		/* extra check: we could rely on core_mmu_get_mem_by_type() */
+		if (!tee_pbuf_is_sec(ps, size))
+			panic("Extra TA RAM is not secure");
+
+		/* remove previous config and init Extra TA ddr memory pool */
+		tee_mm_final(&tee_mm_sec_ddr_extra);
+		tee_mm_init(&tee_mm_sec_ddr_extra, ps, size,
+			    CORE_MMU_USER_CODE_SHIFT, TEE_MM_POOL_NO_FLAGS);
+	}
 }

core/mm/fobj.c

@@ -759,7 +759,7 @@ struct fobj *fobj_sec_mem_alloc(unsigned int num_pages)
 	if (MUL_OVERFLOW(num_pages, SMALL_PAGE_SIZE, &size))
 		goto err;
 
-	f->mm = tee_mm_alloc(&tee_mm_sec_ddr, size);
+	f->mm = tee_mm_alloc_sec_mem(size);
 	if (!f->mm)
 		goto err;
 

core/mm/mobj.c

@@ -22,6 +22,7 @@
 #include <util.h>
 
 struct mobj *mobj_sec_ddr;
+struct mobj *mobj_sec_ddr_extra;
 struct mobj *mobj_tee_ram_rx;
 struct mobj *mobj_tee_ram_rw;
 
@@ -505,6 +506,15 @@ static TEE_Result mobj_init(void)
 	if (!mobj_sec_ddr)
 		panic("Failed to register secure ta ram");
 
+	if (tee_mm_sec_ddr_extra.lo) {
+		mobj_sec_ddr_extra = mobj_phys_alloc(tee_mm_sec_ddr_extra.lo,
+						     tee_mm_sec_ddr_extra.size,
+						     TEE_MATTR_MEM_TYPE_CACHED,
+						     CORE_MEM_TA_RAM);
+		if (!mobj_sec_ddr_extra)
+			panic("Failed to register extra TA ram");
+	}
+
 	if (IS_ENABLED(CFG_CORE_RWDATA_NOEXEC)) {
 		mobj_tee_ram_rx = mobj_phys_init(0,
 						 VCORE_UNPG_RX_SZ,

core/mm/tee_mm.c

@@ -225,6 +225,17 @@ tee_mm_entry_t *tee_mm_alloc(tee_mm_pool_t *pool, size_t size)
 	return NULL;
 }
 
+tee_mm_entry_t* tee_mm_alloc_sec_mem(size_t size)
+{
+	tee_mm_entry_t *mm = NULL;
+
+	mm = tee_mm_alloc(&tee_mm_sec_ddr, size);
+	if (!mm)
+		return tee_mm_alloc(&tee_mm_sec_ddr_extra, size);
+
+	return mm;
+}
+
 static inline bool fit_in_gap(tee_mm_pool_t *pool, tee_mm_entry_t *e,
 			      paddr_t offslo, paddr_t offshi)
 {
@@ -357,6 +368,9 @@ bool tee_mm_is_empty(tee_mm_pool_t *pool)
 /* Physical Secure DDR pool */
 tee_mm_pool_t tee_mm_sec_ddr;
 
+/* Extra physical Secure DDR pool */
+tee_mm_pool_t tee_mm_sec_ddr_extra;
+
 /* Virtual eSRAM pool */
 tee_mm_pool_t tee_mm_vcore;