add callbacks to redirect system and video ram access (v2)

(v2): Add opaque pointer and register read/write callbacks

Signed-off-by: Tom St Denis <tom.stdenis@amd.com>

3 files changed, 85 insertions, 9 deletions

diff --git a/src/lib/lowlevel/linux/mmio.c b/src/lib/lowlevel/linux/mmio.c
index af14cab..1fee5f4 100644
--- a/src/lib/lowlevel/linux/mmio.c
+++ b/src/lib/lowlevel/linux/mmio.c
@@ -104,6 +104,9 @@ uint32_t umr_read_reg(struct umr_asic *asic, uint64_t addr, enum regclass type)
 	if (addr == 0xFFFFFFFF)
 		fprintf(stderr, "[BUG]: reading from addr==0xFFFFFFFF is likely a bug\n");
 
+	if (asic->reg_funcs.read_reg)
+		return asic->reg_funcs.read_reg(asic, addr, type);
+
 	// lop off top bits in no-kernel mode
 	if (asic->options.no_kernel)
 		addr &= 0xFFFFFF;
@@ -138,6 +141,9 @@ int umr_write_reg(struct umr_asic *asic, uint64_t addr, uint32_t value, enum reg
 	if (addr == 0xFFFFFFFF)
 		fprintf(stderr, "[BUG]: reading from addr==0xFFFFFFFF is likely a bug\n");
 
+	if (asic->reg_funcs.write_reg)
+		return asic->reg_funcs.write_reg(asic, addr, value, type);
+
 	// lop off top bits in no-kernel mode
 	if (asic->options.no_kernel)
 		addr &= 0xFFFFFF;
diff --git a/src/lib/read_vram.c b/src/lib/read_vram.c
index 5003db5..34062e1 100644
--- a/src/lib/read_vram.c
+++ b/src/lib/read_vram.c
@@ -265,7 +265,12 @@ next_page:
 		// allow destination to be NULL to simply use decoder
 		if (pdst) {
 			if (pte_fields.system) {
-				if (umr_access_sram(asic, start_addr, chunk_size, pdst, write_en) < 0) {
+				int r;
+				if (asic->mem_funcs.access_sram)
+					r = asic->mem_funcs.access_sram(asic, start_addr, chunk_size, pdst, write_en);
+				else
+					r = umr_access_sram(asic, start_addr, chunk_size, pdst, write_en);
+				if (r < 0) {
 					fprintf(stderr, "[ERROR]: Cannot access system ram, perhaps CONFIG_STRICT_DEVMEM is set in your kernel config?\n");
 					fprintf(stderr, "[ERROR]: Alternatively download and install /dev/fmem\n");
 					return -1;
@@ -489,7 +494,12 @@ static int umr_access_vram_ai(struct umr_asic *asic, uint32_t vmid,
 					if (umr_read_vram(asic, UMR_LINEAR_HUB, pde_address + pde_idx * 8, 8, &pde_entry) < 0)
 						return -1;
 				} else {
-					if (umr_access_sram(asic, pde_address + pde_idx * 8, 8, &pde_entry, 0) < 0)
+					int r;
+					if (asic->mem_funcs.access_sram)
+						r = asic->mem_funcs.access_sram(asic, pde_address + pde_idx * 8, 8, &pde_entry, 0);
+					else
+						r = umr_access_sram(asic, pde_address + pde_idx * 8, 8, &pde_entry, 0);
+					if (r < 0)
 						return -1;
 				}
 
@@ -546,7 +556,12 @@ pte_further:
 				if (umr_read_vram(asic, UMR_LINEAR_HUB, pde_fields.pte_base_addr + pte_idx*8, 8, &pte_entry) < 0)
 					return -1;
 			} else {
-				if (umr_access_sram(asic, pde_fields.pte_base_addr + pte_idx*8, 8, &pte_entry, 0) < 0)
+				int r;
+				if (asic->mem_funcs.access_sram)
+					r = asic->mem_funcs.access_sram(asic, pde_fields.pte_base_addr + pte_idx*8, 8, &pte_entry, 0);
+				else
+					r = umr_access_sram(asic, pde_fields.pte_base_addr + pte_idx*8, 8, &pte_entry, 0);
+				if (r < 0)
 					return -1;
 			}
 
@@ -654,7 +669,12 @@ next_page:
 		if (pte_fields.valid) {
 			if (pdst) {
 				if (pte_fields.system) {
-					if (umr_access_sram(asic, start_addr, chunk_size, pdst, write_en) < 0) {
+					int r;
+					if (asic->mem_funcs.access_sram)
+						r = asic->mem_funcs.access_sram(asic, start_addr, chunk_size, pdst, write_en);
+					else
+						r = umr_access_sram(asic, start_addr, chunk_size, pdst, write_en);
+					if (r < 0) {
 						fprintf(stderr, "[ERROR]: Cannot access system ram, perhaps CONFIG_STRICT_DEVMEM is set in your kernel config?\n");
 						fprintf(stderr, "[ERROR]: Alternatively download and install /dev/fmem\n");
 						return -1;
@@ -736,11 +756,16 @@ int umr_access_vram(struct umr_asic *asic, uint32_t vmid, uint64_t address, uint
 			}
 		}
 
-		// addressing is physical
-		if (asic->options.use_pci == 0)
-			umr_access_linear_vram(asic, address, size, data, write_en);
-		else
-			access_vram_via_mmio(asic, address, size, data, write_en);
+		// use callback for linear access if applicable
+		if (asic->mem_funcs.access_linear_vram == NULL) {
+			// addressing is physical
+			if (asic->options.use_pci == 0)
+				umr_access_linear_vram(asic, address, size, data, write_en);
+			else
+				access_vram_via_mmio(asic, address, size, data, write_en);
+		} else {
+			asic->mem_funcs.access_linear_vram(asic, address, size, data, write_en);
+		}
 		return 0;
 	}
 
diff --git a/src/umr.h b/src/umr.h
index 71c067c..d9620e8 100644
--- a/src/umr.h
+++ b/src/umr.h
@@ -257,6 +257,49 @@ struct umr_options {
 	} xgmi_devices[UMR_MAX_XGMI_DEVICES];
 };
 
+struct umr_memory_access_funcs {
+	/** access_sram -- Access System RAM
+	 * @asic:  The device the memory is bound to
+	 * @address: The address relative to the GPUs bus (might not be a physical system memory access)
+	 * @size: Number of bytes
+	 * @dst: Buffer to read/write
+	 * @write_en: true for write, false for read
+	 */
+	int (*access_sram)(struct umr_asic *asic, uint64_t address, uint32_t size, void *dst, int write_en);
+
+	/** access_linear_vram -- Access Video RAM
+	 * @asic:  The device the memory is bound to
+	 * @address: The address relative to the GPUs start of VRAM (or relative to the start of an XGMI map)
+	 * @size: Number of bytes
+	 * @dst: Buffer to read/write
+	 * @write_en: true for write, false for read
+	 */
+	int (*access_linear_vram)(struct umr_asic *asic, uint64_t address, uint32_t size, void *data, int write_en);
+
+	/** data -- opaque pointer the callbacks can use for state tracking */
+	void *data;
+};
+
+struct umr_register_access_funcs {
+	/** read_reg -- Read a register
+	 * @asic: The device the register is from
+	 * @addr:  The byte address of the register to read
+	 * @type:  REG_MMIO or REG_SMC
+	 */
+	uint32_t (*read_reg)(struct umr_asic *asic, uint64_t addr, enum regclass type);
+
+	/** write_reg -- Write a register
+	 * @asic: The device the register is from
+	 * @addr: The byte address of the register to write
+	 * @value: The 32-bit value to write
+	 * @type: REG_MMIO or REG_SMC
+	 */
+	int (*write_reg)(struct umr_asic *asic, uint64_t addr, uint32_t value, enum regclass type);
+
+	/** data -- opaque pointer the callbacks can use for state tracking */
+	void *data;
+};
+
 struct umr_asic {
 	char *asicname;
 	int no_blocks;
@@ -294,6 +337,8 @@ struct umr_asic {
 		struct umr_reg **reglist;
 	} mmio_accel;
 	struct umr_dma_maps *maps;
+	struct umr_memory_access_funcs mem_funcs;
+	struct umr_register_access_funcs reg_funcs;
 };
 
 struct umr_wave_status {