scan xgmi configuration from sysfs

Signed-off-by: Tom St Denis <tom.stdenis@amd.com>

10 files changed, 101 insertions, 81 deletions

diff --git a/doc/umr.1 b/doc/umr.1
index 7a1d04b..34a2209 100644
--- a/doc/umr.1
+++ b/doc/umr.1
@@ -22,9 +22,6 @@ for instance, '.vega10'.
 Force a specific PCI device using the domain:bus:slot.function format in hex.
 This is useful when more than one GPU is available. If the amdgpu driver is
 loaded the corresponding instance will be automatically detected.
-.IP "--xgmi vramsize,instance[,vramsize,instance,...]"
-Create a contiguous XGMI map for VRAM accesses.  Specify the VRAM size in MiB in order of
-appearance in the XGMI map.
 .IP "--update, -u" <filename>
 Specify update file to add, change, or delete registers from the register
 database.  Useful for adding registers that are not including in the kernel headers.
diff --git a/src/app/enum.c b/src/app/enum.c
index 05a2ed2..77743a4 100644
--- a/src/app/enum.c
+++ b/src/app/enum.c
@@ -98,7 +98,7 @@ void umr_enumerate_devices(void)
 					    asics[x].pcopy.func == func) {
 						asics[x].instance = y;
 						asics[x].asic->instance = y;
-						umr_scan_config(asics[x].asic);
+						umr_scan_config(asics[x].asic, 0);
 					}
 
 				}
diff --git a/src/app/main.c b/src/app/main.c
index 28b731c..efeb2a6 100644
--- a/src/app/main.c
+++ b/src/app/main.c
@@ -44,6 +44,7 @@ static struct umr_asic *get_asic(void)
 		printf("ASIC not found (instance=%d, did=%08lx)\n", options.instance, (unsigned long)options.forcedid);
 		exit(EXIT_FAILURE);
 	}
+	umr_scan_config(asic, 1);
 	return asic;
 }
 
@@ -71,49 +72,6 @@ static char *get_block_name(struct umr_asic *asic, char *path)
 	return block;
 }
 
-// size,instance,size,instance...
-static void parse_xgmi(struct umr_options *options, char *str)
-{
-	uint64_t base;
-	int x, n;
-	char buf[32];
-	uint64_t nums[UMR_MAX_XGMI_DEVICES*2];
-	struct umr_options opts;
-
-	// parse numbers first
-	memset(buf, 0, sizeof buf);
-	n = x = 0;
-	while (*str) {
-		if (*str == ',') {
-			sscanf(buf, "%" SCNu64, &nums[n++]);
-			memset(buf, 0, sizeof buf);
-			x = 0;
-			++str;
-		} else {
-			buf[x++] = *str++;
-		}
-	}
-
-	if (x)
-		sscanf(buf, "%" SCNu64, &nums[n++]);
-
-	base = 0;
-	for (x = 0; x < n; x += 2) {
-		uint64_t size = nums[x + 0] * 1024ULL * 1024ULL;
-		options->xgmi_devices[x/2].base_addr = base;
-		options->xgmi_devices[x/2].size      = size;
-		options->xgmi_devices[x/2].instance  = nums[x + 1];
-		base += size;
-
-		// open instance for xgmi devices (TODO: no_kernel support)
-		memset(&opts, 0, sizeof opts);
-		opts.instance = nums[x + 1];
-		options->xgmi_devices[x/2].asic = umr_discover_asic(&opts);
-	}
-	options->use_xgmi = x/2;
-}
-
-
 static void parse_options(char *str)
 {
 	char option[64], *p;
@@ -252,14 +210,6 @@ int main(int argc, char **argv)
 				printf("--pci requires domain:bus:slot.function\n");
 				return EXIT_FAILURE;
 			}
-		} else if (!strcmp(argv[i], "--xgmi")) {
-			if (i + 1 < argc) {
-				parse_xgmi(&options, argv[i + 1]);
-				++i;
-			} else {
-				printf("--xgmi requires vramsize,instance[,vramsize,instance,...]\n");
-				return EXIT_FAILURE;
-			}
 		} else if (!strcmp(argv[i], "--config") || !strcmp(argv[i], "-c")) {
 			if (!asic)
 				asic = get_asic();
@@ -661,9 +611,6 @@ int main(int argc, char **argv)
 	"\n\t\tForce a specific PCI device using the domain:bus:slot.function format in hex."
 	"\n\t\tThis is useful when more than one GPU is available. If the amdgpu driver is"
 	"\n\t\tloaded the corresponding instance will be automatically detected.\n"
-"\n\t--xgmi vramsize,instance[,vramsize,instance,...]"
-	"\n\t\tCreate a contiguous XGMI map for VRAM accesses.  Specify the VRAM size in MiB"
-	"\n\t\tin order of appearance in the XGMI map.\n"
 "\n\t--update, -u <filename>"
 	"\n\t\tSpecify update file to add, change, or delete registers from the register"
 	"\n\t\tdatabase.  Useful for adding registers that are not including in the kernel headers.  See"
@@ -756,10 +703,12 @@ printf(
 	if (options.print)
 		umr_print_asic(asic, "");
 
-	umr_close_asic(asic);
 	if (options.use_xgmi) {
+		// the parent 'asic' is included in the nodes array
 		int n;
-		for (n = 0; n < options.use_xgmi; n++)
-			umr_close_asic(options.xgmi_devices[n].asic);
+		for (n = 0; asic->config.xgmi.nodes[n].asic; n++)
+			umr_close_asic(asic->config.xgmi.nodes[n].asic);
+	} else {
+		umr_close_asic(asic);
 	}
 }
diff --git a/src/app/print_config.c b/src/app/print_config.c
index a8263b9..5bd0022 100644
--- a/src/app/print_config.c
+++ b/src/app/print_config.c
@@ -108,7 +108,18 @@ void umr_print_config(struct umr_asic *asic)
 
 	printf("\n\tasic.gtt_size == %llu\n", (unsigned long long)asic->config.gtt_size);
 	printf("\tasic.vis_vram_size == %llu\n", (unsigned long long)asic->config.vis_vram_size);
-	printf("\tasic.vram_size == %llu\n", (unsigned long long)asic->config.vram_size);
+	printf("\tasic.vram_size == %llu\n\n", (unsigned long long)asic->config.vram_size);
+
+	if (asic->options.use_xgmi) {
+		printf("\tasic.xgmi_hive_id == %llu\n", (unsigned long long)asic->config.xgmi.hive_id);
+		printf("\tasic.xgmi_device_id == %llu\n", (unsigned long long)asic->config.xgmi.device_id);
+		for (x = 0; asic->config.xgmi.nodes[x].asic; x++) {
+			printf("\tasic.xgmi.node[%d].asicname == %s\n", x, asic->config.xgmi.nodes[x].asic->asicname);
+			printf("\tasic.xgmi.node[%d].devname == %s\n", x, asic->config.xgmi.nodes[x].asic->options.pci.name);
+			printf("\tasic.xgmi.node[%d].device_id == %llu\n", x, (unsigned long long)asic->config.xgmi.nodes[x].node_id);
+			printf("\tasic.xgmi.node[%d].vram_mib == %llu\n", x, (unsigned long long)asic->config.xgmi.nodes[x].asic->config.vram_size >> 20ULL);
+		}
+	}
 
 	printf("\n\tumr.version == %s\n\n", UMR_BUILD_REV);
 
diff --git a/src/lib/discover_by_did.c b/src/lib/discover_by_did.c
index da2b0b3..57247f4 100644
--- a/src/lib/discover_by_did.c
+++ b/src/lib/discover_by_did.c
@@ -315,7 +315,7 @@ struct umr_asic *umr_discover_asic_by_did(struct umr_options *options, long did)
 	if (asic) {
 		asic->did = did;
 		asic->instance = find_first_did(did, options->instance);
-		umr_scan_config(asic);
+		umr_scan_config(asic, 0);
 
 		// set all file handles to -1 (so a call to close_asic won't close handle 0)
 		asic->fd.mmio = -1;
diff --git a/src/lib/discover_by_name.c b/src/lib/discover_by_name.c
index 2fb40e0..a24d4ef 100644
--- a/src/lib/discover_by_name.c
+++ b/src/lib/discover_by_name.c
@@ -101,7 +101,7 @@ struct umr_asic *umr_discover_asic_by_name(struct umr_options *options, char *na
 		} else {
 			asic->instance = options->instance;
 		}
-		umr_scan_config(asic);
+		umr_scan_config(asic, 0);
 	} else {
 		printf("ERROR: Device %s not found in UMR device table\n", name);
 	}
diff --git a/src/lib/lowlevel/linux/discover.c b/src/lib/lowlevel/linux/discover.c
index f81947d..4d3f2c1 100644
--- a/src/lib/lowlevel/linux/discover.c
+++ b/src/lib/lowlevel/linux/discover.c
@@ -169,7 +169,7 @@ struct umr_asic *umr_discover_asic(struct umr_options *options)
 		if (asic) {
 			asic->instance = options->instance;
 			asic->options  = *options;
-			if (!umr_scan_config(asic) && asic->config.pci.device)
+			if (!umr_scan_config(asic, 0) && asic->config.pci.device)
 				trydid = asic->config.pci.device;
 			umr_free_asic(asic);
 			asic = NULL;
@@ -370,7 +370,7 @@ struct umr_asic *umr_discover_asic(struct umr_options *options)
 	}
 
 	if (asic && need_config_scan)
-		umr_scan_config(asic);
+		umr_scan_config(asic, 0);
 
 	return asic;
 err_pci:
diff --git a/src/lib/read_vram.c b/src/lib/read_vram.c
index 34062e1..4df511c 100644
--- a/src/lib/read_vram.c
+++ b/src/lib/read_vram.c
@@ -745,15 +745,20 @@ int umr_access_vram(struct umr_asic *asic, uint32_t vmid, uint64_t address, uint
 		// if we are using xgmi let's find the device for this address
 		if (asic->options.use_xgmi) {
 			int n;
-			for (n = 0; n < asic->options.use_xgmi; n++) {
-				if ((address >= asic->options.xgmi_devices[n].base_addr) &&
-				    (address < (asic->options.xgmi_devices[n].base_addr + asic->options.xgmi_devices[n].size))) {
-					address -= asic->options.xgmi_devices[n].base_addr;
-					DEBUG("Found address in XGMI device %d, using relative address 0x%" PRIx64 "\n", n, address);
-					asic = asic->options.xgmi_devices[n].asic;
+			uint64_t addr = address;
+			for (n = 0; asic->config.xgmi.nodes[n].asic; n++) {
+				// if remaining address is within this nodes VRAM size use it
+				if (addr < asic->config.xgmi.nodes[n].asic->config.vram_size) {
+					asic = asic->config.xgmi.nodes[n].asic;
+					address = addr;
 					break;
+				} else {
+					// otherwise subtract this vram size from the address and go to the next device
+					addr -= asic->config.xgmi.nodes[n].asic->config.vram_size;
 				}
 			}
+			// now {asic, address} are the device and it's relative address
+			// that corresponds to the hive address the caller passed
 		}
 
 		// use callback for linear access if applicable
diff --git a/src/lib/scan_config.c b/src/lib/scan_config.c
index 43ff50d..fb71f2f 100644
--- a/src/lib/scan_config.c
+++ b/src/lib/scan_config.c
@@ -99,10 +99,27 @@ static uint64_t read_int(char *pci_name, char *fname)
 	return 0;
 }
 
+static uint64_t read_int_drm(int cardno, char *fname)
+{
+	char buf[256];
+	FILE *f;
+	uint64_t n;
+
+	snprintf(buf, sizeof(buf)-1, "/sys/class/drm/card%d/device/%s", cardno, fname);
+	f = fopen(buf, "r");
+	if (f) {
+		fscanf(f, "%"SCNu64"\n", &n);
+		fclose(f);
+		return n;
+	}
+	return 0;
+}
+
+
 /**
  * umr_scan_config - Scan the debugfs confiruration data
  */
-int umr_scan_config(struct umr_asic *asic)
+int umr_scan_config(struct umr_asic *asic, int xgmi_scan)
 {
 	uint32_t data[512];
 	FILE *f;
@@ -117,6 +134,43 @@ int umr_scan_config(struct umr_asic *asic)
 	asic->config.vis_vram_size = read_int(asic->options.pci.name, "mem_info_vis_vram_total");
 	asic->config.vram_size = read_int(asic->options.pci.name, "mem_info_vram_total");
 
+	// try to read xgmi info
+	asic->config.xgmi.device_id = read_int_drm(asic->instance, "xgmi_device_id");
+	if (xgmi_scan && asic->config.xgmi.device_id) {
+		int x, y;
+
+		asic->config.xgmi.hive_id = read_int_drm(asic->instance, "xgmi_hive_info/xgmi_hive_id");
+		for (x =  0; x < UMR_MAX_XGMI_DEVICES; x++) {
+			char buf[64];
+			snprintf(buf, sizeof(buf)-1, "xgmi_hive_info/node%d/xgmi_device_id", x+1);
+			asic->config.xgmi.nodes[x].node_id = read_int_drm(asic->instance, buf);
+		}
+
+		// now map instances to node ids
+		for (x = 0; asic->config.xgmi.nodes[x].node_id; x++) {
+			for (y = 0; y < UMR_MAX_XGMI_DEVICES; y++) {
+				uint64_t z;
+				z = read_int_drm(y, "xgmi_device_id");
+				if (z == asic->config.xgmi.nodes[x].node_id) {
+					asic->config.xgmi.nodes[x].instance = y;
+					break;
+				}
+			}
+		}
+
+		// now load all of the devices other than this one ...
+		for (x = 0; asic->config.xgmi.nodes[x].node_id; x++) {
+			if (asic->instance != asic->config.xgmi.nodes[x].instance) {
+				struct umr_options options;
+				memset(&options, 0, sizeof options);
+				options.instance = asic->config.xgmi.nodes[x].instance;
+				asic->config.xgmi.nodes[x].asic = umr_discover_asic(&options);
+			} else {
+				asic->config.xgmi.nodes[x].asic = asic;
+			}
+		}
+		asic->options.use_xgmi = 1;
+	}
 	// read vbios version
 	snprintf(fname, sizeof(fname)-1, "/sys/bus/pci/devices/%s/vbios_version", asic->options.pci.name);
 	f = fopen(fname, "r");
diff --git a/src/umr.h b/src/umr.h
index bf8225b..218b95b 100644
--- a/src/umr.h
+++ b/src/umr.h
@@ -248,13 +248,6 @@ struct umr_options {
 		    func;
 		char name[32];
 	} pci;
-
-	struct {
-		uint64_t base_addr,
-			 size;
-		int instance;
-		struct umr_asic *asic;
-	} xgmi_devices[UMR_MAX_XGMI_DEVICES];
 };
 
 struct umr_memory_access_funcs {
@@ -315,6 +308,17 @@ struct umr_asic {
 		uint64_t vram_size,
 			 vis_vram_size,
 			 gtt_size;
+
+		struct {
+			uint64_t
+				device_id,
+				hive_id;
+			struct {
+				uint64_t node_id;
+				int instance;
+				struct umr_asic *asic;
+			} nodes[UMR_MAX_XGMI_DEVICES];
+		} xgmi;
 	} config;
 	struct {
 		int mmio,
@@ -905,4 +909,4 @@ int umr_access_linear_vram(struct umr_asic *asic, uint64_t address, uint32_t siz
 #define RST     (asic->options.use_colour ? "\x1b[0m" : "")
 
 void umr_bitfield_default(struct umr_asic *asic, char *asicname, char *ipname, char *regname, char *bitname, int start, int stop, uint32_t value);
-int umr_scan_config(struct umr_asic *asic);
+int umr_scan_config(struct umr_asic *asic, int xgmi_scan);