Solaris: improve support for sparc platform

(1) added prom property retrieval on sparc
(2) added multiple domain support on sparc
(3) use kernel device as mapping device
(4) performance improvements by removing redundant ioctl

Signed-off-by: Henry Zhao <henry.zhao@oracle.com>
Reviewed-by: Alan Coopersmith <alan.coopersmith@oracle.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

1 files changed, 217 insertions, 48 deletions

diff --git a/src/solx_devfs.c b/src/solx_devfs.c
index ea91479..b668318 100644
--- a/src/solx_devfs.c
+++ b/src/solx_devfs.c
@@ -66,6 +66,11 @@ typedef struct nexus {
     char *path;			/* for errors/debugging; fd is all we need */
     char *dev_path;
     struct nexus *next;
+#ifdef __sparc
+    struct pci_device **devlist;
+    volatile size_t num_allocated_elems;
+    volatile size_t num_devices;
+#endif
 } nexus_t;
 
 typedef struct probe_info {
@@ -75,7 +80,13 @@ typedef struct probe_info {
 } probe_info_t;
 
 static nexus_t *nexus_list = NULL;
+#if !defined(__sparc)
 static int xsvc_fd = -1;
+#endif
+
+#ifdef __sparc
+static di_prom_handle_t di_phdl;
+#endif
 
 /*
  * Read config space in native processor endianness.  Endian-neutral
@@ -91,6 +102,10 @@ static int xsvc_fd = -1;
 # error "ISA is neither __sparc nor __x86"
 #endif
 
+#ifdef __sparc
+#define MAPPING_DEV_PATH(dev)	 (((struct pci_device_private *) dev)->device_string)
+#endif
+
 /*
  * Identify problematic southbridges.  These have device id 0x5249 and
  * vendor id 0x10b9.  Check for revision ID 0 and class code 060400 as well.
@@ -153,6 +168,22 @@ static const struct pci_system_methods solx_devfs_methods = {
     .fill_capabilities = pci_fill_capabilities_generic
 };
 
+#ifdef __sparc
+static nexus_t *
+find_nexus_for_dev(struct pci_device *dev)
+{
+    nexus_t *nexus;
+    int i;
+
+    for (nexus = nexus_list ; nexus != NULL ; nexus = nexus->next) {
+	for (i = 0; i < nexus->num_devices; i++) {
+	    if (nexus->devlist[i] == dev)
+		return nexus;
+	}
+    }
+    return NULL;
+}
+#else
 static nexus_t *
 find_nexus_for_bus( int domain, int bus )
 {
@@ -166,6 +197,7 @@ find_nexus_for_bus( int domain, int bus )
     }
     return NULL;
 }
+#endif
 
 #define GET_CONFIG_VAL_8(offset) (config_hdr.bytes[offset])
 #define GET_CONFIG_VAL_16(offset) \
@@ -195,14 +227,32 @@ pci_system_solx_devfs_destroy( void )
 	close(nexus->fd);
 	free(nexus->path);
 	free(nexus->dev_path);
+#ifdef __sparc
+	{
+	    struct pci_device *dev;
+	    int i;
+
+	    for (i = 0; i < nexus->num_devices; i++) {
+		dev = nexus->devlist[i];
+		if (MAPPING_DEV_PATH(dev))
+		    di_devfs_path_free((char *) MAPPING_DEV_PATH(dev));
+	    }
+	}
+	free(nexus->devlist);
+#endif
 	free(nexus);
     }
     nexus_list = NULL;
 
+#ifdef __sparc
+    if (di_phdl != DI_PROM_HANDLE_NIL)
+	(void) di_prom_fini(di_phdl);
+#else
     if (xsvc_fd >= 0) {
 	close(xsvc_fd);
 	xsvc_fd = -1;
     }
+#endif
 }
 
 /*
@@ -235,10 +285,16 @@ pci_system_solx_devfs_create( void )
 	return (err);
     }
 
+#ifdef __sparc
+    if ((di_phdl = di_prom_init()) == DI_PROM_HANDLE_NIL)
+	(void) fprintf(stderr, "di_prom_init failed: %s\n", strerror(errno));
+#endif
+
     pinfo.num_allocated_elems = INITIAL_NUM_DEVICES;
     pinfo.num_devices = 0;
     pinfo.devices = devices;
     (void) di_walk_minor(di_node, DDI_NT_REGACC, 0, &pinfo, probe_nexus_node);
+
     di_fini(di_node);
 
     if ((pci_sys = calloc(1, sizeof (struct pci_system))) == NULL) {
@@ -246,6 +302,7 @@ pci_system_solx_devfs_create( void )
 	free(devices);
 	return (err);
     }
+
     pci_sys->methods = &solx_devfs_methods;
     pci_sys->devices = pinfo.devices;
     pci_sys->num_devices = pinfo.num_devices;
@@ -369,6 +426,10 @@ probe_dev(nexus_t *nexus, pcitool_reg_t *prg_p, probe_info_t *pinfo)
 	    else if (((errno != EFAULT) ||
 		      (prg_p->status != PCITOOL_INVALID_ADDRESS)) &&
 		     (prg_p->data != 0xffffffff)) {
+#ifdef __sparc
+/* on sparc, devices can be enumerated discontiguously. Do not quit */
+		rval = 0;
+#endif
 		break;
 	    }
 
@@ -435,6 +496,7 @@ probe_dev(nexus_t *nexus, pcitool_reg_t *prg_p, probe_info_t *pinfo)
 	    pci_base->device_id		= GET_CONFIG_VAL_16(PCI_CONF_DEVID);
 	    pci_base->subvendor_id 	= GET_CONFIG_VAL_16(PCI_CONF_SUBVENID);
 	    pci_base->subdevice_id 	= GET_CONFIG_VAL_16(PCI_CONF_SUBSYSID);
+	    pci_base->irq		= GET_CONFIG_VAL_8(PCI_CONF_ILINE);
 
 	    pinfo->devices[pinfo->num_devices].header_type
 					= GET_CONFIG_VAL_8(PCI_CONF_HEADER);
@@ -466,6 +528,25 @@ probe_dev(nexus_t *nexus, pcitool_reg_t *prg_p, probe_info_t *pinfo)
 		pinfo->devices = new_devs;
 	    }
 
+#ifdef __sparc
+	    nexus->devlist[nexus->num_devices++] = pci_base;
+
+	    if (nexus->num_devices == nexus->num_allocated_elems) {
+		struct pci_device **new_devs;
+		size_t new_num_elems = nexus->num_allocated_elems * 2;
+
+		new_devs = realloc(nexus->devlist,
+			new_num_elems * sizeof (struct pci_device *));
+		if (new_devs == NULL)
+		    return (rval);
+		(void) memset(&new_devs[nexus->num_devices], 0,
+			nexus->num_allocated_elems *
+			sizeof (struct pci_device *));
+		nexus->num_allocated_elems = new_num_elems;
+		nexus->devlist = new_devs;
+	    }
+#endif
+
 	    /*
 	     * Accommodate devices which state their
 	     * multi-functionality only in their function 0 config
@@ -500,6 +581,12 @@ probe_nexus_node(di_node_t di_node, di_minor_t minor, void *arg)
     int pci_node = 0;
     int first_bus = 0, last_bus = PCI_REG_BUS_G(PCI_REG_BUS_M);
     int domain = 0;
+#ifdef __sparc
+    int bus_range_found = 0;
+    int device_type_found = 0;
+    di_prom_prop_t prom_prop;
+#endif
+
 
 #ifdef DEBUG
     nexus_name = di_devfs_minor_path(minor);
@@ -517,11 +604,17 @@ probe_nexus_node(di_node_t di_node, di_minor_t minor, void *arg)
 
 	if (strcmp(prop_name, "device_type") == 0) {
 	    numval = di_prop_strings(prop, &strings);
-	    if (numval != 1 || strncmp(strings, "pci", 3) != 0) {
-		/* not a PCI node, bail */
-		return (DI_WALK_CONTINUE);
+	    if (numval == 1) {
+		if (strncmp(strings, "pci", 3) != 0)
+		    /* not a PCI node, bail */
+		    return (DI_WALK_CONTINUE);
+		else {
+		    pci_node = 1;
+#ifdef __sparc
+		    device_type_found =  1;
+#endif
+		}
 	    }
-	    pci_node = 1;
 	}
 	else if (strcmp(prop_name, "class-code") == 0) {
 	    /* not a root bus node, bail */
@@ -532,6 +625,9 @@ probe_nexus_node(di_node_t di_node, di_minor_t minor, void *arg)
 	    if (numval == 2) {
 		first_bus = ints[0];
 		last_bus = ints[1];
+#ifdef __sparc
+		bus_range_found = 1;
+#endif
 	    }
 	}
 	else if (strcmp(prop_name, "pciseg") == 0) {
@@ -542,10 +638,30 @@ probe_nexus_node(di_node_t di_node, di_minor_t minor, void *arg)
 	}
     }
 
-#ifdef __x86  /* sparc pci nodes don't have the device_type set */
+#ifdef __sparc
+    if ((!device_type_found) && di_phdl) {
+	numval = di_prom_prop_lookup_strings(di_phdl, di_node,
+	    "device_type", &strings);
+	if (numval == 1) {
+	    if (strncmp(strings, "pci", 3) != 0)
+		return (DI_WALK_CONTINUE);
+	    else
+		pci_node = 1;
+	}
+    }
+
+    if ((!bus_range_found) && di_phdl) {
+	numval = di_prom_prop_lookup_ints(di_phdl, di_node,
+	    "bus-range", &ints);
+	if (numval == 2) {
+	    first_bus = ints[0];
+	    last_bus = ints[1];
+	}
+    }
+#endif
+
     if (pci_node != 1)
 	return (DI_WALK_CONTINUE);
-#endif
 
     /* we have a PCI root bus node. */
     nexus = calloc(1, sizeof(nexus_t));
@@ -558,6 +674,18 @@ probe_nexus_node(di_node_t di_node, di_minor_t minor, void *arg)
     nexus->last_bus = last_bus;
     nexus->domain = domain;
 
+#ifdef __sparc
+    if ((nexus->devlist = calloc(INITIAL_NUM_DEVICES,
+			sizeof (struct pci_device *))) == NULL) {
+	(void) fprintf(stderr, "Error allocating memory for nexus devlist: %s\n",
+                       strerror(errno));
+	free (nexus);
+	return (DI_WALK_TERMINATE);
+    }
+    nexus->num_allocated_elems = INITIAL_NUM_DEVICES;
+    nexus->num_devices = 0;
+#endif
+
     nexus_name = di_devfs_minor_path(minor);
     if (nexus_name == NULL) {
 	(void) fprintf(stderr, "Error getting nexus path: %s\n",
@@ -690,6 +818,11 @@ find_target_node(di_node_t node, void *arg)
 
     len = di_prop_lookup_ints(DDI_DEV_T_ANY, node, "reg", &regbuf);
 
+#ifdef __sparc
+    if ((len <= 0) && di_phdl)
+	len = di_prom_prop_lookup_ints(di_phdl, node, "reg", &regbuf);
+#endif
+
     if (len <= 0) {
 #ifdef DEBUG
 	fprintf(stderr, "error = %x\n", errno);
@@ -719,59 +852,50 @@ find_target_node(di_node_t node, void *arg)
 static int
 pci_device_solx_devfs_probe( struct pci_device * dev )
 {
-    uint8_t  config[256];
-    int err;
+    int err = 0;
     di_node_t rnode = DI_NODE_NIL;
     i_devnode_t args = { 0, 0, 0, DI_NODE_NIL };
     int *regbuf;
     pci_regspec_t *reg;
     int i;
-    pciaddr_t bytes;
     int len = 0;
     uint ent = 0;
-
-    err = pci_device_solx_devfs_read( dev, config, 0, 256, & bytes );
-
-    if ( bytes >= 64 ) {
-	struct pci_device_private *priv =
-	    (struct pci_device_private *) dev;
     nexus_t *nexus;
 
+#ifdef __sparc
+    if ( (nexus = find_nexus_for_dev(dev)) == NULL )
+#else
     if ( (nexus = find_nexus_for_bus(dev->domain, dev->bus)) == NULL )
+#endif
 	return ENODEV;
 
-	dev->vendor_id = (uint16_t)config[0] + ((uint16_t)config[1] << 8);
-	dev->device_id = (uint16_t)config[2] + ((uint16_t)config[3] << 8);
-	dev->device_class = (uint32_t)config[9] +
-	    ((uint32_t)config[10] << 8) +
-	    ((uint16_t)config[11] << 16);
-
-	/*
-	 * device class code is already there.
-	 * see probe_dev function.
-	 */
-	dev->revision = config[8];
-	dev->subvendor_id = (uint16_t)config[44] + ((uint16_t)config[45] << 8);
-	dev->subdevice_id = (uint16_t)config[46] + ((uint16_t)config[47] << 8);
-	dev->irq = config[60];
-
-	priv->header_type = config[14];
-	/*
-	 * starting to find if it is MEM/MEM64/IO
-	 * using libdevinfo
-	 */
-	if ((rnode = di_init(nexus->dev_path, DINFOCPYALL)) == DI_NODE_NIL) {
-	    err = errno;
-	    (void) fprintf(stderr, "di_init failed: %s\n", strerror(errno));
-	} else {
-	    args.bus = dev->bus;
-	    args.dev = dev->dev;
-	    args.func = dev->func;
-	    (void) di_walk_node(rnode, DI_WALK_CLDFIRST,
-				(void *)&args, find_target_node);
-	}
+    /*
+     * starting to find if it is MEM/MEM64/IO
+     * using libdevinfo
+     */
+    if ((rnode = di_init(nexus->dev_path, DINFOCPYALL)) == DI_NODE_NIL) {
+	err = errno;
+	(void) fprintf(stderr, "di_init failed: %s\n", strerror(errno));
+    } else {
+	args.bus = dev->bus;
+	args.dev = dev->dev;
+	args.func = dev->func;
+	(void) di_walk_node(rnode, DI_WALK_CLDFIRST,
+		(void *)&args, find_target_node);
     }
+
     if (args.node != DI_NODE_NIL) {
+#ifdef __sparc
+	di_minor_t minor;
+#endif
+
+#ifdef __sparc
+	if (minor = di_minor_next(args.node, DI_MINOR_NIL))
+	    MAPPING_DEV_PATH(dev) = di_devfs_minor_path (minor);
+	else
+	    MAPPING_DEV_PATH(dev) = NULL;
+#endif
+
 	/*
 	 * It will succeed for sure, because it was
 	 * successfully called in find_target_node
@@ -780,6 +904,12 @@ pci_device_solx_devfs_probe( struct pci_device * dev )
 				  "assigned-addresses",
 				  &regbuf);
 
+#ifdef __sparc
+	if ((len <= 0) && di_phdl) {
+	    len = di_prom_prop_lookup_ints(di_phdl, args.node,
+				"assigned-addresses", &regbuf);
+	}
+#endif
     }
 
     if (len <= 0)
@@ -907,7 +1037,13 @@ pci_device_solx_devfs_read( struct pci_device * dev, void * data,
     pcitool_reg_t cfg_prg;
     int err = 0;
     int i = 0;
-    nexus_t *nexus = find_nexus_for_bus(dev->domain, dev->bus);
+    nexus_t *nexus;
+
+#ifdef __sparc
+    nexus = find_nexus_for_dev(dev);
+#else
+    nexus = find_nexus_for_bus(dev->domain, dev->bus);
+#endif
 
     *bytes_read = 0;
 
@@ -959,7 +1095,13 @@ pci_device_solx_devfs_write( struct pci_device * dev, const void * data,
     pcitool_reg_t cfg_prg;
     int err = 0;
     int cmd;
-    nexus_t *nexus = find_nexus_for_bus(dev->domain, dev->bus);
+    nexus_t *nexus;
+
+#ifdef __sparc
+    nexus = find_nexus_for_dev(dev);
+#else
+    nexus = find_nexus_for_bus(dev->domain, dev->bus);
+#endif
 
     if ( bytes_written != NULL ) {
 	*bytes_written = 0;
@@ -973,15 +1115,19 @@ pci_device_solx_devfs_write( struct pci_device * dev, const void * data,
     switch (size) {
         case 1:
 	    cfg_prg.acc_attr = PCITOOL_ACC_ATTR_SIZE_1 + NATIVE_ENDIAN;
+	    cfg_prg.data = *((uint8_t *)data);
 	    break;
         case 2:
 	    cfg_prg.acc_attr = PCITOOL_ACC_ATTR_SIZE_2 + NATIVE_ENDIAN;
+	    cfg_prg.data = *((uint16_t *)data);
 	    break;
         case 4:
 	    cfg_prg.acc_attr = PCITOOL_ACC_ATTR_SIZE_4 + NATIVE_ENDIAN;
+	    cfg_prg.data = *((uint32_t *)data);
 	    break;
         case 8:
 	    cfg_prg.acc_attr = PCITOOL_ACC_ATTR_SIZE_8 + NATIVE_ENDIAN;
+	    cfg_prg.data = *((uint64_t *)data);
 	    break;
         default:
 	    return EINVAL;
@@ -991,7 +1137,6 @@ pci_device_solx_devfs_write( struct pci_device * dev, const void * data,
     cfg_prg.func_no = dev->func;
     cfg_prg.barnum = 0;
     cfg_prg.user_version = PCITOOL_USER_VERSION;
-    cfg_prg.data = *((uint64_t *)data);
 
     /*
      * Check if this device is bridge device.
@@ -1028,6 +1173,24 @@ pci_device_solx_devfs_map_range(struct pci_device *dev,
 			? (PROT_READ | PROT_WRITE) : PROT_READ;
     int err = 0;
 
+#ifdef __sparc
+    char	map_dev[128];
+    int		map_fd;
+
+    if (MAPPING_DEV_PATH(dev))
+	snprintf(map_dev, sizeof (map_dev), "%s%s", "/devices", MAPPING_DEV_PATH(dev));
+    else
+	strcpy (map_dev, "/dev/fb0");
+
+    if ((map_fd = open(map_dev, O_RDWR)) < 0) {
+	err = errno;
+	(void) fprintf(stderr, "can not open %s: %s\n", map_dev,
+			   strerror(errno));
+	return err;
+    }
+
+    map->memory = mmap(NULL, map->size, prot, MAP_SHARED, map_fd, map->base);
+#else
     /*
      * Still used xsvc to do the user space mapping
      */
@@ -1041,6 +1204,8 @@ pci_device_solx_devfs_map_range(struct pci_device *dev,
     }
 
     map->memory = mmap(NULL, map->size, prot, MAP_SHARED, xsvc_fd, map->base);
+#endif
+
     if (map->memory == MAP_FAILED) {
 	err = errno;
 
@@ -1048,5 +1213,9 @@ pci_device_solx_devfs_map_range(struct pci_device *dev,
 		       map->base, strerror(errno));
     }
 
+#ifdef __sparc
+    close (map_fd);
+#endif
+
     return err;
 }