path: root/drivers/cxl/core/cdat.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2023 Intel Corporation. All rights reserved. */
#include <linux/acpi.h>
#include <linux/xarray.h>
#include <linux/fw_table.h>
#include <linux/node.h>
#include <linux/overflow.h>
#include "cxlpci.h"
#include "cxlmem.h"
#include "core.h"
#include "cxl.h"

struct dsmas_entry {
	struct range dpa_range;
	u8 handle;
	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
	struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
	int entries;
	int qos_class;
};

static u32 cdat_normalize(u16 entry, u64 base, u8 type)
{
	u32 value;

	/*
	 * Check for invalid and overflow values
	 */
	if (entry == 0xffff || !entry)
		return 0;
	else if (base > (UINT_MAX / (entry)))
		return 0;

	/*
	 * CDAT fields follow the format of HMAT fields. See table 5 Device
	 * Scoped Latency and Bandwidth Information Structure in Coherent Device
	 * Attribute Table (CDAT) Specification v1.01.
	 */
	value = entry * base;
	switch (type) {
	case ACPI_HMAT_ACCESS_LATENCY:
	case ACPI_HMAT_READ_LATENCY:
	case ACPI_HMAT_WRITE_LATENCY:
		value = DIV_ROUND_UP(value, 1000);
		break;
	default:
		break;
	}
	return value;
}

static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg,
			      const unsigned long end)
{
	struct acpi_cdat_header *hdr = &header->cdat;
	struct acpi_cdat_dsmas *dsmas;
	int size = sizeof(*hdr) + sizeof(*dsmas);
	struct xarray *dsmas_xa = arg;
	struct dsmas_entry *dent;
	u16 len;
	int rc;

	len = le16_to_cpu((__force __le16)hdr->length);
	if (len != size || (unsigned long)hdr + len > end) {
		pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len);
		return -EINVAL;
	}

	/* Skip common header */
	dsmas = (struct acpi_cdat_dsmas *)(hdr + 1);

	dent = kzalloc(sizeof(*dent), GFP_KERNEL);
	if (!dent)
		return -ENOMEM;

	dent->handle = dsmas->dsmad_handle;
	dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address);
	dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) +
			      le64_to_cpu((__force __le64)dsmas->dpa_length) - 1;

	rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL);
	if (rc) {
		kfree(dent);
		return rc;
	}

	return 0;
}

static void __cxl_access_coordinate_set(struct access_coordinate *coord,
					int access, unsigned int val)
{
	switch (access) {
	case ACPI_HMAT_ACCESS_LATENCY:
		coord->read_latency = val;
		coord->write_latency = val;
		break;
	case ACPI_HMAT_READ_LATENCY:
		coord->read_latency = val;
		break;
	case ACPI_HMAT_WRITE_LATENCY:
		coord->write_latency = val;
		break;
	case ACPI_HMAT_ACCESS_BANDWIDTH:
		coord->read_bandwidth = val;
		coord->write_bandwidth = val;
		break;
	case ACPI_HMAT_READ_BANDWIDTH:
		coord->read_bandwidth = val;
		break;
	case ACPI_HMAT_WRITE_BANDWIDTH:
		coord->write_bandwidth = val;
		break;
	}
}

static void cxl_access_coordinate_set(struct access_coordinate *coord,
				      int access, unsigned int val)
{
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
		__cxl_access_coordinate_set(&coord[i], access, val);
}

static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg,
			       const unsigned long end)
{
	struct acpi_cdat_header *hdr = &header->cdat;
	struct acpi_cdat_dslbis *dslbis;
	int size = sizeof(*hdr) + sizeof(*dslbis);
	struct xarray *dsmas_xa = arg;
	struct dsmas_entry *dent;
	__le64 le_base;
	__le16 le_val;
	u64 val;
	u16 len;

	len = le16_to_cpu((__force __le16)hdr->length);
	if (len != size || (unsigned long)hdr + len > end) {
		pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len);
		return -EINVAL;
	}

	/* Skip common header */
	dslbis = (struct acpi_cdat_dslbis *)(hdr + 1);

	/* Skip unrecognized data type */
	if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
		return 0;

	/* Not a memory type, skip */
	if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY)
		return 0;

	dent = xa_load(dsmas_xa, dslbis->handle);
	if (!dent) {
		pr_warn("No matching DSMAS entry for DSLBIS entry.\n");
		return 0;
	}

	le_base = (__force __le64)dslbis->entry_base_unit;
	le_val = (__force __le16)dslbis->entry[0];
	val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),
			     dslbis->data_type);

	cxl_access_coordinate_set(dent->cdat_coord, dslbis->data_type, val);

	return 0;
}

static int cdat_table_parse_output(int rc)
{
	if (rc < 0)
		return rc;
	if (rc == 0)
		return -ENOENT;

	return 0;
}

static int cxl_cdat_endpoint_process(struct cxl_port *port,
				     struct xarray *dsmas_xa)
{
	int rc;

	rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler,
			      dsmas_xa, port->cdat.table, port->cdat.length);
	rc = cdat_table_parse_output(rc);
	if (rc)
		return rc;

	rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler,
			      dsmas_xa, port->cdat.table, port->cdat.length);
	return cdat_table_parse_output(rc);
}

static int cxl_port_perf_data_calculate(struct cxl_port *port,
					struct xarray *dsmas_xa)
{
	struct access_coordinate ep_c[ACCESS_COORDINATE_MAX];
	struct dsmas_entry *dent;
	int valid_entries = 0;
	unsigned long index;
	int rc;

	rc = cxl_endpoint_get_perf_coordinates(port, ep_c);
	if (rc) {
		dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n");
		return rc;
	}

	struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);

	if (!cxl_root)
		return -ENODEV;

	if (!cxl_root->ops || !cxl_root->ops->qos_class)
		return -EOPNOTSUPP;

	xa_for_each(dsmas_xa, index, dent) {
		int qos_class;

		cxl_coordinates_combine(dent->coord, dent->cdat_coord, ep_c);
		dent->entries = 1;
		rc = cxl_root->ops->qos_class(cxl_root,
					      &dent->coord[ACCESS_COORDINATE_CPU],
					      1, &qos_class);
		if (rc != 1)
			continue;

		valid_entries++;
		dent->qos_class = qos_class;
	}

	if (!valid_entries)
		return -ENOENT;

	return 0;
}

static void update_perf_entry(struct device *dev, struct dsmas_entry *dent,
			      struct cxl_dpa_perf *dpa_perf)
{
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
		dpa_perf->coord[i] = dent->coord[i];
		dpa_perf->cdat_coord[i] = dent->cdat_coord[i];
	}
	dpa_perf->dpa_range = dent->dpa_range;
	dpa_perf->qos_class = dent->qos_class;
	dev_dbg(dev,
		"DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n",
		dent->dpa_range.start, dpa_perf->qos_class,
		dent->coord[ACCESS_COORDINATE_CPU].read_bandwidth,
		dent->coord[ACCESS_COORDINATE_CPU].write_bandwidth,
		dent->coord[ACCESS_COORDINATE_CPU].read_latency,
		dent->coord[ACCESS_COORDINATE_CPU].write_latency);
}

static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds,
				     struct xarray *dsmas_xa)
{
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
	struct device *dev = cxlds->dev;
	struct range pmem_range = {
		.start = cxlds->pmem_res.start,
		.end = cxlds->pmem_res.end,
	};
	struct range ram_range = {
		.start = cxlds->ram_res.start,
		.end = cxlds->ram_res.end,
	};
	struct dsmas_entry *dent;
	unsigned long index;

	xa_for_each(dsmas_xa, index, dent) {
		if (resource_size(&cxlds->ram_res) &&
		    range_contains(&ram_range, &dent->dpa_range))
			update_perf_entry(dev, dent, &mds->ram_perf);
		else if (resource_size(&cxlds->pmem_res) &&
			 range_contains(&pmem_range, &dent->dpa_range))
			update_perf_entry(dev, dent, &mds->pmem_perf);
		else
			dev_dbg(dev, "no partition for dsmas dpa: %#llx\n",
				dent->dpa_range.start);
	}
}

static int match_cxlrd_qos_class(struct device *dev, void *data)
{
	int dev_qos_class = *(int *)data;
	struct cxl_root_decoder *cxlrd;

	if (!is_root_decoder(dev))
		return 0;

	cxlrd = to_cxl_root_decoder(dev);
	if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID)
		return 0;

	if (cxlrd->qos_class == dev_qos_class)
		return 1;

	return 0;
}

static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf)
{
	*dpa_perf = (struct cxl_dpa_perf) {
		.qos_class = CXL_QOS_CLASS_INVALID,
	};
}

static bool cxl_qos_match(struct cxl_port *root_port,
			  struct cxl_dpa_perf *dpa_perf)
{
	if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID)
		return false;

	if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class,
				   match_cxlrd_qos_class))
		return false;

	return true;
}

static int match_cxlrd_hb(struct device *dev, void *data)
{
	struct device *host_bridge = data;
	struct cxl_switch_decoder *cxlsd;
	struct cxl_root_decoder *cxlrd;

	if (!is_root_decoder(dev))
		return 0;

	cxlrd = to_cxl_root_decoder(dev);
	cxlsd = &cxlrd->cxlsd;

	guard(rwsem_read)(&cxl_region_rwsem);
	for (int i = 0; i < cxlsd->nr_targets; i++) {
		if (host_bridge == cxlsd->target[i]->dport_dev)
			return 1;
	}

	return 0;
}

static int cxl_qos_class_verify(struct cxl_memdev *cxlmd)
{
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
	struct cxl_port *root_port;
	int rc;

	struct cxl_root *cxl_root __free(put_cxl_root) =
		find_cxl_root(cxlmd->endpoint);

	if (!cxl_root)
		return -ENODEV;

	root_port = &cxl_root->port;

	/* Check that the QTG IDs are all sane between end device and root decoders */
	if (!cxl_qos_match(root_port, &mds->ram_perf))
		reset_dpa_perf(&mds->ram_perf);
	if (!cxl_qos_match(root_port, &mds->pmem_perf))
		reset_dpa_perf(&mds->pmem_perf);

	/* Check to make sure that the device's host bridge is under a root decoder */
	rc = device_for_each_child(&root_port->dev,
				   cxlmd->endpoint->host_bridge, match_cxlrd_hb);
	if (!rc) {
		reset_dpa_perf(&mds->ram_perf);
		reset_dpa_perf(&mds->pmem_perf);
	}

	return rc;
}

static void discard_dsmas(struct xarray *xa)
{
	unsigned long index;
	void *ent;

	xa_for_each(xa, index, ent) {
		xa_erase(xa, index);
		kfree(ent);
	}
	xa_destroy(xa);
}
DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T))

void cxl_endpoint_parse_cdat(struct cxl_port *port)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct xarray __dsmas_xa;
	struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa;
	int rc;

	xa_init(&__dsmas_xa);
	if (!port->cdat.table)
		return;

	rc = cxl_cdat_endpoint_process(port, dsmas_xa);
	if (rc < 0) {
		dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc);
		return;
	}

	rc = cxl_port_perf_data_calculate(port, dsmas_xa);
	if (rc) {
		dev_dbg(&port->dev, "Failed to do perf coord calculations.\n");
		return;
	}

	cxl_memdev_set_qos_class(cxlds, dsmas_xa);
	cxl_qos_class_verify(cxlmd);
	cxl_memdev_update_perf(cxlmd);
}
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL);

static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg,
			       const unsigned long end)
{
	struct acpi_cdat_sslbis_table {
		struct acpi_cdat_header header;
		struct acpi_cdat_sslbis sslbis_header;
		struct acpi_cdat_sslbe entries[];
	} *tbl = (struct acpi_cdat_sslbis_table *)header;
	int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header);
	struct acpi_cdat_sslbis *sslbis;
	struct cxl_port *port = arg;
	struct device *dev = &port->dev;
	int remain, entries, i;
	u16 len;

	len = le16_to_cpu((__force __le16)header->cdat.length);
	remain = len - size;
	if (!remain || remain % sizeof(tbl->entries[0]) ||
	    (unsigned long)header + len > end) {
		dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len);
		return -EINVAL;
	}

	sslbis = &tbl->sslbis_header;
	/* Unrecognized data type, we can skip */
	if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
		return 0;

	entries = remain / sizeof(tbl->entries[0]);
	if (struct_size(tbl, entries, entries) != len)
		return -EINVAL;

	for (i = 0; i < entries; i++) {
		u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id);
		u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id);
		__le64 le_base;
		__le16 le_val;
		struct cxl_dport *dport;
		unsigned long index;
		u16 dsp_id;
		u64 val;

		switch (x) {
		case ACPI_CDAT_SSLBIS_US_PORT:
			dsp_id = y;
			break;
		case ACPI_CDAT_SSLBIS_ANY_PORT:
			switch (y) {
			case ACPI_CDAT_SSLBIS_US_PORT:
				dsp_id = x;
				break;
			case ACPI_CDAT_SSLBIS_ANY_PORT:
				dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT;
				break;
			default:
				dsp_id = y;
				break;
			}
			break;
		default:
			dsp_id = x;
			break;
		}

		le_base = (__force __le64)tbl->sslbis_header.entry_base_unit;
		le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth;
		val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),
				     sslbis->data_type);

		xa_for_each(&port->dports, index, dport) {
			if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT ||
			    dsp_id == dport->port_id) {
				cxl_access_coordinate_set(dport->coord,
							  sslbis->data_type,
							  val);
			}
		}
	}

	return 0;
}

void cxl_switch_parse_cdat(struct cxl_port *port)
{
	int rc;

	if (!port->cdat.table)
		return;

	rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler,
			      port, port->cdat.table, port->cdat.length);
	rc = cdat_table_parse_output(rc);
	if (rc)
		dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc);
}
EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL);

static void __cxl_coordinates_combine(struct access_coordinate *out,
				      struct access_coordinate *c1,
				      struct access_coordinate *c2)
{
		if (c1->write_bandwidth && c2->write_bandwidth)
			out->write_bandwidth = min(c1->write_bandwidth,
						   c2->write_bandwidth);
		out->write_latency = c1->write_latency + c2->write_latency;

		if (c1->read_bandwidth && c2->read_bandwidth)
			out->read_bandwidth = min(c1->read_bandwidth,
						  c2->read_bandwidth);
		out->read_latency = c1->read_latency + c2->read_latency;
}

/**
 * cxl_coordinates_combine - Combine the two input coordinates
 *
 * @out: Output coordinate of c1 and c2 combined
 * @c1: input coordinates
 * @c2: input coordinates
 */
void cxl_coordinates_combine(struct access_coordinate *out,
			     struct access_coordinate *c1,
			     struct access_coordinate *c2)
{
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
		__cxl_coordinates_combine(&out[i], &c1[i], &c2[i]);
}

MODULE_IMPORT_NS(CXL);

static void cxl_bandwidth_add(struct access_coordinate *coord,
			      struct access_coordinate *c1,
			      struct access_coordinate *c2)
{
	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
		coord[i].read_bandwidth = c1[i].read_bandwidth +
					  c2[i].read_bandwidth;
		coord[i].write_bandwidth = c1[i].write_bandwidth +
					   c2[i].write_bandwidth;
	}
}

static bool dpa_perf_contains(struct cxl_dpa_perf *perf,
			      struct resource *dpa_res)
{
	struct range dpa = {
		.start = dpa_res->start,
		.end = dpa_res->end,
	};

	return range_contains(&perf->dpa_range, &dpa);
}

static struct cxl_dpa_perf *cxled_get_dpa_perf(struct cxl_endpoint_decoder *cxled,
					       enum cxl_decoder_mode mode)
{
	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
	struct cxl_dpa_perf *perf;

	switch (mode) {
	case CXL_DECODER_RAM:
		perf = &mds->ram_perf;
		break;
	case CXL_DECODER_PMEM:
		perf = &mds->pmem_perf;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}

	if (!dpa_perf_contains(perf, cxled->dpa_res))
		return ERR_PTR(-EINVAL);

	return perf;
}

/*
 * Transient context for containing the current calculation of bandwidth when
 * doing walking the port hierarchy to deal with shared upstream link.
 */
struct cxl_perf_ctx {
	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
	struct cxl_port *port;
};

/**
 * cxl_endpoint_gather_bandwidth - collect all the endpoint bandwidth in an xarray
 * @cxlr: CXL region for the bandwidth calculation
 * @cxled: endpoint decoder to start on
 * @usp_xa: (output) the xarray that collects all the bandwidth coordinates
 *          indexed by the upstream device with data of 'struct cxl_perf_ctx'.
 * @gp_is_root: (output) bool of whether the grandparent is cxl root.
 *
 * Return: 0 for success or -errno
 *
 * Collects aggregated endpoint bandwidth and store the bandwidth in
 * an xarray indexed by the upstream device of the switch or the RP
 * device. Each endpoint consists the minimum of the bandwidth from DSLBIS
 * from the endpoint CDAT, the endpoint upstream link bandwidth, and the
 * bandwidth from the SSLBIS of the switch CDAT for the switch upstream port to
 * the downstream port that's associated with the endpoint. If the
 * device is directly connected to a RP, then no SSLBIS is involved.
 */
static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr,
					 struct cxl_endpoint_decoder *cxled,
					 struct xarray *usp_xa,
					 bool *gp_is_root)
{
	struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent);
	struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent);
	struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
	struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX];
	struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX];
	struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX];
	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
	struct cxl_perf_ctx *perf_ctx;
	struct cxl_dpa_perf *perf;
	unsigned long index;
	void *ptr;
	int rc;

	if (cxlds->rcd)
		return -ENODEV;

	perf = cxled_get_dpa_perf(cxled, cxlr->mode);
	if (IS_ERR(perf))
		return PTR_ERR(perf);

	gp_port = to_cxl_port(parent_port->dev.parent);
	*gp_is_root = is_cxl_root(gp_port);

	/*
	 * If the grandparent is cxl root, then index is the root port,
	 * otherwise it's the parent switch upstream device.
	 */
	if (*gp_is_root)
		index = (unsigned long)endpoint->parent_dport->dport_dev;
	else
		index = (unsigned long)parent_port->uport_dev;

	perf_ctx = xa_load(usp_xa, index);
	if (!perf_ctx) {
		struct cxl_perf_ctx *c __free(kfree) =
			kzalloc(sizeof(*perf_ctx), GFP_KERNEL);

		if (!c)
			return -ENOMEM;
		ptr = xa_store(usp_xa, index, c, GFP_KERNEL);
		if (xa_is_err(ptr))
			return xa_err(ptr);
		perf_ctx = no_free_ptr(c);
		perf_ctx->port = parent_port;
	}

	/* Direct upstream link from EP bandwidth */
	rc = cxl_pci_get_bandwidth(pdev, pci_coord);
	if (rc < 0)
		return rc;

	/*
	 * Min of upstream link bandwidth and Endpoint CDAT bandwidth from
	 * DSLBIS.
	 */
	cxl_coordinates_combine(ep_coord, pci_coord, perf->cdat_coord);

	/*
	 * If grandparent port is root, then there's no switch involved and
	 * the endpoint is connected to a root port.
	 */
	if (!*gp_is_root) {
		/*
		 * Retrieve the switch SSLBIS for switch downstream port
		 * associated with the endpoint bandwidth.
		 */
		rc = cxl_port_get_switch_dport_bandwidth(endpoint, sw_coord);
		if (rc)
			return rc;

		/*
		 * Min of the earlier coordinates with the switch SSLBIS
		 * bandwidth
		 */
		cxl_coordinates_combine(ep_coord, ep_coord, sw_coord);
	}

	/*
	 * Aggregate the computed bandwidth with the current aggregated bandwidth
	 * of the endpoints with the same switch upstream device or RP.
	 */
	cxl_bandwidth_add(perf_ctx->coord, perf_ctx->coord, ep_coord);

	return 0;
}

static void free_perf_xa(struct xarray *xa)
{
	struct cxl_perf_ctx *ctx;
	unsigned long index;

	if (!xa)
		return;

	xa_for_each(xa, index, ctx)
		kfree(ctx);
	xa_destroy(xa);
	kfree(xa);
}
DEFINE_FREE(free_perf_xa, struct xarray *, if (_T) free_perf_xa(_T))

/**
 * cxl_switch_gather_bandwidth - collect all the bandwidth at switch level in an xarray
 * @cxlr: The region being operated on
 * @input_xa: xarray indexed by upstream device of a switch with data of 'struct
 *	      cxl_perf_ctx'
 * @gp_is_root: (output) bool of whether the grandparent is cxl root.
 *
 * Return: a xarray of resulting cxl_perf_ctx per parent switch or root port
 *         or ERR_PTR(-errno)
 *
 * Iterate through the xarray. Take the minimum of the downstream calculated
 * bandwidth, the upstream link bandwidth, and the SSLBIS of the upstream
 * switch if exists. Sum the resulting bandwidth under the switch upstream
 * device or a RP device. The function can be iterated over multiple switches
 * if the switches are present.
 */
static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr,
						  struct xarray *input_xa,
						  bool *gp_is_root)
{
	struct xarray *res_xa __free(free_perf_xa) =
		kzalloc(sizeof(*res_xa), GFP_KERNEL);
	struct access_coordinate coords[ACCESS_COORDINATE_MAX];
	struct cxl_perf_ctx *ctx, *us_ctx;
	unsigned long index, us_index;
	int dev_count = 0;
	int gp_count = 0;
	void *ptr;
	int rc;

	if (!res_xa)
		return ERR_PTR(-ENOMEM);
	xa_init(res_xa);

	xa_for_each(input_xa, index, ctx) {
		struct device *dev = (struct device *)index;
		struct cxl_port *port = ctx->port;
		struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
		struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
		struct cxl_dport *dport = port->parent_dport;
		bool is_root = false;

		dev_count++;
		if (is_cxl_root(gp_port)) {
			is_root = true;
			gp_count++;
		}

		/*
		 * If the grandparent is cxl root, then index is the root port,
		 * otherwise it's the parent switch upstream device.
		 */
		if (is_root)
			us_index = (unsigned long)port->parent_dport->dport_dev;
		else
			us_index = (unsigned long)parent_port->uport_dev;

		us_ctx = xa_load(res_xa, us_index);
		if (!us_ctx) {
			struct cxl_perf_ctx *n __free(kfree) =
				kzalloc(sizeof(*n), GFP_KERNEL);

			if (!n)
				return ERR_PTR(-ENOMEM);

			ptr = xa_store(res_xa, us_index, n, GFP_KERNEL);
			if (xa_is_err(ptr))
				return ERR_PTR(xa_err(ptr));
			us_ctx = no_free_ptr(n);
			us_ctx->port = parent_port;
		}

		/*
		 * If the device isn't an upstream PCIe port, there's something
		 * wrong with the topology.
		 */
		if (!dev_is_pci(dev))
			return ERR_PTR(-EINVAL);

		/* Retrieve the upstream link bandwidth */
		rc = cxl_pci_get_bandwidth(to_pci_dev(dev), coords);
		if (rc)
			return ERR_PTR(-ENXIO);

		/*
		 * Take the min of downstream bandwidth and the upstream link
		 * bandwidth.
		 */
		cxl_coordinates_combine(coords, coords, ctx->coord);

		/*
		 * Take the min of the calculated bandwdith and the upstream
		 * switch SSLBIS bandwidth if there's a parent switch
		 */
		if (!is_root)
			cxl_coordinates_combine(coords, coords, dport->coord);

		/*
		 * Aggregate the calculated bandwidth common to an upstream
		 * switch.
		 */
		cxl_bandwidth_add(us_ctx->coord, us_ctx->coord, coords);
	}

	/* Asymmetric topology detected. */
	if (gp_count) {
		if (gp_count != dev_count) {
			dev_dbg(&cxlr->dev,
				"Asymmetric hierarchy detected, bandwidth not updated\n");
			return ERR_PTR(-EOPNOTSUPP);
		}
		*gp_is_root = true;
	}

	return no_free_ptr(res_xa);
}

/**
 * cxl_rp_gather_bandwidth - handle the root port level bandwidth collection
 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated
 *      below each root port device.
 *
 * Return: xarray that holds cxl_perf_ctx per host bridge or ERR_PTR(-errno)
 */
static struct xarray *cxl_rp_gather_bandwidth(struct xarray *xa)
{
	struct xarray *hb_xa __free(free_perf_xa) =
		kzalloc(sizeof(*hb_xa), GFP_KERNEL);
	struct cxl_perf_ctx *ctx;
	unsigned long index;

	if (!hb_xa)
		return ERR_PTR(-ENOMEM);
	xa_init(hb_xa);

	xa_for_each(xa, index, ctx) {
		struct cxl_port *port = ctx->port;
		unsigned long hb_index = (unsigned long)port->uport_dev;
		struct cxl_perf_ctx *hb_ctx;
		void *ptr;

		hb_ctx = xa_load(hb_xa, hb_index);
		if (!hb_ctx) {
			struct cxl_perf_ctx *n __free(kfree) =
				kzalloc(sizeof(*n), GFP_KERNEL);

			if (!n)
				return ERR_PTR(-ENOMEM);
			ptr = xa_store(hb_xa, hb_index, n, GFP_KERNEL);
			if (xa_is_err(ptr))
				return ERR_PTR(xa_err(ptr));
			hb_ctx = no_free_ptr(n);
			hb_ctx->port = port;
		}

		cxl_bandwidth_add(hb_ctx->coord, hb_ctx->coord, ctx->coord);
	}

	return no_free_ptr(hb_xa);
}

/**
 * cxl_hb_gather_bandwidth - handle the host bridge level bandwidth collection
 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated
 *      below each host bridge.
 *
 * Return: xarray that holds cxl_perf_ctx per ACPI0017 device or ERR_PTR(-errno)
 */
static struct xarray *cxl_hb_gather_bandwidth(struct xarray *xa)
{
	struct xarray *mw_xa __free(free_perf_xa) =
		kzalloc(sizeof(*mw_xa), GFP_KERNEL);
	struct cxl_perf_ctx *ctx;
	unsigned long index;

	if (!mw_xa)
		return ERR_PTR(-ENOMEM);
	xa_init(mw_xa);

	xa_for_each(xa, index, ctx) {
		struct cxl_port *port = ctx->port;
		struct cxl_port *parent_port;
		struct cxl_perf_ctx *mw_ctx;
		struct cxl_dport *dport;
		unsigned long mw_index;
		void *ptr;

		parent_port = to_cxl_port(port->dev.parent);
		mw_index = (unsigned long)parent_port->uport_dev;

		mw_ctx = xa_load(mw_xa, mw_index);
		if (!mw_ctx) {
			struct cxl_perf_ctx *n __free(kfree) =
				kzalloc(sizeof(*n), GFP_KERNEL);

			if (!n)
				return ERR_PTR(-ENOMEM);
			ptr = xa_store(mw_xa, mw_index, n, GFP_KERNEL);
			if (xa_is_err(ptr))
				return ERR_PTR(xa_err(ptr));
			mw_ctx = no_free_ptr(n);
		}

		dport = port->parent_dport;
		cxl_coordinates_combine(ctx->coord, ctx->coord, dport->coord);
		cxl_bandwidth_add(mw_ctx->coord, mw_ctx->coord, ctx->coord);
	}

	return no_free_ptr(mw_xa);
}

/**
 * cxl_region_update_bandwidth - Update the bandwidth access coordinates of a region
 * @cxlr: The region being operated on
 * @input_xa: xarray holds cxl_perf_ctx wht calculated bandwidth per ACPI0017 instance
 */
static void cxl_region_update_bandwidth(struct cxl_region *cxlr,
					struct xarray *input_xa)
{
	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
	struct cxl_perf_ctx *ctx;
	unsigned long index;

	memset(coord, 0, sizeof(coord));
	xa_for_each(input_xa, index, ctx)
		cxl_bandwidth_add(coord, coord, ctx->coord);

	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
		cxlr->coord[i].read_bandwidth = coord[i].read_bandwidth;
		cxlr->coord[i].write_bandwidth = coord[i].write_bandwidth;
	}
}

/**
 * cxl_region_shared_upstream_bandwidth_update - Recalculate the bandwidth for
 *						 the region
 * @cxlr: the cxl region to recalculate
 *
 * The function walks the topology from bottom up and calculates the bandwidth. It
 * starts at the endpoints, processes at the switches if any, processes at the rootport
 * level, at the host bridge level, and finally aggregates at the region.
 */
void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr)
{
	struct xarray *working_xa;
	int root_count = 0;
	bool is_root;
	int rc;

	lockdep_assert_held(&cxl_dpa_rwsem);

	struct xarray *usp_xa __free(free_perf_xa) =
		kzalloc(sizeof(*usp_xa), GFP_KERNEL);

	if (!usp_xa)
		return;

	xa_init(usp_xa);

	/* Collect bandwidth data from all the endpoints. */
	for (int i = 0; i < cxlr->params.nr_targets; i++) {
		struct cxl_endpoint_decoder *cxled = cxlr->params.targets[i];

		is_root = false;
		rc = cxl_endpoint_gather_bandwidth(cxlr, cxled, usp_xa, &is_root);
		if (rc)
			return;
		root_count += is_root;
	}

	/* Detect asymmetric hierarchy with some direct attached endpoints. */
	if (root_count && root_count != cxlr->params.nr_targets) {
		dev_dbg(&cxlr->dev,
			"Asymmetric hierarchy detected, bandwidth not updated\n");
		return;
	}

	/*
	 * Walk up one or more switches to deal with the bandwidth of the
	 * switches if they exist. Endpoints directly attached to RPs skip
	 * over this part.
	 */
	if (!root_count) {
		do {
			working_xa = cxl_switch_gather_bandwidth(cxlr, usp_xa,
								 &is_root);
			if (IS_ERR(working_xa))
				return;
			free_perf_xa(usp_xa);
			usp_xa = working_xa;
		} while (!is_root);
	}

	/* Handle the bandwidth at the root port of the hierarchy */
	working_xa = cxl_rp_gather_bandwidth(usp_xa);
	if (IS_ERR(working_xa))
		return;
	free_perf_xa(usp_xa);
	usp_xa = working_xa;

	/* Handle the bandwidth at the host bridge of the hierarchy */
	working_xa = cxl_hb_gather_bandwidth(usp_xa);
	if (IS_ERR(working_xa))
		return;
	free_perf_xa(usp_xa);
	usp_xa = working_xa;

	/*
	 * Aggregate all the bandwidth collected per CFMWS (ACPI0017) and
	 * update the region bandwidth with the final calculated values.
	 */
	cxl_region_update_bandwidth(cxlr, usp_xa);
}

void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
				    struct cxl_endpoint_decoder *cxled)
{
	struct cxl_dpa_perf *perf;

	lockdep_assert_held(&cxl_dpa_rwsem);

	perf = cxled_get_dpa_perf(cxled, cxlr->mode);
	if (IS_ERR(perf))
		return;

	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
		/* Get total bandwidth and the worst latency for the cxl region */
		cxlr->coord[i].read_latency = max_t(unsigned int,
						    cxlr->coord[i].read_latency,
						    perf->coord[i].read_latency);
		cxlr->coord[i].write_latency = max_t(unsigned int,
						     cxlr->coord[i].write_latency,
						     perf->coord[i].write_latency);
		cxlr->coord[i].read_bandwidth += perf->coord[i].read_bandwidth;
		cxlr->coord[i].write_bandwidth += perf->coord[i].write_bandwidth;
	}
}

int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr,
				       enum access_coordinate_class access)
{
	return hmat_update_target_coordinates(nid, &cxlr->coord[access], access);
}

bool cxl_need_node_perf_attrs_update(int nid)
{
	return !acpi_node_backed_by_real_pxm(nid);
}