diff options
Diffstat (limited to 'drivers/net/ipa/gsi.c')
| -rw-r--r-- | drivers/net/ipa/gsi.c | 486 |
1 files changed, 305 insertions, 181 deletions
diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index bea2da1c4c51..9a0b1fe4a93a 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2022 Linaro Ltd. + * Copyright (C) 2018-2023 Linaro Ltd. */ #include <linux/types.h> @@ -16,6 +16,7 @@ #include <linux/netdevice.h> #include "gsi.h" +#include "reg.h" #include "gsi_reg.h" #include "gsi_private.h" #include "gsi_trans.h" @@ -162,12 +163,6 @@ static void gsi_validate_build(void) * ensure the elements themselves meet the requirement. */ BUILD_BUG_ON(!is_power_of_2(GSI_RING_ELEMENT_SIZE)); - - /* The channel element size must fit in this field */ - BUILD_BUG_ON(GSI_RING_ELEMENT_SIZE > field_max(ELEMENT_SIZE_FMASK)); - - /* The event ring element size must fit in this field */ - BUILD_BUG_ON(GSI_RING_ELEMENT_SIZE > field_max(EV_ELEMENT_SIZE_FMASK)); } /* Return the channel id associated with a given channel */ @@ -182,21 +177,39 @@ static bool gsi_channel_initialized(struct gsi_channel *channel) return !!channel->gsi; } +/* Encode the channel protocol for the CH_C_CNTXT_0 register */ +static u32 ch_c_cntxt_0_type_encode(enum ipa_version version, + const struct reg *reg, + enum gsi_channel_type type) +{ + u32 val; + + val = reg_encode(reg, CHTYPE_PROTOCOL, type); + if (version < IPA_VERSION_4_5 || version >= IPA_VERSION_5_0) + return val; + + type >>= hweight32(reg_fmask(reg, CHTYPE_PROTOCOL)); + + return val | reg_encode(reg, CHTYPE_PROTOCOL_MSB, type); +} + /* Update the GSI IRQ type register with the cached value */ static void gsi_irq_type_update(struct gsi *gsi, u32 val) { + const struct reg *reg = gsi_reg(gsi, CNTXT_TYPE_IRQ_MSK); + gsi->type_enabled_bitmap = val; - iowrite32(val, gsi->virt + GSI_CNTXT_TYPE_IRQ_MSK_OFFSET); + iowrite32(val, gsi->virt + reg_offset(reg)); } static void gsi_irq_type_enable(struct gsi *gsi, enum gsi_irq_type_id type_id) { - gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | BIT(type_id)); + gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | type_id); } static void gsi_irq_type_disable(struct gsi *gsi, enum gsi_irq_type_id type_id) { - gsi_irq_type_update(gsi, gsi->type_enabled_bitmap & ~BIT(type_id)); + gsi_irq_type_update(gsi, gsi->type_enabled_bitmap & ~type_id); } /* Event ring commands are performed one at a time. Their completion @@ -207,22 +220,29 @@ static void gsi_irq_type_disable(struct gsi *gsi, enum gsi_irq_type_id type_id) static void gsi_irq_ev_ctrl_enable(struct gsi *gsi, u32 evt_ring_id) { u32 val = BIT(evt_ring_id); + const struct reg *reg; /* There's a small chance that a previous command completed * after the interrupt was disabled, so make sure we have no * pending interrupts before we enable them. */ - iowrite32(~0, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET); + reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_CLR); + iowrite32(~0, gsi->virt + reg_offset(reg)); - iowrite32(val, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET); + reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK); + iowrite32(val, gsi->virt + reg_offset(reg)); gsi_irq_type_enable(gsi, GSI_EV_CTRL); } /* Disable event ring control interrupts */ static void gsi_irq_ev_ctrl_disable(struct gsi *gsi) { + const struct reg *reg; + gsi_irq_type_disable(gsi, GSI_EV_CTRL); - iowrite32(0, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET); + + reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK); + iowrite32(0, gsi->virt + reg_offset(reg)); } /* Channel commands are performed one at a time. Their completion is @@ -233,32 +253,43 @@ static void gsi_irq_ev_ctrl_disable(struct gsi *gsi) static void gsi_irq_ch_ctrl_enable(struct gsi *gsi, u32 channel_id) { u32 val = BIT(channel_id); + const struct reg *reg; /* There's a small chance that a previous command completed * after the interrupt was disabled, so make sure we have no * pending interrupts before we enable them. */ - iowrite32(~0, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET); + reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_CLR); + iowrite32(~0, gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK); + iowrite32(val, gsi->virt + reg_offset(reg)); - iowrite32(val, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET); gsi_irq_type_enable(gsi, GSI_CH_CTRL); } /* Disable channel control interrupts */ static void gsi_irq_ch_ctrl_disable(struct gsi *gsi) { + const struct reg *reg; + gsi_irq_type_disable(gsi, GSI_CH_CTRL); - iowrite32(0, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET); + + reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK); + iowrite32(0, gsi->virt + reg_offset(reg)); } static void gsi_irq_ieob_enable_one(struct gsi *gsi, u32 evt_ring_id) { bool enable_ieob = !gsi->ieob_enabled_bitmap; + const struct reg *reg; u32 val; gsi->ieob_enabled_bitmap |= BIT(evt_ring_id); + + reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK); val = gsi->ieob_enabled_bitmap; - iowrite32(val, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET); + iowrite32(val, gsi->virt + reg_offset(reg)); /* Enable the interrupt type if this is the first channel enabled */ if (enable_ieob) @@ -267,6 +298,7 @@ static void gsi_irq_ieob_enable_one(struct gsi *gsi, u32 evt_ring_id) static void gsi_irq_ieob_disable(struct gsi *gsi, u32 event_mask) { + const struct reg *reg; u32 val; gsi->ieob_enabled_bitmap &= ~event_mask; @@ -275,8 +307,9 @@ static void gsi_irq_ieob_disable(struct gsi *gsi, u32 event_mask) if (!gsi->ieob_enabled_bitmap) gsi_irq_type_disable(gsi, GSI_IEOB); + reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK); val = gsi->ieob_enabled_bitmap; - iowrite32(val, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET); + iowrite32(val, gsi->virt + reg_offset(reg)); } static void gsi_irq_ieob_disable_one(struct gsi *gsi, u32 evt_ring_id) @@ -287,34 +320,44 @@ static void gsi_irq_ieob_disable_one(struct gsi *gsi, u32 evt_ring_id) /* Enable all GSI_interrupt types */ static void gsi_irq_enable(struct gsi *gsi) { + const struct reg *reg; u32 val; /* Global interrupts include hardware error reports. Enable * that so we can at least report the error should it occur. */ - iowrite32(BIT(ERROR_INT), gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); - gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | BIT(GSI_GLOB_EE)); + reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); + iowrite32(ERROR_INT, gsi->virt + reg_offset(reg)); + + gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | GSI_GLOB_EE); /* General GSI interrupts are reported to all EEs; if they occur * they are unrecoverable (without reset). A breakpoint interrupt * also exists, but we don't support that. We want to be notified * of errors so we can report them, even if they can't be handled. */ - val = BIT(BUS_ERROR); - val |= BIT(CMD_FIFO_OVRFLOW); - val |= BIT(MCS_STACK_OVRFLOW); - iowrite32(val, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); - gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | BIT(GSI_GENERAL)); + reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN); + val = BUS_ERROR; + val |= CMD_FIFO_OVRFLOW; + val |= MCS_STACK_OVRFLOW; + iowrite32(val, gsi->virt + reg_offset(reg)); + + gsi_irq_type_update(gsi, gsi->type_enabled_bitmap | GSI_GENERAL); } /* Disable all GSI interrupt types */ static void gsi_irq_disable(struct gsi *gsi) { + const struct reg *reg; + gsi_irq_type_update(gsi, 0); /* Clear the type-specific interrupt masks set by gsi_irq_enable() */ - iowrite32(0, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); - iowrite32(0, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); + reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN); + iowrite32(0, gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); + iowrite32(0, gsi->virt + reg_offset(reg)); } /* Return the virtual address associated with a ring index */ @@ -356,11 +399,12 @@ static bool gsi_command(struct gsi *gsi, u32 reg, u32 val) static enum gsi_evt_ring_state gsi_evt_ring_state(struct gsi *gsi, u32 evt_ring_id) { + const struct reg *reg = gsi_reg(gsi, EV_CH_E_CNTXT_0); u32 val; - val = ioread32(gsi->virt + GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id)); + val = ioread32(gsi->virt + reg_n_offset(reg, evt_ring_id)); - return u32_get_bits(val, EV_CHSTATE_FMASK); + return reg_decode(reg, EV_CHSTATE, val); } /* Issue an event ring command and wait for it to complete */ @@ -368,16 +412,18 @@ static void gsi_evt_ring_command(struct gsi *gsi, u32 evt_ring_id, enum gsi_evt_cmd_opcode opcode) { struct device *dev = gsi->dev; + const struct reg *reg; bool timeout; u32 val; /* Enable the completion interrupt for the command */ gsi_irq_ev_ctrl_enable(gsi, evt_ring_id); - val = u32_encode_bits(evt_ring_id, EV_CHID_FMASK); - val |= u32_encode_bits(opcode, EV_OPCODE_FMASK); + reg = gsi_reg(gsi, EV_CH_CMD); + val = reg_encode(reg, EV_CHID, evt_ring_id); + val |= reg_encode(reg, EV_OPCODE, opcode); - timeout = !gsi_command(gsi, GSI_EV_CH_CMD_OFFSET, val); + timeout = !gsi_command(gsi, reg_offset(reg), val); gsi_irq_ev_ctrl_disable(gsi); @@ -464,13 +510,16 @@ static void gsi_evt_ring_de_alloc_command(struct gsi *gsi, u32 evt_ring_id) /* Fetch the current state of a channel from hardware */ static enum gsi_channel_state gsi_channel_state(struct gsi_channel *channel) { + const struct reg *reg = gsi_reg(channel->gsi, CH_C_CNTXT_0); u32 channel_id = gsi_channel_id(channel); - void __iomem *virt = channel->gsi->virt; + struct gsi *gsi = channel->gsi; + void __iomem *virt = gsi->virt; u32 val; - val = ioread32(virt + GSI_CH_C_CNTXT_0_OFFSET(channel_id)); + reg = gsi_reg(gsi, CH_C_CNTXT_0); + val = ioread32(virt + reg_n_offset(reg, channel_id)); - return u32_get_bits(val, CHSTATE_FMASK); + return reg_decode(reg, CHSTATE, val); } /* Issue a channel command and wait for it to complete */ @@ -480,15 +529,18 @@ gsi_channel_command(struct gsi_channel *channel, enum gsi_ch_cmd_opcode opcode) u32 channel_id = gsi_channel_id(channel); struct gsi *gsi = channel->gsi; struct device *dev = gsi->dev; + const struct reg *reg; bool timeout; u32 val; /* Enable the completion interrupt for the command */ gsi_irq_ch_ctrl_enable(gsi, channel_id); - val = u32_encode_bits(channel_id, CH_CHID_FMASK); - val |= u32_encode_bits(opcode, CH_OPCODE_FMASK); - timeout = !gsi_command(gsi, GSI_CH_CMD_OFFSET, val); + reg = gsi_reg(gsi, CH_CMD); + val = reg_encode(reg, CH_CHID, channel_id); + val |= reg_encode(reg, CH_OPCODE, opcode); + + timeout = !gsi_command(gsi, reg_offset(reg), val); gsi_irq_ch_ctrl_disable(gsi); @@ -651,6 +703,7 @@ static void gsi_channel_de_alloc_command(struct gsi *gsi, u32 channel_id) */ static void gsi_evt_ring_doorbell(struct gsi *gsi, u32 evt_ring_id, u32 index) { + const struct reg *reg = gsi_reg(gsi, EV_CH_E_DOORBELL_0); struct gsi_ring *ring = &gsi->evt_ring[evt_ring_id].ring; u32 val; @@ -658,7 +711,7 @@ static void gsi_evt_ring_doorbell(struct gsi *gsi, u32 evt_ring_id, u32 index) /* Note: index *must* be used modulo the ring count here */ val = gsi_ring_addr(ring, (index - 1) % ring->count); - iowrite32(val, gsi->virt + GSI_EV_CH_E_DOORBELL_0_OFFSET(evt_ring_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); } /* Program an event ring for use */ @@ -666,41 +719,56 @@ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) { struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; struct gsi_ring *ring = &evt_ring->ring; - size_t size; + const struct reg *reg; u32 val; + reg = gsi_reg(gsi, EV_CH_E_CNTXT_0); /* We program all event rings as GPI type/protocol */ - val = u32_encode_bits(GSI_CHANNEL_TYPE_GPI, EV_CHTYPE_FMASK); - val |= EV_INTYPE_FMASK; - val |= u32_encode_bits(GSI_RING_ELEMENT_SIZE, EV_ELEMENT_SIZE_FMASK); - iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id)); + val = reg_encode(reg, EV_CHTYPE, GSI_CHANNEL_TYPE_GPI); + /* EV_EE field is 0 (GSI_EE_AP) */ + val |= reg_bit(reg, EV_INTYPE); + val |= reg_encode(reg, EV_ELEMENT_SIZE, GSI_RING_ELEMENT_SIZE); + iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); - size = ring->count * GSI_RING_ELEMENT_SIZE; - val = ev_r_length_encoded(gsi->version, size); - iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_1_OFFSET(evt_ring_id)); + reg = gsi_reg(gsi, EV_CH_E_CNTXT_1); + val = reg_encode(reg, R_LENGTH, ring->count * GSI_RING_ELEMENT_SIZE); + iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); /* The context 2 and 3 registers store the low-order and * high-order 32 bits of the address of the event ring, * respectively. */ + reg = gsi_reg(gsi, EV_CH_E_CNTXT_2); val = lower_32_bits(ring->addr); - iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_2_OFFSET(evt_ring_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); + + reg = gsi_reg(gsi, EV_CH_E_CNTXT_3); val = upper_32_bits(ring->addr); - iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_3_OFFSET(evt_ring_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); /* Enable interrupt moderation by setting the moderation delay */ - val = u32_encode_bits(GSI_EVT_RING_INT_MODT, MODT_FMASK); - val |= u32_encode_bits(1, MODC_FMASK); /* comes from channel */ - iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_8_OFFSET(evt_ring_id)); + reg = gsi_reg(gsi, EV_CH_E_CNTXT_8); + val = reg_encode(reg, EV_MODT, GSI_EVT_RING_INT_MODT); + val |= reg_encode(reg, EV_MODC, 1); /* comes from channel */ + /* EV_MOD_CNT is 0 (no counter-based interrupt coalescing) */ + iowrite32(val, gsi->virt + reg_n_offset(reg, evt_ring_id)); + + /* No MSI write data, and MSI high and low address is 0 */ + reg = gsi_reg(gsi, EV_CH_E_CNTXT_9); + iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); - /* No MSI write data, and MSI address high and low address is 0 */ - iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_9_OFFSET(evt_ring_id)); - iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_10_OFFSET(evt_ring_id)); - iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_11_OFFSET(evt_ring_id)); + reg = gsi_reg(gsi, EV_CH_E_CNTXT_10); + iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); + + reg = gsi_reg(gsi, EV_CH_E_CNTXT_11); + iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); /* We don't need to get event read pointer updates */ - iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_12_OFFSET(evt_ring_id)); - iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_13_OFFSET(evt_ring_id)); + reg = gsi_reg(gsi, EV_CH_E_CNTXT_12); + iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); + + reg = gsi_reg(gsi, EV_CH_E_CNTXT_13); + iowrite32(0, gsi->virt + reg_n_offset(reg, evt_ring_id)); /* Finally, tell the hardware our "last processed" event (arbitrary) */ gsi_evt_ring_doorbell(gsi, evt_ring_id, ring->index); @@ -761,39 +829,52 @@ static void gsi_channel_program(struct gsi_channel *channel, bool doorbell) union gsi_channel_scratch scr = { }; struct gsi_channel_scratch_gpi *gpi; struct gsi *gsi = channel->gsi; + const struct reg *reg; u32 wrr_weight = 0; + u32 offset; u32 val; + reg = gsi_reg(gsi, CH_C_CNTXT_0); + /* We program all channels as GPI type/protocol */ - val = chtype_protocol_encoded(gsi->version, GSI_CHANNEL_TYPE_GPI); + val = ch_c_cntxt_0_type_encode(gsi->version, reg, GSI_CHANNEL_TYPE_GPI); if (channel->toward_ipa) - val |= CHTYPE_DIR_FMASK; - val |= u32_encode_bits(channel->evt_ring_id, ERINDEX_FMASK); - val |= u32_encode_bits(GSI_RING_ELEMENT_SIZE, ELEMENT_SIZE_FMASK); - iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_0_OFFSET(channel_id)); - - val = r_length_encoded(gsi->version, size); - iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_1_OFFSET(channel_id)); + val |= reg_bit(reg, CHTYPE_DIR); + if (gsi->version < IPA_VERSION_5_0) + val |= reg_encode(reg, ERINDEX, channel->evt_ring_id); + val |= reg_encode(reg, ELEMENT_SIZE, GSI_RING_ELEMENT_SIZE); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); + + reg = gsi_reg(gsi, CH_C_CNTXT_1); + val = reg_encode(reg, CH_R_LENGTH, size); + if (gsi->version >= IPA_VERSION_5_0) + val |= reg_encode(reg, CH_ERINDEX, channel->evt_ring_id); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); /* The context 2 and 3 registers store the low-order and * high-order 32 bits of the address of the channel ring, * respectively. */ + reg = gsi_reg(gsi, CH_C_CNTXT_2); val = lower_32_bits(channel->tre_ring.addr); - iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_2_OFFSET(channel_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); + + reg = gsi_reg(gsi, CH_C_CNTXT_3); val = upper_32_bits(channel->tre_ring.addr); - iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_3_OFFSET(channel_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); + + reg = gsi_reg(gsi, CH_C_QOS); /* Command channel gets low weighted round-robin priority */ if (channel->command) - wrr_weight = field_max(WRR_WEIGHT_FMASK); - val = u32_encode_bits(wrr_weight, WRR_WEIGHT_FMASK); + wrr_weight = reg_field_max(reg, WRR_WEIGHT); + val = reg_encode(reg, WRR_WEIGHT, wrr_weight); /* Max prefetch is 1 segment (do not set MAX_PREFETCH_FMASK) */ /* No need to use the doorbell engine starting at IPA v4.0 */ if (gsi->version < IPA_VERSION_4_0 && doorbell) - val |= USE_DB_ENG_FMASK; + val |= reg_bit(reg, USE_DB_ENG); /* v4.0 introduces an escape buffer for prefetch. We use it * on all but the AP command channel. @@ -801,16 +882,15 @@ static void gsi_channel_program(struct gsi_channel *channel, bool doorbell) if (gsi->version >= IPA_VERSION_4_0 && !channel->command) { /* If not otherwise set, prefetch buffers are used */ if (gsi->version < IPA_VERSION_4_5) - val |= USE_ESCAPE_BUF_ONLY_FMASK; + val |= reg_bit(reg, USE_ESCAPE_BUF_ONLY); else - val |= u32_encode_bits(GSI_ESCAPE_BUF_ONLY, - PREFETCH_MODE_FMASK); + val |= reg_encode(reg, PREFETCH_MODE, ESCAPE_BUF_ONLY); } /* All channels set DB_IN_BYTES */ if (gsi->version >= IPA_VERSION_4_9) - val |= DB_IN_BYTES; + val |= reg_bit(reg, DB_IN_BYTES); - iowrite32(val, gsi->virt + GSI_CH_C_QOS_OFFSET(channel_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); /* Now update the scratch registers for GPI protocol */ gpi = &scr.gpi; @@ -818,22 +898,27 @@ static void gsi_channel_program(struct gsi_channel *channel, bool doorbell) GSI_RING_ELEMENT_SIZE; gpi->outstanding_threshold = 2 * GSI_RING_ELEMENT_SIZE; + reg = gsi_reg(gsi, CH_C_SCRATCH_0); val = scr.data.word1; - iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_0_OFFSET(channel_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); + reg = gsi_reg(gsi, CH_C_SCRATCH_1); val = scr.data.word2; - iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_1_OFFSET(channel_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); + reg = gsi_reg(gsi, CH_C_SCRATCH_2); val = scr.data.word3; - iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_2_OFFSET(channel_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); /* We must preserve the upper 16 bits of the last scratch register. * The next sequence assumes those bits remain unchanged between the * read and the write. */ - val = ioread32(gsi->virt + GSI_CH_C_SCRATCH_3_OFFSET(channel_id)); + reg = gsi_reg(gsi, CH_C_SCRATCH_3); + offset = reg_n_offset(reg, channel_id); + val = ioread32(gsi->virt + offset); val = (scr.data.word4 & GENMASK(31, 16)) | (val & GENMASK(15, 0)); - iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_3_OFFSET(channel_id)); + iowrite32(val, gsi->virt + offset); /* All done! */ } @@ -1049,10 +1134,14 @@ static void gsi_trans_tx_completed(struct gsi_trans *trans) /* Channel control interrupt handler */ static void gsi_isr_chan_ctrl(struct gsi *gsi) { + const struct reg *reg; u32 channel_mask; - channel_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_CH_IRQ_OFFSET); - iowrite32(channel_mask, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET); + reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ); + channel_mask = ioread32(gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_CLR); + iowrite32(channel_mask, gsi->virt + reg_offset(reg)); while (channel_mask) { u32 channel_id = __ffs(channel_mask); @@ -1066,10 +1155,14 @@ static void gsi_isr_chan_ctrl(struct gsi *gsi) /* Event ring control interrupt handler */ static void gsi_isr_evt_ctrl(struct gsi *gsi) { + const struct reg *reg; u32 event_mask; - event_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_OFFSET); - iowrite32(event_mask, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET); + reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ); + event_mask = ioread32(gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_CLR); + iowrite32(event_mask, gsi->virt + reg_offset(reg)); while (event_mask) { u32 evt_ring_id = __ffs(event_mask); @@ -1117,21 +1210,29 @@ gsi_isr_glob_evt_err(struct gsi *gsi, u32 err_ee, u32 evt_ring_id, u32 code) /* Global error interrupt handler */ static void gsi_isr_glob_err(struct gsi *gsi) { + const struct reg *log_reg; + const struct reg *clr_reg; enum gsi_err_type type; enum gsi_err_code code; + u32 offset; u32 which; u32 val; u32 ee; /* Get the logged error, then reinitialize the log */ - val = ioread32(gsi->virt + GSI_ERROR_LOG_OFFSET); - iowrite32(0, gsi->virt + GSI_ERROR_LOG_OFFSET); - iowrite32(~0, gsi->virt + GSI_ERROR_LOG_CLR_OFFSET); + log_reg = gsi_reg(gsi, ERROR_LOG); + offset = reg_offset(log_reg); + val = ioread32(gsi->virt + offset); + iowrite32(0, gsi->virt + offset); + + clr_reg = gsi_reg(gsi, ERROR_LOG_CLR); + iowrite32(~0, gsi->virt + reg_offset(clr_reg)); - ee = u32_get_bits(val, ERR_EE_FMASK); - type = u32_get_bits(val, ERR_TYPE_FMASK); - which = u32_get_bits(val, ERR_VIRT_IDX_FMASK); - code = u32_get_bits(val, ERR_CODE_FMASK); + /* Parse the error value */ + ee = reg_decode(log_reg, ERR_EE, val); + type = reg_decode(log_reg, ERR_TYPE, val); + which = reg_decode(log_reg, ERR_VIRT_IDX, val); + code = reg_decode(log_reg, ERR_CODE, val); if (type == GSI_ERR_TYPE_CHAN) gsi_isr_glob_chan_err(gsi, ee, which, code); @@ -1144,6 +1245,7 @@ static void gsi_isr_glob_err(struct gsi *gsi) /* Generic EE interrupt handler */ static void gsi_isr_gp_int1(struct gsi *gsi) { + const struct reg *reg; u32 result; u32 val; @@ -1166,8 +1268,9 @@ static void gsi_isr_gp_int1(struct gsi *gsi) * In either case, we silently ignore a INCORRECT_CHANNEL_STATE * error if we receive it. */ - val = ioread32(gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET); - result = u32_get_bits(val, GENERIC_EE_RESULT_FMASK); + reg = gsi_reg(gsi, CNTXT_SCRATCH_0); + val = ioread32(gsi->virt + reg_offset(reg)); + result = reg_decode(reg, GENERIC_EE_RESULT, val); switch (result) { case GENERIC_EE_SUCCESS: @@ -1191,19 +1294,22 @@ static void gsi_isr_gp_int1(struct gsi *gsi) /* Inter-EE interrupt handler */ static void gsi_isr_glob_ee(struct gsi *gsi) { + const struct reg *reg; u32 val; - val = ioread32(gsi->virt + GSI_CNTXT_GLOB_IRQ_STTS_OFFSET); + reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_STTS); + val = ioread32(gsi->virt + reg_offset(reg)); - if (val & BIT(ERROR_INT)) + if (val & ERROR_INT) gsi_isr_glob_err(gsi); - iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_CLR_OFFSET); + reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_CLR); + iowrite32(val, gsi->virt + reg_offset(reg)); - val &= ~BIT(ERROR_INT); + val &= ~ERROR_INT; - if (val & BIT(GP_INT1)) { - val ^= BIT(GP_INT1); + if (val & GP_INT1) { + val ^= GP_INT1; gsi_isr_gp_int1(gsi); } @@ -1214,11 +1320,16 @@ static void gsi_isr_glob_ee(struct gsi *gsi) /* I/O completion interrupt event */ static void gsi_isr_ieob(struct gsi *gsi) { + const struct reg *reg; u32 event_mask; - event_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_OFFSET); + reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ); + event_mask = ioread32(gsi->virt + reg_offset(reg)); + gsi_irq_ieob_disable(gsi, event_mask); - iowrite32(event_mask, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET); + + reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_CLR); + iowrite32(event_mask, gsi->virt + reg_offset(reg)); while (event_mask) { u32 evt_ring_id = __ffs(event_mask); @@ -1233,10 +1344,14 @@ static void gsi_isr_ieob(struct gsi *gsi) static void gsi_isr_general(struct gsi *gsi) { struct device *dev = gsi->dev; + const struct reg *reg; u32 val; - val = ioread32(gsi->virt + GSI_CNTXT_GSI_IRQ_STTS_OFFSET); - iowrite32(val, gsi->virt + GSI_CNTXT_GSI_IRQ_CLR_OFFSET); + reg = gsi_reg(gsi, CNTXT_GSI_IRQ_STTS); + val = ioread32(gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_GSI_IRQ_CLR); + iowrite32(val, gsi->virt + reg_offset(reg)); dev_err(dev, "unexpected general interrupt 0x%08x\n", val); } @@ -1252,31 +1367,39 @@ static void gsi_isr_general(struct gsi *gsi) static irqreturn_t gsi_isr(int irq, void *dev_id) { struct gsi *gsi = dev_id; + const struct reg *reg; u32 intr_mask; u32 cnt = 0; + u32 offset; + + reg = gsi_reg(gsi, CNTXT_TYPE_IRQ); + offset = reg_offset(reg); /* enum gsi_irq_type_id defines GSI interrupt types */ - while ((intr_mask = ioread32(gsi->virt + GSI_CNTXT_TYPE_IRQ_OFFSET))) { + while ((intr_mask = ioread32(gsi->virt + offset))) { /* intr_mask contains bitmask of pending GSI interrupts */ do { u32 gsi_intr = BIT(__ffs(intr_mask)); intr_mask ^= gsi_intr; + /* Note: the IRQ condition for each type is cleared + * when the type-specific register is updated. + */ switch (gsi_intr) { - case BIT(GSI_CH_CTRL): + case GSI_CH_CTRL: gsi_isr_chan_ctrl(gsi); break; - case BIT(GSI_EV_CTRL): + case GSI_EV_CTRL: gsi_isr_evt_ctrl(gsi); break; - case BIT(GSI_GLOB_EE): + case GSI_GLOB_EE: gsi_isr_glob_ee(gsi); break; - case BIT(GSI_IEOB): + case GSI_IEOB: gsi_isr_ieob(gsi); break; - case BIT(GSI_GENERAL): + case GSI_GENERAL: gsi_isr_general(gsi); break; default: @@ -1467,11 +1590,13 @@ void gsi_channel_doorbell(struct gsi_channel *channel) struct gsi_ring *tre_ring = &channel->tre_ring; u32 channel_id = gsi_channel_id(channel); struct gsi *gsi = channel->gsi; + const struct reg *reg; u32 val; + reg = gsi_reg(gsi, CH_C_DOORBELL_0); /* Note: index *must* be used modulo the ring count here */ val = gsi_ring_addr(tre_ring, tre_ring->index % tre_ring->count); - iowrite32(val, gsi->virt + GSI_CH_C_DOORBELL_0_OFFSET(channel_id)); + iowrite32(val, gsi->virt + reg_n_offset(reg, channel_id)); } /* Consult hardware, move newly completed transactions to completed state */ @@ -1482,6 +1607,7 @@ void gsi_channel_update(struct gsi_channel *channel) struct gsi_evt_ring *evt_ring; struct gsi_trans *trans; struct gsi_ring *ring; + const struct reg *reg; u32 offset; u32 index; @@ -1491,7 +1617,8 @@ void gsi_channel_update(struct gsi_channel *channel) /* See if there's anything new to process; if not, we're done. Note * that index always refers to an entry *within* the event ring. */ - offset = GSI_EV_CH_E_CNTXT_4_OFFSET(evt_ring_id); + reg = gsi_reg(gsi, EV_CH_E_CNTXT_4); + offset = reg_n_offset(reg, evt_ring_id); index = gsi_ring_index(ring, ioread32(gsi->virt + offset)); if (index == ring->index % ring->count) return; @@ -1642,7 +1769,9 @@ static int gsi_generic_command(struct gsi *gsi, u32 channel_id, enum gsi_generic_cmd_opcode opcode, u8 params) { + const struct reg *reg; bool timeout; + u32 offset; u32 val; /* The error global interrupt type is always enabled (until we tear @@ -1654,24 +1783,31 @@ static int gsi_generic_command(struct gsi *gsi, u32 channel_id, * channel), and only from this function. So we enable the GP_INT1 * IRQ type here, and disable it again after the command completes. */ - val = BIT(ERROR_INT) | BIT(GP_INT1); - iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); + reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); + val = ERROR_INT | GP_INT1; + iowrite32(val, gsi->virt + reg_offset(reg)); /* First zero the result code field */ - val = ioread32(gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET); - val &= ~GENERIC_EE_RESULT_FMASK; - iowrite32(val, gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET); + reg = gsi_reg(gsi, CNTXT_SCRATCH_0); + offset = reg_offset(reg); + val = ioread32(gsi->virt + offset); + + val &= ~reg_fmask(reg, GENERIC_EE_RESULT); + iowrite32(val, gsi->virt + offset); /* Now issue the command */ - val = u32_encode_bits(opcode, GENERIC_OPCODE_FMASK); - val |= u32_encode_bits(channel_id, GENERIC_CHID_FMASK); - val |= u32_encode_bits(GSI_EE_MODEM, GENERIC_EE_FMASK); - val |= u32_encode_bits(params, GENERIC_PARAMS_FMASK); + reg = gsi_reg(gsi, GENERIC_CMD); + val = reg_encode(reg, GENERIC_OPCODE, opcode); + val |= reg_encode(reg, GENERIC_CHID, channel_id); + val |= reg_encode(reg, GENERIC_EE, GSI_EE_MODEM); + if (gsi->version >= IPA_VERSION_4_11) + val |= reg_encode(reg, GENERIC_PARAMS, params); - timeout = !gsi_command(gsi, GSI_GENERIC_CMD_OFFSET, val); + timeout = !gsi_command(gsi, reg_offset(reg), val); /* Disable the GP_INT1 IRQ type again */ - iowrite32(BIT(ERROR_INT), gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); + reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); + iowrite32(ERROR_INT, gsi->virt + reg_offset(reg)); if (!timeout) return gsi->result; @@ -1828,32 +1964,40 @@ static void gsi_channel_teardown(struct gsi *gsi) /* Turn off all GSI interrupts initially */ static int gsi_irq_setup(struct gsi *gsi) { + const struct reg *reg; int ret; /* Writing 1 indicates IRQ interrupts; 0 would be MSI */ - iowrite32(1, gsi->virt + GSI_CNTXT_INTSET_OFFSET); + reg = gsi_reg(gsi, CNTXT_INTSET); + iowrite32(reg_bit(reg, INTYPE), gsi->virt + reg_offset(reg)); /* Disable all interrupt types */ gsi_irq_type_update(gsi, 0); /* Clear all type-specific interrupt masks */ - iowrite32(0, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET); - iowrite32(0, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET); - iowrite32(0, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); - iowrite32(0, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET); + reg = gsi_reg(gsi, CNTXT_SRC_CH_IRQ_MSK); + iowrite32(0, gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_SRC_EV_CH_IRQ_MSK); + iowrite32(0, gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_GLOB_IRQ_EN); + iowrite32(0, gsi->virt + reg_offset(reg)); + + reg = gsi_reg(gsi, CNTXT_SRC_IEOB_IRQ_MSK); + iowrite32(0, gsi->virt + reg_offset(reg)); /* The inter-EE interrupts are not supported for IPA v3.0-v3.1 */ if (gsi->version > IPA_VERSION_3_1) { - u32 offset; + reg = gsi_reg(gsi, INTER_EE_SRC_CH_IRQ_MSK); + iowrite32(0, gsi->virt + reg_offset(reg)); - /* These registers are in the non-adjusted address range */ - offset = GSI_INTER_EE_SRC_CH_IRQ_MSK_OFFSET; - iowrite32(0, gsi->virt_raw + offset); - offset = GSI_INTER_EE_SRC_EV_CH_IRQ_MSK_OFFSET; - iowrite32(0, gsi->virt_raw + offset); + reg = gsi_reg(gsi, INTER_EE_SRC_EV_CH_IRQ_MSK); + iowrite32(0, gsi->virt + reg_offset(reg)); } - iowrite32(0, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); + reg = gsi_reg(gsi, CNTXT_GSI_IRQ_EN); + iowrite32(0, gsi->virt + reg_offset(reg)); ret = request_irq(gsi->irq, gsi_isr, 0, "gsi", gsi); if (ret) @@ -1871,6 +2015,7 @@ static void gsi_irq_teardown(struct gsi *gsi) static int gsi_ring_setup(struct gsi *gsi) { struct device *dev = gsi->dev; + const struct reg *reg; u32 count; u32 val; @@ -1882,9 +2027,10 @@ static int gsi_ring_setup(struct gsi *gsi) return 0; } - val = ioread32(gsi->virt + GSI_GSI_HW_PARAM_2_OFFSET); + reg = gsi_reg(gsi, HW_PARAM_2); + val = ioread32(gsi->virt + reg_offset(reg)); - count = u32_get_bits(val, NUM_CH_PER_EE_FMASK); + count = reg_decode(reg, NUM_CH_PER_EE, val); if (!count) { dev_err(dev, "GSI reports zero channels supported\n"); return -EINVAL; @@ -1896,7 +2042,12 @@ static int gsi_ring_setup(struct gsi *gsi) } gsi->channel_count = count; - count = u32_get_bits(val, NUM_EV_PER_EE_FMASK); + if (gsi->version < IPA_VERSION_5_0) { + count = reg_decode(reg, NUM_EV_PER_EE, val); + } else { + reg = gsi_reg(gsi, HW_PARAM_4); + count = reg_decode(reg, EV_PER_EE, val); + } if (!count) { dev_err(dev, "GSI reports zero event rings supported\n"); return -EINVAL; @@ -1915,12 +2066,14 @@ static int gsi_ring_setup(struct gsi *gsi) /* Setup function for GSI. GSI firmware must be loaded and initialized */ int gsi_setup(struct gsi *gsi) { + const struct reg *reg; u32 val; int ret; /* Here is where we first touch the GSI hardware */ - val = ioread32(gsi->virt + GSI_GSI_STATUS_OFFSET); - if (!(val & ENABLED_FMASK)) { + reg = gsi_reg(gsi, GSI_STATUS); + val = ioread32(gsi->virt + reg_offset(reg)); + if (!(val & reg_bit(reg, ENABLED))) { dev_err(gsi->dev, "GSI has not been enabled\n"); return -EIO; } @@ -1934,7 +2087,8 @@ int gsi_setup(struct gsi *gsi) goto err_irq_teardown; /* Initialize the error log */ - iowrite32(0, gsi->virt + GSI_ERROR_LOG_OFFSET); + reg = gsi_reg(gsi, ERROR_LOG); + iowrite32(0, gsi->virt + reg_offset(reg)); ret = gsi_channel_setup(gsi); if (ret) @@ -2205,67 +2359,37 @@ int gsi_init(struct gsi *gsi, struct platform_device *pdev, enum ipa_version version, u32 count, const struct ipa_gsi_endpoint_data *data) { - struct device *dev = &pdev->dev; - struct resource *res; - resource_size_t size; - u32 adjust; int ret; gsi_validate_build(); - gsi->dev = dev; + gsi->dev = &pdev->dev; gsi->version = version; /* GSI uses NAPI on all channels. Create a dummy network device * for the channel NAPI contexts to be associated with. */ init_dummy_netdev(&gsi->dummy_dev); - - /* Get GSI memory range and map it */ - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "gsi"); - if (!res) { - dev_err(dev, "DT error getting \"gsi\" memory property\n"); - return -ENODEV; - } - - size = resource_size(res); - if (res->start > U32_MAX || size > U32_MAX - res->start) { - dev_err(dev, "DT memory resource \"gsi\" out of range\n"); - return -EINVAL; - } - - /* Make sure we can make our pointer adjustment if necessary */ - adjust = gsi->version < IPA_VERSION_4_5 ? 0 : GSI_EE_REG_ADJUST; - if (res->start < adjust) { - dev_err(dev, "DT memory resource \"gsi\" too low (< %u)\n", - adjust); - return -EINVAL; - } - - gsi->virt_raw = ioremap(res->start, size); - if (!gsi->virt_raw) { - dev_err(dev, "unable to remap \"gsi\" memory\n"); - return -ENOMEM; - } - /* Most registers are accessed using an adjusted register range */ - gsi->virt = gsi->virt_raw - adjust; - init_completion(&gsi->completion); + ret = gsi_reg_init(gsi, pdev); + if (ret) + return ret; + ret = gsi_irq_init(gsi, pdev); /* No matching exit required */ if (ret) - goto err_iounmap; + goto err_reg_exit; ret = gsi_channel_init(gsi, count, data); if (ret) - goto err_iounmap; + goto err_reg_exit; mutex_init(&gsi->mutex); return 0; -err_iounmap: - iounmap(gsi->virt_raw); +err_reg_exit: + gsi_reg_exit(gsi); return ret; } @@ -2275,7 +2399,7 @@ void gsi_exit(struct gsi *gsi) { mutex_destroy(&gsi->mutex); gsi_channel_exit(gsi); - iounmap(gsi->virt_raw); + gsi_reg_exit(gsi); } /* The maximum number of outstanding TREs on a channel. This limits |