diff options
author | Rodrigo Vivi <rodrigo.vivi@intel.com> | 2024-10-18 11:02:35 -0400 |
---|---|---|
committer | Rodrigo Vivi <rodrigo.vivi@intel.com> | 2024-10-18 11:02:35 -0400 |
commit | f81fe3d56c7cedf85f7d21fee74bb7e52d669f74 (patch) | |
tree | b48769a64d6b9acd47950d56d74fe1d53a0cf560 | |
parent | 1c78650811ae23d8119e1ec7e3f4b965456c0ae8 (diff) |
2024y-10m-18d-15h-01m-04s UTC: drm-tip rerere cache update
git version 2.46.2
5 files changed, 0 insertions, 3970 deletions
diff --git a/rr-cache/a80de0cebef501a1f1530fc1d4fd99bfb0136a1a/postimage b/rr-cache/a80de0cebef501a1f1530fc1d4fd99bfb0136a1a/postimage deleted file mode 100644 index 3fd462fda625..000000000000 --- a/rr-cache/a80de0cebef501a1f1530fc1d4fd99bfb0136a1a/postimage +++ /dev/null @@ -1,433 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021-2023 Intel Corporation - */ - -#include "xe_mmio.h" - -#include <linux/delay.h> -#include <linux/io-64-nonatomic-lo-hi.h> -#include <linux/minmax.h> -#include <linux/pci.h> - -#include <drm/drm_managed.h> -#include <drm/drm_print.h> - -#include "regs/xe_bars.h" -#include "regs/xe_regs.h" -#include "xe_device.h" -#include "xe_gt.h" -#include "xe_gt_printk.h" -#include "xe_gt_sriov_vf.h" -#include "xe_macros.h" -#include "xe_sriov.h" -#include "xe_trace.h" - -static void tiles_fini(void *arg) -{ - struct xe_device *xe = arg; - struct xe_tile *tile; - int id; - - for_each_remote_tile(tile, xe, id) - tile->mmio.regs = NULL; -} - -/* - * On multi-tile devices, partition the BAR space for MMIO on each tile, - * possibly accounting for register override on the number of tiles available. - * Resulting memory layout is like below: - * - * .----------------------. <- tile_count * tile_mmio_size - * | .... | - * |----------------------| <- 2 * tile_mmio_size - * | tile1->mmio.regs | - * |----------------------| <- 1 * tile_mmio_size - * | tile0->mmio.regs | - * '----------------------' <- 0MB - */ -static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size) -{ - struct xe_tile *tile; - void __iomem *regs; - u8 id; - - /* - * Nothing to be done as tile 0 has already been setup earlier with the - * entire BAR mapped - see xe_mmio_init() - */ - if (xe->info.tile_count == 1) - return; - - /* Possibly override number of tile based on configuration register */ - if (!xe->info.skip_mtcfg) { - struct xe_gt *gt = xe_root_mmio_gt(xe); - u8 tile_count; - u32 mtcfg; - - /* - * Although the per-tile mmio regs are not yet initialized, this - * is fine as it's going to the root gt, that's guaranteed to be - * initialized earlier in xe_mmio_init() - */ - mtcfg = xe_mmio_read64_2x32(gt, XEHP_MTCFG_ADDR); - tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; - - if (tile_count < xe->info.tile_count) { - drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", - xe->info.tile_count, tile_count); - xe->info.tile_count = tile_count; - - /* - * FIXME: Needs some work for standalone media, but - * should be impossible with multi-tile for now: - * multi-tile platform with standalone media doesn't - * exist - */ - xe->info.gt_count = xe->info.tile_count; - } - } - - regs = xe->mmio.regs; - for_each_tile(tile, xe, id) { - tile->mmio.size = tile_mmio_size; - tile->mmio.regs = regs; - regs += tile_mmio_size; - } -} - -/* - * On top of all the multi-tile MMIO space there can be a platform-dependent - * extension for each tile, resulting in a layout like below: - * - * .----------------------. <- ext_base + tile_count * tile_mmio_ext_size - * | .... | - * |----------------------| <- ext_base + 2 * tile_mmio_ext_size - * | tile1->mmio_ext.regs | - * |----------------------| <- ext_base + 1 * tile_mmio_ext_size - * | tile0->mmio_ext.regs | - * |======================| <- ext_base = tile_count * tile_mmio_size - * | | - * | mmio.regs | - * | | - * '----------------------' <- 0MB - * - * Set up the tile[]->mmio_ext pointers/sizes. - */ -static void mmio_extension_setup(struct xe_device *xe, size_t tile_mmio_size, - size_t tile_mmio_ext_size) -{ - struct xe_tile *tile; - void __iomem *regs; - u8 id; - - if (!xe->info.has_mmio_ext) - return; - - regs = xe->mmio.regs + tile_mmio_size * xe->info.tile_count; - for_each_tile(tile, xe, id) { - tile->mmio_ext.size = tile_mmio_ext_size; - tile->mmio_ext.regs = regs; - regs += tile_mmio_ext_size; - } -} - -int xe_mmio_probe_tiles(struct xe_device *xe) -{ - size_t tile_mmio_size = SZ_16M; - size_t tile_mmio_ext_size = xe->info.tile_mmio_ext_size; - - mmio_multi_tile_setup(xe, tile_mmio_size); - mmio_extension_setup(xe, tile_mmio_size, tile_mmio_ext_size); - - return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe); -} - -static void mmio_fini(void *arg) -{ - struct xe_device *xe = arg; - struct xe_tile *root_tile = xe_device_get_root_tile(xe); - - pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs); - xe->mmio.regs = NULL; - root_tile->mmio.regs = NULL; -} - -int xe_mmio_init(struct xe_device *xe) -{ - struct xe_tile *root_tile = xe_device_get_root_tile(xe); - struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - const int mmio_bar = 0; - - /* - * Map the entire BAR. - * The first 16MB of the BAR, belong to the root tile, and include: - * registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB). - */ - xe->mmio.size = pci_resource_len(pdev, mmio_bar); - xe->mmio.regs = pci_iomap(pdev, mmio_bar, GTTMMADR_BAR); - if (xe->mmio.regs == NULL) { - drm_err(&xe->drm, "failed to map registers\n"); - return -EIO; - } - - /* Setup first tile; other tiles (if present) will be setup later. */ - root_tile->mmio.size = SZ_16M; - root_tile->mmio.regs = xe->mmio.regs; - - return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe); -} - -static void mmio_flush_pending_writes(struct xe_gt *gt) -{ -#define DUMMY_REG_OFFSET 0x130030 - struct xe_tile *tile = gt_to_tile(gt); - int i; - - if (tile->xe->info.platform != XE_LUNARLAKE) - return; - - /* 4 dummy writes */ - for (i = 0; i < 4; i++) - writel(0, tile->mmio.regs + DUMMY_REG_OFFSET); -} - -u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - u8 val; - - /* Wa_15015404425 */ - mmio_flush_pending_writes(gt); - - val = readb((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); - trace_xe_reg_rw(gt, false, addr, val, sizeof(val)); - - return val; -} - -u16 xe_mmio_read16(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - u16 val; - - /* Wa_15015404425 */ - mmio_flush_pending_writes(gt); - - val = readw((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); - trace_xe_reg_rw(gt, false, addr, val, sizeof(val)); - - return val; -} - -void xe_mmio_write32(struct xe_gt *gt, struct xe_reg reg, u32 val) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - - trace_xe_reg_rw(gt, true, addr, val, sizeof(val)); - - if (!reg.vf && IS_SRIOV_VF(gt_to_xe(gt))) - xe_gt_sriov_vf_write32(gt, reg, val); - else - writel(val, (reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); -} - -u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - u32 val; - - /* Wa_15015404425 */ - mmio_flush_pending_writes(gt); - - if (!reg.vf && IS_SRIOV_VF(gt_to_xe(gt))) - val = xe_gt_sriov_vf_read32(gt, reg); - else - val = readl((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); - - trace_xe_reg_rw(gt, false, addr, val, sizeof(val)); - - return val; -} - -u32 xe_mmio_rmw32(struct xe_gt *gt, struct xe_reg reg, u32 clr, u32 set) -{ - u32 old, reg_val; - - old = xe_mmio_read32(gt, reg); - reg_val = (old & ~clr) | set; - xe_mmio_write32(gt, reg, reg_val); - - return old; -} - -int xe_mmio_write32_and_verify(struct xe_gt *gt, - struct xe_reg reg, u32 val, u32 mask, u32 eval) -{ - u32 reg_val; - - xe_mmio_write32(gt, reg, val); - reg_val = xe_mmio_read32(gt, reg); - - return (reg_val & mask) != eval ? -EINVAL : 0; -} - -bool xe_mmio_in_range(const struct xe_gt *gt, - const struct xe_mmio_range *range, - struct xe_reg reg) -{ - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - - return range && addr >= range->start && addr <= range->end; -} - -/** - * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads - * @gt: MMIO target GT - * @reg: register to read value from - * - * Although Intel GPUs have some 64-bit registers, the hardware officially - * only supports GTTMMADR register reads of 32 bits or smaller. Even if - * a readq operation may return a reasonable value, that violation of the - * spec shouldn't be relied upon and all 64-bit register reads should be - * performed as two 32-bit reads of the upper and lower dwords. - * - * When reading registers that may be changing (such as - * counters), a rollover of the lower dword between the two 32-bit reads - * can be problematic. This function attempts to ensure the upper dword has - * stabilized before returning the 64-bit value. - * - * Note that because this function may re-read the register multiple times - * while waiting for the value to stabilize it should not be used to read - * any registers where read operations have side effects. - * - * Returns the value of the 64-bit register. - */ -u64 xe_mmio_read64_2x32(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; - u32 ldw, udw, oldudw, retries; - - reg.addr = xe_mmio_adjusted_addr(gt, reg.addr); - reg_udw.addr = xe_mmio_adjusted_addr(gt, reg_udw.addr); - - /* we shouldn't adjust just one register address */ - xe_gt_assert(gt, reg_udw.addr == reg.addr + 0x4); - - oldudw = xe_mmio_read32(gt, reg_udw); - for (retries = 5; retries; --retries) { - ldw = xe_mmio_read32(gt, reg); - udw = xe_mmio_read32(gt, reg_udw); - - if (udw == oldudw) - break; - - oldudw = udw; - } - - xe_gt_WARN(gt, retries == 0, - "64-bit read of %#x did not stabilize\n", reg.addr); - - return (u64)udw << 32 | ldw; -} - -static int __xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, - u32 *out_val, bool atomic, bool expect_match) -{ - ktime_t cur = ktime_get_raw(); - const ktime_t end = ktime_add_us(cur, timeout_us); - int ret = -ETIMEDOUT; - s64 wait = 10; - u32 read; - bool check; - - for (;;) { - read = xe_mmio_read32(gt, reg); - - check = (read & mask) == val; - if (!expect_match) - check = !check; - - if (check) { - ret = 0; - break; - } - - cur = ktime_get_raw(); - if (!ktime_before(cur, end)) - break; - - if (ktime_after(ktime_add_us(cur, wait), end)) - wait = ktime_us_delta(end, cur); - - if (atomic) - udelay(wait); - else - usleep_range(wait, wait << 1); - wait <<= 1; - } - - if (ret != 0) { - read = xe_mmio_read32(gt, reg); - - check = (read & mask) == val; - if (!expect_match) - check = !check; - - if (check) - ret = 0; - } - - if (out_val) - *out_val = read; - - return ret; -} - -/** - * xe_mmio_wait32() - Wait for a register to match the desired masked value - * @gt: MMIO target GT - * @reg: register to read value from - * @mask: mask to be applied to the value read from the register - * @val: desired value after applying the mask - * @timeout_us: time out after this period of time. Wait logic tries to be - * smart, applying an exponential backoff until @timeout_us is reached. - * @out_val: if not NULL, points where to store the last unmasked value - * @atomic: needs to be true if calling from an atomic context - * - * This function polls for the desired masked value and returns zero on success - * or -ETIMEDOUT if timed out. - * - * Note that @timeout_us represents the minimum amount of time to wait before - * giving up. The actual time taken by this function can be a little more than - * @timeout_us for different reasons, specially in non-atomic contexts. Thus, - * it is possible that this function succeeds even after @timeout_us has passed. - */ -int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, - u32 *out_val, bool atomic) -{ - return __xe_mmio_wait32(gt, reg, mask, val, timeout_us, out_val, atomic, true); -} - -/** - * xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value - * @gt: MMIO target GT - * @reg: register to read value from - * @mask: mask to be applied to the value read from the register - * @val: value not to be matched after applying the mask - * @timeout_us: time out after this period of time - * @out_val: if not NULL, points where to store the last unmasked value - * @atomic: needs to be true if calling from an atomic context - * - * This function works exactly like xe_mmio_wait32() with the exception that - * @val is expected not to be matched. - */ -int xe_mmio_wait32_not(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, - u32 *out_val, bool atomic) -{ - return __xe_mmio_wait32(gt, reg, mask, val, timeout_us, out_val, atomic, false); -} diff --git a/rr-cache/a80de0cebef501a1f1530fc1d4fd99bfb0136a1a/preimage b/rr-cache/a80de0cebef501a1f1530fc1d4fd99bfb0136a1a/preimage deleted file mode 100644 index 3510c81e1956..000000000000 --- a/rr-cache/a80de0cebef501a1f1530fc1d4fd99bfb0136a1a/preimage +++ /dev/null @@ -1,439 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021-2023 Intel Corporation - */ - -#include "xe_mmio.h" - -#include <linux/delay.h> -#include <linux/io-64-nonatomic-lo-hi.h> -#include <linux/minmax.h> -#include <linux/pci.h> - -#include <drm/drm_managed.h> -#include <drm/drm_print.h> - -#include "regs/xe_bars.h" -#include "regs/xe_regs.h" -#include "xe_device.h" -#include "xe_gt.h" -#include "xe_gt_printk.h" -#include "xe_gt_sriov_vf.h" -#include "xe_macros.h" -#include "xe_sriov.h" -#include "xe_trace.h" - -static void tiles_fini(void *arg) -{ - struct xe_device *xe = arg; - struct xe_tile *tile; - int id; - -<<<<<<< - for_each_remote_tile(tile, xe, id) - tile->mmio.regs = NULL; -======= - for_each_tile(tile, xe, id) - if (tile != xe_device_get_root_tile(xe)) - tile->mmio.regs = NULL; ->>>>>>> -} - -/* - * On multi-tile devices, partition the BAR space for MMIO on each tile, - * possibly accounting for register override on the number of tiles available. - * Resulting memory layout is like below: - * - * .----------------------. <- tile_count * tile_mmio_size - * | .... | - * |----------------------| <- 2 * tile_mmio_size - * | tile1->mmio.regs | - * |----------------------| <- 1 * tile_mmio_size - * | tile0->mmio.regs | - * '----------------------' <- 0MB - */ -static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size) -{ - struct xe_tile *tile; - void __iomem *regs; - u8 id; - - /* - * Nothing to be done as tile 0 has already been setup earlier with the - * entire BAR mapped - see xe_mmio_init() - */ - if (xe->info.tile_count == 1) - return; - - /* Possibly override number of tile based on configuration register */ - if (!xe->info.skip_mtcfg) { - struct xe_gt *gt = xe_root_mmio_gt(xe); - u8 tile_count; - u32 mtcfg; - - /* - * Although the per-tile mmio regs are not yet initialized, this - * is fine as it's going to the root gt, that's guaranteed to be - * initialized earlier in xe_mmio_init() - */ - mtcfg = xe_mmio_read64_2x32(gt, XEHP_MTCFG_ADDR); - tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; - - if (tile_count < xe->info.tile_count) { - drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", - xe->info.tile_count, tile_count); - xe->info.tile_count = tile_count; - - /* - * FIXME: Needs some work for standalone media, but - * should be impossible with multi-tile for now: - * multi-tile platform with standalone media doesn't - * exist - */ - xe->info.gt_count = xe->info.tile_count; - } - } - - regs = xe->mmio.regs; - for_each_tile(tile, xe, id) { - tile->mmio.size = tile_mmio_size; - tile->mmio.regs = regs; - regs += tile_mmio_size; - } -} - -/* - * On top of all the multi-tile MMIO space there can be a platform-dependent - * extension for each tile, resulting in a layout like below: - * - * .----------------------. <- ext_base + tile_count * tile_mmio_ext_size - * | .... | - * |----------------------| <- ext_base + 2 * tile_mmio_ext_size - * | tile1->mmio_ext.regs | - * |----------------------| <- ext_base + 1 * tile_mmio_ext_size - * | tile0->mmio_ext.regs | - * |======================| <- ext_base = tile_count * tile_mmio_size - * | | - * | mmio.regs | - * | | - * '----------------------' <- 0MB - * - * Set up the tile[]->mmio_ext pointers/sizes. - */ -static void mmio_extension_setup(struct xe_device *xe, size_t tile_mmio_size, - size_t tile_mmio_ext_size) -{ - struct xe_tile *tile; - void __iomem *regs; - u8 id; - - if (!xe->info.has_mmio_ext) - return; - - regs = xe->mmio.regs + tile_mmio_size * xe->info.tile_count; - for_each_tile(tile, xe, id) { - tile->mmio_ext.size = tile_mmio_ext_size; - tile->mmio_ext.regs = regs; - regs += tile_mmio_ext_size; - } -} - -int xe_mmio_probe_tiles(struct xe_device *xe) -{ - size_t tile_mmio_size = SZ_16M; - size_t tile_mmio_ext_size = xe->info.tile_mmio_ext_size; - - mmio_multi_tile_setup(xe, tile_mmio_size); - mmio_extension_setup(xe, tile_mmio_size, tile_mmio_ext_size); - - return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe); -} - -static void mmio_fini(void *arg) -{ - struct xe_device *xe = arg; - struct xe_tile *root_tile = xe_device_get_root_tile(xe); - - pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs); - xe->mmio.regs = NULL; - root_tile->mmio.regs = NULL; -} - -int xe_mmio_init(struct xe_device *xe) -{ - struct xe_tile *root_tile = xe_device_get_root_tile(xe); - struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - const int mmio_bar = 0; - - /* - * Map the entire BAR. - * The first 16MB of the BAR, belong to the root tile, and include: - * registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB). - */ - xe->mmio.size = pci_resource_len(pdev, mmio_bar); - xe->mmio.regs = pci_iomap(pdev, mmio_bar, GTTMMADR_BAR); - if (xe->mmio.regs == NULL) { - drm_err(&xe->drm, "failed to map registers\n"); - return -EIO; - } - - /* Setup first tile; other tiles (if present) will be setup later. */ - root_tile->mmio.size = SZ_16M; - root_tile->mmio.regs = xe->mmio.regs; - - return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe); -} - -static void mmio_flush_pending_writes(struct xe_gt *gt) -{ -#define DUMMY_REG_OFFSET 0x130030 - struct xe_tile *tile = gt_to_tile(gt); - int i; - - if (tile->xe->info.platform != XE_LUNARLAKE) - return; - - /* 4 dummy writes */ - for (i = 0; i < 4; i++) - writel(0, tile->mmio.regs + DUMMY_REG_OFFSET); -} - -u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - u8 val; - - /* Wa_15015404425 */ - mmio_flush_pending_writes(gt); - - val = readb((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); - trace_xe_reg_rw(gt, false, addr, val, sizeof(val)); - - return val; -} - -u16 xe_mmio_read16(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - u16 val; - - /* Wa_15015404425 */ - mmio_flush_pending_writes(gt); - - val = readw((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); - trace_xe_reg_rw(gt, false, addr, val, sizeof(val)); - - return val; -} - -void xe_mmio_write32(struct xe_gt *gt, struct xe_reg reg, u32 val) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - - trace_xe_reg_rw(gt, true, addr, val, sizeof(val)); - - if (!reg.vf && IS_SRIOV_VF(gt_to_xe(gt))) - xe_gt_sriov_vf_write32(gt, reg, val); - else - writel(val, (reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); -} - -u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_tile *tile = gt_to_tile(gt); - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - u32 val; - - /* Wa_15015404425 */ - mmio_flush_pending_writes(gt); - - if (!reg.vf && IS_SRIOV_VF(gt_to_xe(gt))) - val = xe_gt_sriov_vf_read32(gt, reg); - else - val = readl((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr); - - trace_xe_reg_rw(gt, false, addr, val, sizeof(val)); - - return val; -} - -u32 xe_mmio_rmw32(struct xe_gt *gt, struct xe_reg reg, u32 clr, u32 set) -{ - u32 old, reg_val; - - old = xe_mmio_read32(gt, reg); - reg_val = (old & ~clr) | set; - xe_mmio_write32(gt, reg, reg_val); - - return old; -} - -int xe_mmio_write32_and_verify(struct xe_gt *gt, - struct xe_reg reg, u32 val, u32 mask, u32 eval) -{ - u32 reg_val; - - xe_mmio_write32(gt, reg, val); - reg_val = xe_mmio_read32(gt, reg); - - return (reg_val & mask) != eval ? -EINVAL : 0; -} - -bool xe_mmio_in_range(const struct xe_gt *gt, - const struct xe_mmio_range *range, - struct xe_reg reg) -{ - u32 addr = xe_mmio_adjusted_addr(gt, reg.addr); - - return range && addr >= range->start && addr <= range->end; -} - -/** - * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads - * @gt: MMIO target GT - * @reg: register to read value from - * - * Although Intel GPUs have some 64-bit registers, the hardware officially - * only supports GTTMMADR register reads of 32 bits or smaller. Even if - * a readq operation may return a reasonable value, that violation of the - * spec shouldn't be relied upon and all 64-bit register reads should be - * performed as two 32-bit reads of the upper and lower dwords. - * - * When reading registers that may be changing (such as - * counters), a rollover of the lower dword between the two 32-bit reads - * can be problematic. This function attempts to ensure the upper dword has - * stabilized before returning the 64-bit value. - * - * Note that because this function may re-read the register multiple times - * while waiting for the value to stabilize it should not be used to read - * any registers where read operations have side effects. - * - * Returns the value of the 64-bit register. - */ -u64 xe_mmio_read64_2x32(struct xe_gt *gt, struct xe_reg reg) -{ - struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; - u32 ldw, udw, oldudw, retries; - - reg.addr = xe_mmio_adjusted_addr(gt, reg.addr); - reg_udw.addr = xe_mmio_adjusted_addr(gt, reg_udw.addr); - - /* we shouldn't adjust just one register address */ - xe_gt_assert(gt, reg_udw.addr == reg.addr + 0x4); - - oldudw = xe_mmio_read32(gt, reg_udw); - for (retries = 5; retries; --retries) { - ldw = xe_mmio_read32(gt, reg); - udw = xe_mmio_read32(gt, reg_udw); - - if (udw == oldudw) - break; - - oldudw = udw; - } - - xe_gt_WARN(gt, retries == 0, - "64-bit read of %#x did not stabilize\n", reg.addr); - - return (u64)udw << 32 | ldw; -} - -static int __xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, - u32 *out_val, bool atomic, bool expect_match) -{ - ktime_t cur = ktime_get_raw(); - const ktime_t end = ktime_add_us(cur, timeout_us); - int ret = -ETIMEDOUT; - s64 wait = 10; - u32 read; - bool check; - - for (;;) { - read = xe_mmio_read32(gt, reg); - - check = (read & mask) == val; - if (!expect_match) - check = !check; - - if (check) { - ret = 0; - break; - } - - cur = ktime_get_raw(); - if (!ktime_before(cur, end)) - break; - - if (ktime_after(ktime_add_us(cur, wait), end)) - wait = ktime_us_delta(end, cur); - - if (atomic) - udelay(wait); - else - usleep_range(wait, wait << 1); - wait <<= 1; - } - - if (ret != 0) { - read = xe_mmio_read32(gt, reg); - - check = (read & mask) == val; - if (!expect_match) - check = !check; - - if (check) - ret = 0; - } - - if (out_val) - *out_val = read; - - return ret; -} - -/** - * xe_mmio_wait32() - Wait for a register to match the desired masked value - * @gt: MMIO target GT - * @reg: register to read value from - * @mask: mask to be applied to the value read from the register - * @val: desired value after applying the mask - * @timeout_us: time out after this period of time. Wait logic tries to be - * smart, applying an exponential backoff until @timeout_us is reached. - * @out_val: if not NULL, points where to store the last unmasked value - * @atomic: needs to be true if calling from an atomic context - * - * This function polls for the desired masked value and returns zero on success - * or -ETIMEDOUT if timed out. - * - * Note that @timeout_us represents the minimum amount of time to wait before - * giving up. The actual time taken by this function can be a little more than - * @timeout_us for different reasons, specially in non-atomic contexts. Thus, - * it is possible that this function succeeds even after @timeout_us has passed. - */ -int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, - u32 *out_val, bool atomic) -{ - return __xe_mmio_wait32(gt, reg, mask, val, timeout_us, out_val, atomic, true); -} - -/** - * xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value - * @gt: MMIO target GT - * @reg: register to read value from - * @mask: mask to be applied to the value read from the register - * @val: value not to be matched after applying the mask - * @timeout_us: time out after this period of time - * @out_val: if not NULL, points where to store the last unmasked value - * @atomic: needs to be true if calling from an atomic context - * - * This function works exactly like xe_mmio_wait32() with the exception that - * @val is expected not to be matched. - */ -int xe_mmio_wait32_not(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, - u32 *out_val, bool atomic) -{ - return __xe_mmio_wait32(gt, reg, mask, val, timeout_us, out_val, atomic, false); -} diff --git a/rr-cache/b30fea06d38e3f9da0193354a33197e40cdf9ac9/postimage b/rr-cache/b30fea06d38e3f9da0193354a33197e40cdf9ac9/postimage deleted file mode 100644 index 1188ab83cfae..000000000000 --- a/rr-cache/b30fea06d38e3f9da0193354a33197e40cdf9ac9/postimage +++ /dev/null @@ -1,423 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2023 Intel Corporation - */ - -#include "xe_display.h" -#include "regs/xe_regs.h" - -#include <linux/fb.h> - -#include <drm/drm_drv.h> -#include <drm/drm_managed.h> -#include <drm/xe_drm.h> - -#include "soc/intel_dram.h" -#include "i915_drv.h" /* FIXME: HAS_DISPLAY() depends on this */ -#include "intel_acpi.h" -#include "intel_audio.h" -#include "intel_bw.h" -#include "intel_display.h" -#include "intel_display_driver.h" -#include "intel_display_irq.h" -#include "intel_display_types.h" -#include "intel_dmc.h" -#include "intel_dp.h" -#include "intel_encoder.h" -#include "intel_fbdev.h" -#include "intel_hdcp.h" -#include "intel_hotplug.h" -#include "intel_opregion.h" -#include "xe_module.h" - -/* Xe device functions */ - -static bool has_display(struct xe_device *xe) -{ - return HAS_DISPLAY(xe); -} - -/** - * xe_display_driver_probe_defer - Detect if we need to wait for other drivers - * early on - * @pdev: PCI device - * - * Returns: true if probe needs to be deferred, false otherwise - */ -bool xe_display_driver_probe_defer(struct pci_dev *pdev) -{ - if (!xe_modparam.enable_display) - return 0; - - return intel_display_driver_probe_defer(pdev); -} - -/** - * xe_display_driver_set_hooks - Add driver flags and hooks for display - * @driver: DRM device driver - * - * Set features and function hooks in @driver that are needed for driving the - * display IP. This sets the driver's capability of driving display, regardless - * if the device has it enabled - */ -void xe_display_driver_set_hooks(struct drm_driver *driver) -{ - if (!xe_modparam.enable_display) - return; - - driver->driver_features |= DRIVER_MODESET | DRIVER_ATOMIC; -} - -static void unset_display_features(struct xe_device *xe) -{ - xe->drm.driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC); -} - -static void display_destroy(struct drm_device *dev, void *dummy) -{ - struct xe_device *xe = to_xe_device(dev); - - destroy_workqueue(xe->display.hotplug.dp_wq); -} - -/** - * xe_display_create - create display struct - * @xe: XE device instance - * - * Initialize all fields used by the display part. - * - * TODO: once everything can be inside a single struct, make the struct opaque - * to the rest of xe and return it to be xe->display. - * - * Returns: 0 on success - */ -int xe_display_create(struct xe_device *xe) -{ - spin_lock_init(&xe->display.fb_tracking.lock); - - xe->display.hotplug.dp_wq = alloc_ordered_workqueue("xe-dp", 0); - - return drmm_add_action_or_reset(&xe->drm, display_destroy, NULL); -} - -static void xe_display_fini_nommio(struct drm_device *dev, void *dummy) -{ - struct xe_device *xe = to_xe_device(dev); - - if (!xe->info.enable_display) - return; - - intel_power_domains_cleanup(xe); -} - -int xe_display_init_nommio(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return 0; - - /* Fake uncore lock */ - spin_lock_init(&xe->uncore.lock); - - /* This must be called before any calls to HAS_PCH_* */ - intel_detect_pch(xe); - - return drmm_add_action_or_reset(&xe->drm, xe_display_fini_nommio, xe); -} - -static void xe_display_fini_noirq(void *arg) -{ - struct xe_device *xe = arg; - struct intel_display *display = &xe->display; - - if (!xe->info.enable_display) - return; - - intel_display_driver_remove_noirq(xe); - intel_opregion_cleanup(display); -} - -int xe_display_init_noirq(struct xe_device *xe) -{ - struct intel_display *display = &xe->display; - int err; - - if (!xe->info.enable_display) - return 0; - - intel_display_driver_early_probe(xe); - - /* Early display init.. */ - intel_opregion_setup(display); - - /* - * Fill the dram structure to get the system dram info. This will be - * used for memory latency calculation. - */ - intel_dram_detect(xe); - - intel_bw_init_hw(xe); - - intel_display_device_info_runtime_init(xe); - - err = intel_display_driver_probe_noirq(xe); - if (err) { - intel_opregion_cleanup(display); - return err; - } - - return devm_add_action_or_reset(xe->drm.dev, xe_display_fini_noirq, xe); -} - -static void xe_display_fini_noaccel(void *arg) -{ - struct xe_device *xe = arg; - - if (!xe->info.enable_display) - return; - - intel_display_driver_remove_nogem(xe); -} - -int xe_display_init_noaccel(struct xe_device *xe) -{ - int err; - - if (!xe->info.enable_display) - return 0; - - err = intel_display_driver_probe_nogem(xe); - if (err) - return err; - - return devm_add_action_or_reset(xe->drm.dev, xe_display_fini_noaccel, xe); -} - -int xe_display_init(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return 0; - - return intel_display_driver_probe(xe); -} - -void xe_display_fini(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_hpd_poll_fini(xe); - - intel_hdcp_component_fini(xe); - intel_audio_deinit(xe); -} - -void xe_display_register(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_display_driver_register(xe); - intel_register_dsm_handler(); - intel_power_domains_enable(xe); -} - -void xe_display_unregister(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_unregister_dsm_handler(); - intel_power_domains_disable(xe); - intel_display_driver_unregister(xe); -} - -void xe_display_driver_remove(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_display_driver_remove(xe); -} - -/* IRQ-related functions */ - -void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl) -{ - if (!xe->info.enable_display) - return; - - if (master_ctl & DISPLAY_IRQ) - gen11_display_irq_handler(xe); -} - -void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir) -{ - struct intel_display *display = &xe->display; - - if (!xe->info.enable_display) - return; - - if (gu_misc_iir & GU_MISC_GSE) - intel_opregion_asle_intr(display); -} - -void xe_display_irq_reset(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - gen11_display_irq_reset(xe); -} - -void xe_display_irq_postinstall(struct xe_device *xe, struct xe_gt *gt) -{ - if (!xe->info.enable_display) - return; - - if (gt->info.id == XE_GT0) - gen11_de_irq_postinstall(xe); -} - -static bool suspend_to_idle(void) -{ -#if IS_ENABLED(CONFIG_ACPI_SLEEP) - if (acpi_target_system_state() < ACPI_STATE_S3) - return true; -#endif - return false; -} - -static void xe_display_flush_cleanup_work(struct xe_device *xe) -{ - struct intel_crtc *crtc; - - for_each_intel_crtc(&xe->drm, crtc) { - struct drm_crtc_commit *commit; - - spin_lock(&crtc->base.commit_lock); - commit = list_first_entry_or_null(&crtc->base.commit_list, - struct drm_crtc_commit, commit_entry); - if (commit) - drm_crtc_commit_get(commit); - spin_unlock(&crtc->base.commit_lock); - - if (commit) { - wait_for_completion(&commit->cleanup_done); - drm_crtc_commit_put(commit); - } - } -} - -void xe_display_pm_suspend(struct xe_device *xe, bool runtime) -{ - struct intel_display *display = &xe->display; - bool s2idle = suspend_to_idle(); - if (!xe->info.enable_display) - return; - - /* - * We do a lot of poking in a lot of registers, make sure they work - * properly. - */ - intel_power_domains_disable(xe); - if (has_display(xe)) - drm_kms_helper_poll_disable(&xe->drm); - - if (!runtime) - intel_display_driver_suspend(xe); - - xe_display_flush_cleanup_work(xe); - - intel_dp_mst_suspend(xe); - - intel_hpd_cancel_work(xe); - - intel_encoder_suspend_all(&xe->display); - - intel_opregion_suspend(display, s2idle ? PCI_D1 : PCI_D3cold); - - intel_fbdev_set_suspend(&xe->drm, FBINFO_STATE_SUSPENDED, true); - - intel_dmc_suspend(xe); -} - -void xe_display_pm_suspend_late(struct xe_device *xe) -{ - bool s2idle = suspend_to_idle(); - if (!xe->info.enable_display) - return; - - intel_power_domains_suspend(xe, s2idle); - - intel_display_power_suspend_late(xe); -} - -void xe_display_pm_resume_early(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_display_power_resume_early(xe); - - intel_power_domains_resume(xe); -} - -void xe_display_pm_resume(struct xe_device *xe, bool runtime) -{ - struct intel_display *display = &xe->display; - - if (!xe->info.enable_display) - return; - - intel_dmc_resume(xe); - - if (has_display(xe)) - drm_mode_config_reset(&xe->drm); - - intel_display_driver_init_hw(xe); - intel_hpd_init(xe); - - /* MST sideband requires HPD interrupts enabled */ - intel_dp_mst_resume(xe); - if (!runtime) - intel_display_driver_resume(xe); - - intel_hpd_poll_disable(xe); - if (has_display(xe)) - drm_kms_helper_poll_enable(&xe->drm); - - intel_opregion_resume(display); - - intel_fbdev_set_suspend(&xe->drm, FBINFO_STATE_RUNNING, false); - - intel_power_domains_enable(xe); -} - -static void display_device_remove(struct drm_device *dev, void *arg) -{ - struct xe_device *xe = arg; - - intel_display_device_remove(xe); -} - -int xe_display_probe(struct xe_device *xe) -{ - int err; - - if (!xe->info.enable_display) - goto no_display; - - intel_display_device_probe(xe); - - err = drmm_add_action_or_reset(&xe->drm, display_device_remove, xe); - if (err) - return err; - - if (has_display(xe)) - return 0; - -no_display: - xe->info.enable_display = false; - unset_display_features(xe); - return 0; -} diff --git a/rr-cache/b30fea06d38e3f9da0193354a33197e40cdf9ac9/preimage b/rr-cache/b30fea06d38e3f9da0193354a33197e40cdf9ac9/preimage deleted file mode 100644 index 50ee89fb5f7b..000000000000 --- a/rr-cache/b30fea06d38e3f9da0193354a33197e40cdf9ac9/preimage +++ /dev/null @@ -1,431 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2023 Intel Corporation - */ - -#include "xe_display.h" -#include "regs/xe_regs.h" - -#include <linux/fb.h> - -#include <drm/drm_drv.h> -#include <drm/drm_managed.h> -#include <drm/xe_drm.h> - -#include "soc/intel_dram.h" -#include "i915_drv.h" /* FIXME: HAS_DISPLAY() depends on this */ -#include "intel_acpi.h" -#include "intel_audio.h" -#include "intel_bw.h" -#include "intel_display.h" -#include "intel_display_driver.h" -#include "intel_display_irq.h" -#include "intel_display_types.h" -#include "intel_dmc.h" -#include "intel_dp.h" -#include "intel_encoder.h" -#include "intel_fbdev.h" -#include "intel_hdcp.h" -#include "intel_hotplug.h" -#include "intel_opregion.h" -#include "xe_module.h" - -/* Xe device functions */ - -static bool has_display(struct xe_device *xe) -{ - return HAS_DISPLAY(xe); -} - -/** - * xe_display_driver_probe_defer - Detect if we need to wait for other drivers - * early on - * @pdev: PCI device - * - * Returns: true if probe needs to be deferred, false otherwise - */ -bool xe_display_driver_probe_defer(struct pci_dev *pdev) -{ - if (!xe_modparam.enable_display) - return 0; - - return intel_display_driver_probe_defer(pdev); -} - -/** - * xe_display_driver_set_hooks - Add driver flags and hooks for display - * @driver: DRM device driver - * - * Set features and function hooks in @driver that are needed for driving the - * display IP. This sets the driver's capability of driving display, regardless - * if the device has it enabled - */ -void xe_display_driver_set_hooks(struct drm_driver *driver) -{ - if (!xe_modparam.enable_display) - return; - - driver->driver_features |= DRIVER_MODESET | DRIVER_ATOMIC; -} - -static void unset_display_features(struct xe_device *xe) -{ - xe->drm.driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC); -} - -static void display_destroy(struct drm_device *dev, void *dummy) -{ - struct xe_device *xe = to_xe_device(dev); - - destroy_workqueue(xe->display.hotplug.dp_wq); -} - -/** - * xe_display_create - create display struct - * @xe: XE device instance - * - * Initialize all fields used by the display part. - * - * TODO: once everything can be inside a single struct, make the struct opaque - * to the rest of xe and return it to be xe->display. - * - * Returns: 0 on success - */ -int xe_display_create(struct xe_device *xe) -{ - spin_lock_init(&xe->display.fb_tracking.lock); - - xe->display.hotplug.dp_wq = alloc_ordered_workqueue("xe-dp", 0); - - return drmm_add_action_or_reset(&xe->drm, display_destroy, NULL); -} - -static void xe_display_fini_nommio(struct drm_device *dev, void *dummy) -{ - struct xe_device *xe = to_xe_device(dev); - - if (!xe->info.enable_display) - return; - - intel_power_domains_cleanup(xe); -} - -int xe_display_init_nommio(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return 0; - - /* Fake uncore lock */ - spin_lock_init(&xe->uncore.lock); - - /* This must be called before any calls to HAS_PCH_* */ - intel_detect_pch(xe); - - return drmm_add_action_or_reset(&xe->drm, xe_display_fini_nommio, xe); -} - -static void xe_display_fini_noirq(void *arg) -{ - struct xe_device *xe = arg; - struct intel_display *display = &xe->display; - - if (!xe->info.enable_display) - return; - - intel_display_driver_remove_noirq(xe); -<<<<<<< - intel_opregion_cleanup(display); -======= - intel_opregion_cleanup(xe); ->>>>>>> -} - -int xe_display_init_noirq(struct xe_device *xe) -{ - struct intel_display *display = &xe->display; - int err; - - if (!xe->info.enable_display) - return 0; - - intel_display_driver_early_probe(xe); - - /* Early display init.. */ - intel_opregion_setup(display); - - /* - * Fill the dram structure to get the system dram info. This will be - * used for memory latency calculation. - */ - intel_dram_detect(xe); - - intel_bw_init_hw(xe); - - intel_display_device_info_runtime_init(xe); - - err = intel_display_driver_probe_noirq(xe); - if (err) { -<<<<<<< - intel_opregion_cleanup(display); -======= - intel_opregion_cleanup(xe); ->>>>>>> - return err; - } - - return devm_add_action_or_reset(xe->drm.dev, xe_display_fini_noirq, xe); -} - -static void xe_display_fini_noaccel(void *arg) -{ - struct xe_device *xe = arg; - - if (!xe->info.enable_display) - return; - - intel_display_driver_remove_nogem(xe); -} - -int xe_display_init_noaccel(struct xe_device *xe) -{ - int err; - - if (!xe->info.enable_display) - return 0; - - err = intel_display_driver_probe_nogem(xe); - if (err) - return err; - - return devm_add_action_or_reset(xe->drm.dev, xe_display_fini_noaccel, xe); -} - -int xe_display_init(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return 0; - - return intel_display_driver_probe(xe); -} - -void xe_display_fini(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_hpd_poll_fini(xe); - - intel_hdcp_component_fini(xe); - intel_audio_deinit(xe); -} - -void xe_display_register(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_display_driver_register(xe); - intel_register_dsm_handler(); - intel_power_domains_enable(xe); -} - -void xe_display_unregister(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_unregister_dsm_handler(); - intel_power_domains_disable(xe); - intel_display_driver_unregister(xe); -} - -void xe_display_driver_remove(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_display_driver_remove(xe); -} - -/* IRQ-related functions */ - -void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl) -{ - if (!xe->info.enable_display) - return; - - if (master_ctl & DISPLAY_IRQ) - gen11_display_irq_handler(xe); -} - -void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir) -{ - struct intel_display *display = &xe->display; - - if (!xe->info.enable_display) - return; - - if (gu_misc_iir & GU_MISC_GSE) - intel_opregion_asle_intr(display); -} - -void xe_display_irq_reset(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - gen11_display_irq_reset(xe); -} - -void xe_display_irq_postinstall(struct xe_device *xe, struct xe_gt *gt) -{ - if (!xe->info.enable_display) - return; - - if (gt->info.id == XE_GT0) - gen11_de_irq_postinstall(xe); -} - -static bool suspend_to_idle(void) -{ -#if IS_ENABLED(CONFIG_ACPI_SLEEP) - if (acpi_target_system_state() < ACPI_STATE_S3) - return true; -#endif - return false; -} - -static void xe_display_flush_cleanup_work(struct xe_device *xe) -{ - struct intel_crtc *crtc; - - for_each_intel_crtc(&xe->drm, crtc) { - struct drm_crtc_commit *commit; - - spin_lock(&crtc->base.commit_lock); - commit = list_first_entry_or_null(&crtc->base.commit_list, - struct drm_crtc_commit, commit_entry); - if (commit) - drm_crtc_commit_get(commit); - spin_unlock(&crtc->base.commit_lock); - - if (commit) { - wait_for_completion(&commit->cleanup_done); - drm_crtc_commit_put(commit); - } - } -} - -void xe_display_pm_suspend(struct xe_device *xe, bool runtime) -{ - struct intel_display *display = &xe->display; - bool s2idle = suspend_to_idle(); - if (!xe->info.enable_display) - return; - - /* - * We do a lot of poking in a lot of registers, make sure they work - * properly. - */ - intel_power_domains_disable(xe); - if (has_display(xe)) - drm_kms_helper_poll_disable(&xe->drm); - - if (!runtime) - intel_display_driver_suspend(xe); - - xe_display_flush_cleanup_work(xe); - - intel_dp_mst_suspend(xe); - - intel_hpd_cancel_work(xe); - - intel_encoder_suspend_all(&xe->display); - - intel_opregion_suspend(display, s2idle ? PCI_D1 : PCI_D3cold); - - intel_fbdev_set_suspend(&xe->drm, FBINFO_STATE_SUSPENDED, true); - - intel_dmc_suspend(xe); -} - -void xe_display_pm_suspend_late(struct xe_device *xe) -{ - bool s2idle = suspend_to_idle(); - if (!xe->info.enable_display) - return; - - intel_power_domains_suspend(xe, s2idle); - - intel_display_power_suspend_late(xe); -} - -void xe_display_pm_resume_early(struct xe_device *xe) -{ - if (!xe->info.enable_display) - return; - - intel_display_power_resume_early(xe); - - intel_power_domains_resume(xe); -} - -void xe_display_pm_resume(struct xe_device *xe, bool runtime) -{ - struct intel_display *display = &xe->display; - - if (!xe->info.enable_display) - return; - - intel_dmc_resume(xe); - - if (has_display(xe)) - drm_mode_config_reset(&xe->drm); - - intel_display_driver_init_hw(xe); - intel_hpd_init(xe); - - /* MST sideband requires HPD interrupts enabled */ - intel_dp_mst_resume(xe); - if (!runtime) - intel_display_driver_resume(xe); - - intel_hpd_poll_disable(xe); - if (has_display(xe)) - drm_kms_helper_poll_enable(&xe->drm); - - intel_opregion_resume(display); - - intel_fbdev_set_suspend(&xe->drm, FBINFO_STATE_RUNNING, false); - - intel_power_domains_enable(xe); -} - -static void display_device_remove(struct drm_device *dev, void *arg) -{ - struct xe_device *xe = arg; - - intel_display_device_remove(xe); -} - -int xe_display_probe(struct xe_device *xe) -{ - int err; - - if (!xe->info.enable_display) - goto no_display; - - intel_display_device_probe(xe); - - err = drmm_add_action_or_reset(&xe->drm, display_device_remove, xe); - if (err) - return err; - - if (has_display(xe)) - return 0; - -no_display: - xe->info.enable_display = false; - unset_display_features(xe); - return 0; -} diff --git a/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.3 b/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.3 deleted file mode 100644 index cb13209846cb..000000000000 --- a/rr-cache/e7c9aafc2297a37f89715cfeed48ccbfb82f76bb/preimage.3 +++ /dev/null @@ -1,2244 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2022 Intel Corporation - */ - -#include "xe_guc_submit.h" - -#include <linux/bitfield.h> -#include <linux/bitmap.h> -#include <linux/circ_buf.h> -#include <linux/delay.h> -#include <linux/dma-fence-array.h> -#include <linux/math64.h> - -#include <drm/drm_managed.h> - -#include "abi/guc_actions_abi.h" -#include "abi/guc_klvs_abi.h" -#include "regs/xe_lrc_layout.h" -#include "xe_assert.h" -#include "xe_devcoredump.h" -#include "xe_device.h" -#include "xe_exec_queue.h" -#include "xe_force_wake.h" -#include "xe_gpu_scheduler.h" -#include "xe_gt.h" -#include "xe_gt_clock.h" -#include "xe_gt_printk.h" -#include "xe_guc.h" -#include "xe_guc_ct.h" -#include "xe_guc_exec_queue_types.h" -#include "xe_guc_id_mgr.h" -#include "xe_guc_submit_types.h" -#include "xe_hw_engine.h" -#include "xe_hw_fence.h" -#include "xe_lrc.h" -#include "xe_macros.h" -#include "xe_map.h" -#include "xe_mocs.h" -#include "xe_pm.h" -#include "xe_ring_ops_types.h" -#include "xe_sched_job.h" -#include "xe_trace.h" -#include "xe_vm.h" - -static struct xe_guc * -exec_queue_to_guc(struct xe_exec_queue *q) -{ - return &q->gt->uc.guc; -} - -/* - * Helpers for engine state, using an atomic as some of the bits can transition - * as the same time (e.g. a suspend can be happning at the same time as schedule - * engine done being processed). - */ -#define EXEC_QUEUE_STATE_REGISTERED (1 << 0) -#define EXEC_QUEUE_STATE_ENABLED (1 << 1) -#define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) -#define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) -#define EXEC_QUEUE_STATE_DESTROYED (1 << 4) -#define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) -#define EXEC_QUEUE_STATE_RESET (1 << 6) -#define EXEC_QUEUE_STATE_KILLED (1 << 7) -#define EXEC_QUEUE_STATE_WEDGED (1 << 8) -#define EXEC_QUEUE_STATE_BANNED (1 << 9) -#define EXEC_QUEUE_STATE_CHECK_TIMEOUT (1 << 10) -#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11) - -static bool exec_queue_registered(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; -} - -static void set_exec_queue_registered(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); -} - -static void clear_exec_queue_registered(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); -} - -static bool exec_queue_enabled(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; -} - -static void set_exec_queue_enabled(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); -} - -static void clear_exec_queue_enabled(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); -} - -static bool exec_queue_pending_enable(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; -} - -static void set_exec_queue_pending_enable(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); -} - -static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); -} - -static bool exec_queue_pending_disable(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; -} - -static void set_exec_queue_pending_disable(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); -} - -static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); -} - -static bool exec_queue_destroyed(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; -} - -static void set_exec_queue_destroyed(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); -} - -static bool exec_queue_banned(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; -} - -static void set_exec_queue_banned(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); -} - -static bool exec_queue_suspended(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; -} - -static void set_exec_queue_suspended(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); -} - -static void clear_exec_queue_suspended(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); -} - -static bool exec_queue_reset(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; -} - -static void set_exec_queue_reset(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); -} - -static bool exec_queue_killed(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; -} - -static void set_exec_queue_killed(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); -} - -static bool exec_queue_wedged(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; -} - -static void set_exec_queue_wedged(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); -} - -static bool exec_queue_check_timeout(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_CHECK_TIMEOUT; -} - -static void set_exec_queue_check_timeout(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); -} - -static void clear_exec_queue_check_timeout(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); -} - -static bool exec_queue_extra_ref(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_EXTRA_REF; -} - -static void set_exec_queue_extra_ref(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_EXTRA_REF, &q->guc->state); -} - -static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) -{ - return (atomic_read(&q->guc->state) & - (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | - EXEC_QUEUE_STATE_BANNED)); -} - -#ifdef CONFIG_PROVE_LOCKING -static int alloc_submit_wq(struct xe_guc *guc) -{ - int i; - - for (i = 0; i < NUM_SUBMIT_WQ; ++i) { - guc->submission_state.submit_wq_pool[i] = - alloc_ordered_workqueue("submit_wq", 0); - if (!guc->submission_state.submit_wq_pool[i]) - goto err_free; - } - - return 0; - -err_free: - while (i) - destroy_workqueue(guc->submission_state.submit_wq_pool[--i]); - - return -ENOMEM; -} - -static void free_submit_wq(struct xe_guc *guc) -{ - int i; - - for (i = 0; i < NUM_SUBMIT_WQ; ++i) - destroy_workqueue(guc->submission_state.submit_wq_pool[i]); -} - -static struct workqueue_struct *get_submit_wq(struct xe_guc *guc) -{ - int idx = guc->submission_state.submit_wq_idx++ % NUM_SUBMIT_WQ; - - return guc->submission_state.submit_wq_pool[idx]; -} -#else -static int alloc_submit_wq(struct xe_guc *guc) -{ - return 0; -} - -static void free_submit_wq(struct xe_guc *guc) -{ - -} - -static struct workqueue_struct *get_submit_wq(struct xe_guc *guc) -{ - return NULL; -} -#endif - -static void guc_submit_fini(struct drm_device *drm, void *arg) -{ - struct xe_guc *guc = arg; - - xa_destroy(&guc->submission_state.exec_queue_lookup); - free_submit_wq(guc); -} - -static void guc_submit_wedged_fini(struct drm_device *drm, void *arg) -{ - struct xe_guc *guc = arg; - struct xe_exec_queue *q; - unsigned long index; - - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - if (exec_queue_wedged(q)) - xe_exec_queue_put(q); -} - -static const struct xe_exec_queue_ops guc_exec_queue_ops; - -static void primelockdep(struct xe_guc *guc) -{ - if (!IS_ENABLED(CONFIG_LOCKDEP)) - return; - - fs_reclaim_acquire(GFP_KERNEL); - - mutex_lock(&guc->submission_state.lock); - mutex_unlock(&guc->submission_state.lock); - - fs_reclaim_release(GFP_KERNEL); -} - -/** - * xe_guc_submit_init() - Initialize GuC submission. - * @guc: the &xe_guc to initialize - * @num_ids: number of GuC context IDs to use - * - * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all - * GuC context IDs supported by the GuC firmware should be used for submission. - * - * Only VF drivers will have to provide explicit number of GuC context IDs - * that they can use for submission. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_gt *gt = guc_to_gt(guc); - int err; - - err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); - if (err) - return err; - - err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); - if (err) - return err; - - err = alloc_submit_wq(guc); - if (err) - return err; - - gt->exec_queue_ops = &guc_exec_queue_ops; - - xa_init(&guc->submission_state.exec_queue_lookup); - - primelockdep(guc); - - return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc); -} - -static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count) -{ - int i; - - lockdep_assert_held(&guc->submission_state.lock); - - for (i = 0; i < xa_count; ++i) - xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i); - - xe_guc_id_mgr_release_locked(&guc->submission_state.idm, - q->guc->id, q->width); -} - -static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) -{ - int ret; - void *ptr; - int i; - - /* - * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, - * worse case user gets -ENOMEM on engine create and has to try again. - * - * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent - * failure. - */ - lockdep_assert_held(&guc->submission_state.lock); - - ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, - q->width); - if (ret < 0) - return ret; - - q->guc->id = ret; - - for (i = 0; i < q->width; ++i) { - ptr = xa_store(&guc->submission_state.exec_queue_lookup, - q->guc->id + i, q, GFP_NOWAIT); - if (IS_ERR(ptr)) { - ret = PTR_ERR(ptr); - goto err_release; - } - } - - return 0; - -err_release: - __release_guc_id(guc, q, i); - - return ret; -} - -static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) -{ - mutex_lock(&guc->submission_state.lock); - __release_guc_id(guc, q, q->width); - mutex_unlock(&guc->submission_state.lock); -} - -struct exec_queue_policy { - u32 count; - struct guc_update_exec_queue_policy h2g; -}; - -static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) -{ - size_t bytes = sizeof(policy->h2g.header) + - (sizeof(policy->h2g.klv[0]) * policy->count); - - return bytes / sizeof(u32); -} - -static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, - u16 guc_id) -{ - policy->h2g.header.action = - XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; - policy->h2g.header.guc_id = guc_id; - policy->count = 0; -} - -#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ -static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ - u32 data) \ -{ \ - XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ -\ - policy->h2g.klv[policy->count].kl = \ - FIELD_PREP(GUC_KLV_0_KEY, \ - GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ - FIELD_PREP(GUC_KLV_0_LEN, 1); \ - policy->h2g.klv[policy->count].value = data; \ - policy->count++; \ -} - -MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) -MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) -MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) -#undef MAKE_EXEC_QUEUE_POLICY_ADD - -static const int xe_exec_queue_prio_to_guc[] = { - [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, - [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, - [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, - [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, -}; - -static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct exec_queue_policy policy; - struct xe_device *xe = guc_to_xe(guc); - enum xe_exec_queue_priority prio = q->sched_props.priority; - u32 timeslice_us = q->sched_props.timeslice_us; - u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; - - xe_assert(xe, exec_queue_registered(q)); - - __guc_exec_queue_policy_start_klv(&policy, q->guc->id); - __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); - __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); - __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); - - xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_exec_queue_policy_action_size(&policy), 0, 0); -} - -static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct exec_queue_policy policy; - - __guc_exec_queue_policy_start_klv(&policy, q->guc->id); - __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); - - xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_exec_queue_policy_action_size(&policy), 0, 0); -} - -#define parallel_read(xe_, map_, field_) \ - xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ - field_) -#define parallel_write(xe_, map_, field_, val_) \ - xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ - field_, val_) - -static void __register_mlrc_exec_queue(struct xe_guc *guc, - struct xe_exec_queue *q, - struct guc_ctxt_registration_info *info) -{ -#define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) - struct xe_device *xe = guc_to_xe(guc); - u32 action[MAX_MLRC_REG_SIZE]; - int len = 0; - int i; - - xe_assert(xe, xe_exec_queue_is_parallel(q)); - - action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; - action[len++] = info->flags; - action[len++] = info->context_idx; - action[len++] = info->engine_class; - action[len++] = info->engine_submit_mask; - action[len++] = info->wq_desc_lo; - action[len++] = info->wq_desc_hi; - action[len++] = info->wq_base_lo; - action[len++] = info->wq_base_hi; - action[len++] = info->wq_size; - action[len++] = q->width; - action[len++] = info->hwlrca_lo; - action[len++] = info->hwlrca_hi; - - for (i = 1; i < q->width; ++i) { - struct xe_lrc *lrc = q->lrc[i]; - - action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); - action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); - } - - xe_assert(xe, len <= MAX_MLRC_REG_SIZE); -#undef MAX_MLRC_REG_SIZE - - xe_guc_ct_send(&guc->ct, action, len, 0, 0); -} - -static void __register_exec_queue(struct xe_guc *guc, - struct guc_ctxt_registration_info *info) -{ - u32 action[] = { - XE_GUC_ACTION_REGISTER_CONTEXT, - info->flags, - info->context_idx, - info->engine_class, - info->engine_submit_mask, - info->wq_desc_lo, - info->wq_desc_hi, - info->wq_base_lo, - info->wq_base_hi, - info->wq_size, - info->hwlrca_lo, - info->hwlrca_hi, - }; - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); -} - -static void register_exec_queue(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_lrc *lrc = q->lrc[0]; - struct guc_ctxt_registration_info info; - - xe_assert(xe, !exec_queue_registered(q)); - - memset(&info, 0, sizeof(info)); - info.context_idx = q->guc->id; - info.engine_class = xe_engine_class_to_guc_class(q->class); - info.engine_submit_mask = q->logical_mask; - info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); - info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); - info.flags = CONTEXT_REGISTRATION_FLAG_KMD; - - if (xe_exec_queue_is_parallel(q)) { - u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); - struct iosys_map map = xe_lrc_parallel_map(lrc); - - info.wq_desc_lo = lower_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq_desc)); - info.wq_desc_hi = upper_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq_desc)); - info.wq_base_lo = lower_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq[0])); - info.wq_base_hi = upper_32_bits(ggtt_addr + - offsetof(struct guc_submit_parallel_scratch, wq[0])); - info.wq_size = WQ_SIZE; - - q->guc->wqi_head = 0; - q->guc->wqi_tail = 0; - xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); - parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); - } - - /* - * We must keep a reference for LR engines if engine is registered with - * the GuC as jobs signal immediately and can't destroy an engine if the - * GuC has a reference to it. - */ - if (xe_exec_queue_is_lr(q)) - xe_exec_queue_get(q); - - set_exec_queue_registered(q); - trace_xe_exec_queue_register(q); - if (xe_exec_queue_is_parallel(q)) - __register_mlrc_exec_queue(guc, q, &info); - else - __register_exec_queue(guc, &info); - init_policies(guc, q); -} - -static u32 wq_space_until_wrap(struct xe_exec_queue *q) -{ - return (WQ_SIZE - q->guc->wqi_tail); -} - -static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - unsigned int sleep_period_ms = 1; - -#define AVAILABLE_SPACE \ - CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) - if (wqi_size > AVAILABLE_SPACE) { -try_again: - q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); - if (wqi_size > AVAILABLE_SPACE) { - if (sleep_period_ms == 1024) { - xe_gt_reset_async(q->gt); - return -ENODEV; - } - - msleep(sleep_period_ms); - sleep_period_ms <<= 1; - goto try_again; - } - } -#undef AVAILABLE_SPACE - - return 0; -} - -static int wq_noop_append(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; - - if (wq_wait_for_space(q, wq_space_until_wrap(q))) - return -ENODEV; - - xe_assert(xe, FIELD_FIT(WQ_LEN_MASK, len_dw)); - - parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], - FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | - FIELD_PREP(WQ_LEN_MASK, len_dw)); - q->guc->wqi_tail = 0; - - return 0; -} - -static void wq_item_append(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); -#define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ - u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; - u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); - u32 len_dw = (wqi_size / sizeof(u32)) - 1; - int i = 0, j; - - if (wqi_size > wq_space_until_wrap(q)) { - if (wq_noop_append(q)) - return; - } - if (wq_wait_for_space(q, wqi_size)) - return; - - wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | - FIELD_PREP(WQ_LEN_MASK, len_dw); - wqi[i++] = xe_lrc_descriptor(q->lrc[0]); - wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | - FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); - wqi[i++] = 0; - for (j = 1; j < q->width; ++j) { - struct xe_lrc *lrc = q->lrc[j]; - - wqi[i++] = lrc->ring.tail / sizeof(u64); - } - - xe_assert(xe, i == wqi_size / sizeof(u32)); - - iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, - wq[q->guc->wqi_tail / sizeof(u32)])); - xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); - q->guc->wqi_tail += wqi_size; - xe_assert(xe, q->guc->wqi_tail <= WQ_SIZE); - - xe_device_wmb(xe); - - map = xe_lrc_parallel_map(q->lrc[0]); - parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); -} - -#define RESUME_PENDING ~0x0ull -static void submit_exec_queue(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_lrc *lrc = q->lrc[0]; - u32 action[3]; - u32 g2h_len = 0; - u32 num_g2h = 0; - int len = 0; - bool extra_submit = false; - - xe_assert(xe, exec_queue_registered(q)); - - if (xe_exec_queue_is_parallel(q)) - wq_item_append(q); - else - xe_lrc_set_ring_tail(lrc, lrc->ring.tail); - - if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) - return; - - if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; - action[len++] = q->guc->id; - action[len++] = GUC_CONTEXT_ENABLE; - g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; - num_g2h = 1; - if (xe_exec_queue_is_parallel(q)) - extra_submit = true; - - q->guc->resume_time = RESUME_PENDING; - set_exec_queue_pending_enable(q); - set_exec_queue_enabled(q); - trace_xe_exec_queue_scheduling_enable(q); - } else { - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = q->guc->id; - trace_xe_exec_queue_submit(q); - } - - xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); - - if (extra_submit) { - len = 0; - action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = q->guc->id; - trace_xe_exec_queue_submit(q); - - xe_guc_ct_send(&guc->ct, action, len, 0, 0); - } -} - -static struct dma_fence * -guc_exec_queue_run_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_exec_queue *q = job->q; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - bool lr = xe_exec_queue_is_lr(q); - - xe_assert(xe, !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || - exec_queue_banned(q) || exec_queue_suspended(q)); - - trace_xe_sched_job_run(job); - - if (!exec_queue_killed_or_banned_or_wedged(q) && !xe_sched_job_is_error(job)) { - if (!exec_queue_registered(q)) - register_exec_queue(q); - if (!lr) /* LR jobs are emitted in the exec IOCTL */ - q->ring_ops->emit_job(job); - submit_exec_queue(q); - } - - if (lr) { - xe_sched_job_set_error(job, -EOPNOTSUPP); - return NULL; - } else if (test_and_set_bit(JOB_FLAG_SUBMIT, &job->fence->flags)) { - return job->fence; - } else { - return dma_fence_get(job->fence); - } -} - -static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - - xe_exec_queue_update_run_ticks(job->q); - - trace_xe_sched_job_free(job); - xe_sched_job_put(job); -} - -static int guc_read_stopped(struct xe_guc *guc) -{ - return atomic_read(&guc->submission_state.stopped); -} - -#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ - u32 action[] = { \ - XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ - q->guc->id, \ - GUC_CONTEXT_##enable_disable, \ - } - -static void disable_scheduling_deregister(struct xe_guc *guc, - struct xe_exec_queue *q) -{ - MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); - struct xe_device *xe = guc_to_xe(guc); - int ret; - - set_min_preemption_timeout(guc, q); - smp_rmb(); - ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret) { - struct xe_gpu_scheduler *sched = &q->guc->sched; - - drm_warn(&xe->drm, "Pending enable failed to respond"); - xe_sched_submission_start(sched); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(sched); - return; - } - - clear_exec_queue_enabled(q); - set_exec_queue_pending_disable(q); - set_exec_queue_destroyed(q); - trace_xe_exec_queue_scheduling_disable(q); - - /* - * Reserve space for both G2H here as the 2nd G2H is sent from a G2H - * handler and we are not allowed to reserved G2H space in handlers. - */ - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + - G2H_LEN_DW_DEREGISTER_CONTEXT, 2); -} - -static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ - wake_up_all(&xe->ufence_wq); - - if (xe_exec_queue_is_lr(q)) - queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); - else - xe_sched_tdr_queue_imm(&q->guc->sched); -} - -/** - * xe_guc_submit_wedge() - Wedge GuC submission - * @guc: the GuC object - * - * Save exec queue's registered with GuC state by taking a ref to each queue. - * Register a DRMM handler to drop refs upon driver unload. - */ -void xe_guc_submit_wedge(struct xe_guc *guc) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - unsigned long index; - int err; - - xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); - - err = drmm_add_action_or_reset(&guc_to_xe(guc)->drm, - guc_submit_wedged_fini, guc); - if (err) { - drm_err(&xe->drm, "Failed to register xe_guc_submit clean-up on wedged.mode=2. Although device is wedged.\n"); - return; - } - - mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - if (xe_exec_queue_get_unless_zero(q)) - set_exec_queue_wedged(q); - mutex_unlock(&guc->submission_state.lock); -} - -static bool guc_submit_hint_wedged(struct xe_guc *guc) -{ - struct xe_device *xe = guc_to_xe(guc); - - if (xe->wedged.mode != 2) - return false; - - if (xe_device_wedged(xe)) - return true; - - xe_device_declare_wedged(xe); - - return true; -} - -static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w) -{ - struct xe_guc_exec_queue *ge = - container_of(w, struct xe_guc_exec_queue, lr_tdr); - struct xe_exec_queue *q = ge->q; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_gpu_scheduler *sched = &ge->sched; - bool wedged; - - xe_assert(xe, xe_exec_queue_is_lr(q)); - trace_xe_exec_queue_lr_cleanup(q); - - wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); - - /* Kill the run_job / process_msg entry points */ - xe_sched_submission_stop(sched); - - /* - * Engine state now mostly stable, disable scheduling / deregister if - * needed. This cleanup routine might be called multiple times, where - * the actual async engine deregister drops the final engine ref. - * Calling disable_scheduling_deregister will mark the engine as - * destroyed and fire off the CT requests to disable scheduling / - * deregister, which we only want to do once. We also don't want to mark - * the engine as pending_disable again as this may race with the - * xe_guc_deregister_done_handler() which treats it as an unexpected - * state. - */ - if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { - struct xe_guc *guc = exec_queue_to_guc(q); - int ret; - - set_exec_queue_banned(q); - disable_scheduling_deregister(guc, q); - - /* - * Must wait for scheduling to be disabled before signalling - * any fences, if GT broken the GT reset code should signal us. - */ - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_disable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret) { - drm_warn(&xe->drm, "Schedule disable failed to respond"); - xe_sched_submission_start(sched); - xe_gt_reset_async(q->gt); - return; - } - } - - xe_sched_submission_start(sched); -} - -#define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) - -static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) -{ - struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); - u32 ctx_timestamp = xe_lrc_ctx_timestamp(q->lrc[0]); - u32 ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); - u32 timeout_ms = q->sched_props.job_timeout_ms; - u32 diff; - u64 running_time_ms; - - /* - * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch - * possible overflows with a high timeout. - */ - xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); - - if (ctx_timestamp < ctx_job_timestamp) - diff = ctx_timestamp + U32_MAX - ctx_job_timestamp; - else - diff = ctx_timestamp - ctx_job_timestamp; - - /* - * Ensure timeout is within 5% to account for an GuC scheduling latency - */ - running_time_ms = - ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); - - xe_gt_dbg(gt, - "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, running_time_ms, timeout_ms, diff); - - return running_time_ms >= timeout_ms; -} - -static void enable_scheduling(struct xe_exec_queue *q) -{ - MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); - struct xe_guc *guc = exec_queue_to_guc(q); - int ret; - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - - set_exec_queue_pending_enable(q); - set_exec_queue_enabled(q); - trace_xe_exec_queue_scheduling_enable(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); - - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) { - xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); - set_exec_queue_banned(q); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(&q->guc->sched); - } -} - -static void disable_scheduling(struct xe_exec_queue *q, bool immediate) -{ - MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); - struct xe_guc *guc = exec_queue_to_guc(q); - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - - if (immediate) - set_min_preemption_timeout(guc, q); - clear_exec_queue_enabled(q); - set_exec_queue_pending_disable(q); - trace_xe_exec_queue_scheduling_disable(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); -} - -static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) -{ - u32 action[] = { - XE_GUC_ACTION_DEREGISTER_CONTEXT, - q->guc->id, - }; - - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - - set_exec_queue_destroyed(q); - trace_xe_exec_queue_deregister(q); - - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_DEREGISTER_CONTEXT, 1); -} - -static enum drm_gpu_sched_stat -guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) -{ - struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_sched_job *tmp_job; - struct xe_exec_queue *q = job->q; - struct xe_gpu_scheduler *sched = &q->guc->sched; - struct xe_guc *guc = exec_queue_to_guc(q); -<<<<<<< -======= - const char *process_name = "no process"; ->>>>>>> - int err = -ETIME; - int i = 0; - bool wedged, skip_timeout_check; - - /* - * TDR has fired before free job worker. Common if exec queue - * immediately closed after last fence signaled. - */ - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) { - guc_exec_queue_free_job(drm_job); - - return DRM_GPU_SCHED_STAT_NOMINAL; - } - - /* Kill the run_job entry point */ - xe_sched_submission_stop(sched); - - /* Must check all state after stopping scheduler */ - skip_timeout_check = exec_queue_reset(q) || - exec_queue_killed_or_banned_or_wedged(q) || - exec_queue_destroyed(q); - - /* Job hasn't started, can't be timed out */ - if (!skip_timeout_check && !xe_sched_job_started(job)) - goto rearm; - - /* - * XXX: Sampling timeout doesn't work in wedged mode as we have to - * modify scheduling state to read timestamp. We could read the - * timestamp from a register to accumulate current running time but this - * doesn't work for SRIOV. For now assuming timeouts in wedged mode are - * genuine timeouts. - */ - wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); - - /* Engine state now stable, disable scheduling to check timestamp */ - if (!wedged && exec_queue_registered(q)) { - int ret; - - if (exec_queue_reset(q)) - err = -EIO; - - if (!exec_queue_destroyed(q)) { - /* - * Wait for any pending G2H to flush out before - * modifying state - */ - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_enable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) - goto trigger_reset; - - /* - * Flag communicates to G2H handler that schedule - * disable originated from a timeout check. The G2H then - * avoid triggering cleanup or deregistering the exec - * queue. - */ - set_exec_queue_check_timeout(q); - disable_scheduling(q, skip_timeout_check); - } - - /* - * Must wait for scheduling to be disabled before signalling - * any fences, if GT broken the GT reset code should signal us. - * - * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault - * error) messages which can cause the schedule disable to get - * lost. If this occurs, trigger a GT reset to recover. - */ - smp_rmb(); - ret = wait_event_timeout(guc->ct.wq, - !exec_queue_pending_disable(q) || - guc_read_stopped(guc), HZ * 5); - if (!ret || guc_read_stopped(guc)) { -trigger_reset: - if (!ret) - xe_gt_warn(guc_to_gt(guc), "Schedule disable failed to respond"); - set_exec_queue_extra_ref(q); - xe_exec_queue_get(q); /* GT reset owns this */ - set_exec_queue_banned(q); - xe_gt_reset_async(q->gt); - xe_sched_tdr_queue_imm(sched); - goto rearm; - } - } - - /* - * Check if job is actually timed out, if so restart job execution and TDR - */ - if (!wedged && !skip_timeout_check && !check_timeout(q, job) && - !exec_queue_reset(q) && exec_queue_registered(q)) { - clear_exec_queue_check_timeout(q); - goto sched_enable; - } - -<<<<<<< - if (q->vm && q->vm->xef) { - process_name = q->vm->xef->process_name; - pid = q->vm->xef->pid; - } - xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, q->flags, process_name, pid); - -======= - xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", - xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), - q->guc->id, q->flags); ->>>>>>> - trace_xe_sched_job_timedout(job); - - if (!exec_queue_killed(q)) - xe_devcoredump(job); - - /* - * Kernel jobs should never fail, nor should VM jobs if they do - * somethings has gone wrong and the GT needs a reset - */ - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL, - "Kernel-submitted job timed out\n"); - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q), - "VM job timed out on non-killed execqueue\n"); - if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL || - (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) { - if (!xe_sched_invalidate_job(job, 2)) { - clear_exec_queue_check_timeout(q); - xe_gt_reset_async(q->gt); - goto rearm; - } - } - - /* Finish cleaning up exec queue via deregister */ - set_exec_queue_banned(q); - if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { - set_exec_queue_extra_ref(q); - xe_exec_queue_get(q); - __deregister_exec_queue(guc, q); - } - - /* Stop fence signaling */ - xe_hw_fence_irq_stop(q->fence_irq); - - /* - * Fence state now stable, stop / start scheduler which cleans up any - * fences that are complete - */ - xe_sched_add_pending_job(sched, job); - xe_sched_submission_start(sched); - - xe_guc_exec_queue_trigger_cleanup(q); - - /* Mark all outstanding jobs as bad, thus completing them */ - spin_lock(&sched->base.job_list_lock); - list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list) - xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED); - spin_unlock(&sched->base.job_list_lock); - - /* Start fence signaling */ - xe_hw_fence_irq_start(q->fence_irq); - - return DRM_GPU_SCHED_STAT_NOMINAL; - -sched_enable: - enable_scheduling(q); -rearm: - /* - * XXX: Ideally want to adjust timeout based on current exection time - * but there is not currently an easy way to do in DRM scheduler. With - * some thought, do this in a follow up. - */ - xe_sched_add_pending_job(sched, job); - xe_sched_submission_start(sched); - - return DRM_GPU_SCHED_STAT_NOMINAL; -} - -static void __guc_exec_queue_fini_async(struct work_struct *w) -{ - struct xe_guc_exec_queue *ge = - container_of(w, struct xe_guc_exec_queue, fini_async); - struct xe_exec_queue *q = ge->q; - struct xe_guc *guc = exec_queue_to_guc(q); - - xe_pm_runtime_get(guc_to_xe(guc)); - trace_xe_exec_queue_destroy(q); - - if (xe_exec_queue_is_lr(q)) - cancel_work_sync(&ge->lr_tdr); - release_guc_id(guc, q); - xe_sched_entity_fini(&ge->entity); - xe_sched_fini(&ge->sched); - - kfree(ge); - xe_exec_queue_fini(q); - xe_pm_runtime_put(guc_to_xe(guc)); -} - -static void guc_exec_queue_fini_async(struct xe_exec_queue *q) -{ - INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async); - - /* We must block on kernel engines so slabs are empty on driver unload */ - if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) - __guc_exec_queue_fini_async(&q->guc->fini_async); - else - queue_work(system_wq, &q->guc->fini_async); -} - -static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q) -{ - /* - * Might be done from within the GPU scheduler, need to do async as we - * fini the scheduler when the engine is fini'd, the scheduler can't - * complete fini within itself (circular dependency). Async resolves - * this we and don't really care when everything is fini'd, just that it - * is. - */ - guc_exec_queue_fini_async(q); -} - -static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); - trace_xe_exec_queue_cleanup_entity(q); - - if (exec_queue_registered(q)) - disable_scheduling_deregister(guc, q); - else - __guc_exec_queue_fini(guc, q); -} - -static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) -{ - return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); -} - -static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - - if (guc_exec_queue_allowed_to_change_state(q)) - init_policies(guc, q); - kfree(msg); -} - -static void suspend_fence_signal(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, exec_queue_suspended(q) || exec_queue_killed(q) || - guc_read_stopped(guc)); - xe_assert(xe, q->guc->suspend_pending); - - q->guc->suspend_pending = false; - smp_wmb(); - wake_up(&q->guc->suspend_wait); -} - -static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - struct xe_guc *guc = exec_queue_to_guc(q); - - if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && - exec_queue_enabled(q)) { - wait_event(guc->ct.wq, q->guc->resume_time != RESUME_PENDING || - guc_read_stopped(guc)); - - if (!guc_read_stopped(guc)) { - s64 since_resume_ms = - ktime_ms_delta(ktime_get(), - q->guc->resume_time); - s64 wait_ms = q->vm->preempt.min_run_period_ms - - since_resume_ms; - - if (wait_ms > 0 && q->guc->resume_time) - msleep(wait_ms); - - set_exec_queue_suspended(q); - disable_scheduling(q, false); - } - } else if (q->guc->suspend_pending) { - set_exec_queue_suspended(q); - suspend_fence_signal(q); - } -} - -static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) -{ - struct xe_exec_queue *q = msg->private_data; - - if (guc_exec_queue_allowed_to_change_state(q)) { - q->guc->resume_time = RESUME_PENDING; - clear_exec_queue_suspended(q); - enable_scheduling(q); - } else { - clear_exec_queue_suspended(q); - } -} - -#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ -#define SET_SCHED_PROPS 2 -#define SUSPEND 3 -#define RESUME 4 - -static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) -{ - trace_xe_sched_msg_recv(msg); - - switch (msg->opcode) { - case CLEANUP: - __guc_exec_queue_process_msg_cleanup(msg); - break; - case SET_SCHED_PROPS: - __guc_exec_queue_process_msg_set_sched_props(msg); - break; - case SUSPEND: - __guc_exec_queue_process_msg_suspend(msg); - break; - case RESUME: - __guc_exec_queue_process_msg_resume(msg); - break; - default: - XE_WARN_ON("Unknown message type"); - } - - xe_pm_runtime_put(guc_to_xe(exec_queue_to_guc(msg->private_data))); -} - -static const struct drm_sched_backend_ops drm_sched_ops = { - .run_job = guc_exec_queue_run_job, - .free_job = guc_exec_queue_free_job, - .timedout_job = guc_exec_queue_timedout_job, -}; - -static const struct xe_sched_backend_ops xe_sched_ops = { - .process_msg = guc_exec_queue_process_msg, -}; - -static int guc_exec_queue_init(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct xe_guc_exec_queue *ge; - long timeout; - int err; - - xe_assert(xe, xe_device_uc_enabled(guc_to_xe(guc))); - - ge = kzalloc(sizeof(*ge), GFP_KERNEL); - if (!ge) - return -ENOMEM; - - q->guc = ge; - ge->q = q; - init_waitqueue_head(&ge->suspend_wait); - - timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : - msecs_to_jiffies(q->sched_props.job_timeout_ms); - err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, - get_submit_wq(guc), - q->lrc[0]->ring.size / MAX_JOB_SIZE_BYTES, 64, - timeout, guc_to_gt(guc)->ordered_wq, NULL, - q->name, gt_to_xe(q->gt)->drm.dev); - if (err) - goto err_free; - - sched = &ge->sched; - err = xe_sched_entity_init(&ge->entity, sched); - if (err) - goto err_sched; - - if (xe_exec_queue_is_lr(q)) - INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup); - - mutex_lock(&guc->submission_state.lock); - - err = alloc_guc_id(guc, q); - if (err) - goto err_entity; - - q->entity = &ge->entity; - - if (guc_read_stopped(guc)) - xe_sched_stop(sched); - - mutex_unlock(&guc->submission_state.lock); - - xe_exec_queue_assign_name(q, q->guc->id); - - trace_xe_exec_queue_create(q); - - return 0; - -err_entity: - mutex_unlock(&guc->submission_state.lock); - xe_sched_entity_fini(&ge->entity); -err_sched: - xe_sched_fini(&ge->sched); -err_free: - kfree(ge); - - return err; -} - -static void guc_exec_queue_kill(struct xe_exec_queue *q) -{ - trace_xe_exec_queue_kill(q); - set_exec_queue_killed(q); - xe_guc_exec_queue_trigger_cleanup(q); -} - -static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, - u32 opcode) -{ - xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q))); - - INIT_LIST_HEAD(&msg->link); - msg->opcode = opcode; - msg->private_data = q; - - trace_xe_sched_msg_add(msg); - xe_sched_add_msg(&q->guc->sched, msg); -} - -#define STATIC_MSG_CLEANUP 0 -#define STATIC_MSG_SUSPEND 1 -#define STATIC_MSG_RESUME 2 -static void guc_exec_queue_fini(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; - - if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q)) - guc_exec_queue_add_msg(q, msg, CLEANUP); - else - __guc_exec_queue_fini(exec_queue_to_guc(q), q); -} - -static int guc_exec_queue_set_priority(struct xe_exec_queue *q, - enum xe_exec_queue_priority priority) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.priority == priority || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.priority = priority; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.timeslice_us == timeslice_us || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.timeslice_us = timeslice_us; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, - u32 preempt_timeout_us) -{ - struct xe_sched_msg *msg; - - if (q->sched_props.preempt_timeout_us == preempt_timeout_us || - exec_queue_killed_or_banned_or_wedged(q)) - return 0; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (!msg) - return -ENOMEM; - - q->sched_props.preempt_timeout_us = preempt_timeout_us; - guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); - - return 0; -} - -static int guc_exec_queue_suspend(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; - - if (exec_queue_killed_or_banned_or_wedged(q) || q->guc->suspend_pending) - return -EINVAL; - - q->guc->suspend_pending = true; - guc_exec_queue_add_msg(q, msg, SUSPEND); - - return 0; -} - -static void guc_exec_queue_suspend_wait(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - - wait_event(q->guc->suspend_wait, !q->guc->suspend_pending || - guc_read_stopped(guc)); -} - -static void guc_exec_queue_resume(struct xe_exec_queue *q) -{ - struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME; - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, !q->guc->suspend_pending); - - guc_exec_queue_add_msg(q, msg, RESUME); -} - -static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) -{ - return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); -} - -/* - * All of these functions are an abstraction layer which other parts of XE can - * use to trap into the GuC backend. All of these functions, aside from init, - * really shouldn't do much other than trap into the DRM scheduler which - * synchronizes these operations. - */ -static const struct xe_exec_queue_ops guc_exec_queue_ops = { - .init = guc_exec_queue_init, - .kill = guc_exec_queue_kill, - .fini = guc_exec_queue_fini, - .set_priority = guc_exec_queue_set_priority, - .set_timeslice = guc_exec_queue_set_timeslice, - .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, - .suspend = guc_exec_queue_suspend, - .suspend_wait = guc_exec_queue_suspend_wait, - .resume = guc_exec_queue_resume, - .reset_status = guc_exec_queue_reset_status, -}; - -static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - - /* Stop scheduling + flush any DRM scheduler operations */ - xe_sched_submission_stop(sched); - - /* Clean up lost G2H + reset engine state */ - if (exec_queue_registered(q)) { - if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) - xe_exec_queue_put(q); - else if (exec_queue_destroyed(q)) - __guc_exec_queue_fini(guc, q); - } - if (q->guc->suspend_pending) { - set_exec_queue_suspended(q); - suspend_fence_signal(q); - } - atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | - EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | - EXEC_QUEUE_STATE_SUSPENDED, - &q->guc->state); - q->guc->resume_time = 0; - trace_xe_exec_queue_stop(q); - - /* - * Ban any engine (aside from kernel and engines used for VM ops) with a - * started but not complete job or if a job has gone through a GT reset - * more than twice. - */ - if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { - struct xe_sched_job *job = xe_sched_first_pending_job(sched); - bool ban = false; - - if (job) { - if ((xe_sched_job_started(job) && - !xe_sched_job_completed(job)) || - xe_sched_invalidate_job(job, 2)) { - trace_xe_sched_job_ban(job); - ban = true; - } - } else if (xe_exec_queue_is_lr(q) && - (xe_lrc_ring_head(q->lrc[0]) != xe_lrc_ring_tail(q->lrc[0]))) { - ban = true; - } - - if (ban) { - set_exec_queue_banned(q); - xe_guc_exec_queue_trigger_cleanup(q); - } - } -} - -int xe_guc_submit_reset_prepare(struct xe_guc *guc) -{ - int ret; - - /* - * Using an atomic here rather than submission_state.lock as this - * function can be called while holding the CT lock (engine reset - * failure). submission_state.lock needs the CT lock to resubmit jobs. - * Atomic is not ideal, but it works to prevent against concurrent reset - * and releasing any TDRs waiting on guc->submission_state.stopped. - */ - ret = atomic_fetch_or(1, &guc->submission_state.stopped); - smp_wmb(); - wake_up_all(&guc->ct.wq); - - return ret; -} - -void xe_guc_submit_reset_wait(struct xe_guc *guc) -{ - wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || - !guc_read_stopped(guc)); -} - -void xe_guc_submit_stop(struct xe_guc *guc) -{ - struct xe_exec_queue *q; - unsigned long index; - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, guc_read_stopped(guc) == 1); - - mutex_lock(&guc->submission_state.lock); - - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_stop(guc, q); - - mutex_unlock(&guc->submission_state.lock); - - /* - * No one can enter the backend at this point, aside from new engine - * creation which is protected by guc->submission_state.lock. - */ - -} - -static void guc_exec_queue_start(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - - if (!exec_queue_killed_or_banned_or_wedged(q)) { - int i; - - trace_xe_exec_queue_resubmit(q); - for (i = 0; i < q->width; ++i) - xe_lrc_set_ring_head(q->lrc[i], q->lrc[i]->ring.tail); - xe_sched_resubmit_jobs(sched); - } - - xe_sched_submission_start(sched); -} - -int xe_guc_submit_start(struct xe_guc *guc) -{ - struct xe_exec_queue *q; - unsigned long index; - struct xe_device *xe = guc_to_xe(guc); - - xe_assert(xe, guc_read_stopped(guc) == 1); - - mutex_lock(&guc->submission_state.lock); - atomic_dec(&guc->submission_state.stopped); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_start(q); - mutex_unlock(&guc->submission_state.lock); - - wake_up_all(&guc->ct.wq); - - return 0; -} - -static struct xe_exec_queue * -g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - - if (unlikely(guc_id >= GUC_ID_MAX)) { - drm_err(&xe->drm, "Invalid guc_id %u", guc_id); - return NULL; - } - - q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); - if (unlikely(!q)) { - drm_err(&xe->drm, "Not engine present for guc_id %u", guc_id); - return NULL; - } - - xe_assert(xe, guc_id >= q->guc->id); - xe_assert(xe, guc_id < (q->guc->id + q->width)); - - return q; -} - -static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) -{ - u32 action[] = { - XE_GUC_ACTION_DEREGISTER_CONTEXT, - q->guc->id, - }; - - xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); - - trace_xe_exec_queue_deregister(q); - - xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); -} - -static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, - u32 runnable_state) -{ - trace_xe_exec_queue_scheduling_done(q); - - if (runnable_state == 1) { - xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); - - q->guc->resume_time = ktime_get(); - clear_exec_queue_pending_enable(q); - smp_wmb(); - wake_up_all(&guc->ct.wq); - } else { - bool check_timeout = exec_queue_check_timeout(q); - - xe_gt_assert(guc_to_gt(guc), runnable_state == 0); - xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); - - clear_exec_queue_pending_disable(q); - if (q->guc->suspend_pending) { - suspend_fence_signal(q); - } else { - if (exec_queue_banned(q) || check_timeout) { - smp_wmb(); - wake_up_all(&guc->ct.wq); - } - if (!check_timeout) - deregister_exec_queue(guc, q); - } - } -} - -int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - u32 runnable_state = msg[1]; - - if (unlikely(len < 2)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - if (unlikely(!exec_queue_pending_enable(q) && - !exec_queue_pending_disable(q))) { - xe_gt_err(guc_to_gt(guc), - "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", - atomic_read(&q->guc->state), q->guc->id, - runnable_state); - return -EPROTO; - } - - handle_sched_done(guc, q, runnable_state); - - return 0; -} - -static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) -{ - trace_xe_exec_queue_deregister_done(q); - - clear_exec_queue_registered(q); - - if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) - xe_exec_queue_put(q); - else - __guc_exec_queue_fini(guc, q); -} - -int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || - exec_queue_pending_enable(q) || exec_queue_enabled(q)) { - xe_gt_err(guc_to_gt(guc), - "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", - atomic_read(&q->guc->state), q->guc->id); - return -EPROTO; - } - - handle_deregister_done(guc, q); - - return 0; -} - -int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_gt *gt = guc_to_gt(guc); - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); - - /* FIXME: Do error capture, most likely async */ - - trace_xe_exec_queue_reset(q); - - /* - * A banned engine is a NOP at this point (came from - * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel - * jobs by setting timeout of the job to the minimum value kicking - * guc_exec_queue_timedout_job. - */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); - - return 0; -} - -int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, - u32 len) -{ - struct xe_gt *gt = guc_to_gt(guc); - struct xe_device *xe = guc_to_xe(guc); - struct xe_exec_queue *q; - u32 guc_id = msg[0]; - - if (unlikely(len < 1)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - q = g2h_exec_queue_lookup(guc, guc_id); - if (unlikely(!q)) - return -EPROTO; - - xe_gt_dbg(gt, "Engine memory cat error: engine_class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); - - trace_xe_exec_queue_memory_cat_error(q); - - /* Treat the same as engine reset */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); - - return 0; -} - -int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_device *xe = guc_to_xe(guc); - u8 guc_class, instance; - u32 reason; - - if (unlikely(len != 3)) { - drm_err(&xe->drm, "Invalid length %u", len); - return -EPROTO; - } - - guc_class = msg[0]; - instance = msg[1]; - reason = msg[2]; - - /* Unexpected failure of a hardware feature, log an actual error */ - drm_err(&xe->drm, "GuC engine reset request failed on %d:%d because 0x%08X", - guc_class, instance, reason); - - xe_gt_reset_async(guc_to_gt(guc)); - - return 0; -} - -static void -guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, - struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - int i; - - snapshot->guc.wqi_head = q->guc->wqi_head; - snapshot->guc.wqi_tail = q->guc->wqi_tail; - snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); - snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); - snapshot->parallel.wq_desc.status = parallel_read(xe, map, - wq_desc.wq_status); - - if (snapshot->parallel.wq_desc.head != - snapshot->parallel.wq_desc.tail) { - for (i = snapshot->parallel.wq_desc.head; - i != snapshot->parallel.wq_desc.tail; - i = (i + sizeof(u32)) % WQ_SIZE) - snapshot->parallel.wq[i / sizeof(u32)] = - parallel_read(xe, map, wq[i / sizeof(u32)]); - } -} - -static void -guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, - struct drm_printer *p) -{ - int i; - - drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", - snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); - drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", - snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); - drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); - - if (snapshot->parallel.wq_desc.head != - snapshot->parallel.wq_desc.tail) { - for (i = snapshot->parallel.wq_desc.head; - i != snapshot->parallel.wq_desc.tail; - i = (i + sizeof(u32)) % WQ_SIZE) - drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), - snapshot->parallel.wq[i / sizeof(u32)]); - } -} - -/** - * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. - * @q: faulty exec queue - * - * This can be printed out in a later stage like during dev_coredump - * analysis. - * - * Returns: a GuC Submit Engine snapshot object that must be freed by the - * caller, using `xe_guc_exec_queue_snapshot_free`. - */ -struct xe_guc_submit_exec_queue_snapshot * -xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) -{ - struct xe_gpu_scheduler *sched = &q->guc->sched; - struct xe_guc_submit_exec_queue_snapshot *snapshot; - int i; - - snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC); - - if (!snapshot) - return NULL; - - snapshot->guc.id = q->guc->id; - memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); - snapshot->class = q->class; - snapshot->logical_mask = q->logical_mask; - snapshot->width = q->width; - snapshot->refcount = kref_read(&q->refcount); - snapshot->sched_timeout = sched->base.timeout; - snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; - snapshot->sched_props.preempt_timeout_us = - q->sched_props.preempt_timeout_us; - - snapshot->lrc = kmalloc_array(q->width, sizeof(struct xe_lrc_snapshot *), - GFP_ATOMIC); - - if (snapshot->lrc) { - for (i = 0; i < q->width; ++i) { - struct xe_lrc *lrc = q->lrc[i]; - - snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); - } - } - - snapshot->schedule_state = atomic_read(&q->guc->state); - snapshot->exec_queue_flags = q->flags; - - snapshot->parallel_execution = xe_exec_queue_is_parallel(q); - if (snapshot->parallel_execution) - guc_exec_queue_wq_snapshot_capture(q, snapshot); - - spin_lock(&sched->base.job_list_lock); - snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list); - snapshot->pending_list = kmalloc_array(snapshot->pending_list_size, - sizeof(struct pending_list_snapshot), - GFP_ATOMIC); - - if (snapshot->pending_list) { - struct xe_sched_job *job_iter; - - i = 0; - list_for_each_entry(job_iter, &sched->base.pending_list, drm.list) { - snapshot->pending_list[i].seqno = - xe_sched_job_seqno(job_iter); - snapshot->pending_list[i].fence = - dma_fence_is_signaled(job_iter->fence) ? 1 : 0; - snapshot->pending_list[i].finished = - dma_fence_is_signaled(&job_iter->drm.s_fence->finished) - ? 1 : 0; - i++; - } - } - - spin_unlock(&sched->base.job_list_lock); - - return snapshot; -} - -/** - * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. - * @snapshot: Previously captured snapshot of job. - * - * This captures some data that requires taking some locks, so it cannot be done in signaling path. - */ -void -xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - int i; - - if (!snapshot || !snapshot->lrc) - return; - - for (i = 0; i < snapshot->width; ++i) - xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); -} - -/** - * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. - * @snapshot: GuC Submit Engine snapshot object. - * @p: drm_printer where it will be printed out. - * - * This function prints out a given GuC Submit Engine snapshot object. - */ -void -xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, - struct drm_printer *p) -{ - int i; - - if (!snapshot) - return; - - drm_printf(p, "\nGuC ID: %d\n", snapshot->guc.id); - drm_printf(p, "\tName: %s\n", snapshot->name); - drm_printf(p, "\tClass: %d\n", snapshot->class); - drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); - drm_printf(p, "\tWidth: %d\n", snapshot->width); - drm_printf(p, "\tRef: %d\n", snapshot->refcount); - drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); - drm_printf(p, "\tTimeslice: %u (us)\n", - snapshot->sched_props.timeslice_us); - drm_printf(p, "\tPreempt timeout: %u (us)\n", - snapshot->sched_props.preempt_timeout_us); - - for (i = 0; snapshot->lrc && i < snapshot->width; ++i) - xe_lrc_snapshot_print(snapshot->lrc[i], p); - - drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); - drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); - - if (snapshot->parallel_execution) - guc_exec_queue_wq_snapshot_print(snapshot, p); - - for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size; - i++) - drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n", - snapshot->pending_list[i].seqno, - snapshot->pending_list[i].fence, - snapshot->pending_list[i].finished); -} - -/** - * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given - * snapshot. - * @snapshot: GuC Submit Engine snapshot object. - * - * This function free all the memory that needed to be allocated at capture - * time. - */ -void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) -{ - int i; - - if (!snapshot) - return; - - if (snapshot->lrc) { - for (i = 0; i < snapshot->width; i++) - xe_lrc_snapshot_free(snapshot->lrc[i]); - kfree(snapshot->lrc); - } - kfree(snapshot->pending_list); - kfree(snapshot); -} - -static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) -{ - struct xe_guc_submit_exec_queue_snapshot *snapshot; - - snapshot = xe_guc_exec_queue_snapshot_capture(q); - xe_guc_exec_queue_snapshot_print(snapshot, p); - xe_guc_exec_queue_snapshot_free(snapshot); -} - -/** - * xe_guc_submit_print - GuC Submit Print. - * @guc: GuC. - * @p: drm_printer where it will be printed out. - * - * This function capture and prints snapshots of **all** GuC Engines. - */ -void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) -{ - struct xe_exec_queue *q; - unsigned long index; - - if (!xe_device_uc_enabled(guc_to_xe(guc))) - return; - - mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - guc_exec_queue_print(q, p); - mutex_unlock(&guc->submission_state.lock); -} |