diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
141 files changed, 3918 insertions, 1416 deletions
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index d0d936d9137b..6947495bf34b 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -201,10 +201,10 @@ i915-y += \ display/intel_combo_phy.o \ display/intel_connector.o \ display/intel_crtc.o \ - display/intel_csr.o \ display/intel_cursor.o \ display/intel_display.o \ display/intel_display_power.o \ + display/intel_dmc.o \ display/intel_dpio_phy.o \ display/intel_dpll.o \ display/intel_dpll_mgr.o \ @@ -263,6 +263,7 @@ i915-y += \ display/intel_lvds.o \ display/intel_panel.o \ display/intel_pps.o \ + display/intel_qp_tables.o \ display/intel_sdvo.o \ display/intel_tv.o \ display/intel_vdsc.o \ diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index ce544e20f35c..16812488c5dd 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -363,10 +363,19 @@ static void gen11_dsi_program_esc_clk_div(struct intel_encoder *encoder, struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); enum port port; int afe_clk_khz; - u32 esc_clk_div_m; + int theo_word_clk, act_word_clk; + u32 esc_clk_div_m, esc_clk_div_m_phy; afe_clk_khz = afe_clk(encoder, crtc_state); - esc_clk_div_m = DIV_ROUND_UP(afe_clk_khz, DSI_MAX_ESC_CLK); + + if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) { + theo_word_clk = DIV_ROUND_UP(afe_clk_khz, 8 * DSI_MAX_ESC_CLK); + act_word_clk = max(3, theo_word_clk + (theo_word_clk + 1) % 2); + esc_clk_div_m = act_word_clk * 8; + esc_clk_div_m_phy = (act_word_clk - 1) / 2; + } else { + esc_clk_div_m = DIV_ROUND_UP(afe_clk_khz, DSI_MAX_ESC_CLK); + } for_each_dsi_port(port, intel_dsi->ports) { intel_de_write(dev_priv, ICL_DSI_ESC_CLK_DIV(port), @@ -379,6 +388,14 @@ static void gen11_dsi_program_esc_clk_div(struct intel_encoder *encoder, esc_clk_div_m & ICL_ESC_CLK_DIV_MASK); intel_de_posting_read(dev_priv, ICL_DPHY_ESC_CLK_DIV(port)); } + + if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) { + for_each_dsi_port(port, intel_dsi->ports) { + intel_de_write(dev_priv, ADL_MIPIO_DW(port, 8), + esc_clk_div_m_phy & TX_ESC_CLK_DIV_PHY); + intel_de_posting_read(dev_priv, ADL_MIPIO_DW(port, 8)); + } + } } static void get_dsi_io_power_domains(struct drm_i915_private *dev_priv, diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c index 88f424020a5f..b4e7ac51aa31 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.c +++ b/drivers/gpu/drm/i915/display/intel_atomic.c @@ -187,6 +187,26 @@ intel_connector_needs_modeset(struct intel_atomic_state *state, new_conn_state->crtc))); } +/** + * intel_any_crtc_needs_modeset - check if any CRTC needs a modeset + * @state: the atomic state corresponding to this modeset + * + * Returns true if any CRTC in @state needs a modeset. + */ +bool intel_any_crtc_needs_modeset(struct intel_atomic_state *state) +{ + struct intel_crtc *crtc; + struct intel_crtc_state *crtc_state; + int i; + + for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { + if (intel_crtc_needs_modeset(crtc_state)) + return true; + } + + return false; +} + struct intel_digital_connector_state * intel_atomic_get_digital_connector_state(struct intel_atomic_state *state, struct intel_connector *connector) diff --git a/drivers/gpu/drm/i915/display/intel_atomic.h b/drivers/gpu/drm/i915/display/intel_atomic.h index 62a3365ed5e6..d2700c74c9da 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.h +++ b/drivers/gpu/drm/i915/display/intel_atomic.h @@ -35,6 +35,7 @@ struct drm_connector_state * intel_digital_connector_duplicate_state(struct drm_connector *connector); bool intel_connector_needs_modeset(struct intel_atomic_state *state, struct drm_connector *connector); +bool intel_any_crtc_needs_modeset(struct intel_atomic_state *state); struct intel_digital_connector_state * intel_atomic_get_digital_connector_state(struct intel_atomic_state *state, struct intel_connector *connector); diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c index a35435083b60..bfb398f0432e 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.c +++ b/drivers/gpu/drm/i915/display/intel_bw.c @@ -162,7 +162,7 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel { struct intel_qgv_info qi = {}; bool is_y_tile = true; /* assume y tile may be used */ - int num_channels = dev_priv->dram_info.num_channels; + int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels); int deinterleave; int ipqdepth, ipqdepthpch; int dclk_max; @@ -267,7 +267,7 @@ void intel_bw_init_hw(struct drm_i915_private *dev_priv) if (!HAS_DISPLAY(dev_priv)) return; - if (IS_ALDERLAKE_S(dev_priv)) + if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) icl_get_bw_info(dev_priv, &adls_sa_info); else if (IS_ROCKETLAKE(dev_priv)) icl_get_bw_info(dev_priv, &rkl_sa_info); diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 4656a6edc3be..613ffcc68eba 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -28,6 +28,7 @@ #include "intel_cdclk.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_psr.h" #include "intel_sideband.h" /** @@ -1547,6 +1548,35 @@ static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco) dev_priv->cdclk.hw.vco = vco; } +static bool has_cdclk_crawl(struct drm_i915_private *i915) +{ + return INTEL_INFO(i915)->has_cdclk_crawl; +} + +static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco) +{ + int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); + u32 val; + + /* Write PLL ratio without disabling */ + val = CNL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE; + intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val); + + /* Submit freq change request */ + val |= BXT_DE_PLL_FREQ_REQ; + intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val); + + /* Timeout 200us */ + if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, + BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1)) + DRM_ERROR("timeout waiting for FREQ change request ack\n"); + + val &= ~BXT_DE_PLL_FREQ_REQ; + intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val); + + dev_priv->cdclk.hw.vco = vco; +} + static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) { if (DISPLAY_VER(dev_priv) >= 12) { @@ -1619,14 +1649,16 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, return; } - if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) { + if (has_cdclk_crawl(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) { + if (dev_priv->cdclk.hw.vco != vco) + adlp_cdclk_pll_crawl(dev_priv, vco); + } else if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) { if (dev_priv->cdclk.hw.vco != 0 && dev_priv->cdclk.hw.vco != vco) cnl_cdclk_pll_disable(dev_priv); if (dev_priv->cdclk.hw.vco != vco) cnl_cdclk_pll_enable(dev_priv, vco); - } else { if (dev_priv->cdclk.hw.vco != 0 && dev_priv->cdclk.hw.vco != vco) @@ -1819,6 +1851,28 @@ void intel_cdclk_uninit_hw(struct drm_i915_private *i915) skl_cdclk_uninit_hw(i915); } +static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv, + const struct intel_cdclk_config *a, + const struct intel_cdclk_config *b) +{ + int a_div, b_div; + + if (!has_cdclk_crawl(dev_priv)) + return false; + + /* + * The vco and cd2x divider will change independently + * from each, so we disallow cd2x change when crawling. + */ + a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk); + b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk); + + return a->vco != 0 && b->vco != 0 && + a->vco != b->vco && + a_div == b_div && + a->ref == b->ref; +} + /** * intel_cdclk_needs_modeset - Determine if changong between the CDCLK * configurations requires a modeset on all pipes @@ -1908,6 +1962,12 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to"); + for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + intel_psr_pause(intel_dp); + } + /* * Lock aux/gmbus while we change cdclk in case those * functions use cdclk. Not all platforms/ports do, @@ -1930,6 +1990,12 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, } mutex_unlock(&dev_priv->gmbus_mutex); + for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + intel_psr_resume(intel_dp); + } + if (drm_WARN(&dev_priv->drm, intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config), "cdclk state doesn't match!\n")) { @@ -2462,7 +2528,7 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state) struct drm_i915_private *dev_priv = to_i915(state->base.dev); const struct intel_cdclk_state *old_cdclk_state; struct intel_cdclk_state *new_cdclk_state; - enum pipe pipe; + enum pipe pipe = INVALID_PIPE; int ret; new_cdclk_state = intel_atomic_get_cdclk_state(state); @@ -2514,15 +2580,18 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state) if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi)) pipe = INVALID_PIPE; - } else { - pipe = INVALID_PIPE; } - if (pipe != INVALID_PIPE) { + if (intel_cdclk_can_crawl(dev_priv, + &old_cdclk_state->actual, + &new_cdclk_state->actual)) { + drm_dbg_kms(&dev_priv->drm, + "Can change cdclk via crawl\n"); + } else if (pipe != INVALID_PIPE) { new_cdclk_state->pipe = pipe; drm_dbg_kms(&dev_priv->drm, - "Can change cdclk with pipe %c active\n", + "Can change cdclk cd2x divider with pipe %c active\n", pipe_name(pipe)); } else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual, &new_cdclk_state->actual)) { @@ -2531,8 +2600,6 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state) if (ret) return ret; - new_cdclk_state->pipe = INVALID_PIPE; - drm_dbg_kms(&dev_priv->drm, "Modeset required for cdclk change\n"); } diff --git a/drivers/gpu/drm/i915/display/intel_csr.h b/drivers/gpu/drm/i915/display/intel_csr.h deleted file mode 100644 index 03c64f8af7ab..000000000000 --- a/drivers/gpu/drm/i915/display/intel_csr.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ - -#ifndef __INTEL_CSR_H__ -#define __INTEL_CSR_H__ - -struct drm_i915_private; - -#define CSR_VERSION(major, minor) ((major) << 16 | (minor)) -#define CSR_VERSION_MAJOR(version) ((version) >> 16) -#define CSR_VERSION_MINOR(version) ((version) & 0xffff) - -void intel_csr_ucode_init(struct drm_i915_private *i915); -void intel_csr_load_program(struct drm_i915_private *i915); -void intel_csr_ucode_fini(struct drm_i915_private *i915); -void intel_csr_ucode_suspend(struct drm_i915_private *i915); -void intel_csr_ucode_resume(struct drm_i915_private *i915); - -#endif /* __INTEL_CSR_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index 2ab389b38694..966e020331fb 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -383,6 +383,10 @@ static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state, if (plane_state->hw.rotation & DRM_MODE_ROTATE_180) cntl |= MCURSOR_ROTATE_180; + /* Wa_22012358565:adlp */ + if (DISPLAY_VER(dev_priv) == 13) + cntl |= MCURSOR_ARB_SLOTS(1); + return cntl; } diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index eccbdd42d223..390869bd6b63 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -249,15 +249,48 @@ static u32 icl_pll_to_ddi_clk_sel(struct intel_encoder *encoder, } } +static u32 ddi_buf_phy_link_rate(int port_clock) +{ + switch (port_clock) { + case 162000: + return DDI_BUF_PHY_LINK_RATE(0); + case 216000: + return DDI_BUF_PHY_LINK_RATE(4); + case 243000: + return DDI_BUF_PHY_LINK_RATE(5); + case 270000: + return DDI_BUF_PHY_LINK_RATE(1); + case 324000: + return DDI_BUF_PHY_LINK_RATE(6); + case 432000: + return DDI_BUF_PHY_LINK_RATE(7); + case 540000: + return DDI_BUF_PHY_LINK_RATE(2); + case 810000: + return DDI_BUF_PHY_LINK_RATE(3); + default: + MISSING_CASE(port_clock); + return DDI_BUF_PHY_LINK_RATE(0); + } +} + static void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_digital_port *dig_port = enc_to_dig_port(encoder); + enum phy phy = intel_port_to_phy(i915, encoder->port); intel_dp->DP = dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0); intel_dp->DP |= DDI_PORT_WIDTH(crtc_state->lane_count); + + if (IS_ALDERLAKE_P(i915) && intel_phy_is_tc(i915, phy)) { + intel_dp->DP |= ddi_buf_phy_link_rate(crtc_state->port_clock); + if (dig_port->tc_mode != TC_PORT_TBT_ALT) + intel_dp->DP |= DDI_BUF_CTL_TC_PHY_OWNERSHIP; + } } static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv, @@ -979,6 +1012,8 @@ static u8 intel_ddi_dp_voltage_max(struct intel_dp *intel_dp, if (DISPLAY_VER(dev_priv) >= 12) { if (intel_phy_is_combo(dev_priv, phy)) tgl_get_combo_buf_trans(encoder, crtc_state, &n_entries); + else if (IS_ALDERLAKE_P(dev_priv)) + adlp_get_dkl_buf_trans(encoder, crtc_state, &n_entries); else tgl_get_dkl_buf_trans(encoder, crtc_state, &n_entries); } else if (DISPLAY_VER(dev_priv) == 11) { @@ -1425,7 +1460,10 @@ tgl_dkl_phy_ddi_vswing_sequence(struct intel_encoder *encoder, if (enc_to_dig_port(encoder)->tc_mode == TC_PORT_TBT_ALT) return; - ddi_translations = tgl_get_dkl_buf_trans(encoder, crtc_state, &n_entries); + if (IS_ALDERLAKE_P(dev_priv)) + ddi_translations = adlp_get_dkl_buf_trans(encoder, crtc_state, &n_entries); + else + ddi_translations = tgl_get_dkl_buf_trans(encoder, crtc_state, &n_entries); if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) return; @@ -2772,7 +2810,6 @@ static void intel_ddi_pre_enable(struct intel_atomic_state *state, conn_state); /* FIXME precompute everything properly */ - /* FIXME how do we turn infoframes off again? */ if (dig_port->lspcon.active && dig_port->dp.has_hdmi_sink) dig_port->set_infoframes(encoder, crtc_state->has_infoframe, @@ -3157,6 +3194,9 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state, /* In HDMI/DVI mode, the port width, and swing/emphasis values * are ignored so nothing special needs to be done besides * enabling the port. + * + * On ADL_P the PHY link rate and lane count must be programmed but + * these are both 0 for HDMI. */ intel_de_write(dev_priv, DDI_BUF_CTL(port), dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE); @@ -4022,9 +4062,11 @@ static int intel_ddi_compute_config_late(struct intel_encoder *encoder, static void intel_ddi_encoder_destroy(struct drm_encoder *encoder) { + struct drm_i915_private *i915 = to_i915(encoder->dev); struct intel_digital_port *dig_port = enc_to_dig_port(to_intel_encoder(encoder)); intel_dp_encoder_flush_work(encoder); + intel_display_power_flush_work(i915); drm_encoder_cleanup(encoder); if (dig_port) @@ -4688,9 +4730,12 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) dig_port->hpd_pulse = intel_dp_hpd_pulse; - /* Splitter enable for eDP MSO is supported for pipe A only. */ - if (dig_port->dp.mso_link_count) + /* Splitter enable for eDP MSO is limited to certain pipes. */ + if (dig_port->dp.mso_link_count) { encoder->pipe_mask = BIT(PIPE_A); + if (IS_ALDERLAKE_P(dev_priv)) + encoder->pipe_mask |= BIT(PIPE_B); + } } /* In theory we don't need the encoder->type check, but leave it just in diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c index 7bcdd5c12028..8bfd00f49f2a 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c +++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c @@ -735,6 +735,34 @@ static const struct cnl_ddi_buf_trans rkl_combo_phy_ddi_translations_dp_hbr2_hbr { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ }; +static const struct tgl_dkl_phy_ddi_buf_trans adlp_dkl_phy_dp_ddi_trans_hbr[] = { + /* VS pre-emp Non-trans mV Pre-emph dB */ + { 0x7, 0x0, 0x01 }, /* 0 0 400mV 0 dB */ + { 0x5, 0x0, 0x06 }, /* 0 1 400mV 3.5 dB */ + { 0x2, 0x0, 0x0B }, /* 0 2 400mV 6 dB */ + { 0x0, 0x0, 0x17 }, /* 0 3 400mV 9.5 dB */ + { 0x5, 0x0, 0x00 }, /* 1 0 600mV 0 dB */ + { 0x2, 0x0, 0x08 }, /* 1 1 600mV 3.5 dB */ + { 0x0, 0x0, 0x14 }, /* 1 2 600mV 6 dB */ + { 0x2, 0x0, 0x00 }, /* 2 0 800mV 0 dB */ + { 0x0, 0x0, 0x0B }, /* 2 1 800mV 3.5 dB */ + { 0x0, 0x0, 0x00 }, /* 3 0 1200mV 0 dB */ +}; + +static const struct tgl_dkl_phy_ddi_buf_trans adlp_dkl_phy_dp_ddi_trans_hbr2_hbr3[] = { + /* VS pre-emp Non-trans mV Pre-emph dB */ + { 0x7, 0x0, 0x00 }, /* 0 0 400mV 0 dB */ + { 0x5, 0x0, 0x04 }, /* 0 1 400mV 3.5 dB */ + { 0x2, 0x0, 0x0A }, /* 0 2 400mV 6 dB */ + { 0x0, 0x0, 0x18 }, /* 0 3 400mV 9.5 dB */ + { 0x5, 0x0, 0x00 }, /* 1 0 600mV 0 dB */ + { 0x2, 0x0, 0x06 }, /* 1 1 600mV 3.5 dB */ + { 0x0, 0x0, 0x14 }, /* 1 2 600mV 6 dB */ + { 0x2, 0x0, 0x00 }, /* 2 0 800mV 0 dB */ + { 0x0, 0x0, 0x09 }, /* 2 1 800mV 3.5 dB */ + { 0x0, 0x0, 0x00 }, /* 3 0 1200mV 0 dB */ +}; + bool is_hobl_buf_trans(const struct cnl_ddi_buf_trans *table) { return table == tgl_combo_phy_ddi_translations_edp_hbr2_hobl; @@ -1348,6 +1376,31 @@ tgl_get_dkl_buf_trans(struct intel_encoder *encoder, return tgl_get_dkl_buf_trans_dp(encoder, crtc_state, n_entries); } +static const struct tgl_dkl_phy_ddi_buf_trans * +adlp_get_dkl_buf_trans_dp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + if (crtc_state->port_clock > 270000) { + *n_entries = ARRAY_SIZE(adlp_dkl_phy_dp_ddi_trans_hbr2_hbr3); + return adlp_dkl_phy_dp_ddi_trans_hbr2_hbr3; + } + + *n_entries = ARRAY_SIZE(adlp_dkl_phy_dp_ddi_trans_hbr); + return adlp_dkl_phy_dp_ddi_trans_hbr; +} + +const struct tgl_dkl_phy_ddi_buf_trans * +adlp_get_dkl_buf_trans(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + return tgl_get_dkl_buf_trans_hdmi(encoder, crtc_state, n_entries); + else + return adlp_get_dkl_buf_trans_dp(encoder, crtc_state, n_entries); +} + int intel_ddi_hdmi_num_entries(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int *default_entry) diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h index f8f0ef87e977..4c2efab38642 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h +++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h @@ -67,6 +67,10 @@ bxt_get_buf_trans(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int *n_entries); +const struct tgl_dkl_phy_ddi_buf_trans * +adlp_get_dkl_buf_trans(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries); const struct cnl_ddi_buf_trans * tgl_get_combo_buf_trans(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index f17c5f54feb6..362bff9beb5c 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -79,9 +79,9 @@ #include "intel_cdclk.h" #include "intel_color.h" #include "intel_crtc.h" -#include "intel_csr.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dmc.h" #include "intel_dp_link_training.h" #include "intel_fbc.h" #include "intel_fdi.h" @@ -975,6 +975,11 @@ void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) /* FIXME: assert CPU port conditions for SNB+ */ } + /* Wa_22012358565:adlp */ + if (DISPLAY_VER(dev_priv) == 13) + intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe), + 0, PIPE_ARB_USE_PROG_SLOTS); + reg = PIPECONF(cpu_transcoder); val = intel_de_read(dev_priv, reg); if (val & PIPECONF_ENABLE) { @@ -1690,7 +1695,8 @@ initial_plane_vma(struct drm_i915_private *i915, * important and we should probably use that space with FBC or other * features. */ - if (size * 2 > i915->stolen_usable_size) + if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) && + size * 2 > i915->stolen_usable_size) return NULL; obj = i915_gem_object_create_stolen_for_preallocated(i915, base, size); @@ -2208,6 +2214,21 @@ static void icl_set_pipe_chicken(struct intel_crtc *crtc) * across pipe */ tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU; + + /* + * "The underrun recovery mechanism should be disabled + * when the following is enabled for this pipe: + * WiDi + * Downscaling (this includes YUV420 fullblend) + * COG + * DSC + * PSR2" + * + * FIXME: enable whenever possible... + */ + if (IS_ALDERLAKE_P(dev_priv)) + tmp |= UNDERRUN_RECOVERY_DISABLE; + intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp); } @@ -3675,7 +3696,9 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy) bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy) { - if (IS_TIGERLAKE(dev_priv)) + if (IS_ALDERLAKE_P(dev_priv)) + return phy >= PHY_F && phy <= PHY_I; + else if (IS_TIGERLAKE(dev_priv)) return phy >= PHY_D && phy <= PHY_I; else if (IS_ICELAKE(dev_priv)) return phy >= PHY_C && phy <= PHY_F; @@ -5714,8 +5737,12 @@ static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state) static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + const struct intel_crtc_scaler_state *scaler_state = + &crtc_state->scaler_state; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 val = 0; + int i; switch (crtc_state->pipe_bpp) { case 18: @@ -5754,6 +5781,23 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state) if (DISPLAY_VER(dev_priv) >= 12) val |= PIPEMISC_PIXEL_ROUNDING_TRUNC; + if (IS_ALDERLAKE_P(dev_priv)) { + bool scaler_in_use = false; + + for (i = 0; i < crtc->num_scalers; i++) { + if (!scaler_state->scalers[i].in_use) + continue; + + scaler_in_use = true; + break; + } + + intel_de_rmw(dev_priv, PIPE_MISC2(crtc->pipe), + PIPE_MISC2_UNDERRUN_BUBBLE_COUNTER_MASK, + scaler_in_use ? PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN : + PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS); + } + intel_de_write(dev_priv, PIPEMISC(crtc->pipe), val); } @@ -7631,10 +7675,11 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, intel_hdmi_infoframe_enable(DP_SDP_VSC)) intel_dump_dp_vsc_sdp(dev_priv, &pipe_config->infoframes.vsc); - drm_dbg_kms(&dev_priv->drm, "vrr: %s, vmin: %d, vmax: %d, pipeline full: %d, flipline: %d, vmin vblank: %d, vmax vblank: %d\n", + drm_dbg_kms(&dev_priv->drm, "vrr: %s, vmin: %d, vmax: %d, pipeline full: %d, guardband: %d flipline: %d, vmin vblank: %d, vmax vblank: %d\n", yesno(pipe_config->vrr.enable), pipe_config->vrr.vmin, pipe_config->vrr.vmax, - pipe_config->vrr.pipeline_full, pipe_config->vrr.flipline, + pipe_config->vrr.pipeline_full, pipe_config->vrr.guardband, + pipe_config->vrr.flipline, intel_vrr_vmin_vblank_start(pipe_config), intel_vrr_vmax_vblank_start(pipe_config)); @@ -8270,6 +8315,16 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, } \ } while (0) +#define PIPE_CONF_CHECK_X_WITH_MASK(name, mask) do { \ + if ((current_config->name & (mask)) != (pipe_config->name & (mask))) { \ + pipe_config_mismatch(fastset, crtc, __stringify(name), \ + "(expected 0x%08x, found 0x%08x)", \ + current_config->name & (mask), \ + pipe_config->name & (mask)); \ + ret = false; \ + } \ +} while (0) + #define PIPE_CONF_CHECK_I(name) do { \ if (current_config->name != pipe_config->name) { \ pipe_config_mismatch(fastset, crtc, __stringify(name), \ @@ -8558,6 +8613,11 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, bp_gamma = intel_color_get_gamma_bit_precision(pipe_config); if (bp_gamma) PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma); + + PIPE_CONF_CHECK_BOOL(has_psr); + PIPE_CONF_CHECK_BOOL(has_psr2); + PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch); + PIPE_CONF_CHECK_I(dc3co_exitline); } PIPE_CONF_CHECK_BOOL(double_wide); @@ -8611,7 +8671,12 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_I(min_voltage_level); } - PIPE_CONF_CHECK_X(infoframes.enable); + if (fastset && (current_config->has_psr || pipe_config->has_psr)) + PIPE_CONF_CHECK_X_WITH_MASK(infoframes.enable, + ~intel_hdmi_infoframe_enable(DP_SDP_VSC)); + else + PIPE_CONF_CHECK_X(infoframes.enable); + PIPE_CONF_CHECK_X(infoframes.gcp); PIPE_CONF_CHECK_INFOFRAME(avi); PIPE_CONF_CHECK_INFOFRAME(spd); @@ -8640,11 +8705,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_I(vrr.vmax); PIPE_CONF_CHECK_I(vrr.flipline); PIPE_CONF_CHECK_I(vrr.pipeline_full); - - PIPE_CONF_CHECK_BOOL(has_psr); - PIPE_CONF_CHECK_BOOL(has_psr2); - PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch); - PIPE_CONF_CHECK_I(dc3co_exitline); + PIPE_CONF_CHECK_I(vrr.guardband); #undef PIPE_CONF_CHECK_X #undef PIPE_CONF_CHECK_I @@ -8750,6 +8811,38 @@ static void verify_wm_state(struct intel_crtc *crtc, hw_wm_level->lines); } + hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0; + sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0; + + if (HAS_HW_SAGV_WM(dev_priv) && + !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { + drm_err(&dev_priv->drm, + "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", + plane->base.base.id, plane->base.name, + sw_wm_level->enable, + sw_wm_level->blocks, + sw_wm_level->lines, + hw_wm_level->enable, + hw_wm_level->blocks, + hw_wm_level->lines); + } + + hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm; + sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm; + + if (HAS_HW_SAGV_WM(dev_priv) && + !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { + drm_err(&dev_priv->drm, + "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", + plane->base.base.id, plane->base.name, + sw_wm_level->enable, + sw_wm_level->blocks, + sw_wm_level->lines, + hw_wm_level->enable, + hw_wm_level->blocks, + hw_wm_level->lines); + } + /* DDB */ hw_ddb_entry = &hw->ddb_y[plane->id]; sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane->id]; @@ -9923,6 +10016,9 @@ static int intel_atomic_check(struct drm_device *dev, if (ret) goto fail; + if (intel_any_crtc_needs_modeset(state)) + any_ms = true; + if (any_ms) { ret = intel_modeset_checks(state); if (ret) @@ -11219,7 +11315,14 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) if (!HAS_DISPLAY(dev_priv)) return; - if (IS_ALDERLAKE_S(dev_priv)) { + if (IS_ALDERLAKE_P(dev_priv)) { + intel_ddi_init(dev_priv, PORT_A); + intel_ddi_init(dev_priv, PORT_B); + intel_ddi_init(dev_priv, PORT_TC1); + intel_ddi_init(dev_priv, PORT_TC2); + intel_ddi_init(dev_priv, PORT_TC3); + intel_ddi_init(dev_priv, PORT_TC4); + } else if (IS_ALDERLAKE_S(dev_priv)) { intel_ddi_init(dev_priv, PORT_A); intel_ddi_init(dev_priv, PORT_TC1); intel_ddi_init(dev_priv, PORT_TC2); @@ -11660,11 +11763,20 @@ intel_user_framebuffer_create(struct drm_device *dev, struct drm_framebuffer *fb; struct drm_i915_gem_object *obj; struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd; + struct drm_i915_private *i915; obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]); if (!obj) return ERR_PTR(-ENOENT); + /* object is backed with LMEM for discrete */ + i915 = to_i915(obj->base.dev); + if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj)) { + /* object is "remote", not in local memory */ + i915_gem_object_put(obj); + return ERR_PTR(-EREMOTE); + } + fb = intel_framebuffer_create(obj, &mode_cmd); i915_gem_object_put(obj); @@ -12183,7 +12295,7 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915) if (!HAS_DISPLAY(i915)) return 0; - intel_csr_ucode_init(i915); + intel_dmc_ucode_init(i915); i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0); i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI | @@ -12191,19 +12303,21 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915) i915->framestart_delay = 1; /* 1-4 */ + i915->window2_delay = 0; /* No DSB so no window2 delay */ + intel_mode_config_init(i915); ret = intel_cdclk_init(i915); if (ret) - goto cleanup_vga_client_pw_domain_csr; + goto cleanup_vga_client_pw_domain_dmc; ret = intel_dbuf_init(i915); if (ret) - goto cleanup_vga_client_pw_domain_csr; + goto cleanup_vga_client_pw_domain_dmc; ret = intel_bw_init(i915); if (ret) - goto cleanup_vga_client_pw_domain_csr; + goto cleanup_vga_client_pw_domain_dmc; init_llist_head(&i915->atomic_helper.free_list); INIT_WORK(&i915->atomic_helper.free_work, @@ -12215,8 +12329,8 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915) return 0; -cleanup_vga_client_pw_domain_csr: - intel_csr_ucode_fini(i915); +cleanup_vga_client_pw_domain_dmc: + intel_dmc_ucode_fini(i915); intel_power_domains_driver_remove(i915); intel_vga_unregister(i915); cleanup_bios: @@ -13295,7 +13409,7 @@ void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915) /* part #3: call after gem init */ void intel_modeset_driver_remove_nogem(struct drm_i915_private *i915) { - intel_csr_ucode_fini(i915); + intel_dmc_ucode_fini(i915); intel_power_domains_driver_remove(i915); diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index d77a0ab5cacf..88bb05d5c483 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -7,11 +7,11 @@ #include <drm/drm_fourcc.h> #include "i915_debugfs.h" -#include "intel_csr.h" #include "intel_display_debugfs.h" #include "intel_display_power.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dmc.h" #include "intel_dp.h" #include "intel_fbc.h" #include "intel_hdcp.h" @@ -532,24 +532,24 @@ static int i915_dmc_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); intel_wakeref_t wakeref; - struct intel_csr *csr; + struct intel_dmc *dmc; i915_reg_t dc5_reg, dc6_reg = {}; - if (!HAS_CSR(dev_priv)) + if (!HAS_DMC(dev_priv)) return -ENODEV; - csr = &dev_priv->csr; + dmc = &dev_priv->dmc; wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); - seq_printf(m, "fw loaded: %s\n", yesno(csr->dmc_payload != NULL)); - seq_printf(m, "path: %s\n", csr->fw_path); + seq_printf(m, "fw loaded: %s\n", yesno(intel_dmc_has_payload(dev_priv))); + seq_printf(m, "path: %s\n", dmc->fw_path); - if (!csr->dmc_payload) + if (!intel_dmc_has_payload(dev_priv)) goto out; - seq_printf(m, "version: %d.%d\n", CSR_VERSION_MAJOR(csr->version), - CSR_VERSION_MINOR(csr->version)); + seq_printf(m, "version: %d.%d\n", DMC_VERSION_MAJOR(dmc->version), + DMC_VERSION_MINOR(dmc->version)); if (DISPLAY_VER(dev_priv) >= 12) { if (IS_DGFX(dev_priv)) { @@ -568,10 +568,10 @@ static int i915_dmc_info(struct seq_file *m, void *unused) seq_printf(m, "DC3CO count: %d\n", intel_de_read(dev_priv, DMC_DEBUG3)); } else { - dc5_reg = IS_BROXTON(dev_priv) ? BXT_CSR_DC3_DC5_COUNT : - SKL_CSR_DC3_DC5_COUNT; + dc5_reg = IS_BROXTON(dev_priv) ? BXT_DMC_DC3_DC5_COUNT : + SKL_DMC_DC3_DC5_COUNT; if (!IS_GEMINILAKE(dev_priv) && !IS_BROXTON(dev_priv)) - dc6_reg = SKL_CSR_DC5_DC6_COUNT; + dc6_reg = SKL_DMC_DC5_DC6_COUNT; } seq_printf(m, "DC3 -> DC5 count: %d\n", @@ -582,10 +582,10 @@ static int i915_dmc_info(struct seq_file *m, void *unused) out: seq_printf(m, "program base: 0x%08x\n", - intel_de_read(dev_priv, CSR_PROGRAM(0))); + intel_de_read(dev_priv, DMC_PROGRAM(0))); seq_printf(m, "ssp base: 0x%08x\n", - intel_de_read(dev_priv, CSR_SSP_BASE)); - seq_printf(m, "htp: 0x%08x\n", intel_de_read(dev_priv, CSR_HTP_SKL)); + intel_de_read(dev_priv, DMC_SSP_BASE)); + seq_printf(m, "htp: 0x%08x\n", intel_de_read(dev_priv, DMC_HTP_SKL)); intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 54c6d65011ee..4298ae684d7d 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -9,10 +9,10 @@ #include "i915_irq.h" #include "intel_cdclk.h" #include "intel_combo_phy.h" -#include "intel_csr.h" #include "intel_display_power.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dmc.h" #include "intel_dpio_phy.h" #include "intel_hotplug.h" #include "intel_pm.h" @@ -291,8 +291,7 @@ static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv, #define ICL_TBT_AUX_PW_TO_CH(pw_idx) \ ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C) -static enum aux_ch icl_tc_phy_aux_ch(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) +static enum aux_ch icl_aux_pw_to_ch(const struct i915_power_well *power_well) { int pw_idx = power_well->desc->hsw.idx; @@ -327,6 +326,15 @@ aux_ch_to_digital_port(struct drm_i915_private *dev_priv, return dig_port; } +static enum phy icl_aux_pw_to_phy(struct drm_i915_private *i915, + const struct i915_power_well *power_well) +{ + enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well); + struct intel_digital_port *dig_port = aux_ch_to_digital_port(i915, aux_ch); + + return intel_port_to_phy(i915, dig_port->base.port); +} + static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well, bool timeout_expected) @@ -468,15 +476,13 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv, hsw_wait_for_power_well_disable(dev_priv, power_well); } -#define ICL_AUX_PW_TO_PHY(pw_idx) ((pw_idx) - ICL_PW_CTL_IDX_AUX_A) - static void icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; int pw_idx = power_well->desc->hsw.idx; - enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); + enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well); u32 val; drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv)); @@ -508,7 +514,7 @@ icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv, { const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; int pw_idx = power_well->desc->hsw.idx; - enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); + enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well); u32 val; drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv)); @@ -595,7 +601,7 @@ static void icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well); + enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well); struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch); const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; bool is_tbt = power_well->desc->hsw.is_tc_tbt; @@ -619,11 +625,9 @@ icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, * or need to enable AUX on a legacy TypeC port as part of the TC-cold * exit sequence. */ - timeout_expected = is_tbt; - if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port) { + timeout_expected = is_tbt || intel_tc_cold_requires_aux_pw(dig_port); + if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port) icl_tc_cold_exit(dev_priv); - timeout_expected = true; - } hsw_wait_for_power_well_enable(dev_priv, power_well, timeout_expected); @@ -645,7 +649,7 @@ static void icl_tc_phy_aux_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well); + enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well); struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch); icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port); @@ -657,11 +661,9 @@ static void icl_aux_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - int pw_idx = power_well->desc->hsw.idx; - enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); /* non-TBT only */ - bool is_tbt = power_well->desc->hsw.is_tc_tbt; + enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well); - if (is_tbt || intel_phy_is_tc(dev_priv, phy)) + if (intel_phy_is_tc(dev_priv, phy)) return icl_tc_phy_aux_power_well_enable(dev_priv, power_well); else if (IS_ICELAKE(dev_priv)) return icl_combo_phy_aux_power_well_enable(dev_priv, @@ -674,11 +676,9 @@ static void icl_aux_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - int pw_idx = power_well->desc->hsw.idx; - enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); /* non-TBT only */ - bool is_tbt = power_well->desc->hsw.is_tc_tbt; + enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well); - if (is_tbt || intel_phy_is_tc(dev_priv, phy)) + if (intel_phy_is_tc(dev_priv, phy)) return icl_tc_phy_aux_power_well_disable(dev_priv, power_well); else if (IS_ICELAKE(dev_priv)) return icl_combo_phy_aux_power_well_disable(dev_priv, @@ -829,8 +829,8 @@ static void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv) drm_dbg_kms(&dev_priv->drm, "Resetting DC state tracking from %02x to %02x\n", - dev_priv->csr.dc_state, val); - dev_priv->csr.dc_state = val; + dev_priv->dmc.dc_state, val); + dev_priv->dmc.dc_state = val; } /** @@ -865,8 +865,8 @@ static void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state) return; if (drm_WARN_ON_ONCE(&dev_priv->drm, - state & ~dev_priv->csr.allowed_dc_mask)) - state &= dev_priv->csr.allowed_dc_mask; + state & ~dev_priv->dmc.allowed_dc_mask)) + state &= dev_priv->dmc.allowed_dc_mask; val = intel_de_read(dev_priv, DC_STATE_EN); mask = gen9_dc_mask(dev_priv); @@ -874,16 +874,16 @@ static void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state) val & mask, state); /* Check if DMC is ignoring our DC state requests */ - if ((val & mask) != dev_priv->csr.dc_state) + if ((val & mask) != dev_priv->dmc.dc_state) drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n", - dev_priv->csr.dc_state, val & mask); + dev_priv->dmc.dc_state, val & mask); val &= ~mask; val |= state; gen9_write_dc_state(dev_priv, val); - dev_priv->csr.dc_state = val & mask; + dev_priv->dmc.dc_state = val & mask; } static u32 @@ -902,7 +902,7 @@ sanitize_target_dc_state(struct drm_i915_private *dev_priv, if (target_dc_state != states[i]) continue; - if (dev_priv->csr.allowed_dc_mask & target_dc_state) + if (dev_priv->dmc.allowed_dc_mask & target_dc_state) break; target_dc_state = states[i + 1]; @@ -958,15 +958,15 @@ static void bxt_disable_dc9(struct drm_i915_private *dev_priv) intel_pps_unlock_regs_wa(dev_priv); } -static void assert_csr_loaded(struct drm_i915_private *dev_priv) +static void assert_dmc_loaded(struct drm_i915_private *dev_priv) { drm_WARN_ONCE(&dev_priv->drm, - !intel_de_read(dev_priv, CSR_PROGRAM(0)), - "CSR program storage start is NULL\n"); - drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, CSR_SSP_BASE), - "CSR SSP Base Not fine\n"); - drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, CSR_HTP_SKL), - "CSR HTP Not fine\n"); + !intel_de_read(dev_priv, DMC_PROGRAM(0)), + "DMC program storage start is NULL\n"); + drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, DMC_SSP_BASE), + "DMC SSP Base Not fine\n"); + drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, DMC_HTP_SKL), + "DMC HTP Not fine\n"); } static struct i915_power_well * @@ -1016,7 +1016,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv, state = sanitize_target_dc_state(dev_priv, state); - if (state == dev_priv->csr.target_dc_state) + if (state == dev_priv->dmc.target_dc_state) goto unlock; dc_off_enabled = power_well->desc->ops->is_enabled(dev_priv, @@ -1028,7 +1028,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv, if (!dc_off_enabled) power_well->desc->ops->enable(dev_priv, power_well); - dev_priv->csr.target_dc_state = state; + dev_priv->dmc.target_dc_state = state; if (!dc_off_enabled) power_well->desc->ops->disable(dev_priv, power_well); @@ -1057,7 +1057,7 @@ static void assert_can_enable_dc5(struct drm_i915_private *dev_priv) "DC5 already programmed to be enabled.\n"); assert_rpm_wakelock_held(&dev_priv->runtime_pm); - assert_csr_loaded(dev_priv); + assert_dmc_loaded(dev_priv); } static void gen9_enable_dc5(struct drm_i915_private *dev_priv) @@ -1084,7 +1084,7 @@ static void assert_can_enable_dc6(struct drm_i915_private *dev_priv) DC_STATE_EN_UPTO_DC6), "DC6 already programmed to be enabled.\n"); - assert_csr_loaded(dev_priv); + assert_dmc_loaded(dev_priv); } static void skl_enable_dc6(struct drm_i915_private *dev_priv) @@ -1181,7 +1181,7 @@ static void gen9_disable_dc_states(struct drm_i915_private *dev_priv) { struct intel_cdclk_config cdclk_config = {}; - if (dev_priv->csr.target_dc_state == DC_STATE_EN_DC3CO) { + if (dev_priv->dmc.target_dc_state == DC_STATE_EN_DC3CO) { tgl_disable_dc3co(dev_priv); return; } @@ -1220,10 +1220,10 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv, static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - if (!dev_priv->csr.dmc_payload) + if (!intel_dmc_has_payload(dev_priv)) return; - switch (dev_priv->csr.target_dc_state) { + switch (dev_priv->dmc.target_dc_state) { case DC_STATE_EN_DC3CO: tgl_enable_dc3co(dev_priv); break; @@ -2265,6 +2265,12 @@ intel_display_power_put_async_work(struct work_struct *work) fetch_and_zero(&power_domains->async_put_domains[1]); queue_async_put_domains_work(power_domains, fetch_and_zero(&new_work_wakeref)); + } else { + /* + * Cancel the work that got queued after this one got dequeued, + * since here we released the corresponding async-put reference. + */ + cancel_delayed_work(&power_domains->async_put_work); } out_verify: @@ -3072,7 +3078,6 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_D_XELPD) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_E_XELPD) | \ - BIT_ULL(POWER_DOMAIN_AUX_C) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) | \ BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC3) | \ @@ -3084,6 +3089,10 @@ intel_display_power_put_mask_in_set(struct drm_i915_private *i915, BIT_ULL(POWER_DOMAIN_AUX_USBC2) | \ BIT_ULL(POWER_DOMAIN_AUX_USBC3) | \ BIT_ULL(POWER_DOMAIN_AUX_USBC4) | \ + BIT_ULL(POWER_DOMAIN_AUX_TBT1) | \ + BIT_ULL(POWER_DOMAIN_AUX_TBT2) | \ + BIT_ULL(POWER_DOMAIN_AUX_TBT3) | \ + BIT_ULL(POWER_DOMAIN_AUX_TBT4) | \ BIT_ULL(POWER_DOMAIN_INIT)) /* @@ -5090,10 +5099,10 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) dev_priv->params.disable_power_well = sanitize_disable_power_well_option(dev_priv, dev_priv->params.disable_power_well); - dev_priv->csr.allowed_dc_mask = + dev_priv->dmc.allowed_dc_mask = get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc); - dev_priv->csr.target_dc_state = + dev_priv->dmc.target_dc_state = sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6); BUILD_BUG_ON(POWER_DOMAIN_NUM > 64); @@ -5245,6 +5254,9 @@ static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv) { enum dbuf_slice slice; + if (IS_ALDERLAKE_P(dev_priv)) + return; + for_each_dbuf_slice(dev_priv, slice) intel_de_rmw(dev_priv, DBUF_CTL_S(slice), DBUF_TRACKER_STATE_SERVICE_MASK, @@ -5256,6 +5268,9 @@ static void icl_mbus_init(struct drm_i915_private *dev_priv) unsigned long abox_regs = INTEL_INFO(dev_priv)->abox_mask; u32 mask, val, i; + if (IS_ALDERLAKE_P(dev_priv)) + return; + mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK | MBUS_ABOX_BT_CREDIT_POOL2_MASK | MBUS_ABOX_B_CREDIT_MASK | @@ -5573,8 +5588,8 @@ static void skl_display_core_init(struct drm_i915_private *dev_priv, gen9_dbuf_enable(dev_priv); - if (resume && dev_priv->csr.dmc_payload) - intel_csr_load_program(dev_priv); + if (resume && intel_dmc_has_payload(dev_priv)) + intel_dmc_load_program(dev_priv); } static void skl_display_core_uninit(struct drm_i915_private *dev_priv) @@ -5640,8 +5655,8 @@ static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume gen9_dbuf_enable(dev_priv); - if (resume && dev_priv->csr.dmc_payload) - intel_csr_load_program(dev_priv); + if (resume && intel_dmc_has_payload(dev_priv)) + intel_dmc_load_program(dev_priv); } static void bxt_display_core_uninit(struct drm_i915_private *dev_priv) @@ -5706,8 +5721,8 @@ static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume /* 6. Enable DBUF */ gen9_dbuf_enable(dev_priv); - if (resume && dev_priv->csr.dmc_payload) - intel_csr_load_program(dev_priv); + if (resume && intel_dmc_has_payload(dev_priv)) + intel_dmc_load_program(dev_priv); } static void cnl_display_core_uninit(struct drm_i915_private *dev_priv) @@ -5863,8 +5878,8 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv, if (DISPLAY_VER(dev_priv) >= 12) tgl_bw_buddy_init(dev_priv); - if (resume && dev_priv->csr.dmc_payload) - intel_csr_load_program(dev_priv); + if (resume && intel_dmc_has_payload(dev_priv)) + intel_dmc_load_program(dev_priv); /* Wa_14011508470 */ if (DISPLAY_VER(dev_priv) == 12) { @@ -6218,13 +6233,13 @@ void intel_power_domains_suspend(struct drm_i915_private *i915, /* * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9 * support don't manually deinit the power domains. This also means the - * CSR/DMC firmware will stay active, it will power down any HW + * DMC firmware will stay active, it will power down any HW * resources as required and also enable deeper system power states * that would be blocked if the firmware was inactive. */ - if (!(i915->csr.allowed_dc_mask & DC_STATE_EN_DC9) && + if (!(i915->dmc.allowed_dc_mask & DC_STATE_EN_DC9) && suspend_mode == I915_DRM_SUSPEND_IDLE && - i915->csr.dmc_payload) { + intel_dmc_has_payload(i915)) { intel_display_power_flush_work(i915); intel_power_domains_verify_state(i915); return; @@ -6414,19 +6429,19 @@ void intel_display_power_resume(struct drm_i915_private *i915) if (DISPLAY_VER(i915) >= 11) { bxt_disable_dc9(i915); icl_display_core_init(i915, true); - if (i915->csr.dmc_payload) { - if (i915->csr.allowed_dc_mask & + if (intel_dmc_has_payload(i915)) { + if (i915->dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC6) skl_enable_dc6(i915); - else if (i915->csr.allowed_dc_mask & + else if (i915->dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5) gen9_enable_dc5(i915); } } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) { bxt_disable_dc9(i915); bxt_display_core_init(i915, true); - if (i915->csr.dmc_payload && - (i915->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) + if (intel_dmc_has_payload(i915) && + (i915->dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) gen9_enable_dc5(i915); } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { hsw_disable_pc8(i915); diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 8696b39da2fe..15e91a99c8b9 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -1202,7 +1202,7 @@ struct intel_crtc_state { struct { bool enable; u8 pipeline_full; - u16 flipline, vmin, vmax; + u16 flipline, vmin, vmax, guardband; } vrr; /* Stream Splitter for eDP MSO */ @@ -1482,6 +1482,7 @@ struct intel_psr { bool sink_support; bool source_support; bool enabled; + bool paused; enum pipe pipe; enum transcoder transcoder; bool active; @@ -1498,7 +1499,7 @@ struct intel_psr { bool sink_not_reliable; bool irq_aux_error; u16 su_x_granularity; - bool dc3co_enabled; + u32 dc3co_exitline; u32 dc3co_exit_delay; struct delayed_work dc3co_work; struct drm_dp_vsc_sdp vsc; diff --git a/drivers/gpu/drm/i915/display/intel_csr.c b/drivers/gpu/drm/i915/display/intel_dmc.c index 26a922d34263..97308da28059 100644 --- a/drivers/gpu/drm/i915/display/intel_csr.c +++ b/drivers/gpu/drm/i915/display/intel_dmc.c @@ -26,14 +26,13 @@ #include "i915_drv.h" #include "i915_reg.h" -#include "intel_csr.h" #include "intel_de.h" +#include "intel_dmc.h" /** - * DOC: csr support for dmc + * DOC: DMC Firmware Support * - * Display Context Save and Restore (CSR) firmware support added from gen9 - * onwards to drive newly added DMC (Display microcontroller) in display + * From gen9 onwards we have newly added DMC (Display microcontroller) in display * engine to save and restore the state of display engine when it enter into * low-power state and comes back to normal. */ @@ -44,55 +43,55 @@ __stringify(major) "_" \ __stringify(minor) ".bin" -#define GEN12_CSR_MAX_FW_SIZE ICL_CSR_MAX_FW_SIZE +#define GEN12_DMC_MAX_FW_SIZE ICL_DMC_MAX_FW_SIZE -#define ADLS_CSR_PATH DMC_PATH(adls, 2, 01) -#define ADLS_CSR_VERSION_REQUIRED CSR_VERSION(2, 1) -MODULE_FIRMWARE(ADLS_CSR_PATH); +#define ADLS_DMC_PATH DMC_PATH(adls, 2, 01) +#define ADLS_DMC_VERSION_REQUIRED DMC_VERSION(2, 1) +MODULE_FIRMWARE(ADLS_DMC_PATH); -#define DG1_CSR_PATH DMC_PATH(dg1, 2, 02) -#define DG1_CSR_VERSION_REQUIRED CSR_VERSION(2, 2) -MODULE_FIRMWARE(DG1_CSR_PATH); +#define DG1_DMC_PATH DMC_PATH(dg1, 2, 02) +#define DG1_DMC_VERSION_REQUIRED DMC_VERSION(2, 2) +MODULE_FIRMWARE(DG1_DMC_PATH); -#define RKL_CSR_PATH DMC_PATH(rkl, 2, 02) -#define RKL_CSR_VERSION_REQUIRED CSR_VERSION(2, 2) -MODULE_FIRMWARE(RKL_CSR_PATH); +#define RKL_DMC_PATH DMC_PATH(rkl, 2, 02) +#define RKL_DMC_VERSION_REQUIRED DMC_VERSION(2, 2) +MODULE_FIRMWARE(RKL_DMC_PATH); -#define TGL_CSR_PATH DMC_PATH(tgl, 2, 08) -#define TGL_CSR_VERSION_REQUIRED CSR_VERSION(2, 8) -MODULE_FIRMWARE(TGL_CSR_PATH); +#define TGL_DMC_PATH DMC_PATH(tgl, 2, 08) +#define TGL_DMC_VERSION_REQUIRED DMC_VERSION(2, 8) +MODULE_FIRMWARE(TGL_DMC_PATH); -#define ICL_CSR_PATH DMC_PATH(icl, 1, 09) -#define ICL_CSR_VERSION_REQUIRED CSR_VERSION(1, 9) -#define ICL_CSR_MAX_FW_SIZE 0x6000 -MODULE_FIRMWARE(ICL_CSR_PATH); +#define ICL_DMC_PATH DMC_PATH(icl, 1, 09) +#define ICL_DMC_VERSION_REQUIRED DMC_VERSION(1, 9) +#define ICL_DMC_MAX_FW_SIZE 0x6000 +MODULE_FIRMWARE(ICL_DMC_PATH); -#define CNL_CSR_PATH DMC_PATH(cnl, 1, 07) -#define CNL_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) -#define CNL_CSR_MAX_FW_SIZE GLK_CSR_MAX_FW_SIZE -MODULE_FIRMWARE(CNL_CSR_PATH); +#define CNL_DMC_PATH DMC_PATH(cnl, 1, 07) +#define CNL_DMC_VERSION_REQUIRED DMC_VERSION(1, 7) +#define CNL_DMC_MAX_FW_SIZE GLK_DMC_MAX_FW_SIZE +MODULE_FIRMWARE(CNL_DMC_PATH); -#define GLK_CSR_PATH DMC_PATH(glk, 1, 04) -#define GLK_CSR_VERSION_REQUIRED CSR_VERSION(1, 4) -#define GLK_CSR_MAX_FW_SIZE 0x4000 -MODULE_FIRMWARE(GLK_CSR_PATH); +#define GLK_DMC_PATH DMC_PATH(glk, 1, 04) +#define GLK_DMC_VERSION_REQUIRED DMC_VERSION(1, 4) +#define GLK_DMC_MAX_FW_SIZE 0x4000 +MODULE_FIRMWARE(GLK_DMC_PATH); -#define KBL_CSR_PATH DMC_PATH(kbl, 1, 04) -#define KBL_CSR_VERSION_REQUIRED CSR_VERSION(1, 4) -#define KBL_CSR_MAX_FW_SIZE BXT_CSR_MAX_FW_SIZE -MODULE_FIRMWARE(KBL_CSR_PATH); +#define KBL_DMC_PATH DMC_PATH(kbl, 1, 04) +#define KBL_DMC_VERSION_REQUIRED DMC_VERSION(1, 4) +#define KBL_DMC_MAX_FW_SIZE BXT_DMC_MAX_FW_SIZE +MODULE_FIRMWARE(KBL_DMC_PATH); -#define SKL_CSR_PATH DMC_PATH(skl, 1, 27) -#define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 27) -#define SKL_CSR_MAX_FW_SIZE BXT_CSR_MAX_FW_SIZE -MODULE_FIRMWARE(SKL_CSR_PATH); +#define SKL_DMC_PATH DMC_PATH(skl, 1, 27) +#define SKL_DMC_VERSION_REQUIRED DMC_VERSION(1, 27) +#define SKL_DMC_MAX_FW_SIZE BXT_DMC_MAX_FW_SIZE +MODULE_FIRMWARE(SKL_DMC_PATH); -#define BXT_CSR_PATH DMC_PATH(bxt, 1, 07) -#define BXT_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) -#define BXT_CSR_MAX_FW_SIZE 0x3000 -MODULE_FIRMWARE(BXT_CSR_PATH); +#define BXT_DMC_PATH DMC_PATH(bxt, 1, 07) +#define BXT_DMC_VERSION_REQUIRED DMC_VERSION(1, 7) +#define BXT_DMC_MAX_FW_SIZE 0x3000 +MODULE_FIRMWARE(BXT_DMC_PATH); -#define CSR_DEFAULT_FW_OFFSET 0xFFFFFFFF +#define DMC_DEFAULT_FW_OFFSET 0xFFFFFFFF #define PACKAGE_MAX_FW_INFO_ENTRIES 20 #define PACKAGE_V2_MAX_FW_INFO_ENTRIES 32 #define DMC_V1_MAX_MMIO_COUNT 8 @@ -238,6 +237,11 @@ struct stepping_info { char substepping; }; +bool intel_dmc_has_payload(struct drm_i915_private *i915) +{ + return i915->dmc.dmc_payload; +} + static const struct stepping_info skl_stepping_info[] = { {'A', '0'}, {'B', '0'}, {'C', '0'}, {'D', '0'}, {'E', '0'}, {'F', '0'}, @@ -303,47 +307,47 @@ static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv) } /** - * intel_csr_load_program() - write the firmware from memory to register. + * intel_dmc_load_program() - write the firmware from memory to register. * @dev_priv: i915 drm device. * - * CSR firmware is read from a .bin file and kept in internal memory one time. + * DMC firmware is read from a .bin file and kept in internal memory one time. * Everytime display comes back from low power state this function is called to * copy the firmware from internal memory to registers. */ -void intel_csr_load_program(struct drm_i915_private *dev_priv) +void intel_dmc_load_program(struct drm_i915_private *dev_priv) { - u32 *payload = dev_priv->csr.dmc_payload; + u32 *payload = dev_priv->dmc.dmc_payload; u32 i, fw_size; - if (!HAS_CSR(dev_priv)) { + if (!HAS_DMC(dev_priv)) { drm_err(&dev_priv->drm, - "No CSR support available for this platform\n"); + "No DMC support available for this platform\n"); return; } - if (!dev_priv->csr.dmc_payload) { + if (!intel_dmc_has_payload(dev_priv)) { drm_err(&dev_priv->drm, "Tried to program CSR with empty payload\n"); return; } - fw_size = dev_priv->csr.dmc_fw_size; + fw_size = dev_priv->dmc.dmc_fw_size; assert_rpm_wakelock_held(&dev_priv->runtime_pm); preempt_disable(); for (i = 0; i < fw_size; i++) - intel_uncore_write_fw(&dev_priv->uncore, CSR_PROGRAM(i), + intel_uncore_write_fw(&dev_priv->uncore, DMC_PROGRAM(i), payload[i]); preempt_enable(); - for (i = 0; i < dev_priv->csr.mmio_count; i++) { - intel_de_write(dev_priv, dev_priv->csr.mmioaddr[i], - dev_priv->csr.mmiodata[i]); + for (i = 0; i < dev_priv->dmc.mmio_count; i++) { + intel_de_write(dev_priv, dev_priv->dmc.mmioaddr[i], + dev_priv->dmc.mmiodata[i]); } - dev_priv->csr.dc_state = 0; + dev_priv->dmc.dc_state = 0; gen9_set_dc_state_debugmask(dev_priv); } @@ -357,7 +361,7 @@ static u32 find_dmc_fw_offset(const struct intel_fw_info *fw_info, const struct stepping_info *si, u8 package_ver) { - u32 dmc_offset = CSR_DEFAULT_FW_OFFSET; + u32 dmc_offset = DMC_DEFAULT_FW_OFFSET; unsigned int i; for (i = 0; i < num_entries; i++) { @@ -392,17 +396,18 @@ static u32 find_dmc_fw_offset(const struct intel_fw_info *fw_info, return dmc_offset; } -static u32 parse_csr_fw_dmc(struct intel_csr *csr, - const struct intel_dmc_header_base *dmc_header, - size_t rem_size) +static u32 parse_dmc_fw_header(struct intel_dmc *dmc, + const struct intel_dmc_header_base *dmc_header, + size_t rem_size) { + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); unsigned int header_len_bytes, dmc_header_size, payload_size, i; const u32 *mmioaddr, *mmiodata; u32 mmio_count, mmio_count_max; u8 *payload; - BUILD_BUG_ON(ARRAY_SIZE(csr->mmioaddr) < DMC_V3_MAX_MMIO_COUNT || - ARRAY_SIZE(csr->mmioaddr) < DMC_V1_MAX_MMIO_COUNT); + BUILD_BUG_ON(ARRAY_SIZE(dmc->mmioaddr) < DMC_V3_MAX_MMIO_COUNT || + ARRAY_SIZE(dmc->mmioaddr) < DMC_V1_MAX_MMIO_COUNT); /* * Check if we can access common fields, we will checkc again below @@ -440,34 +445,34 @@ static u32 parse_csr_fw_dmc(struct intel_csr *csr, header_len_bytes = dmc_header->header_len; dmc_header_size = sizeof(*v1); } else { - DRM_ERROR("Unknown DMC fw header version: %u\n", - dmc_header->header_ver); + drm_err(&i915->drm, "Unknown DMC fw header version: %u\n", + dmc_header->header_ver); return 0; } if (header_len_bytes != dmc_header_size) { - DRM_ERROR("DMC firmware has wrong dmc header length " - "(%u bytes)\n", header_len_bytes); + drm_err(&i915->drm, "DMC firmware has wrong dmc header length " + "(%u bytes)\n", header_len_bytes); return 0; } /* Cache the dmc header info. */ if (mmio_count > mmio_count_max) { - DRM_ERROR("DMC firmware has wrong mmio count %u\n", mmio_count); + drm_err(&i915->drm, "DMC firmware has wrong mmio count %u\n", mmio_count); return 0; } for (i = 0; i < mmio_count; i++) { - if (mmioaddr[i] < CSR_MMIO_START_RANGE || - mmioaddr[i] > CSR_MMIO_END_RANGE) { - DRM_ERROR("DMC firmware has wrong mmio address 0x%x\n", - mmioaddr[i]); + if (mmioaddr[i] < DMC_MMIO_START_RANGE || + mmioaddr[i] > DMC_MMIO_END_RANGE) { + drm_err(&i915->drm, "DMC firmware has wrong mmio address 0x%x\n", + mmioaddr[i]); return 0; } - csr->mmioaddr[i] = _MMIO(mmioaddr[i]); - csr->mmiodata[i] = mmiodata[i]; + dmc->mmioaddr[i] = _MMIO(mmioaddr[i]); + dmc->mmiodata[i] = mmiodata[i]; } - csr->mmio_count = mmio_count; + dmc->mmio_count = mmio_count; rem_size -= header_len_bytes; @@ -476,34 +481,33 @@ static u32 parse_csr_fw_dmc(struct intel_csr *csr, if (rem_size < payload_size) goto error_truncated; - if (payload_size > csr->max_fw_size) { - DRM_ERROR("DMC FW too big (%u bytes)\n", payload_size); + if (payload_size > dmc->max_fw_size) { + drm_err(&i915->drm, "DMC FW too big (%u bytes)\n", payload_size); return 0; } - csr->dmc_fw_size = dmc_header->fw_size; + dmc->dmc_fw_size = dmc_header->fw_size; - csr->dmc_payload = kmalloc(payload_size, GFP_KERNEL); - if (!csr->dmc_payload) { - DRM_ERROR("Memory allocation failed for dmc payload\n"); + dmc->dmc_payload = kmalloc(payload_size, GFP_KERNEL); + if (!dmc->dmc_payload) return 0; - } payload = (u8 *)(dmc_header) + header_len_bytes; - memcpy(csr->dmc_payload, payload, payload_size); + memcpy(dmc->dmc_payload, payload, payload_size); return header_len_bytes + payload_size; error_truncated: - DRM_ERROR("Truncated DMC firmware, refusing.\n"); + drm_err(&i915->drm, "Truncated DMC firmware, refusing.\n"); return 0; } static u32 -parse_csr_fw_package(struct intel_csr *csr, +parse_dmc_fw_package(struct intel_dmc *dmc, const struct intel_package_header *package_header, const struct stepping_info *si, size_t rem_size) { + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); u32 package_size = sizeof(struct intel_package_header); u32 num_entries, max_entries, dmc_offset; const struct intel_fw_info *fw_info; @@ -516,8 +520,8 @@ parse_csr_fw_package(struct intel_csr *csr, } else if (package_header->header_ver == 2) { max_entries = PACKAGE_V2_MAX_FW_INFO_ENTRIES; } else { - DRM_ERROR("DMC firmware has unknown header version %u\n", - package_header->header_ver); + drm_err(&i915->drm, "DMC firmware has unknown header version %u\n", + package_header->header_ver); return 0; } @@ -530,8 +534,8 @@ parse_csr_fw_package(struct intel_csr *csr, goto error_truncated; if (package_header->header_len * 4 != package_size) { - DRM_ERROR("DMC firmware has wrong package header length " - "(%u bytes)\n", package_size); + drm_err(&i915->drm, "DMC firmware has wrong package header length " + "(%u bytes)\n", package_size); return 0; } @@ -543,9 +547,9 @@ parse_csr_fw_package(struct intel_csr *csr, ((u8 *)package_header + sizeof(*package_header)); dmc_offset = find_dmc_fw_offset(fw_info, num_entries, si, package_header->header_ver); - if (dmc_offset == CSR_DEFAULT_FW_OFFSET) { - DRM_ERROR("DMC firmware not supported for %c stepping\n", - si->stepping); + if (dmc_offset == DMC_DEFAULT_FW_OFFSET) { + drm_err(&i915->drm, "DMC firmware not supported for %c stepping\n", + si->stepping); return 0; } @@ -553,51 +557,53 @@ parse_csr_fw_package(struct intel_csr *csr, return package_size + dmc_offset * 4; error_truncated: - DRM_ERROR("Truncated DMC firmware, refusing.\n"); + drm_err(&i915->drm, "Truncated DMC firmware, refusing.\n"); return 0; } /* Return number of bytes parsed or 0 on error */ -static u32 parse_csr_fw_css(struct intel_csr *csr, +static u32 parse_dmc_fw_css(struct intel_dmc *dmc, struct intel_css_header *css_header, size_t rem_size) { + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); + if (rem_size < sizeof(struct intel_css_header)) { - DRM_ERROR("Truncated DMC firmware, refusing.\n"); + drm_err(&i915->drm, "Truncated DMC firmware, refusing.\n"); return 0; } if (sizeof(struct intel_css_header) != (css_header->header_len * 4)) { - DRM_ERROR("DMC firmware has wrong CSS header length " - "(%u bytes)\n", - (css_header->header_len * 4)); + drm_err(&i915->drm, "DMC firmware has wrong CSS header length " + "(%u bytes)\n", + (css_header->header_len * 4)); return 0; } - if (csr->required_version && - css_header->version != csr->required_version) { - DRM_INFO("Refusing to load DMC firmware v%u.%u," + if (dmc->required_version && + css_header->version != dmc->required_version) { + drm_info(&i915->drm, "Refusing to load DMC firmware v%u.%u," " please use v%u.%u\n", - CSR_VERSION_MAJOR(css_header->version), - CSR_VERSION_MINOR(css_header->version), - CSR_VERSION_MAJOR(csr->required_version), - CSR_VERSION_MINOR(csr->required_version)); + DMC_VERSION_MAJOR(css_header->version), + DMC_VERSION_MINOR(css_header->version), + DMC_VERSION_MAJOR(dmc->required_version), + DMC_VERSION_MINOR(dmc->required_version)); return 0; } - csr->version = css_header->version; + dmc->version = css_header->version; return sizeof(struct intel_css_header); } -static void parse_csr_fw(struct drm_i915_private *dev_priv, +static void parse_dmc_fw(struct drm_i915_private *dev_priv, const struct firmware *fw) { struct intel_css_header *css_header; struct intel_package_header *package_header; struct intel_dmc_header_base *dmc_header; - struct intel_csr *csr = &dev_priv->csr; + struct intel_dmc *dmc = &dev_priv->dmc; const struct stepping_info *si = intel_get_stepping_info(dev_priv); u32 readcount = 0; u32 r; @@ -607,7 +613,7 @@ static void parse_csr_fw(struct drm_i915_private *dev_priv, /* Extract CSS Header information */ css_header = (struct intel_css_header *)fw->data; - r = parse_csr_fw_css(csr, css_header, fw->size); + r = parse_dmc_fw_css(dmc, css_header, fw->size); if (!r) return; @@ -615,7 +621,7 @@ static void parse_csr_fw(struct drm_i915_private *dev_priv, /* Extract Package Header information */ package_header = (struct intel_package_header *)&fw->data[readcount]; - r = parse_csr_fw_package(csr, package_header, si, fw->size - readcount); + r = parse_dmc_fw_package(dmc, package_header, si, fw->size - readcount); if (!r) return; @@ -623,49 +629,49 @@ static void parse_csr_fw(struct drm_i915_private *dev_priv, /* Extract dmc_header information */ dmc_header = (struct intel_dmc_header_base *)&fw->data[readcount]; - parse_csr_fw_dmc(csr, dmc_header, fw->size - readcount); + parse_dmc_fw_header(dmc, dmc_header, fw->size - readcount); } -static void intel_csr_runtime_pm_get(struct drm_i915_private *dev_priv) +static void intel_dmc_runtime_pm_get(struct drm_i915_private *dev_priv) { - drm_WARN_ON(&dev_priv->drm, dev_priv->csr.wakeref); - dev_priv->csr.wakeref = + drm_WARN_ON(&dev_priv->drm, dev_priv->dmc.wakeref); + dev_priv->dmc.wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); } -static void intel_csr_runtime_pm_put(struct drm_i915_private *dev_priv) +static void intel_dmc_runtime_pm_put(struct drm_i915_private *dev_priv) { intel_wakeref_t wakeref __maybe_unused = - fetch_and_zero(&dev_priv->csr.wakeref); + fetch_and_zero(&dev_priv->dmc.wakeref); intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); } -static void csr_load_work_fn(struct work_struct *work) +static void dmc_load_work_fn(struct work_struct *work) { struct drm_i915_private *dev_priv; - struct intel_csr *csr; + struct intel_dmc *dmc; const struct firmware *fw = NULL; - dev_priv = container_of(work, typeof(*dev_priv), csr.work); - csr = &dev_priv->csr; + dev_priv = container_of(work, typeof(*dev_priv), dmc.work); + dmc = &dev_priv->dmc; - request_firmware(&fw, dev_priv->csr.fw_path, dev_priv->drm.dev); - parse_csr_fw(dev_priv, fw); + request_firmware(&fw, dev_priv->dmc.fw_path, dev_priv->drm.dev); + parse_dmc_fw(dev_priv, fw); - if (dev_priv->csr.dmc_payload) { - intel_csr_load_program(dev_priv); - intel_csr_runtime_pm_put(dev_priv); + if (intel_dmc_has_payload(dev_priv)) { + intel_dmc_load_program(dev_priv); + intel_dmc_runtime_pm_put(dev_priv); drm_info(&dev_priv->drm, "Finished loading DMC firmware %s (v%u.%u)\n", - dev_priv->csr.fw_path, CSR_VERSION_MAJOR(csr->version), - CSR_VERSION_MINOR(csr->version)); + dev_priv->dmc.fw_path, DMC_VERSION_MAJOR(dmc->version), + DMC_VERSION_MINOR(dmc->version)); } else { drm_notice(&dev_priv->drm, "Failed to load DMC firmware %s." " Disabling runtime power management.\n", - csr->fw_path); + dmc->fw_path); drm_notice(&dev_priv->drm, "DMC firmware homepage: %s", INTEL_UC_FIRMWARE_URL); } @@ -674,152 +680,152 @@ static void csr_load_work_fn(struct work_struct *work) } /** - * intel_csr_ucode_init() - initialize the firmware loading. + * intel_dmc_ucode_init() - initialize the firmware loading. * @dev_priv: i915 drm device. * * This function is called at the time of loading the display driver to read * firmware from a .bin file and copied into a internal memory. */ -void intel_csr_ucode_init(struct drm_i915_private *dev_priv) +void intel_dmc_ucode_init(struct drm_i915_private *dev_priv) { - struct intel_csr *csr = &dev_priv->csr; + struct intel_dmc *dmc = &dev_priv->dmc; - INIT_WORK(&dev_priv->csr.work, csr_load_work_fn); + INIT_WORK(&dev_priv->dmc.work, dmc_load_work_fn); - if (!HAS_CSR(dev_priv)) + if (!HAS_DMC(dev_priv)) return; /* - * Obtain a runtime pm reference, until CSR is loaded, to avoid entering + * Obtain a runtime pm reference, until DMC is loaded, to avoid entering * runtime-suspend. * * On error, we return with the rpm wakeref held to prevent runtime - * suspend as runtime suspend *requires* a working CSR for whatever + * suspend as runtime suspend *requires* a working DMC for whatever * reason. */ - intel_csr_runtime_pm_get(dev_priv); + intel_dmc_runtime_pm_get(dev_priv); if (IS_ALDERLAKE_S(dev_priv)) { - csr->fw_path = ADLS_CSR_PATH; - csr->required_version = ADLS_CSR_VERSION_REQUIRED; - csr->max_fw_size = GEN12_CSR_MAX_FW_SIZE; + dmc->fw_path = ADLS_DMC_PATH; + dmc->required_version = ADLS_DMC_VERSION_REQUIRED; + dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; } else if (IS_DG1(dev_priv)) { - csr->fw_path = DG1_CSR_PATH; - csr->required_version = DG1_CSR_VERSION_REQUIRED; - csr->max_fw_size = GEN12_CSR_MAX_FW_SIZE; + dmc->fw_path = DG1_DMC_PATH; + dmc->required_version = DG1_DMC_VERSION_REQUIRED; + dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; } else if (IS_ROCKETLAKE(dev_priv)) { - csr->fw_path = RKL_CSR_PATH; - csr->required_version = RKL_CSR_VERSION_REQUIRED; - csr->max_fw_size = GEN12_CSR_MAX_FW_SIZE; + dmc->fw_path = RKL_DMC_PATH; + dmc->required_version = RKL_DMC_VERSION_REQUIRED; + dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; } else if (DISPLAY_VER(dev_priv) >= 12) { - csr->fw_path = TGL_CSR_PATH; - csr->required_version = TGL_CSR_VERSION_REQUIRED; - csr->max_fw_size = GEN12_CSR_MAX_FW_SIZE; + dmc->fw_path = TGL_DMC_PATH; + dmc->required_version = TGL_DMC_VERSION_REQUIRED; + dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; } else if (DISPLAY_VER(dev_priv) == 11) { - csr->fw_path = ICL_CSR_PATH; - csr->required_version = ICL_CSR_VERSION_REQUIRED; - csr->max_fw_size = ICL_CSR_MAX_FW_SIZE; + dmc->fw_path = ICL_DMC_PATH; + dmc->required_version = ICL_DMC_VERSION_REQUIRED; + dmc->max_fw_size = ICL_DMC_MAX_FW_SIZE; } else if (IS_CANNONLAKE(dev_priv)) { - csr->fw_path = CNL_CSR_PATH; - csr->required_version = CNL_CSR_VERSION_REQUIRED; - csr->max_fw_size = CNL_CSR_MAX_FW_SIZE; + dmc->fw_path = CNL_DMC_PATH; + dmc->required_version = CNL_DMC_VERSION_REQUIRED; + dmc->max_fw_size = CNL_DMC_MAX_FW_SIZE; } else if (IS_GEMINILAKE(dev_priv)) { - csr->fw_path = GLK_CSR_PATH; - csr->required_version = GLK_CSR_VERSION_REQUIRED; - csr->max_fw_size = GLK_CSR_MAX_FW_SIZE; + dmc->fw_path = GLK_DMC_PATH; + dmc->required_version = GLK_DMC_VERSION_REQUIRED; + dmc->max_fw_size = GLK_DMC_MAX_FW_SIZE; } else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) { - csr->fw_path = KBL_CSR_PATH; - csr->required_version = KBL_CSR_VERSION_REQUIRED; - csr->max_fw_size = KBL_CSR_MAX_FW_SIZE; + dmc->fw_path = KBL_DMC_PATH; + dmc->required_version = KBL_DMC_VERSION_REQUIRED; + dmc->max_fw_size = KBL_DMC_MAX_FW_SIZE; } else if (IS_SKYLAKE(dev_priv)) { - csr->fw_path = SKL_CSR_PATH; - csr->required_version = SKL_CSR_VERSION_REQUIRED; - csr->max_fw_size = SKL_CSR_MAX_FW_SIZE; + dmc->fw_path = SKL_DMC_PATH; + dmc->required_version = SKL_DMC_VERSION_REQUIRED; + dmc->max_fw_size = SKL_DMC_MAX_FW_SIZE; } else if (IS_BROXTON(dev_priv)) { - csr->fw_path = BXT_CSR_PATH; - csr->required_version = BXT_CSR_VERSION_REQUIRED; - csr->max_fw_size = BXT_CSR_MAX_FW_SIZE; + dmc->fw_path = BXT_DMC_PATH; + dmc->required_version = BXT_DMC_VERSION_REQUIRED; + dmc->max_fw_size = BXT_DMC_MAX_FW_SIZE; } if (dev_priv->params.dmc_firmware_path) { if (strlen(dev_priv->params.dmc_firmware_path) == 0) { - csr->fw_path = NULL; + dmc->fw_path = NULL; drm_info(&dev_priv->drm, - "Disabling CSR firmware and runtime PM\n"); + "Disabling DMC firmware and runtime PM\n"); return; } - csr->fw_path = dev_priv->params.dmc_firmware_path; + dmc->fw_path = dev_priv->params.dmc_firmware_path; /* Bypass version check for firmware override. */ - csr->required_version = 0; + dmc->required_version = 0; } - if (csr->fw_path == NULL) { + if (!dmc->fw_path) { drm_dbg_kms(&dev_priv->drm, - "No known CSR firmware for platform, disabling runtime PM\n"); + "No known DMC firmware for platform, disabling runtime PM\n"); return; } - drm_dbg_kms(&dev_priv->drm, "Loading %s\n", csr->fw_path); - schedule_work(&dev_priv->csr.work); + drm_dbg_kms(&dev_priv->drm, "Loading %s\n", dmc->fw_path); + schedule_work(&dev_priv->dmc.work); } /** - * intel_csr_ucode_suspend() - prepare CSR firmware before system suspend + * intel_dmc_ucode_suspend() - prepare DMC firmware before system suspend * @dev_priv: i915 drm device * * Prepare the DMC firmware before entering system suspend. This includes * flushing pending work items and releasing any resources acquired during * init. */ -void intel_csr_ucode_suspend(struct drm_i915_private *dev_priv) +void intel_dmc_ucode_suspend(struct drm_i915_private *dev_priv) { - if (!HAS_CSR(dev_priv)) + if (!HAS_DMC(dev_priv)) return; - flush_work(&dev_priv->csr.work); + flush_work(&dev_priv->dmc.work); /* Drop the reference held in case DMC isn't loaded. */ - if (!dev_priv->csr.dmc_payload) - intel_csr_runtime_pm_put(dev_priv); + if (!intel_dmc_has_payload(dev_priv)) + intel_dmc_runtime_pm_put(dev_priv); } /** - * intel_csr_ucode_resume() - init CSR firmware during system resume + * intel_dmc_ucode_resume() - init DMC firmware during system resume * @dev_priv: i915 drm device * * Reinitialize the DMC firmware during system resume, reacquiring any - * resources released in intel_csr_ucode_suspend(). + * resources released in intel_dmc_ucode_suspend(). */ -void intel_csr_ucode_resume(struct drm_i915_private *dev_priv) +void intel_dmc_ucode_resume(struct drm_i915_private *dev_priv) { - if (!HAS_CSR(dev_priv)) + if (!HAS_DMC(dev_priv)) return; /* * Reacquire the reference to keep RPM disabled in case DMC isn't * loaded. */ - if (!dev_priv->csr.dmc_payload) - intel_csr_runtime_pm_get(dev_priv); + if (!intel_dmc_has_payload(dev_priv)) + intel_dmc_runtime_pm_get(dev_priv); } /** - * intel_csr_ucode_fini() - unload the CSR firmware. + * intel_dmc_ucode_fini() - unload the DMC firmware. * @dev_priv: i915 drm device. * * Firmmware unloading includes freeing the internal memory and reset the * firmware loading status. */ -void intel_csr_ucode_fini(struct drm_i915_private *dev_priv) +void intel_dmc_ucode_fini(struct drm_i915_private *dev_priv) { - if (!HAS_CSR(dev_priv)) + if (!HAS_DMC(dev_priv)) return; - intel_csr_ucode_suspend(dev_priv); - drm_WARN_ON(&dev_priv->drm, dev_priv->csr.wakeref); + intel_dmc_ucode_suspend(dev_priv); + drm_WARN_ON(&dev_priv->drm, dev_priv->dmc.wakeref); - kfree(dev_priv->csr.dmc_payload); + kfree(dev_priv->dmc.dmc_payload); } diff --git a/drivers/gpu/drm/i915/display/intel_dmc.h b/drivers/gpu/drm/i915/display/intel_dmc.h new file mode 100644 index 000000000000..4c22f567b61b --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_dmc.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef __INTEL_DMC_H__ +#define __INTEL_DMC_H__ + +#include "i915_reg.h" +#include "intel_wakeref.h" +#include <linux/workqueue.h> + +struct drm_i915_private; + +#define DMC_VERSION(major, minor) ((major) << 16 | (minor)) +#define DMC_VERSION_MAJOR(version) ((version) >> 16) +#define DMC_VERSION_MINOR(version) ((version) & 0xffff) + +struct intel_dmc { + struct work_struct work; + const char *fw_path; + u32 required_version; + u32 max_fw_size; /* bytes */ + u32 *dmc_payload; + u32 dmc_fw_size; /* dwords */ + u32 version; + u32 mmio_count; + i915_reg_t mmioaddr[20]; + u32 mmiodata[20]; + u32 dc_state; + u32 target_dc_state; + u32 allowed_dc_mask; + intel_wakeref_t wakeref; +}; + +void intel_dmc_ucode_init(struct drm_i915_private *i915); +void intel_dmc_load_program(struct drm_i915_private *i915); +void intel_dmc_ucode_fini(struct drm_i915_private *i915); +void intel_dmc_ucode_suspend(struct drm_i915_private *i915); +void intel_dmc_ucode_resume(struct drm_i915_private *i915); +bool intel_dmc_has_payload(struct drm_i915_private *i915); + +#endif /* __INTEL_DMC_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index 6bf6f1ec13ed..08bceae40aa8 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -128,49 +128,13 @@ intel_dp_set_lttpr_transparent_mode(struct intel_dp *intel_dp, bool enable) return drm_dp_dpcd_write(&intel_dp->aux, DP_PHY_REPEATER_MODE, &val, 1) == 1; } -/** - * intel_dp_init_lttpr_and_dprx_caps - detect LTTPR and DPRX caps, init the LTTPR link training mode - * @intel_dp: Intel DP struct - * - * Read the LTTPR common and DPRX capabilities and switch to non-transparent - * link training mode if any is detected and read the PHY capabilities for all - * detected LTTPRs. In case of an LTTPR detection error or if the number of - * LTTPRs is more than is supported (8), fall back to the no-LTTPR, - * transparent mode link training mode. - * - * Returns: - * >0 if LTTPRs were detected and the non-transparent LT mode was set. The - * DPRX capabilities are read out. - * 0 if no LTTPRs or more than 8 LTTPRs were detected or in case of a - * detection failure and the transparent LT mode was set. The DPRX - * capabilities are read out. - * <0 Reading out the DPRX capabilities failed. - */ -int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp) +static int intel_dp_init_lttpr(struct intel_dp *intel_dp) { int lttpr_count; - bool ret; int i; - ret = intel_dp_read_lttpr_common_caps(intel_dp); - - /* The DPTX shall read the DPRX caps after LTTPR detection. */ - if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd)) { - intel_dp_reset_lttpr_common_caps(intel_dp); - return -EIO; - } - - if (!ret) - return 0; - - /* - * The 0xF0000-0xF02FF range is only valid if the DPCD revision is - * at least 1.4. - */ - if (intel_dp->dpcd[DP_DPCD_REV] < 0x14) { - intel_dp_reset_lttpr_common_caps(intel_dp); + if (!intel_dp_read_lttpr_common_caps(intel_dp)) return 0; - } lttpr_count = drm_dp_lttpr_count(intel_dp->lttpr_common_caps); /* @@ -211,6 +175,37 @@ int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp) return lttpr_count; } + +/** + * intel_dp_init_lttpr_and_dprx_caps - detect LTTPR and DPRX caps, init the LTTPR link training mode + * @intel_dp: Intel DP struct + * + * Read the LTTPR common and DPRX capabilities and switch to non-transparent + * link training mode if any is detected and read the PHY capabilities for all + * detected LTTPRs. In case of an LTTPR detection error or if the number of + * LTTPRs is more than is supported (8), fall back to the no-LTTPR, + * transparent mode link training mode. + * + * Returns: + * >0 if LTTPRs were detected and the non-transparent LT mode was set. The + * DPRX capabilities are read out. + * 0 if no LTTPRs or more than 8 LTTPRs were detected or in case of a + * detection failure and the transparent LT mode was set. The DPRX + * capabilities are read out. + * <0 Reading out the DPRX capabilities failed. + */ +int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp) +{ + int lttpr_count = intel_dp_init_lttpr(intel_dp); + + /* The DPTX shall read the DPRX caps after LTTPR detection. */ + if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd)) { + intel_dp_reset_lttpr_common_caps(intel_dp); + return -EIO; + } + + return lttpr_count; +} EXPORT_SYMBOL(intel_dp_init_lttpr_and_dprx_caps); static u8 dp_voltage_max(u8 preemph) diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index 18bfe8d09277..71ac57670043 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -149,6 +149,16 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv, pll->info->name, onoff(state), onoff(cur_state)); } +static enum tc_port icl_pll_id_to_tc_port(enum intel_dpll_id id) +{ + return TC_PORT_1 + id - DPLL_ID_ICL_MGPLL1; +} + +enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port) +{ + return tc_port - TC_PORT_1 + DPLL_ID_ICL_MGPLL1; +} + static i915_reg_t intel_combo_pll_enable_reg(struct drm_i915_private *i915, struct intel_shared_dpll *pll) @@ -161,6 +171,19 @@ intel_combo_pll_enable_reg(struct drm_i915_private *i915, return CNL_DPLL_ENABLE(pll->info->id); } +static i915_reg_t +intel_tc_pll_enable_reg(struct drm_i915_private *i915, + struct intel_shared_dpll *pll) +{ + const enum intel_dpll_id id = pll->info->id; + enum tc_port tc_port = icl_pll_id_to_tc_port(id); + + if (IS_ALDERLAKE_P(i915)) + return ADLP_PORTTC_PLL_ENABLE(tc_port); + + return MG_PLL_ENABLE(tc_port); +} + /** * intel_prepare_shared_dpll - call a dpll's prepare hook * @crtc_state: CRTC, and its state, which has a shared dpll @@ -3120,16 +3143,6 @@ static void icl_calc_dpll_state(struct drm_i915_private *i915, pll_state->cfgcr1 |= DPLL_CFGCR1_CENTRAL_FREQ_8400; } -static enum tc_port icl_pll_id_to_tc_port(enum intel_dpll_id id) -{ - return id - DPLL_ID_ICL_MGPLL1; -} - -enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port) -{ - return tc_port + DPLL_ID_ICL_MGPLL1; -} - static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc, u32 *target_dco_khz, struct intel_dpll_hw_state *state, @@ -3728,12 +3741,14 @@ static bool mg_pll_get_hw_state(struct drm_i915_private *dev_priv, bool ret = false; u32 val; + i915_reg_t enable_reg = intel_tc_pll_enable_reg(dev_priv, pll); + wakeref = intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_DISPLAY_CORE); if (!wakeref) return false; - val = intel_de_read(dev_priv, MG_PLL_ENABLE(tc_port)); + val = intel_de_read(dev_priv, enable_reg); if (!(val & PLL_ENABLE)) goto out; @@ -3797,7 +3812,7 @@ static bool dkl_pll_get_hw_state(struct drm_i915_private *dev_priv, if (!wakeref) return false; - val = intel_de_read(dev_priv, MG_PLL_ENABLE(tc_port)); + val = intel_de_read(dev_priv, intel_tc_pll_enable_reg(dev_priv, pll)); if (!(val & PLL_ENABLE)) goto out; @@ -4169,8 +4184,7 @@ static void tbt_pll_enable(struct drm_i915_private *dev_priv, static void mg_pll_enable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { - i915_reg_t enable_reg = - MG_PLL_ENABLE(icl_pll_id_to_tc_port(pll->info->id)); + i915_reg_t enable_reg = intel_tc_pll_enable_reg(dev_priv, pll); icl_pll_power_enable(dev_priv, pll, enable_reg); @@ -4249,8 +4263,7 @@ static void tbt_pll_disable(struct drm_i915_private *dev_priv, static void mg_pll_disable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { - i915_reg_t enable_reg = - MG_PLL_ENABLE(icl_pll_id_to_tc_port(pll->info->id)); + i915_reg_t enable_reg = intel_tc_pll_enable_reg(dev_priv, pll); icl_pll_disable(dev_priv, pll, enable_reg); } @@ -4416,6 +4429,26 @@ static const struct intel_dpll_mgr adls_pll_mgr = { .dump_hw_state = icl_dump_hw_state, }; +static const struct dpll_info adlp_plls[] = { + { "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 }, + { "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 }, + { "TBT PLL", &tbt_pll_funcs, DPLL_ID_ICL_TBTPLL, 0 }, + { "TC PLL 1", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 }, + { "TC PLL 2", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 }, + { "TC PLL 3", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 }, + { "TC PLL 4", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 }, + { }, +}; + +static const struct intel_dpll_mgr adlp_pll_mgr = { + .dpll_info = adlp_plls, + .get_dplls = icl_get_dplls, + .put_dplls = icl_put_dplls, + .update_active_dpll = icl_update_active_dpll, + .update_ref_clks = icl_update_dpll_ref_clks, + .dump_hw_state = icl_dump_hw_state, +}; + /** * intel_shared_dpll_init - Initialize shared DPLLs * @dev: drm device @@ -4429,7 +4462,9 @@ void intel_shared_dpll_init(struct drm_device *dev) const struct dpll_info *dpll_info; int i; - if (IS_ALDERLAKE_S(dev_priv)) + if (IS_ALDERLAKE_P(dev_priv)) + dpll_mgr = &adlp_pll_mgr; + else if (IS_ALDERLAKE_S(dev_priv)) dpll_mgr = &adls_pll_mgr; else if (IS_DG1(dev_priv)) dpll_mgr = &dg1_pll_mgr; diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c index a005c68889e7..c60a81a81c09 100644 --- a/drivers/gpu/drm/i915/display/intel_fb.c +++ b/drivers/gpu/drm/i915/display/intel_fb.c @@ -355,8 +355,17 @@ static int intel_fb_offset_to_xy(int *x, int *y, unsigned int height; u32 alignment; - if (DISPLAY_VER(i915) >= 12 && - is_semiplanar_uv_plane(fb, color_plane)) + /* + * All DPT color planes must be 512*4k aligned (the amount mapped by a + * single DPT page). For ADL_P CCS FBs this only works by requiring + * the allocated offsets to be 2MB aligned. Once supoort to remap + * such FBs is added we can remove this requirement, as then all the + * planes can be remapped to an aligned offset. + */ + if (IS_ALDERLAKE_P(i915) && is_ccs_modifier(fb->modifier)) + alignment = 512 * 4096; + else if (DISPLAY_VER(i915) >= 12 && + is_semiplanar_uv_plane(fb, color_plane)) alignment = intel_tile_row_size(fb, color_plane); else if (fb->modifier != DRM_FORMAT_MOD_LINEAR) alignment = intel_tile_size(i915); diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index ccd00e65a5fe..4af40229f5ec 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -41,6 +41,8 @@ #include <drm/drm_fb_helper.h> #include <drm/drm_fourcc.h> +#include "gem/i915_gem_lmem.h" + #include "i915_drv.h" #include "intel_display_types.h" #include "intel_fbdev.h" @@ -137,14 +139,22 @@ static int intelfb_alloc(struct drm_fb_helper *helper, size = mode_cmd.pitches[0] * mode_cmd.height; size = PAGE_ALIGN(size); - /* If the FB is too big, just don't use it since fbdev is not very - * important and we should probably use that space with FBC or other - * features. */ obj = ERR_PTR(-ENODEV); - if (size * 2 < dev_priv->stolen_usable_size) - obj = i915_gem_object_create_stolen(dev_priv, size); - if (IS_ERR(obj)) - obj = i915_gem_object_create_shmem(dev_priv, size); + if (HAS_LMEM(dev_priv)) { + obj = i915_gem_object_create_lmem(dev_priv, size, + I915_BO_ALLOC_CONTIGUOUS); + } else { + /* + * If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. + */ + if (size * 2 < dev_priv->stolen_usable_size) + obj = i915_gem_object_create_stolen(dev_priv, size); + if (IS_ERR(obj)) + obj = i915_gem_object_create_shmem(dev_priv, size); + } + if (IS_ERR(obj)) { drm_err(&dev_priv->drm, "failed to allocate framebuffer\n"); return PTR_ERR(obj); @@ -178,6 +188,7 @@ static int intelfb_create(struct drm_fb_helper *helper, unsigned long flags = 0; bool prealloc = false; void __iomem *vaddr; + struct drm_i915_gem_object *obj; int ret; if (intel_fb && @@ -232,13 +243,27 @@ static int intelfb_create(struct drm_fb_helper *helper, info->fbops = &intelfb_ops; /* setup aperture base/size for vesafb takeover */ - info->apertures->ranges[0].base = ggtt->gmadr.start; - info->apertures->ranges[0].size = ggtt->mappable_end; + obj = intel_fb_obj(&intel_fb->base); + if (i915_gem_object_is_lmem(obj)) { + struct intel_memory_region *mem = obj->mm.region; + + info->apertures->ranges[0].base = mem->io_start; + info->apertures->ranges[0].size = mem->total; + + /* Use fbdev's framebuffer from lmem for discrete */ + info->fix.smem_start = + (unsigned long)(mem->io_start + + i915_gem_object_get_dma_address(obj, 0)); + info->fix.smem_len = obj->base.size; + } else { + info->apertures->ranges[0].base = ggtt->gmadr.start; + info->apertures->ranges[0].size = ggtt->mappable_end; - /* Our framebuffer is the entirety of fbdev's system memory */ - info->fix.smem_start = - (unsigned long)(ggtt->gmadr.start + vma->node.start); - info->fix.smem_len = vma->node.size; + /* Our framebuffer is the entirety of fbdev's system memory */ + info->fix.smem_start = + (unsigned long)(ggtt->gmadr.start + vma->node.start); + info->fix.smem_len = vma->node.size; + } vaddr = i915_vma_pin_iomap(vma); if (IS_ERR(vaddr)) { diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c index 3315aa1d4d5a..eb841960840d 100644 --- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c @@ -185,15 +185,34 @@ static void ivb_set_fifo_underrun_reporting(struct drm_device *dev, } } +static u32 +icl_pipe_status_underrun_mask(struct drm_i915_private *dev_priv) +{ + u32 mask = PIPE_STATUS_UNDERRUN; + + if (DISPLAY_VER(dev_priv) >= 13) + mask |= PIPE_STATUS_SOFT_UNDERRUN_XELPD | + PIPE_STATUS_HARD_UNDERRUN_XELPD | + PIPE_STATUS_PORT_UNDERRUN_XELPD; + + return mask; +} + static void bdw_set_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { struct drm_i915_private *dev_priv = to_i915(dev); + u32 mask = gen8_de_pipe_underrun_mask(dev_priv); - if (enable) - bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN); - else - bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN); + if (enable) { + if (DISPLAY_VER(dev_priv) >= 11) + intel_de_write(dev_priv, ICL_PIPESTATUS(pipe), + icl_pipe_status_underrun_mask(dev_priv)); + + bdw_enable_pipe_irq(dev_priv, pipe, mask); + } else { + bdw_disable_pipe_irq(dev_priv, pipe, mask); + } } static void ibx_set_fifo_underrun_reporting(struct drm_device *dev, @@ -373,6 +392,7 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe) { struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + u32 underruns = 0; /* We may be called too early in init, thanks BIOS! */ if (crtc == NULL) @@ -383,10 +403,35 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, crtc->cpu_fifo_underrun_disabled) return; + /* + * Starting with display version 11, the PIPE_STAT register records + * whether an underrun has happened, and on XELPD+, it will also record + * whether the underrun was soft/hard and whether it was triggered by + * the downstream port logic. We should clear these bits (which use + * write-1-to-clear logic) too. + * + * Note that although the IIR gives us the same underrun and soft/hard + * information, PIPE_STAT is the only place we can find out whether + * the underrun was caused by the downstream port. + */ + if (DISPLAY_VER(dev_priv) >= 11) { + underruns = intel_de_read(dev_priv, ICL_PIPESTATUS(pipe)) & + icl_pipe_status_underrun_mask(dev_priv); + intel_de_write(dev_priv, ICL_PIPESTATUS(pipe), underruns); + } + if (intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false)) { trace_intel_cpu_fifo_underrun(dev_priv, pipe); - drm_err(&dev_priv->drm, "CPU pipe %c FIFO underrun\n", - pipe_name(pipe)); + + if (DISPLAY_VER(dev_priv) >= 11) + drm_err(&dev_priv->drm, "CPU pipe %c FIFO underrun: %s%s%s%s\n", + pipe_name(pipe), + underruns & PIPE_STATUS_SOFT_UNDERRUN_XELPD ? "soft," : "", + underruns & PIPE_STATUS_HARD_UNDERRUN_XELPD ? "hard," : "", + underruns & PIPE_STATUS_PORT_UNDERRUN_XELPD ? "port," : "", + underruns & PIPE_STATUS_UNDERRUN ? "transcoder," : ""); + else + drm_err(&dev_priv->drm, "CPU pipe %c FIFO underrun\n", pipe_name(pipe)); } intel_fbc_handle_fifo_underrun_irq(dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c index 8161d49e78ba..8e75debcce1a 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c @@ -211,7 +211,6 @@ static int frontbuffer_active(struct i915_active *ref) return 0; } -__i915_active_call static void frontbuffer_retire(struct i915_active *ref) { struct intel_frontbuffer *front = @@ -266,7 +265,8 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj) atomic_set(&front->bits, 0); i915_active_init(&front->write, frontbuffer_active, - i915_active_may_sleep(frontbuffer_retire)); + frontbuffer_retire, + I915_ACTIVE_RETIRE_SLEEPS); spin_lock(&i915->fb_tracking.lock); if (rcu_access_pointer(obj->frontbuffer)) { diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index 4a1b2d863b0c..7e51c98c475e 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -1865,28 +1865,69 @@ static int intel_hdmi_port_clock(int clock, int bpc) return clock * bpc / 8; } +static bool intel_hdmi_bpc_possible(struct drm_connector *connector, + int bpc, bool has_hdmi_sink, bool ycbcr420_output) +{ + struct drm_i915_private *i915 = to_i915(connector->dev); + const struct drm_display_info *info = &connector->display_info; + const struct drm_hdmi_info *hdmi = &info->hdmi; + + switch (bpc) { + case 12: + if (HAS_GMCH(i915)) + return false; + + if (!has_hdmi_sink) + return false; + + if (ycbcr420_output) + return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_36; + else + return info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_36; + case 10: + if (DISPLAY_VER(i915) < 11) + return false; + + if (!has_hdmi_sink) + return false; + + if (ycbcr420_output) + return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_30; + else + return info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_30; + case 8: + return true; + default: + MISSING_CASE(bpc); + return false; + } +} + static enum drm_mode_status -intel_hdmi_mode_clock_valid(struct intel_hdmi *hdmi, int clock, bool has_hdmi_sink) +intel_hdmi_mode_clock_valid(struct drm_connector *connector, int clock, + bool has_hdmi_sink, bool ycbcr420_output) { - struct drm_device *dev = intel_hdmi_to_dev(hdmi); - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_hdmi *hdmi = intel_attached_hdmi(to_intel_connector(connector)); enum drm_mode_status status; + if (ycbcr420_output) + clock /= 2; + /* check if we can do 8bpc */ status = hdmi_port_clock_valid(hdmi, intel_hdmi_port_clock(clock, 8), true, has_hdmi_sink); - if (has_hdmi_sink) { - /* if we can't do 8bpc we may still be able to do 12bpc */ - if (status != MODE_OK && !HAS_GMCH(dev_priv)) - status = hdmi_port_clock_valid(hdmi, intel_hdmi_port_clock(clock, 12), - true, has_hdmi_sink); + /* if we can't do 8bpc we may still be able to do 12bpc */ + if (status != MODE_OK && + intel_hdmi_bpc_possible(connector, 12, has_hdmi_sink, ycbcr420_output)) + status = hdmi_port_clock_valid(hdmi, intel_hdmi_port_clock(clock, 12), + true, has_hdmi_sink); - /* if we can't do 8,12bpc we may still be able to do 10bpc */ - if (status != MODE_OK && DISPLAY_VER(dev_priv) >= 11) - status = hdmi_port_clock_valid(hdmi, intel_hdmi_port_clock(clock, 10), - true, has_hdmi_sink); - } + /* if we can't do 8,12bpc we may still be able to do 10bpc */ + if (status != MODE_OK && + intel_hdmi_bpc_possible(connector, 10, has_hdmi_sink, ycbcr420_output)) + status = hdmi_port_clock_valid(hdmi, intel_hdmi_port_clock(clock, 10), + true, has_hdmi_sink); return status; } @@ -1920,18 +1961,15 @@ intel_hdmi_mode_valid(struct drm_connector *connector, } ycbcr_420_only = drm_mode_is_420_only(&connector->display_info, mode); - if (ycbcr_420_only) - clock /= 2; - status = intel_hdmi_mode_clock_valid(hdmi, clock, has_hdmi_sink); + status = intel_hdmi_mode_clock_valid(connector, clock, has_hdmi_sink, ycbcr_420_only); if (status != MODE_OK) { if (ycbcr_420_only || !connector->ycbcr_420_allowed || !drm_mode_is_420_also(&connector->display_info, mode)) return status; - clock /= 2; - status = intel_hdmi_mode_clock_valid(hdmi, clock, has_hdmi_sink); + status = intel_hdmi_mode_clock_valid(connector, clock, has_hdmi_sink, true); if (status != MODE_OK) return status; } @@ -1950,32 +1988,12 @@ bool intel_hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, if (crtc_state->pipe_bpp < bpc * 3) return false; - if (!has_hdmi_sink) - return false; - for_each_new_connector_in_state(state, connector, connector_state, i) { - const struct drm_display_info *info = &connector->display_info; - if (connector_state->crtc != crtc_state->uapi.crtc) continue; - if (ycbcr420_output) { - const struct drm_hdmi_info *hdmi = &info->hdmi; - - if (bpc == 12 && !(hdmi->y420_dc_modes & - DRM_EDID_YCBCR420_DC_36)) - return false; - else if (bpc == 10 && !(hdmi->y420_dc_modes & - DRM_EDID_YCBCR420_DC_30)) - return false; - } else { - if (bpc == 12 && !(info->edid_hdmi_dc_modes & - DRM_EDID_HDMI_DC_36)) - return false; - else if (bpc == 10 && !(info->edid_hdmi_dc_modes & - DRM_EDID_HDMI_DC_30)) - return false; - } + if (!intel_hdmi_bpc_possible(connector, bpc, has_hdmi_sink, ycbcr420_output)) + return false; } return true; @@ -1989,12 +2007,6 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; - if (HAS_GMCH(dev_priv)) - return false; - - if (bpc == 10 && DISPLAY_VER(dev_priv) < 11) - return false; - /* * HDMI deep color affects the clocks, so it's only possible * when not cloning with other encoder types. diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c index 46cba12be888..7e3f5c6ca484 100644 --- a/drivers/gpu/drm/i915/display/intel_overlay.c +++ b/drivers/gpu/drm/i915/display/intel_overlay.c @@ -384,8 +384,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay) i830_overlay_clock_gating(dev_priv, true); } -__i915_active_call static void -intel_overlay_last_flip_retire(struct i915_active *active) +static void intel_overlay_last_flip_retire(struct i915_active *active) { struct intel_overlay *overlay = container_of(active, typeof(*overlay), last_flip); @@ -1402,7 +1401,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv) overlay->saturation = 146; i915_active_init(&overlay->last_flip, - NULL, intel_overlay_last_flip_retire); + NULL, intel_overlay_last_flip_retire, 0); ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(dev_priv)); if (ret) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index acaf3d459821..77865cf6641f 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -638,7 +638,7 @@ unlock: static void tgl_disallow_dc3co_on_psr2_exit(struct intel_dp *intel_dp) { - if (!intel_dp->psr.dc3co_enabled) + if (!intel_dp->psr.dc3co_exitline) return; cancel_delayed_work(&intel_dp->psr.dc3co_work); @@ -646,12 +646,26 @@ static void tgl_disallow_dc3co_on_psr2_exit(struct intel_dp *intel_dp) tgl_psr2_disable_dc3co(intel_dp); } +static bool +dc3co_is_pipe_port_compatible(struct intel_dp *intel_dp, + struct intel_crtc_state *crtc_state) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + enum port port = dig_port->base.port; + + if (IS_ALDERLAKE_P(dev_priv)) + return pipe <= PIPE_B && port <= PORT_B; + else + return pipe == PIPE_A && port == PORT_A; +} + static void tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state) { const u32 crtc_vdisplay = crtc_state->uapi.adjusted_mode.crtc_vdisplay; - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); u32 exit_scanlines; @@ -669,12 +683,10 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp, if (crtc_state->enable_psr2_sel_fetch) return; - if (!(dev_priv->csr.allowed_dc_mask & DC_STATE_EN_DC3CO)) + if (!(dev_priv->dmc.allowed_dc_mask & DC_STATE_EN_DC3CO)) return; - /* B.Specs:49196 DC3CO only works with pipeA and DDIA.*/ - if (to_intel_crtc(crtc_state->uapi.crtc)->pipe != PIPE_A || - dig_port->base.port != PORT_A) + if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state)) return; /* @@ -753,6 +765,15 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, return false; } + /* + * We are missing the implementation of some workarounds to enabled PSR2 + * in Alderlake_P, until ready PSR2 should be kept disabled. + */ + if (IS_ALDERLAKE_P(dev_priv)) { + drm_dbg_kms(&dev_priv->drm, "PSR2 is missing the implementation of workarounds\n"); + return false; + } + if (!transcoder_has_psr2(dev_priv, crtc_state->cpu_transcoder)) { drm_dbg_kms(&dev_priv->drm, "PSR2 not supported in transcoder %s\n", @@ -969,11 +990,10 @@ static void intel_psr_activate(struct intel_dp *intel_dp) intel_dp->psr.active = true; } -static void intel_psr_enable_source(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) +static void intel_psr_enable_source(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + enum transcoder cpu_transcoder = intel_dp->psr.transcoder; u32 mask; /* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+ @@ -1010,7 +1030,7 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp, psr_irq_control(intel_dp); - if (crtc_state->dc3co_exitline) { + if (intel_dp->psr.dc3co_exitline) { u32 val; /* @@ -1019,7 +1039,7 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp, */ val = intel_de_read(dev_priv, EXITLINE(cpu_transcoder)); val &= ~EXITLINE_MASK; - val |= crtc_state->dc3co_exitline << EXITLINE_SHIFT; + val |= intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT; val |= EXITLINE_ENABLE; intel_de_write(dev_priv, EXITLINE(cpu_transcoder), val); } @@ -1030,27 +1050,11 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp, IGNORE_PSR2_HW_TRACKING : 0); } -static void intel_psr_enable_locked(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static bool psr_interrupt_error_check(struct intel_dp *intel_dp) { - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct intel_encoder *encoder = &dig_port->base; u32 val; - drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled); - - intel_dp->psr.psr2_enabled = crtc_state->has_psr2; - intel_dp->psr.busy_frontbuffer_bits = 0; - intel_dp->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; - intel_dp->psr.dc3co_enabled = !!crtc_state->dc3co_exitline; - intel_dp->psr.transcoder = crtc_state->cpu_transcoder; - /* DC5/DC6 requires at least 6 idle frames */ - val = usecs_to_jiffies(intel_get_frame_time_us(crtc_state) * 6); - intel_dp->psr.dc3co_exit_delay = val; - intel_dp->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch; - /* * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR * will still keep the error set even after the reset done in the @@ -1071,17 +1075,45 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, intel_dp->psr.sink_not_reliable = true; drm_dbg_kms(&dev_priv->drm, "PSR interruption error set, not enabling PSR\n"); - return; + return false; } + return true; +} + +static void intel_psr_enable_locked(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_encoder *encoder = &dig_port->base; + u32 val; + + drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled); + + intel_dp->psr.psr2_enabled = crtc_state->has_psr2; + intel_dp->psr.busy_frontbuffer_bits = 0; + intel_dp->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; + intel_dp->psr.transcoder = crtc_state->cpu_transcoder; + /* DC5/DC6 requires at least 6 idle frames */ + val = usecs_to_jiffies(intel_get_frame_time_us(crtc_state) * 6); + intel_dp->psr.dc3co_exit_delay = val; + intel_dp->psr.dc3co_exitline = crtc_state->dc3co_exitline; + intel_dp->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch; + + if (!psr_interrupt_error_check(intel_dp)) + return; + drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n", intel_dp->psr.psr2_enabled ? "2" : "1"); intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state, &intel_dp->psr.vsc); intel_write_dp_vsc_sdp(encoder, crtc_state, &intel_dp->psr.vsc); intel_psr_enable_sink(intel_dp); - intel_psr_enable_source(intel_dp, crtc_state); + intel_psr_enable_source(intel_dp); intel_dp->psr.enabled = true; + intel_dp->psr.paused = false; intel_psr_activate(intel_dp); } @@ -1151,22 +1183,12 @@ static void intel_psr_exit(struct intel_dp *intel_dp) intel_dp->psr.active = false; } -static void intel_psr_disable_locked(struct intel_dp *intel_dp) +static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); i915_reg_t psr_status; u32 psr_status_mask; - lockdep_assert_held(&intel_dp->psr.lock); - - if (!intel_dp->psr.enabled) - return; - - drm_dbg_kms(&dev_priv->drm, "Disabling PSR%s\n", - intel_dp->psr.psr2_enabled ? "2" : "1"); - - intel_psr_exit(intel_dp); - if (intel_dp->psr.psr2_enabled) { psr_status = EDP_PSR2_STATUS(intel_dp->psr.transcoder); psr_status_mask = EDP_PSR2_STATUS_STATE_MASK; @@ -1179,6 +1201,22 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp) if (intel_de_wait_for_clear(dev_priv, psr_status, psr_status_mask, 2000)) drm_err(&dev_priv->drm, "Timed out waiting PSR idle state\n"); +} + +static void intel_psr_disable_locked(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + lockdep_assert_held(&intel_dp->psr.lock); + + if (!intel_dp->psr.enabled) + return; + + drm_dbg_kms(&dev_priv->drm, "Disabling PSR%s\n", + intel_dp->psr.psr2_enabled ? "2" : "1"); + + intel_psr_exit(intel_dp); + intel_psr_wait_exit_locked(intel_dp); /* WA 1408330847 */ if (intel_dp->psr.psr2_sel_fetch_enabled && @@ -1223,6 +1261,61 @@ void intel_psr_disable(struct intel_dp *intel_dp, cancel_delayed_work_sync(&intel_dp->psr.dc3co_work); } +/** + * intel_psr_pause - Pause PSR + * @intel_dp: Intel DP + * + * This function need to be called after enabling psr. + */ +void intel_psr_pause(struct intel_dp *intel_dp) +{ + struct intel_psr *psr = &intel_dp->psr; + + if (!CAN_PSR(intel_dp)) + return; + + mutex_lock(&psr->lock); + + if (!psr->enabled) { + mutex_unlock(&psr->lock); + return; + } + + intel_psr_exit(intel_dp); + intel_psr_wait_exit_locked(intel_dp); + psr->paused = true; + + mutex_unlock(&psr->lock); + + cancel_work_sync(&psr->work); + cancel_delayed_work_sync(&psr->dc3co_work); +} + +/** + * intel_psr_resume - Resume PSR + * @intel_dp: Intel DP + * + * This function need to be called after pausing psr. + */ +void intel_psr_resume(struct intel_dp *intel_dp) +{ + struct intel_psr *psr = &intel_dp->psr; + + if (!CAN_PSR(intel_dp)) + return; + + mutex_lock(&psr->lock); + + if (!psr->paused) + goto unlock; + + psr->paused = false; + intel_psr_activate(intel_dp); + +unlock: + mutex_unlock(&psr->lock); +} + static void psr_force_hw_tracking_exit(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); @@ -1818,7 +1911,7 @@ tgl_dc3co_flush(struct intel_dp *intel_dp, unsigned int frontbuffer_bits, { mutex_lock(&intel_dp->psr.lock); - if (!intel_dp->psr.dc3co_enabled) + if (!intel_dp->psr.dc3co_exitline) goto unlock; if (!intel_dp->psr.psr2_enabled || !intel_dp->psr.active) @@ -1877,6 +1970,16 @@ void intel_psr_flush(struct drm_i915_private *dev_priv, INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe); intel_dp->psr.busy_frontbuffer_bits &= ~pipe_frontbuffer_bits; + /* + * If the PSR is paused by an explicit intel_psr_paused() call, + * we have to ensure that the PSR is not activated until + * intel_psr_resume() is called. + */ + if (intel_dp->psr.paused) { + mutex_unlock(&intel_dp->psr.lock); + continue; + } + /* By definition flush = invalidate + flush */ if (pipe_frontbuffer_bits) psr_force_hw_tracking_exit(intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h index e3db85e97f4c..641521b101c8 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.h +++ b/drivers/gpu/drm/i915/display/intel_psr.h @@ -51,5 +51,7 @@ void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane, const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, int color_plane); +void intel_psr_pause(struct intel_dp *intel_dp); +void intel_psr_resume(struct intel_dp *intel_dp); #endif /* __INTEL_PSR_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_qp_tables.c b/drivers/gpu/drm/i915/display/intel_qp_tables.c new file mode 100644 index 000000000000..c626a24fe98f --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_qp_tables.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <drm/drm_dsc.h> + +#include "i915_utils.h" +#include "intel_qp_tables.h" + +/* from BPP 6 to 24 in steps of 0.5 */ +#define RC_RANGE_QP444_8BPC_MAX_NUM_BPP 37 + +/* from BPP 6 to 30 in steps of 0.5 */ +#define RC_RANGE_QP444_10BPC_MAX_NUM_BPP 49 + +/* from BPP 6 to 36 in steps of 0.5 */ +#define RC_RANGE_QP444_12BPC_MAX_NUM_BPP 61 + +/* + * These qp tables are as per the C model + * and it has the rows pointing to bpps which increment + * in steps of 0.5 + * We do not support fractional bpps as of today, + * hence we would skip the fractional bpps during + * our references for qp calclulations. + */ +static const u8 rc_range_minqp444_8bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_8BPC_MAX_NUM_BPP] = { + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 3, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 3, 3, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 5, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0 }, + { 5, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0 }, + { 5, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 }, + { 5, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0 }, + { 6, 5, 5, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0 }, + { 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, + 3, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0 }, + { 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0 }, + { 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, + 4, 4, 4, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0 }, + { 9, 9, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 5, + 5, 5, 5, 4, 4, 3, 3, 3, 3, 2, 2, 1, 1, 1 }, + { 14, 14, 13, 13, 12, 12, 12, 12, 11, 11, 10, 10, 10, 10, 9, 9, 9, 8, 8, + 8, 7, 7, 7, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3 } +}; + +static const u8 rc_range_maxqp444_8bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_8BPC_MAX_NUM_BPP] = { + { 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 6, 6, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 }, + { 8, 7, 7, 6, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 2, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 }, + { 8, 8, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 4, 3, 3, 2, 2, 2, 2, 2, + 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 }, + { 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 5, 4, 4, 3, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0 }, + { 9, 8, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 5, 4, 4, 3, 3, 3, 3, 3, + 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1 }, + { 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 5, 4, 4, 3, 3, 3, 3, 3, + 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1 }, + { 10, 10, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8, 8, 7, 7, 6, 5, 5, 4, 4, 4, 4, 3, + 3, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1 }, + { 11, 11, 10, 10, 9, 9, 9, 9, 9, 9, 8, 8, 8, 7, 7, 6, 6, 5, 5, 5, 5, 5, + 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1 }, + { 12, 11, 11, 10, 10, 10, 9, 9, 9, 9, 9, 9, 9, 8, 8, 7, 6, 6, 5, 5, 5, + 5, 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1 }, + { 12, 12, 11, 11, 10, 10, 10, 10, 10, 10, 9, 9, 9, 8, 8, 7, 7, 6, 6, 6, + 5, 5, 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1 }, + { 12, 12, 12, 11, 11, 11, 10, 10, 10, 10, 9, 9, 9, 9, 8, 8, 8, 7, 7, 7, + 6, 6, 5, 5, 5, 5, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1 }, + { 12, 12, 12, 12, 11, 11, 11, 11, 11, 10, 10, 9, 9, 9, 8, 8, 8, 7, 7, 7, + 6, 6, 5, 5, 5, 5, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1 }, + { 13, 13, 13, 13, 12, 12, 11, 11, 11, 11, 10, 10, 10, 10, 9, 9, 8, 8, 8, + 8, 7, 7, 6, 6, 6, 6, 5, 5, 4, 4, 4, 4, 3, 3, 2, 2, 2 }, + { 15, 15, 14, 14, 13, 13, 13, 13, 12, 12, 11, 11, 11, 11, 10, 10, 10, 9, + 9, 9, 8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 4, 4, 4, 4 } +}; + +static const u8 rc_range_minqp444_10bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_10BPC_MAX_NUM_BPP] = { + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }, + { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }, + { 7, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, + 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }, + { 7, 7, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, + 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }, + { 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, + 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }, + { 9, 8, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 4, + 4, 4, 4, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 0, 0, 0, + 0, 0, 0 }, + { 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 5, 5, 5, 5, 5, + 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 1, 1, 1, 1, 0, 0, + 0, 0, 0 }, + { 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 5, + 5, 5, 5, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 0, + 0, 0, 0 }, + { 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 5, + 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 1, 1, 1, + 1, 0, 0 }, + { 10, 9, 9, 8, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 1, 1, + 1, 1, 0 }, + { 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, 8, 8, 7, 7, 7, 7, 7, 6, + 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 1, + 1, 1, 1 }, + { 10, 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, 8, 8, 8, + 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, + 2, 1, 1, 1 }, + { 10, 10, 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, + 8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, + 2, 2, 2, 1 }, + { 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 10, 10, 9, 9, 9, 9, 8, 8, 7, 7, 7, 7, 6, 6, 5, 5, 5, 5, 4, + 4, 3, 3, 3, 3, 2, 2, 2, 2, 1 }, + { 18, 18, 17, 17, 16, 16, 16, 16, 15, 15, 14, 14, 14, 14, 13, 13, 13, + 12, 12, 12, 11, 11, 11, 11, 10, 10, 9, 9, 9, 9, 9, 8, 8, 7, 7, 7, 7, + 7, 6, 6, 5, 5, 5, 5, 4, 4, 3, 3, 3 } +}; + +static const u8 rc_range_maxqp444_10bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_10BPC_MAX_NUM_BPP] = { + { 8, 8, 8, 8, 8, 8, 7, 7, 7, 6, 5, 5, 4, 4, 3, 3, 3, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }, + { 10, 10, 9, 9, 8, 8, 8, 8, 8, 8, 7, 7, 6, 6, 6, 5, 5, 4, 4, 4, 4, 3, 3, + 3, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }, + { 12, 11, 11, 10, 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, 8, 7, 6, 6, 5, 5, 5, 4, + 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0 }, + { 12, 12, 11, 11, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9, 8, 7, 7, 6, + 6, 6, 5, 5, 5, 5, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0 }, + { 13, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 9, 8, 8, 7, + 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3, 3, 2, 2, 2, 2, 1, + 1, 1, 1, 0, 0, 0, 0, 0 }, + { 13, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 10, 9, 8, 8, + 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, + 2, 2, 1, 1, 1, 1, 0, 0 }, + { 13, 13, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 9, 8, 8, + 7, 7, 7, 7, 7, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 2, + 2, 2, 2, 1, 1, 1, 1, 1 }, + { 14, 14, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 10, 9, 9, + 8, 8, 8, 8, 7, 7, 7, 7, 6, 6, 6, 5, 5, 5, 4, 4, 4, 4, 4, 4, 3, 3, 3, + 3, 2, 2, 2, 1, 1, 1, 1 }, + { 15, 15, 14, 14, 13, 13, 13, 13, 13, 13, 12, 12, 12, 11, 11, 10, 10, 9, + 9, 9, 9, 9, 8, 8, 8, 8, 7, 7, 6, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 3, + 3, 3, 2, 2, 2, 2, 1, 1 }, + { 16, 15, 15, 14, 14, 14, 13, 13, 13, 13, 13, 13, 13, 12, 12, 11, 10, + 10, 9, 9, 9, 9, 8, 8, 8, 8, 7, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, + 4, 3, 3, 3, 2, 2, 2, 2, 1 }, + { 16, 16, 15, 15, 14, 14, 14, 14, 14, 14, 13, 13, 13, 12, 12, 11, 11, + 10, 10, 10, 9, 9, 8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, + 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }, + { 16, 16, 16, 15, 15, 15, 14, 14, 14, 14, 13, 13, 13, 13, 12, 12, 12, + 11, 11, 11, 10, 10, 9, 9, 9, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, + 5, 4, 4, 4, 4, 3, 3, 2, 2, 2 }, + { 16, 16, 16, 16, 15, 15, 15, 15, 15, 14, 14, 13, 13, 13, 12, 12, 12, + 11, 11, 11, 10, 10, 9, 9, 9, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, + 5, 4, 4, 4, 4, 3, 3, 3, 3, 2 }, + { 17, 17, 17, 17, 16, 16, 15, 15, 15, 15, 14, 14, 14, 14, 13, 13, 12, + 12, 12, 12, 11, 11, 10, 10, 10, 10, 9, 9, 8, 8, 8, 8, 7, 7, 6, 6, 6, + 6, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2 }, + { 19, 19, 18, 18, 17, 17, 17, 17, 16, 16, 15, 15, 15, 15, 14, 14, 14, + 13, 13, 13, 12, 12, 12, 12, 11, 11, 10, 10, 10, 10, 10, 9, 9, 8, 8, 8, + 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 4, 4, 4 } +}; + +static const u8 rc_range_minqp444_12bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_12BPC_MAX_NUM_BPP] = { + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 11, 10, 10, 9, 8, 8, 8, 8, 8, 8, 8, 8, 7, 7, 7, 6, 5, 5, 4, 4, 4, 3, 3, 3, 3, + 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 11, 11, 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, 8, 7, 6, 6, 6, 6, 6, 5, 5, 5, + 5, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 13, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 9, 9, 9, 8, 7, 7, 7, + 7, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 13, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 9, 9, 8, 8, + 8, 8, 6, 6, 6, 6, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 13, 13, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 9, 9, 9, 9, + 9, 9, 9, 8, 8, 8, 7, 7, 7, 6, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 3, 3, + 3, 2, 2, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 }, + { 13, 13, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 10, + 10, 10, 10, 9, 9, 9, 9, 8, 8, 8, 7, 7, 7, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, + 4, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0 }, + { 13, 13, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 10, 10, 10, 10, 9, 9, 8, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 5, 5, + 5, 4, 4, 4, 4, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 0, 0 }, + { 14, 13, 13, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 10, 10, 10, 10, 9, 9, 8, 8, 8, 8, 8, 8, 7, 7, 7, 7, 7, 6, 6, 5, + 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 1, 1, 0 }, + { 14, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, + 12, 11, 11, 11, 11, 11, 11, 10, 10, 9, 9, 9, 9, 9, 9, 9, 9, 8, 8, 8, 7, 7, 7, + 6, 6, 6, 5, 5, 5, 5, 4, 4, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1 }, + { 14, 14, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 12, 12, 11, 11, 11, 11, 11, 11, 10, 10, 10, 10, 9, 9, 9, 9, 8, 8, 8, 8, + 7, 7, 7, 7, 6, 6, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3, 2, 2, 1, 1, 1 }, + { 14, 14, 14, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 12, 12, 12, 12, 12, 12, 11, 11, 10, 10, 10, 10, 9, 9, 9, 9, 8, 8, 8, 8, + 7, 7, 7, 7, 6, 6, 6, 6, 5, 4, 4, 4, 3, 3, 3, 3, 2, 2, 1, 1, 1 }, + { 17, 17, 17, 17, 16, 16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 14, 14, 13, 13, 13, 13, 12, 12, 11, 11, 11, 11, 10, 10, 9, 9, 9, 9, 8, 8, + 7, 7, 7, 7, 7, 6, 6, 6, 5, 5, 5, 5, 4, 4, 3, 3, 3, 3, 2, 2, 1 }, + { 22, 22, 21, 21, 20, 20, 20, 20, 19, 19, 18, 18, 18, 18, 17, 17, 17, 16, 16, + 16, 15, 15, 15, 15, 14, 14, 13, 13, 13, 13, 13, 12, 12, 11, 11, 11, 11, 11, + 10, 10, 9, 9, 9, 9, 9, 8, 8, 7, 7, 7, 7, 7, 6, 6, 5, 5, 5, 5, 4, 4, 3 } +}; + +static const u8 rc_range_maxqp444_12bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_12BPC_MAX_NUM_BPP] = { + { 12, 12, 12, 12, 12, 12, 11, 11, 11, 10, 9, 9, 6, 6, 5, 5, 5, 4, 4, 4, 4, 4, + 4, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 14, 14, 13, 13, 12, 12, 12, 12, 12, 12, 11, 11, 9, 9, 9, 8, 8, 7, 7, 7, 7, 5, + 5, 5, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 16, 15, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 11, 10, 10, 9, 9, + 9, 7, 7, 7, 7, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 1, 1, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 16, 16, 15, 15, 14, 14, 14, 14, 14, 14, 14, 14, 13, 13, 13, 12, 11, 11, 10, + 10, 10, 8, 8, 8, 8, 8, 7, 7, 6, 5, 5, 5, 5, 5, 5, 5, 4, 4, 3, 3, 3, 3, 3, 2, + 2, 2, 2, 2, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 17, 16, 16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 14, 14, 13, 12, 12, 11, 10, + 10, 10, 10, 8, 8, 8, 8, 8, 8, 7, 7, 7, 6, 6, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0 }, + { 17, 16, 16, 16, 15, 15, 15, 15, 15, 15, 15, 15, 14, 14, 14, 13, 12, 12, 11, + 11, 11, 11, 9, 9, 9, 9, 8, 8, 8, 8, 7, 6, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 3, + 3, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 0 }, + { 17, 17, 16, 16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 14, 14, 13, 12, 12, 11, + 11, 11, 11, 11, 10, 10, 10, 9, 9, 9, 8, 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 5, 5, + 5, 5, 4, 4, 4, 3, 3, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0 }, + { 18, 18, 17, 17, 16, 16, 16, 16, 16, 16, 16, 16, 16, 15, 15, 14, 13, 13, 12, + 12, 12, 12, 11, 11, 11, 11, 10, 10, 10, 8, 8, 8, 7, 7, 7, 7, 7, 7, 6, 6, 6, + 6, 5, 5, 5, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1 }, + { 19, 19, 18, 18, 17, 17, 17, 17, 17, 17, 16, 16, 16, 15, 15, 14, 14, 13, 13, + 13, 13, 13, 12, 12, 12, 12, 11, 11, 10, 9, 8, 8, 8, 8, 7, 7, 7, 7, 7, 7, 6, + 6, 6, 5, 5, 5, 5, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 1, 1 }, + { 20, 19, 19, 18, 18, 18, 17, 17, 17, 17, 17, 17, 17, 16, 16, 15, 14, 14, 13, + 13, 13, 13, 12, 12, 12, 12, 11, 11, 10, 10, 9, 9, 9, 9, 8, 8, 8, 8, 8, 7, 7, + 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 2, 2, 1 }, + { 20, 20, 19, 19, 18, 18, 18, 18, 18, 18, 17, 17, 17, 16, 16, 15, 15, 14, 14, + 14, 13, 13, 12, 12, 12, 12, 11, 11, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9, + 8, 8, 8, 7, 7, 7, 6, 6, 6, 6, 5, 5, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2 }, + { 20, 20, 20, 19, 19, 19, 18, 18, 18, 18, 17, 17, 17, 17, 16, 16, 16, 15, 15, + 15, 14, 14, 13, 13, 13, 13, 12, 12, 11, 11, 11, 11, 10, 10, 10, 10, 9, 9, 9, + 9, 8, 8, 8, 8, 7, 7, 6, 6, 6, 5, 5, 5, 4, 4, 4, 4, 3, 3, 2, 2, 2 }, + { 20, 20, 20, 20, 19, 19, 19, 19, 19, 18, 18, 17, 17, 17, 16, 16, 16, 15, 15, + 15, 14, 14, 13, 13, 13, 13, 12, 12, 11, 11, 11, 11, 10, 10, 10, 10, 9, 9, 9, + 9, 8, 8, 8, 8, 7, 7, 7, 7, 6, 5, 5, 5, 4, 4, 4, 4, 3, 3, 2, 2, 2 }, + { 21, 21, 21, 21, 20, 20, 19, 19, 19, 19, 18, 18, 18, 18, 17, 17, 16, 16, 16, + 16, 15, 15, 14, 14, 14, 14, 13, 13, 12, 12, 12, 12, 11, 11, 10, 10, 10, 10, + 9, 9, 8, 8, 8, 8, 8, 7, 7, 7, 6, 6, 6, 6, 5, 5, 4, 4, 4, 4, 3, 3, 2 }, + { 23, 23, 22, 22, 21, 21, 21, 21, 20, 20, 19, 19, 19, 19, 18, 18, 18, 17, 17, + 17, 16, 16, 16, 16, 15, 15, 14, 14, 14, 14, 14, 13, 13, 12, 12, 12, 12, 12, + 11, 11, 10, 10, 10, 10, 10, 9, 9, 8, 8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 4 } +}; + +#define PARAM_TABLE(_minmax, _bpc, _row, _col) do { \ + if (bpc == (_bpc)) \ + return rc_range_##_minmax##qp444_##_bpc##bpc[_row][_col]; \ +} while (0) + +u8 intel_lookup_range_min_qp(int bpc, int buf_i, int bpp_i) +{ + PARAM_TABLE(min, 8, buf_i, bpp_i); + PARAM_TABLE(min, 10, buf_i, bpp_i); + PARAM_TABLE(min, 12, buf_i, bpp_i); + + MISSING_CASE(bpc); + return 0; +} + +u8 intel_lookup_range_max_qp(int bpc, int buf_i, int bpp_i) +{ + PARAM_TABLE(max, 8, buf_i, bpp_i); + PARAM_TABLE(max, 10, buf_i, bpp_i); + PARAM_TABLE(max, 12, buf_i, bpp_i); + + MISSING_CASE(bpc); + return 0; +} diff --git a/drivers/gpu/drm/i915/display/intel_qp_tables.h b/drivers/gpu/drm/i915/display/intel_qp_tables.h new file mode 100644 index 000000000000..9fb3c36bd7c6 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_qp_tables.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _INTEL_QP_TABLES_H_ +#define _INTEL_QP_TABLES_H_ + +#include <linux/types.h> + +u8 intel_lookup_range_min_qp(int bpc, int buf_i, int bpp_i); +u8 intel_lookup_range_max_qp(int bpc, int buf_i, int bpp_i); + +#endif diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c index e325463acddd..c23c210a55f5 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.c +++ b/drivers/gpu/drm/i915/display/intel_tc.c @@ -26,9 +26,7 @@ static const char *tc_port_mode_name(enum tc_port_mode mode) static enum intel_display_power_domain tc_cold_get_power_domain(struct intel_digital_port *dig_port) { - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - - if (DISPLAY_VER(i915) == 11) + if (intel_tc_cold_requires_aux_pw(dig_port)) return intel_legacy_aux_to_power_domain(dig_port->aux_ch); else return POWER_DOMAIN_TC_COLD_OFF; @@ -205,7 +203,7 @@ static void tc_port_fixup_legacy_flag(struct intel_digital_port *dig_port, dig_port->tc_legacy_port = !dig_port->tc_legacy_port; } -static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port) +static u32 icl_tc_port_live_status_mask(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); struct intel_uncore *uncore = &i915->uncore; @@ -238,6 +236,40 @@ static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port) return mask; } +static u32 adl_tc_port_live_status_mask(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port); + u32 isr_bit = i915->hotplug.pch_hpd[dig_port->base.hpd_pin]; + struct intel_uncore *uncore = &i915->uncore; + u32 val, mask = 0; + + val = intel_uncore_read(uncore, TCSS_DDI_STATUS(tc_port)); + if (val & TCSS_DDI_STATUS_HPD_LIVE_STATUS_ALT) + mask |= BIT(TC_PORT_DP_ALT); + if (val & TCSS_DDI_STATUS_HPD_LIVE_STATUS_TBT) + mask |= BIT(TC_PORT_TBT_ALT); + + if (intel_uncore_read(uncore, SDEISR) & isr_bit) + mask |= BIT(TC_PORT_LEGACY); + + /* The sink can be connected only in a single mode. */ + if (!drm_WARN_ON(&i915->drm, hweight32(mask) > 1)) + tc_port_fixup_legacy_flag(dig_port, mask); + + return mask; +} + +static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + + if (IS_ALDERLAKE_P(i915)) + return adl_tc_port_live_status_mask(dig_port); + + return icl_tc_port_live_status_mask(dig_port); +} + static bool icl_tc_phy_status_complete(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); @@ -256,6 +288,33 @@ static bool icl_tc_phy_status_complete(struct intel_digital_port *dig_port) return val & DP_PHY_MODE_STATUS_COMPLETED(dig_port->tc_phy_fia_idx); } +static bool adl_tc_phy_status_complete(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + struct intel_uncore *uncore = &i915->uncore; + u32 val; + + val = intel_uncore_read(uncore, TCSS_DDI_STATUS(dig_port->tc_phy_fia_idx)); + if (val == 0xffffffff) { + drm_dbg_kms(&i915->drm, + "Port %s: PHY in TCCOLD, assuming not complete\n", + dig_port->tc_port_name); + return false; + } + + return val & TCSS_DDI_STATUS_READY; +} + +static bool tc_phy_status_complete(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + + if (IS_ALDERLAKE_P(i915)) + return adl_tc_phy_status_complete(dig_port); + + return icl_tc_phy_status_complete(dig_port); +} + static bool icl_tc_phy_take_ownership(struct intel_digital_port *dig_port, bool take) { @@ -280,7 +339,7 @@ static bool icl_tc_phy_take_ownership(struct intel_digital_port *dig_port, intel_uncore_write(uncore, PORT_TX_DFLEXDPCSSS(dig_port->tc_phy_fia), val); - if (!take && wait_for(!icl_tc_phy_status_complete(dig_port), 10)) + if (!take && wait_for(!tc_phy_status_complete(dig_port), 10)) drm_dbg_kms(&i915->drm, "Port %s: PHY complete clear timed out\n", dig_port->tc_port_name); @@ -288,6 +347,34 @@ static bool icl_tc_phy_take_ownership(struct intel_digital_port *dig_port, return true; } +static bool adl_tc_phy_take_ownership(struct intel_digital_port *dig_port, + bool take) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + struct intel_uncore *uncore = &i915->uncore; + enum port port = dig_port->base.port; + u32 val; + + val = intel_uncore_read(uncore, DDI_BUF_CTL(port)); + if (take) + val |= DDI_BUF_CTL_TC_PHY_OWNERSHIP; + else + val &= ~DDI_BUF_CTL_TC_PHY_OWNERSHIP; + intel_uncore_write(uncore, DDI_BUF_CTL(port), val); + + return true; +} + +static bool tc_phy_take_ownership(struct intel_digital_port *dig_port, bool take) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + + if (IS_ALDERLAKE_P(i915)) + return adl_tc_phy_take_ownership(dig_port, take); + + return icl_tc_phy_take_ownership(dig_port, take); +} + static bool icl_tc_phy_is_owned(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); @@ -306,6 +393,27 @@ static bool icl_tc_phy_is_owned(struct intel_digital_port *dig_port) return val & DP_PHY_MODE_STATUS_NOT_SAFE(dig_port->tc_phy_fia_idx); } +static bool adl_tc_phy_is_owned(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + struct intel_uncore *uncore = &i915->uncore; + enum port port = dig_port->base.port; + u32 val; + + val = intel_uncore_read(uncore, DDI_BUF_CTL(port)); + return val & DDI_BUF_CTL_TC_PHY_OWNERSHIP; +} + +static bool tc_phy_is_owned(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + + if (IS_ALDERLAKE_P(i915)) + return adl_tc_phy_is_owned(dig_port); + + return icl_tc_phy_is_owned(dig_port); +} + /* * This function implements the first part of the Connect Flow described by our * specification, Gen11 TypeC Programming chapter. The rest of the flow (reading @@ -323,13 +431,13 @@ static void icl_tc_phy_connect(struct intel_digital_port *dig_port, struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); int max_lanes; - if (!icl_tc_phy_status_complete(dig_port)) { + if (!tc_phy_status_complete(dig_port)) { drm_dbg_kms(&i915->drm, "Port %s: PHY not ready\n", dig_port->tc_port_name); goto out_set_tbt_alt_mode; } - if (!icl_tc_phy_take_ownership(dig_port, true) && + if (!tc_phy_take_ownership(dig_port, true) && !drm_WARN_ON(&i915->drm, dig_port->tc_legacy_port)) goto out_set_tbt_alt_mode; @@ -364,7 +472,7 @@ static void icl_tc_phy_connect(struct intel_digital_port *dig_port, return; out_release_phy: - icl_tc_phy_take_ownership(dig_port, false); + tc_phy_take_ownership(dig_port, false); out_set_tbt_alt_mode: dig_port->tc_mode = TC_PORT_TBT_ALT; } @@ -380,7 +488,7 @@ static void icl_tc_phy_disconnect(struct intel_digital_port *dig_port) /* Nothing to do, we never disconnect from legacy mode */ break; case TC_PORT_DP_ALT: - icl_tc_phy_take_ownership(dig_port, false); + tc_phy_take_ownership(dig_port, false); dig_port->tc_mode = TC_PORT_TBT_ALT; break; case TC_PORT_TBT_ALT: @@ -395,13 +503,13 @@ static bool icl_tc_phy_is_connected(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - if (!icl_tc_phy_status_complete(dig_port)) { + if (!tc_phy_status_complete(dig_port)) { drm_dbg_kms(&i915->drm, "Port %s: PHY status not complete\n", dig_port->tc_port_name); return dig_port->tc_mode == TC_PORT_TBT_ALT; } - if (!icl_tc_phy_is_owned(dig_port)) { + if (!tc_phy_is_owned(dig_port)) { drm_dbg_kms(&i915->drm, "Port %s: PHY not owned\n", dig_port->tc_port_name); @@ -419,8 +527,8 @@ intel_tc_port_get_current_mode(struct intel_digital_port *dig_port) u32 live_status_mask = tc_port_live_status_mask(dig_port); enum tc_port_mode mode; - if (!icl_tc_phy_is_owned(dig_port) || - drm_WARN_ON(&i915->drm, !icl_tc_phy_status_complete(dig_port))) + if (!tc_phy_is_owned(dig_port) || + drm_WARN_ON(&i915->drm, !tc_phy_status_complete(dig_port))) return TC_PORT_TBT_ALT; mode = dig_port->tc_legacy_port ? TC_PORT_LEGACY : TC_PORT_DP_ALT; @@ -442,7 +550,7 @@ intel_tc_port_get_target_mode(struct intel_digital_port *dig_port) if (live_status_mask) return fls(live_status_mask) - 1; - return icl_tc_phy_status_complete(dig_port) && + return tc_phy_status_complete(dig_port) && dig_port->tc_legacy_port ? TC_PORT_LEGACY : TC_PORT_TBT_ALT; } @@ -454,7 +562,7 @@ static void intel_tc_port_reset_mode(struct intel_digital_port *dig_port, enum tc_port_mode old_tc_mode = dig_port->tc_mode; intel_display_power_flush_work(i915); - if (DISPLAY_VER(i915) != 11 || !dig_port->tc_legacy_port) { + if (!intel_tc_cold_requires_aux_pw(dig_port)) { enum intel_display_power_domain aux_domain; bool aux_powered; @@ -624,13 +732,11 @@ tc_has_modular_fia(struct drm_i915_private *i915, struct intel_digital_port *dig if (!INTEL_INFO(i915)->display.has_modular_fia) return false; - /* TODO: check if in real HW MODULAR_FIA_MASK is set, if so remove this block */ - if (IS_ALDERLAKE_P(i915)) - return true; - + mutex_lock(&dig_port->tc_lock); wakeref = tc_cold_block(dig_port); val = intel_uncore_read(&i915->uncore, PORT_TX_DFLEXDPSP(FIA1)); tc_cold_unblock(dig_port, wakeref); + mutex_unlock(&dig_port->tc_lock); drm_WARN_ON(&i915->drm, val == 0xffffffff); @@ -673,3 +779,11 @@ void intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy) dig_port->tc_link_refcount = 0; tc_port_load_fia_params(i915, dig_port); } + +bool intel_tc_cold_requires_aux_pw(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + + return (DISPLAY_VER(i915) == 11 && dig_port->tc_legacy_port) || + IS_ALDERLAKE_P(i915); +} diff --git a/drivers/gpu/drm/i915/display/intel_tc.h b/drivers/gpu/drm/i915/display/intel_tc.h index b619e4736f85..0eacbd76ec15 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.h +++ b/drivers/gpu/drm/i915/display/intel_tc.h @@ -29,4 +29,6 @@ bool intel_tc_port_ref_held(struct intel_digital_port *dig_port); void intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy); +bool intel_tc_cold_requires_aux_pw(struct intel_digital_port *dig_port); + #endif /* __INTEL_TC_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c index ce73ebdfc669..aa52af7891f0 100644 --- a/drivers/gpu/drm/i915/display/intel_tv.c +++ b/drivers/gpu/drm/i915/display/intel_tv.c @@ -1307,7 +1307,7 @@ intel_tv_compute_config(struct intel_encoder *encoder, * the active portion. Hence following this formula seems * more trouble that it's worth. * - * if (IS_GEN(dev_priv, 4)) { + * if (GRAPHICS_VER(dev_priv) == 4) { * num = cdclk * (tv_mode->oversample >> !tv_mode->progressive); * den = tv_mode->clock; * } else { diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index efc3184d8315..7121b66bf96d 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -5,12 +5,13 @@ * Author: Gaurav K Singh <gaurav.k.singh@intel.com> * Manasi Navare <manasi.d.navare@intel.com> */ - +#include <linux/limits.h> #include "i915_drv.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_dsi.h" #include "intel_vdsc.h" +#include "intel_qp_tables.h" enum ROW_INDEX_BPP { ROW_INDEX_6BPP = 0, @@ -373,12 +374,81 @@ static bool is_pipe_dsc(const struct intel_crtc_state *crtc_state) return true; } +static void +calculate_rc_params(struct rc_parameters *rc, + struct drm_dsc_config *vdsc_cfg) +{ + int bpc = vdsc_cfg->bits_per_component; + int bpp = vdsc_cfg->bits_per_pixel >> 4; + int ofs_und6[] = { 0, -2, -2, -4, -6, -6, -8, -8, -8, -10, -10, -12, -12, -12, -12 }; + int ofs_und8[] = { 2, 0, 0, -2, -4, -6, -8, -8, -8, -10, -10, -10, -12, -12, -12 }; + int ofs_und12[] = { 2, 0, 0, -2, -4, -6, -8, -8, -8, -10, -10, -10, -12, -12, -12 }; + int ofs_und15[] = { 10, 8, 6, 4, 2, 0, -2, -4, -6, -8, -10, -10, -12, -12, -12 }; + int qp_bpc_modifier = (bpc - 8) * 2; + u32 res, buf_i, bpp_i; + + if (vdsc_cfg->slice_height >= 8) + rc->first_line_bpg_offset = + 12 + DIV_ROUND_UP((9 * min(34, vdsc_cfg->slice_height - 8)), 100); + else + rc->first_line_bpg_offset = 2 * (vdsc_cfg->slice_height - 1); + + /* Our hw supports only 444 modes as of today */ + if (bpp >= 12) + rc->initial_offset = 2048; + else if (bpp >= 10) + rc->initial_offset = 5632 - DIV_ROUND_UP(((bpp - 10) * 3584), 2); + else if (bpp >= 8) + rc->initial_offset = 6144 - DIV_ROUND_UP(((bpp - 8) * 512), 2); + else + rc->initial_offset = 6144; + + /* initial_xmit_delay = rc_model_size/2/compression_bpp */ + rc->initial_xmit_delay = DIV_ROUND_UP(DSC_RC_MODEL_SIZE_CONST, 2 * bpp); + + rc->flatness_min_qp = 3 + qp_bpc_modifier; + rc->flatness_max_qp = 12 + qp_bpc_modifier; + + rc->rc_quant_incr_limit0 = 11 + qp_bpc_modifier; + rc->rc_quant_incr_limit1 = 11 + qp_bpc_modifier; + + bpp_i = (2 * (bpp - 6)); + for (buf_i = 0; buf_i < DSC_NUM_BUF_RANGES; buf_i++) { + /* Read range_minqp and range_max_qp from qp tables */ + rc->rc_range_params[buf_i].range_min_qp = + intel_lookup_range_min_qp(bpc, buf_i, bpp_i); + rc->rc_range_params[buf_i].range_max_qp = + intel_lookup_range_max_qp(bpc, buf_i, bpp_i); + + /* Calculate range_bgp_offset */ + if (bpp <= 6) { + rc->rc_range_params[buf_i].range_bpg_offset = ofs_und6[buf_i]; + } else if (bpp <= 8) { + res = DIV_ROUND_UP(((bpp - 6) * (ofs_und8[buf_i] - ofs_und6[buf_i])), 2); + rc->rc_range_params[buf_i].range_bpg_offset = + ofs_und6[buf_i] + res; + } else if (bpp <= 12) { + rc->rc_range_params[buf_i].range_bpg_offset = + ofs_und8[buf_i]; + } else if (bpp <= 15) { + res = DIV_ROUND_UP(((bpp - 12) * (ofs_und15[buf_i] - ofs_und12[buf_i])), 3); + rc->rc_range_params[buf_i].range_bpg_offset = + ofs_und12[buf_i] + res; + } else { + rc->rc_range_params[buf_i].range_bpg_offset = + ofs_und15[buf_i]; + } + } +} + int intel_dsc_compute_params(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct drm_dsc_config *vdsc_cfg = &pipe_config->dsc.config; u16 compressed_bpp = pipe_config->dsc.compressed_bpp; const struct rc_parameters *rc_params; + struct rc_parameters *rc = NULL; u8 i = 0; vdsc_cfg->pic_width = pipe_config->hw.adjusted_mode.crtc_hdisplay; @@ -413,9 +483,24 @@ int intel_dsc_compute_params(struct intel_encoder *encoder, vdsc_cfg->rc_buf_thresh[13] = 0x7D; } - rc_params = get_rc_params(compressed_bpp, vdsc_cfg->bits_per_component); - if (!rc_params) - return -EINVAL; + /* + * From XE_LPD onwards we supports compression bpps in steps of 1 + * upto uncompressed bpp-1, hence add calculations for all the rc + * parameters + */ + if (DISPLAY_VER(dev_priv) >= 13) { + rc = kmalloc(sizeof(*rc), GFP_KERNEL); + if (!rc) + return -ENOMEM; + + calculate_rc_params(rc, vdsc_cfg); + rc_params = rc; + } else { + rc_params = get_rc_params(compressed_bpp, + vdsc_cfg->bits_per_component); + if (!rc_params) + return -EINVAL; + } vdsc_cfg->first_line_bpg_offset = rc_params->first_line_bpg_offset; vdsc_cfg->initial_xmit_delay = rc_params->initial_xmit_delay; @@ -441,20 +526,20 @@ int intel_dsc_compute_params(struct intel_encoder *encoder, /* * BitsPerComponent value determines mux_word_size: - * When BitsPerComponent is 12bpc, muxWordSize will be equal to 64 bits - * When BitsPerComponent is 8 or 10bpc, muxWordSize will be equal to - * 48 bits + * When BitsPerComponent is less than or 10bpc, muxWordSize will be equal to + * 48 bits otherwise 64 */ - if (vdsc_cfg->bits_per_component == 8 || - vdsc_cfg->bits_per_component == 10) + if (vdsc_cfg->bits_per_component <= 10) vdsc_cfg->mux_word_size = DSC_MUX_WORD_SIZE_8_10_BPC; - else if (vdsc_cfg->bits_per_component == 12) + else vdsc_cfg->mux_word_size = DSC_MUX_WORD_SIZE_12_BPC; /* InitialScaleValue is a 6 bit value with 3 fractional bits (U3.3) */ vdsc_cfg->initial_scale_value = (vdsc_cfg->rc_model_size << 3) / (vdsc_cfg->rc_model_size - vdsc_cfg->initial_offset); + kfree(rc); + return 0; } @@ -1076,12 +1161,12 @@ void intel_dsc_disable(const struct intel_crtc_state *old_crtc_state) struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - if (!(old_crtc_state->dsc.compression_enable && - old_crtc_state->bigjoiner)) - return; - - intel_de_write(dev_priv, dss_ctl1_reg(old_crtc_state), 0); - intel_de_write(dev_priv, dss_ctl2_reg(old_crtc_state), 0); + /* Disable only if either of them is enabled */ + if (old_crtc_state->dsc.compression_enable || + old_crtc_state->bigjoiner) { + intel_de_write(dev_priv, dss_ctl1_reg(old_crtc_state), 0); + intel_de_write(dev_priv, dss_ctl2_reg(old_crtc_state), 0); + } } void intel_uncompressed_joiner_get_config(struct intel_crtc_state *crtc_state) diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c index 046210ae1de0..c335b1dbafcf 100644 --- a/drivers/gpu/drm/i915/display/intel_vrr.c +++ b/drivers/gpu/drm/i915/display/intel_vrr.c @@ -68,7 +68,10 @@ static int intel_vrr_vblank_exit_length(const struct intel_crtc_state *crtc_stat struct drm_i915_private *i915 = to_i915(crtc->base.dev); /* The hw imposes the extra scanline before frame start */ - return crtc_state->vrr.pipeline_full + i915->framestart_delay + 1; + if (DISPLAY_VER(i915) >= 13) + return crtc_state->vrr.guardband + i915->framestart_delay + 1; + else + return crtc_state->vrr.pipeline_full + i915->framestart_delay + 1; } int intel_vrr_vmin_vblank_start(const struct intel_crtc_state *crtc_state) @@ -86,6 +89,8 @@ void intel_vrr_compute_config(struct intel_crtc_state *crtc_state, struct drm_connector_state *conn_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); struct intel_connector *connector = to_intel_connector(conn_state->connector); struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; @@ -124,17 +129,26 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state, crtc_state->vrr.flipline = crtc_state->vrr.vmin + 1; /* - * FIXME: s/4/framestart_delay+1/ to get consistent - * earliest/latest points for register latching regardless - * of the framestart_delay used? - * - * FIXME: this really needs the extra scanline to provide consistent - * behaviour for all framestart_delay values. Otherwise with - * framestart_delay==3 we will end up extending the min vblank by - * one extra line. + * For XE_LPD+, we use guardband and pipeline override + * is deprecated. */ - crtc_state->vrr.pipeline_full = - min(255, crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay - 4 - 1); + if (DISPLAY_VER(i915) >= 13) + crtc_state->vrr.guardband = + crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay - + i915->window2_delay; + else + /* + * FIXME: s/4/framestart_delay+1/ to get consistent + * earliest/latest points for register latching regardless + * of the framestart_delay used? + * + * FIXME: this really needs the extra scanline to provide consistent + * behaviour for all framestart_delay values. Otherwise with + * framestart_delay==3 we will end up extending the min vblank by + * one extra line. + */ + crtc_state->vrr.pipeline_full = + min(255, crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay - 4 - 1); crtc_state->mode_flags |= I915_MODE_FLAG_VRR; } @@ -149,10 +163,15 @@ void intel_vrr_enable(struct intel_encoder *encoder, if (!crtc_state->vrr.enable) return; - trans_vrr_ctl = VRR_CTL_VRR_ENABLE | - VRR_CTL_IGN_MAX_SHIFT | VRR_CTL_FLIP_LINE_EN | - VRR_CTL_PIPELINE_FULL(crtc_state->vrr.pipeline_full) | - VRR_CTL_PIPELINE_FULL_OVERRIDE; + if (DISPLAY_VER(dev_priv) >= 13) + trans_vrr_ctl = VRR_CTL_VRR_ENABLE | + VRR_CTL_IGN_MAX_SHIFT | VRR_CTL_FLIP_LINE_EN | + XELPD_VRR_CTL_VRR_GUARDBAND(crtc_state->vrr.guardband); + else + trans_vrr_ctl = VRR_CTL_VRR_ENABLE | + VRR_CTL_IGN_MAX_SHIFT | VRR_CTL_FLIP_LINE_EN | + VRR_CTL_PIPELINE_FULL(crtc_state->vrr.pipeline_full) | + VRR_CTL_PIPELINE_FULL_OVERRIDE; intel_de_write(dev_priv, TRANS_VRR_VMIN(cpu_transcoder), crtc_state->vrr.vmin - 1); intel_de_write(dev_priv, TRANS_VRR_VMAX(cpu_transcoder), crtc_state->vrr.vmax - 1); @@ -199,8 +218,13 @@ void intel_vrr_get_config(struct intel_crtc *crtc, if (!crtc_state->vrr.enable) return; - if (trans_vrr_ctl & VRR_CTL_PIPELINE_FULL_OVERRIDE) - crtc_state->vrr.pipeline_full = REG_FIELD_GET(VRR_CTL_PIPELINE_FULL_MASK, trans_vrr_ctl); + if (DISPLAY_VER(dev_priv) >= 13) + crtc_state->vrr.guardband = + REG_FIELD_GET(XELPD_VRR_CTL_VRR_GUARDBAND_MASK, trans_vrr_ctl); + else + if (trans_vrr_ctl & VRR_CTL_PIPELINE_FULL_OVERRIDE) + crtc_state->vrr.pipeline_full = + REG_FIELD_GET(VRR_CTL_PIPELINE_FULL_MASK, trans_vrr_ctl); if (trans_vrr_ctl & VRR_CTL_FLIP_LINE_EN) crtc_state->vrr.flipline = intel_de_read(dev_priv, TRANS_VRR_FLIPLINE(cpu_transcoder)) + 1; crtc_state->vrr.vmax = intel_de_read(dev_priv, TRANS_VRR_VMAX(cpu_transcoder)) + 1; diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 59e032f3687a..92a4fd508e92 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -850,6 +850,29 @@ static u32 cnl_plane_ctl_flip(unsigned int reflect) return 0; } +static u32 adlp_plane_ctl_arb_slots(const struct intel_plane_state *plane_state) +{ + const struct drm_framebuffer *fb = plane_state->hw.fb; + + if (intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) { + switch (fb->format->cpp[0]) { + case 2: + return PLANE_CTL_ARB_SLOTS(1); + default: + return PLANE_CTL_ARB_SLOTS(0); + } + } else { + switch (fb->format->cpp[0]) { + case 8: + return PLANE_CTL_ARB_SLOTS(3); + case 4: + return PLANE_CTL_ARB_SLOTS(1); + default: + return PLANE_CTL_ARB_SLOTS(0); + } + } +} + static u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); @@ -903,6 +926,10 @@ static u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, else if (key->flags & I915_SET_COLORKEY_SOURCE) plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE; + /* Wa_22012358565:adlp */ + if (DISPLAY_VER(dev_priv) == 13) + plane_ctl |= adlp_plane_ctl_arb_slots(plane_state); + return plane_ctl; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index fd8ee52e17a4..188dee13e017 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -1046,7 +1046,6 @@ struct context_barrier_task { void *data; }; -__i915_active_call static void cb_retire(struct i915_active *base) { struct context_barrier_task *cb = container_of(base, typeof(*cb), base); @@ -1080,7 +1079,7 @@ static int context_barrier_task(struct i915_gem_context *ctx, if (!cb) return -ENOMEM; - i915_active_init(&cb->base, NULL, cb_retire); + i915_active_init(&cb->base, NULL, cb_retire, 0); err = i915_active_acquire(&cb->base); if (err) { kfree(cb); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_create.c b/drivers/gpu/drm/i915/gem/i915_gem_create.c index 45d60e3d98e3..548ddf39d853 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_create.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_create.c @@ -4,44 +4,102 @@ */ #include "gem/i915_gem_ioctls.h" +#include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" #include "i915_drv.h" +#include "i915_trace.h" +#include "i915_user_extensions.h" + +static u32 object_max_page_size(struct drm_i915_gem_object *obj) +{ + u32 max_page_size = 0; + int i; + + for (i = 0; i < obj->mm.n_placements; i++) { + struct intel_memory_region *mr = obj->mm.placements[i]; + + GEM_BUG_ON(!is_power_of_2(mr->min_page_size)); + max_page_size = max_t(u32, max_page_size, mr->min_page_size); + } + + GEM_BUG_ON(!max_page_size); + return max_page_size; +} + +static void object_set_placements(struct drm_i915_gem_object *obj, + struct intel_memory_region **placements, + unsigned int n_placements) +{ + GEM_BUG_ON(!n_placements); + + /* + * For the common case of one memory region, skip storing an + * allocated array and just point at the region directly. + */ + if (n_placements == 1) { + struct intel_memory_region *mr = placements[0]; + struct drm_i915_private *i915 = mr->i915; + + obj->mm.placements = &i915->mm.regions[mr->id]; + obj->mm.n_placements = 1; + } else { + obj->mm.placements = placements; + obj->mm.n_placements = n_placements; + } +} + +static int i915_gem_publish(struct drm_i915_gem_object *obj, + struct drm_file *file, + u64 *size_p, + u32 *handle_p) +{ + u64 size = obj->base.size; + int ret; + + ret = drm_gem_handle_create(file, &obj->base, handle_p); + /* drop reference from allocate - handle holds it now */ + i915_gem_object_put(obj); + if (ret) + return ret; + + *size_p = size; + return 0; +} static int -i915_gem_create(struct drm_file *file, - struct intel_memory_region *mr, - u64 *size_p, - u32 *handle_p) +i915_gem_setup(struct drm_i915_gem_object *obj, u64 size) { - struct drm_i915_gem_object *obj; - u32 handle; - u64 size; + struct intel_memory_region *mr = obj->mm.placements[0]; + unsigned int flags; int ret; - GEM_BUG_ON(!is_power_of_2(mr->min_page_size)); - size = round_up(*size_p, mr->min_page_size); + size = round_up(size, object_max_page_size(obj)); if (size == 0) return -EINVAL; /* For most of the ABI (e.g. mmap) we think in system pages */ GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); - /* Allocate the new object */ - obj = i915_gem_object_create_region(mr, size, 0); - if (IS_ERR(obj)) - return PTR_ERR(obj); + if (i915_gem_object_size_2big(size)) + return -E2BIG; - GEM_BUG_ON(size != obj->base.size); + /* + * For now resort to CPU based clearing for device local-memory, in the + * near future this will use the blitter engine for accelerated, GPU + * based clearing. + */ + flags = 0; + if (mr->type == INTEL_MEMORY_LOCAL) + flags = I915_BO_ALLOC_CPU_CLEAR; - ret = drm_gem_handle_create(file, &obj->base, &handle); - /* drop reference from allocate - handle holds it now */ - i915_gem_object_put(obj); + ret = mr->ops->init_object(mr, obj, size, flags); if (ret) return ret; - *handle_p = handle; - *size_p = size; + GEM_BUG_ON(size != obj->base.size); + + trace_i915_gem_object_create(obj); return 0; } @@ -50,9 +108,12 @@ i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, struct drm_mode_create_dumb *args) { + struct drm_i915_gem_object *obj; + struct intel_memory_region *mr; enum intel_memory_type mem_type; int cpp = DIV_ROUND_UP(args->bpp, 8); u32 format; + int ret; switch (cpp) { case 1: @@ -85,10 +146,22 @@ i915_gem_dumb_create(struct drm_file *file, if (HAS_LMEM(to_i915(dev))) mem_type = INTEL_MEMORY_LOCAL; - return i915_gem_create(file, - intel_memory_region_by_type(to_i915(dev), - mem_type), - &args->size, &args->handle); + obj = i915_gem_object_alloc(); + if (!obj) + return -ENOMEM; + + mr = intel_memory_region_by_type(to_i915(dev), mem_type); + object_set_placements(obj, &mr, 1); + + ret = i915_gem_setup(obj, args->size); + if (ret) + goto object_free; + + return i915_gem_publish(obj, file, &args->size, &args->handle); + +object_free: + i915_gem_object_free(obj); + return ret; } /** @@ -103,11 +176,229 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, { struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_create *args = data; + struct drm_i915_gem_object *obj; + struct intel_memory_region *mr; + int ret; i915_gem_flush_free_objects(i915); - return i915_gem_create(file, - intel_memory_region_by_type(i915, - INTEL_MEMORY_SYSTEM), - &args->size, &args->handle); + obj = i915_gem_object_alloc(); + if (!obj) + return -ENOMEM; + + mr = intel_memory_region_by_type(i915, INTEL_MEMORY_SYSTEM); + object_set_placements(obj, &mr, 1); + + ret = i915_gem_setup(obj, args->size); + if (ret) + goto object_free; + + return i915_gem_publish(obj, file, &args->size, &args->handle); + +object_free: + i915_gem_object_free(obj); + return ret; +} + +struct create_ext { + struct drm_i915_private *i915; + struct drm_i915_gem_object *vanilla_object; +}; + +static void repr_placements(char *buf, size_t size, + struct intel_memory_region **placements, + int n_placements) +{ + int i; + + buf[0] = '\0'; + + for (i = 0; i < n_placements; i++) { + struct intel_memory_region *mr = placements[i]; + int r; + + r = snprintf(buf, size, "\n %s -> { class: %d, inst: %d }", + mr->name, mr->type, mr->instance); + if (r >= size) + return; + + buf += r; + size -= r; + } +} + +static int set_placements(struct drm_i915_gem_create_ext_memory_regions *args, + struct create_ext *ext_data) +{ + struct drm_i915_private *i915 = ext_data->i915; + struct drm_i915_gem_memory_class_instance __user *uregions = + u64_to_user_ptr(args->regions); + struct drm_i915_gem_object *obj = ext_data->vanilla_object; + struct intel_memory_region **placements; + u32 mask; + int i, ret = 0; + + if (args->pad) { + drm_dbg(&i915->drm, "pad should be zero\n"); + ret = -EINVAL; + } + + if (!args->num_regions) { + drm_dbg(&i915->drm, "num_regions is zero\n"); + ret = -EINVAL; + } + + if (args->num_regions > ARRAY_SIZE(i915->mm.regions)) { + drm_dbg(&i915->drm, "num_regions is too large\n"); + ret = -EINVAL; + } + + if (ret) + return ret; + + placements = kmalloc_array(args->num_regions, + sizeof(struct intel_memory_region *), + GFP_KERNEL); + if (!placements) + return -ENOMEM; + + mask = 0; + for (i = 0; i < args->num_regions; i++) { + struct drm_i915_gem_memory_class_instance region; + struct intel_memory_region *mr; + + if (copy_from_user(®ion, uregions, sizeof(region))) { + ret = -EFAULT; + goto out_free; + } + + mr = intel_memory_region_lookup(i915, + region.memory_class, + region.memory_instance); + if (!mr || mr->private) { + drm_dbg(&i915->drm, "Device is missing region { class: %d, inst: %d } at index = %d\n", + region.memory_class, region.memory_instance, i); + ret = -EINVAL; + goto out_dump; + } + + if (mask & BIT(mr->id)) { + drm_dbg(&i915->drm, "Found duplicate placement %s -> { class: %d, inst: %d } at index = %d\n", + mr->name, region.memory_class, + region.memory_instance, i); + ret = -EINVAL; + goto out_dump; + } + + placements[i] = mr; + mask |= BIT(mr->id); + + ++uregions; + } + + if (obj->mm.placements) { + ret = -EINVAL; + goto out_dump; + } + + object_set_placements(obj, placements, args->num_regions); + if (args->num_regions == 1) + kfree(placements); + + return 0; + +out_dump: + if (1) { + char buf[256]; + + if (obj->mm.placements) { + repr_placements(buf, + sizeof(buf), + obj->mm.placements, + obj->mm.n_placements); + drm_dbg(&i915->drm, + "Placements were already set in previous EXT. Existing placements: %s\n", + buf); + } + + repr_placements(buf, sizeof(buf), placements, i); + drm_dbg(&i915->drm, "New placements(so far validated): %s\n", buf); + } + +out_free: + kfree(placements); + return ret; +} + +static int ext_set_placements(struct i915_user_extension __user *base, + void *data) +{ + struct drm_i915_gem_create_ext_memory_regions ext; + + if (!IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) + return -ENODEV; + + if (copy_from_user(&ext, base, sizeof(ext))) + return -EFAULT; + + return set_placements(&ext, data); +} + +static const i915_user_extension_fn create_extensions[] = { + [I915_GEM_CREATE_EXT_MEMORY_REGIONS] = ext_set_placements, +}; + +/** + * Creates a new mm object and returns a handle to it. + * @dev: drm device pointer + * @data: ioctl data blob + * @file: drm file pointer + */ +int +i915_gem_create_ext_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct drm_i915_private *i915 = to_i915(dev); + struct drm_i915_gem_create_ext *args = data; + struct create_ext ext_data = { .i915 = i915 }; + struct intel_memory_region **placements_ext; + struct drm_i915_gem_object *obj; + int ret; + + if (args->flags) + return -EINVAL; + + i915_gem_flush_free_objects(i915); + + obj = i915_gem_object_alloc(); + if (!obj) + return -ENOMEM; + + ext_data.vanilla_object = obj; + ret = i915_user_extensions(u64_to_user_ptr(args->extensions), + create_extensions, + ARRAY_SIZE(create_extensions), + &ext_data); + placements_ext = obj->mm.placements; + if (ret) + goto object_free; + + if (!placements_ext) { + struct intel_memory_region *mr = + intel_memory_region_by_type(i915, INTEL_MEMORY_SYSTEM); + + object_set_placements(obj, &mr, 1); + } + + ret = i915_gem_setup(obj, args->size); + if (ret) + goto object_free; + + return i915_gem_publish(obj, file, &args->size, &args->handle); + +object_free: + if (obj->mm.n_placements > 1) + kfree(placements_ext); + i915_gem_object_free(obj); + return ret; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h b/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h index 7fd22f3efbef..28d6526e32ab 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_ioctls.h @@ -14,6 +14,8 @@ int i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int i915_gem_create_ext_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); int i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c index ce1c83c13d05..f44bdd08f7cb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c @@ -17,9 +17,27 @@ const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops = { .release = i915_gem_object_release_memory_region, }; +void __iomem * +i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, + unsigned long n, + unsigned long size) +{ + resource_size_t offset; + + GEM_BUG_ON(!i915_gem_object_is_contiguous(obj)); + + offset = i915_gem_object_get_dma_address(obj, n); + offset -= obj->mm.region->region.start; + + return io_mapping_map_wc(&obj->mm.region->iomap, offset, size); +} + bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj) { - return obj->ops == &i915_gem_lmem_obj_ops; + struct intel_memory_region *mr = obj->mm.region; + + return mr && (mr->type == INTEL_MEMORY_LOCAL || + mr->type == INTEL_MEMORY_STOLEN_LOCAL); } struct drm_i915_gem_object * diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h index 036d53c01de9..fac6bc5a5ebb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h @@ -14,6 +14,11 @@ struct intel_memory_region; extern const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops; +void __iomem * +i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, + unsigned long n, + unsigned long size); + bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj); struct drm_i915_gem_object * diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index ea74cbca95be..28144410df86 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -249,6 +249,9 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, if (obj->ops->release) obj->ops->release(obj); + if (obj->mm.n_placements > 1) + kfree(obj->mm.placements); + /* But keep the pointer alive for RCU-protected lookups */ call_rcu(&obj->rcu, __i915_gem_free_object_rcu); cond_resched(); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 8e485cb3343c..0727d0c76aa0 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -172,11 +172,13 @@ struct drm_i915_gem_object { #define I915_BO_ALLOC_CONTIGUOUS BIT(0) #define I915_BO_ALLOC_VOLATILE BIT(1) #define I915_BO_ALLOC_STRUCT_PAGE BIT(2) +#define I915_BO_ALLOC_CPU_CLEAR BIT(3) #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \ I915_BO_ALLOC_VOLATILE | \ - I915_BO_ALLOC_STRUCT_PAGE) -#define I915_BO_READONLY BIT(3) -#define I915_TILING_QUIRK_BIT 4 /* unknown swizzling; do not release! */ + I915_BO_ALLOC_STRUCT_PAGE | \ + I915_BO_ALLOC_CPU_CLEAR) +#define I915_BO_READONLY BIT(4) +#define I915_TILING_QUIRK_BIT 5 /* unknown swizzling; do not release! */ /* * Is the object to be mapped as read-only to the GPU @@ -220,6 +222,12 @@ struct drm_i915_gem_object { atomic_t shrink_pin; /** + * Priority list of potential placements for this object. + */ + struct intel_memory_region **placements; + int n_placements; + + /** * Memory region for this object. */ struct intel_memory_region *region; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index 6a84fb6dde24..ce8fcfc54079 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -95,6 +95,28 @@ i915_gem_object_get_pages_buddy(struct drm_i915_gem_object *obj) sg_mark_end(sg); i915_sg_trim(st); + /* Intended for kernel internal use only */ + if (obj->flags & I915_BO_ALLOC_CPU_CLEAR) { + struct scatterlist *sg; + unsigned long i; + + for_each_sg(st->sgl, sg, st->nents, i) { + unsigned int length; + void __iomem *vaddr; + dma_addr_t daddr; + + daddr = sg_dma_address(sg); + daddr -= mem->region.start; + length = sg_dma_len(sg); + + vaddr = io_mapping_map_wc(&mem->iomap, daddr, length); + memset64((void __force *)vaddr, 0, length / sizeof(u64)); + io_mapping_unmap(vaddr); + } + + wmb(); + } + __i915_gem_object_set_pages(obj, st, sg_page_sizes); return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index 4f9c8d3021ab..f4fb68e8955a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -38,15 +38,17 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) } static bool unsafe_drop_pages(struct drm_i915_gem_object *obj, - unsigned long shrink) + unsigned long shrink, bool trylock_vm) { unsigned long flags; flags = 0; if (shrink & I915_SHRINK_ACTIVE) - flags = I915_GEM_OBJECT_UNBIND_ACTIVE; + flags |= I915_GEM_OBJECT_UNBIND_ACTIVE; if (!(shrink & I915_SHRINK_BOUND)) - flags = I915_GEM_OBJECT_UNBIND_TEST; + flags |= I915_GEM_OBJECT_UNBIND_TEST; + if (trylock_vm) + flags |= I915_GEM_OBJECT_UNBIND_VM_TRYLOCK; if (i915_gem_object_unbind(obj, flags) == 0) return true; @@ -117,6 +119,9 @@ i915_gem_shrink(struct i915_gem_ww_ctx *ww, unsigned long scanned = 0; int err; + /* CHV + VTD workaround use stop_machine(); need to trylock vm->mutex */ + bool trylock_vm = !ww && intel_vm_no_concurrent_access_wa(i915); + trace_i915_gem_shrink(i915, target, shrink); /* @@ -204,7 +209,7 @@ i915_gem_shrink(struct i915_gem_ww_ctx *ww, spin_unlock_irqrestore(&i915->mm.obj_lock, flags); err = 0; - if (unsafe_drop_pages(obj, shrink)) { + if (unsafe_drop_pages(obj, shrink, trylock_vm)) { /* May arrive from get_pages on another bo */ if (!ww) { if (!i915_gem_object_trylock(obj)) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index b0597de206de..b5553fc3ac4d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -10,6 +10,7 @@ #include <drm/drm_mm.h> #include <drm/i915_drm.h> +#include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" #include "i915_drv.h" #include "i915_gem_stolen.h" @@ -122,6 +123,14 @@ static int i915_adjust_stolen(struct drm_i915_private *i915, } /* + * With stolen lmem, we don't need to check if the address range + * overlaps with the non-stolen system memory range, since lmem is local + * to the gpu. + */ + if (HAS_LMEM(i915)) + return 0; + + /* * Verify that nothing else uses this physical address. Stolen * memory should be reserved by the BIOS and hidden from the * kernel. So if the region is already marked as busy, something @@ -374,8 +383,9 @@ static void icl_get_stolen_reserved(struct drm_i915_private *i915, } } -static int i915_gem_init_stolen(struct drm_i915_private *i915) +static int i915_gem_init_stolen(struct intel_memory_region *mem) { + struct drm_i915_private *i915 = mem->i915; struct intel_uncore *uncore = &i915->uncore; resource_size_t reserved_base, stolen_top; resource_size_t reserved_total, reserved_size; @@ -396,10 +406,10 @@ static int i915_gem_init_stolen(struct drm_i915_private *i915) return 0; } - if (resource_size(&intel_graphics_stolen_res) == 0) + if (resource_size(&mem->region) == 0) return 0; - i915->dsm = intel_graphics_stolen_res; + i915->dsm = mem->region; if (i915_adjust_stolen(i915, &i915->dsm)) return 0; @@ -627,10 +637,17 @@ static int __i915_gem_object_create_stolen(struct intel_memory_region *mem, { static struct lock_class_key lock_class; unsigned int cache_level; + unsigned int flags; int err; + /* + * Stolen objects are always physically contiguous since we just + * allocate one big block underneath using the drm_mm range allocator. + */ + flags = I915_BO_ALLOC_CONTIGUOUS; + drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size); - i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, 0); + i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, flags); obj->stolen = stolen; obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; @@ -640,9 +657,11 @@ static int __i915_gem_object_create_stolen(struct intel_memory_region *mem, if (WARN_ON(!i915_gem_object_trylock(obj))) return -EBUSY; + i915_gem_object_init_memory_region(obj, mem); + err = i915_gem_object_pin_pages(obj); - if (!err) - i915_gem_object_init_memory_region(obj, mem); + if (err) + i915_gem_object_release_memory_region(obj); i915_gem_object_unlock(obj); return err; @@ -667,7 +686,8 @@ static int _i915_gem_object_stolen_init(struct intel_memory_region *mem, if (!stolen) return -ENOMEM; - ret = i915_gem_stolen_insert_node(i915, stolen, size, 4096); + ret = i915_gem_stolen_insert_node(i915, stolen, size, + mem->min_page_size); if (ret) goto err_free; @@ -688,39 +708,126 @@ struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_i915_private *i915, resource_size_t size) { - return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_STOLEN_SMEM], - size, I915_BO_ALLOC_CONTIGUOUS); + return i915_gem_object_create_region(i915->mm.stolen_region, size, 0); } -static int init_stolen(struct intel_memory_region *mem) +static int init_stolen_smem(struct intel_memory_region *mem) { - intel_memory_region_set_name(mem, "stolen"); - /* * Initialise stolen early so that we may reserve preallocated * objects for the BIOS to KMS transition. */ - return i915_gem_init_stolen(mem->i915); + return i915_gem_init_stolen(mem); } -static void release_stolen(struct intel_memory_region *mem) +static void release_stolen_smem(struct intel_memory_region *mem) { i915_gem_cleanup_stolen(mem->i915); } -static const struct intel_memory_region_ops i915_region_stolen_ops = { - .init = init_stolen, - .release = release_stolen, +static const struct intel_memory_region_ops i915_region_stolen_smem_ops = { + .init = init_stolen_smem, + .release = release_stolen_smem, .init_object = _i915_gem_object_stolen_init, }; -struct intel_memory_region *i915_gem_stolen_setup(struct drm_i915_private *i915) +static int init_stolen_lmem(struct intel_memory_region *mem) { - return intel_memory_region_create(i915, - intel_graphics_stolen_res.start, - resource_size(&intel_graphics_stolen_res), - PAGE_SIZE, 0, - &i915_region_stolen_ops); + int err; + + if (GEM_WARN_ON(resource_size(&mem->region) == 0)) + return -ENODEV; + + if (!io_mapping_init_wc(&mem->iomap, + mem->io_start, + resource_size(&mem->region))) + return -EIO; + + /* + * TODO: For stolen lmem we mostly just care about populating the dsm + * related bits and setting up the drm_mm allocator for the range. + * Perhaps split up i915_gem_init_stolen() for this. + */ + err = i915_gem_init_stolen(mem); + if (err) + goto err_fini; + + return 0; + +err_fini: + io_mapping_fini(&mem->iomap); + return err; +} + +static void release_stolen_lmem(struct intel_memory_region *mem) +{ + io_mapping_fini(&mem->iomap); + i915_gem_cleanup_stolen(mem->i915); +} + +static const struct intel_memory_region_ops i915_region_stolen_lmem_ops = { + .init = init_stolen_lmem, + .release = release_stolen_lmem, + .init_object = _i915_gem_object_stolen_init, +}; + +struct intel_memory_region * +i915_gem_stolen_lmem_setup(struct drm_i915_private *i915) +{ + struct intel_uncore *uncore = &i915->uncore; + struct pci_dev *pdev = to_pci_dev(i915->drm.dev); + struct intel_memory_region *mem; + resource_size_t io_start; + resource_size_t lmem_size; + u64 lmem_base; + + lmem_base = intel_uncore_read64(uncore, GEN12_DSMBASE); + if (GEM_WARN_ON(lmem_base >= pci_resource_len(pdev, 2))) + return ERR_PTR(-ENODEV); + + lmem_size = pci_resource_len(pdev, 2) - lmem_base; + io_start = pci_resource_start(pdev, 2) + lmem_base; + + mem = intel_memory_region_create(i915, lmem_base, lmem_size, + I915_GTT_PAGE_SIZE_4K, io_start, + &i915_region_stolen_lmem_ops); + if (IS_ERR(mem)) + return mem; + + /* + * TODO: consider creating common helper to just print all the + * interesting stuff from intel_memory_region, which we can use for all + * our probed regions. + */ + + drm_dbg(&i915->drm, "Stolen Local memory IO start: %pa\n", + &mem->io_start); + + intel_memory_region_set_name(mem, "stolen-local"); + + mem->private = true; + + return mem; +} + +struct intel_memory_region* +i915_gem_stolen_smem_setup(struct drm_i915_private *i915) +{ + struct intel_memory_region *mem; + + mem = intel_memory_region_create(i915, + intel_graphics_stolen_res.start, + resource_size(&intel_graphics_stolen_res), + PAGE_SIZE, 0, + &i915_region_stolen_smem_ops); + if (IS_ERR(mem)) + return mem; + + intel_memory_region_set_name(mem, "stolen-system"); + + mem->private = true; + + return mem; } struct drm_i915_gem_object * @@ -728,7 +835,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *i915, resource_size_t stolen_offset, resource_size_t size) { - struct intel_memory_region *mem = i915->mm.regions[INTEL_REGION_STOLEN_SMEM]; + struct intel_memory_region *mem = i915->mm.stolen_region; struct drm_i915_gem_object *obj; struct drm_mm_node *stolen; int ret; @@ -742,8 +849,8 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *i915, /* KISS and expect everything to be page-aligned */ if (GEM_WARN_ON(size == 0) || - GEM_WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) || - GEM_WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT))) + GEM_WARN_ON(!IS_ALIGNED(size, mem->min_page_size)) || + GEM_WARN_ON(!IS_ALIGNED(stolen_offset, mem->min_page_size))) return ERR_PTR(-EINVAL); stolen = kzalloc(sizeof(*stolen), GFP_KERNEL); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h index b03489706796..2bec6c367b9c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h @@ -21,7 +21,8 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, u64 end); void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, struct drm_mm_node *node); -struct intel_memory_region *i915_gem_stolen_setup(struct drm_i915_private *i915); +struct intel_memory_region *i915_gem_stolen_smem_setup(struct drm_i915_private *i915); +struct intel_memory_region *i915_gem_stolen_lmem_setup(struct drm_i915_private *i915); struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, resource_size_t size); diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 5fef592390cb..ce70d0a3afb2 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -1740,7 +1740,6 @@ out: static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) { struct i915_address_space *vm; - struct page *page; u32 *vaddr; int err = 0; @@ -1748,24 +1747,18 @@ static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) if (!vm) return -ENODEV; - page = __px_page(vm->scratch[0]); - if (!page) { + if (!vm->scratch[0]) { pr_err("No scratch page!\n"); return -EINVAL; } - vaddr = kmap(page); - if (!vaddr) { - pr_err("No (mappable) scratch page!\n"); - return -EINVAL; - } + vaddr = __px_vaddr(vm->scratch[0]); memcpy(out, vaddr, sizeof(*out)); if (memchr_inv(vaddr, *out, PAGE_SIZE)) { pr_err("Inconsistent initial state of scratch page!\n"); err = -EINVAL; } - kunmap(page); return err; } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index 5cf6df49c333..05a3b29f545e 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -842,6 +842,24 @@ static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) return true; } +static void object_set_placements(struct drm_i915_gem_object *obj, + struct intel_memory_region **placements, + unsigned int n_placements) +{ + GEM_BUG_ON(!n_placements); + + if (n_placements == 1) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct intel_memory_region *mr = placements[0]; + + obj->mm.placements = &i915->mm.regions[mr->id]; + obj->mm.n_placements = 1; + } else { + obj->mm.placements = placements; + obj->mm.n_placements = n_placements; + } +} + #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) static int __igt_mmap(struct drm_i915_private *i915, struct drm_i915_gem_object *obj, @@ -950,6 +968,8 @@ static int igt_mmap(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC); @@ -1068,6 +1088,8 @@ static int igt_mmap_access(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB); @@ -1211,6 +1233,8 @@ static int igt_mmap_gpu(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC); @@ -1354,6 +1378,8 @@ static int igt_mmap_revoke(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + object_set_placements(obj, &mr, 1); + err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT); if (err == 0) err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC); diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index e08dff376339..1aee5e6b1b23 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -96,9 +96,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, * entries back to scratch. */ - vaddr = kmap_atomic_px(pt); + vaddr = px_vaddr(pt); memset32(vaddr + pte, scratch_pte, count); - kunmap_atomic(vaddr); pte = 0; } @@ -120,7 +119,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, GEM_BUG_ON(!pd->entry[act_pt]); - vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt)); + vaddr = px_vaddr(i915_pt_entry(pd, act_pt)); do { GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE); vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); @@ -136,12 +135,10 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, } if (++act_pte == GEN6_PTES) { - kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt)); + vaddr = px_vaddr(i915_pt_entry(pd, ++act_pt)); act_pte = 0; } } while (1); - kunmap_atomic(vaddr); vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; } @@ -235,7 +232,7 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) goto err_scratch0; } - ret = pin_pt_dma(vm, vm->scratch[1]); + ret = map_pt_dma(vm, vm->scratch[1]); if (ret) goto err_scratch1; @@ -346,7 +343,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size) if (!vma) return ERR_PTR(-ENOMEM); - i915_active_init(&vma->active, NULL, NULL); + i915_active_init(&vma->active, NULL, NULL, 0); kref_init(&vma->ref); mutex_init(&vma->pages_mutex); diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 74bf6fc8461f..e3a8924d2286 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -242,11 +242,10 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, atomic_read(&pt->used)); GEM_BUG_ON(!count || count >= atomic_read(&pt->used)); - vaddr = kmap_atomic_px(pt); + vaddr = px_vaddr(pt); memset64(vaddr + gen8_pd_index(start, 0), vm->scratch[0]->encode, count); - kunmap_atomic(vaddr); atomic_sub(count, &pt->used); start += count; @@ -375,7 +374,7 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, gen8_pte_t *vaddr; pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2)); - vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); + vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1))); do { GEM_BUG_ON(sg_dma_len(iter->sg) < I915_GTT_PAGE_SIZE); vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; @@ -402,12 +401,10 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, } clflush_cache_range(vaddr, PAGE_SIZE); - kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); + vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1))); } } while (1); clflush_cache_range(vaddr, PAGE_SIZE); - kunmap_atomic(vaddr); return idx; } @@ -442,7 +439,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, encode |= GEN8_PDE_PS_2M; page_size = I915_GTT_PAGE_SIZE_2M; - vaddr = kmap_atomic_px(pd); + vaddr = px_vaddr(pd); } else { struct i915_page_table *pt = i915_pt_entry(pd, __gen8_pte_index(start, 1)); @@ -457,7 +454,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)) maybe_64K = __gen8_pte_index(start, 1); - vaddr = kmap_atomic_px(pt); + vaddr = px_vaddr(pt); } do { @@ -491,7 +488,6 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, } while (rem >= page_size && index < I915_PDES); clflush_cache_range(vaddr, PAGE_SIZE); - kunmap_atomic(vaddr); /* * Is it safe to mark the 2M block as 64K? -- Either we have @@ -505,9 +501,8 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, !iter->sg && IS_ALIGNED(vma->node.start + vma->node.size, I915_GTT_PAGE_SIZE_2M)))) { - vaddr = kmap_atomic_px(pd); + vaddr = px_vaddr(pd); vaddr[maybe_64K] |= GEN8_PDE_IPS_64K; - kunmap_atomic(vaddr); page_size = I915_GTT_PAGE_SIZE_64K; /* @@ -523,12 +518,11 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, u16 i; encode = vma->vm->scratch[0]->encode; - vaddr = kmap_atomic_px(i915_pt_entry(pd, maybe_64K)); + vaddr = px_vaddr(i915_pt_entry(pd, maybe_64K)); for (i = 1; i < index; i += 16) memset64(vaddr + i, encode, 15); - kunmap_atomic(vaddr); } } @@ -602,7 +596,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) if (IS_ERR(obj)) goto free_scratch; - ret = pin_pt_dma(vm, obj); + ret = map_pt_dma(vm, obj); if (ret) { i915_gem_object_put(obj); goto free_scratch; @@ -639,7 +633,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) if (IS_ERR(pde)) return PTR_ERR(pde); - err = pin_pt_dma(vm, pde->pt.base); + err = map_pt_dma(vm, pde->pt.base); if (err) { free_pd(vm, pde); return err; @@ -674,7 +668,7 @@ gen8_alloc_top_pd(struct i915_address_space *vm) goto err_pd; } - err = pin_pt_dma(vm, pd->pt.base); + err = map_pt_dma(vm, pd->pt.base); if (err) goto err_pd; @@ -717,7 +711,10 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) */ ppgtt->vm.has_read_only = !IS_GEN_RANGE(gt->i915, 11, 12); - ppgtt->vm.alloc_pt_dma = alloc_pt_dma; + if (HAS_LMEM(gt->i915)) + ppgtt->vm.alloc_pt_dma = alloc_pt_lmem; + else + ppgtt->vm.alloc_pt_dma = alloc_pt_dma; err = gen8_init_scratch(&ppgtt->vm); if (err) diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 17cf2640b082..4033184f13b9 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -326,7 +326,6 @@ void intel_context_unpin(struct intel_context *ce) intel_context_put(ce); } -__i915_active_call static void __intel_context_retire(struct i915_active *active) { struct intel_context *ce = container_of(active, typeof(*ce), active); @@ -385,7 +384,7 @@ intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) mutex_init(&ce->pin_mutex); i915_active_init(&ce->active, - __intel_context_active, __intel_context_retire); + __intel_context_active, __intel_context_retire, 0); } void intel_context_fini(struct intel_context *ce) diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h index 47ee8578e511..8d9184920c51 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine.h +++ b/drivers/gpu/drm/i915/gt/intel_engine.h @@ -13,8 +13,9 @@ #include "i915_reg.h" #include "i915_request.h" #include "i915_selftest.h" -#include "gt/intel_timeline.h" #include "intel_engine_types.h" +#include "intel_gt_types.h" +#include "intel_timeline.h" #include "intel_workarounds.h" struct drm_printer; @@ -262,6 +263,11 @@ void intel_engine_init_active(struct intel_engine_cs *engine, #define ENGINE_MOCK 1 #define ENGINE_VIRTUAL 2 +static inline bool intel_engine_uses_guc(const struct intel_engine_cs *engine) +{ + return engine->gt->submission_method >= INTEL_SUBMISSION_GUC; +} + static inline bool intel_engine_has_preempt_reset(const struct intel_engine_cs *engine) { diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 6dbdbde00f14..3f9a811eb02b 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -255,6 +255,11 @@ static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) intel_engine_set_hwsp_writemask(engine, ~0u); } +static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + GEM_DEBUG_WARN_ON(iir); +} + static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) { const struct engine_info *info = &intel_engines[id]; @@ -292,6 +297,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->hw_id = info->hw_id; engine->guc_id = MAKE_GUC_ID(info->class, info->instance); + engine->irq_handler = nop_irq_handler; + engine->class = info->class; engine->instance = info->instance; __sprint_engine_name(engine); @@ -898,7 +905,7 @@ static int engine_init_common(struct intel_engine_cs *engine) return 0; err_context: - intel_context_put(ce); + destroy_pinned_context(ce); return ret; } @@ -909,12 +916,16 @@ int intel_engines_init(struct intel_gt *gt) enum intel_engine_id id; int err; - if (intel_uc_uses_guc_submission(>->uc)) + if (intel_uc_uses_guc_submission(>->uc)) { + gt->submission_method = INTEL_SUBMISSION_GUC; setup = intel_guc_submission_setup; - else if (HAS_EXECLISTS(gt->i915)) + } else if (HAS_EXECLISTS(gt->i915)) { + gt->submission_method = INTEL_SUBMISSION_ELSP; setup = intel_execlists_submission_setup; - else + } else { + gt->submission_method = INTEL_SUBMISSION_RING; setup = intel_ring_submission_setup; + } for_each_engine(engine, gt, id) { err = engine_setup_common(engine); @@ -1479,7 +1490,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); } - if (intel_engine_in_guc_submission_mode(engine)) { + if (intel_engine_uses_guc(engine)) { /* nothing to print yet */ } else if (HAS_EXECLISTS(dev_priv)) { struct i915_request * const *port, *rq; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index 7c9af86fdb1e..47f4397095e5 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -23,7 +23,7 @@ static void dbg_poison_ce(struct intel_context *ce) if (ce->state) { struct drm_i915_gem_object *obj = ce->state->obj; - int type = i915_coherent_map_type(ce->engine->i915); + int type = i915_coherent_map_type(ce->engine->i915, obj, true); void *map; if (!i915_gem_object_trylock(obj)) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 883bafc44902..9ef349cd5cea 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -402,6 +402,7 @@ struct intel_engine_cs { u32 irq_enable_mask; /* bitmask to enable ring interrupt */ void (*irq_enable)(struct intel_engine_cs *engine); void (*irq_disable)(struct intel_engine_cs *engine); + void (*irq_handler)(struct intel_engine_cs *engine, u16 iir); void (*sanitize)(struct intel_engine_cs *engine); int (*resume)(struct intel_engine_cs *engine); @@ -481,10 +482,9 @@ struct intel_engine_cs { #define I915_ENGINE_HAS_PREEMPTION BIT(2) #define I915_ENGINE_HAS_SEMAPHORES BIT(3) #define I915_ENGINE_HAS_TIMESLICES BIT(4) -#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(5) -#define I915_ENGINE_IS_VIRTUAL BIT(6) -#define I915_ENGINE_HAS_RELATIVE_MMIO BIT(7) -#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(8) +#define I915_ENGINE_IS_VIRTUAL BIT(5) +#define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) +#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) unsigned int flags; /* @@ -594,12 +594,6 @@ intel_engine_has_timeslices(const struct intel_engine_cs *engine) } static inline bool -intel_engine_needs_breadcrumb_tasklet(const struct intel_engine_cs *engine) -{ - return engine->flags & I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; -} - -static inline bool intel_engine_is_virtual(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_IS_VIRTUAL; diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index de124870af44..8db200422950 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -118,6 +118,7 @@ #include "intel_engine_stats.h" #include "intel_execlists_submission.h" #include "intel_gt.h" +#include "intel_gt_irq.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" #include "intel_lrc.h" @@ -1768,7 +1769,6 @@ process_csb(struct intel_engine_cs *engine, struct i915_request **inactive) */ GEM_BUG_ON(!tasklet_is_locked(&execlists->tasklet) && !reset_in_progress(execlists)); - GEM_BUG_ON(!intel_engine_in_execlists_submission_mode(engine)); /* * Note that csb_write, csb_status may be either in HWSP or mmio. @@ -2385,6 +2385,45 @@ static void execlists_submission_tasklet(struct tasklet_struct *t) rcu_read_unlock(); } +static void execlists_irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + bool tasklet = false; + + if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) { + u32 eir; + + /* Upper 16b are the enabling mask, rsvd for internal errors */ + eir = ENGINE_READ(engine, RING_EIR) & GENMASK(15, 0); + ENGINE_TRACE(engine, "CS error: %x\n", eir); + + /* Disable the error interrupt until after the reset */ + if (likely(eir)) { + ENGINE_WRITE(engine, RING_EMR, ~0u); + ENGINE_WRITE(engine, RING_EIR, eir); + WRITE_ONCE(engine->execlists.error_interrupt, eir); + tasklet = true; + } + } + + if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) { + WRITE_ONCE(engine->execlists.yield, + ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI)); + ENGINE_TRACE(engine, "semaphore yield: %08x\n", + engine->execlists.yield); + if (del_timer(&engine->execlists.timer)) + tasklet = true; + } + + if (iir & GT_CONTEXT_SWITCH_INTERRUPT) + tasklet = true; + + if (iir & GT_RENDER_USER_INTERRUPT) + intel_engine_signal_breadcrumbs(engine); + + if (tasklet) + tasklet_hi_schedule(&engine->execlists.tasklet); +} + static void __execlists_kick(struct intel_engine_execlists *execlists) { /* Kick the tasklet for some interrupt coalescing and reset handling */ @@ -3076,29 +3115,6 @@ static void execlists_set_default_submission(struct intel_engine_cs *engine) engine->submit_request = execlists_submit_request; engine->schedule = i915_schedule; engine->execlists.tasklet.callback = execlists_submission_tasklet; - - engine->reset.prepare = execlists_reset_prepare; - engine->reset.rewind = execlists_reset_rewind; - engine->reset.cancel = execlists_reset_cancel; - engine->reset.finish = execlists_reset_finish; - - engine->park = execlists_park; - engine->unpark = NULL; - - engine->flags |= I915_ENGINE_SUPPORTS_STATS; - if (!intel_vgpu_active(engine->i915)) { - engine->flags |= I915_ENGINE_HAS_SEMAPHORES; - if (can_preempt(engine)) { - engine->flags |= I915_ENGINE_HAS_PREEMPTION; - if (IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) - engine->flags |= I915_ENGINE_HAS_TIMESLICES; - } - } - - if (intel_engine_has_preemption(engine)) - engine->emit_bb_start = gen8_emit_bb_start; - else - engine->emit_bb_start = gen8_emit_bb_start_noarb; } static void execlists_shutdown(struct intel_engine_cs *engine) @@ -3129,6 +3145,14 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->cops = &execlists_context_ops; engine->request_alloc = execlists_request_alloc; + engine->reset.prepare = execlists_reset_prepare; + engine->reset.rewind = execlists_reset_rewind; + engine->reset.cancel = execlists_reset_cancel; + engine->reset.finish = execlists_reset_finish; + + engine->park = execlists_park; + engine->unpark = NULL; + engine->emit_flush = gen8_emit_flush_xcs; engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs; @@ -3149,6 +3173,22 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) * until a more refined solution exists. */ } + intel_engine_set_irq_handler(engine, execlists_irq_handler); + + engine->flags |= I915_ENGINE_SUPPORTS_STATS; + if (!intel_vgpu_active(engine->i915)) { + engine->flags |= I915_ENGINE_HAS_SEMAPHORES; + if (can_preempt(engine)) { + engine->flags |= I915_ENGINE_HAS_PREEMPTION; + if (IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + engine->flags |= I915_ENGINE_HAS_TIMESLICES; + } + } + + if (intel_engine_has_preemption(engine)) + engine->emit_bb_start = gen8_emit_bb_start; + else + engine->emit_bb_start = gen8_emit_bb_start_noarb; } static void logical_ring_default_irqs(struct intel_engine_cs *engine) @@ -3884,13 +3924,6 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, spin_unlock_irqrestore(&engine->active.lock, flags); } -bool -intel_engine_in_execlists_submission_mode(const struct intel_engine_cs *engine) -{ - return engine->set_default_submission == - execlists_set_default_submission; -} - #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_execlists.c" #endif diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.h b/drivers/gpu/drm/i915/gt/intel_execlists_submission.h index fd61dae820e9..4ca9b475e252 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.h +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.h @@ -43,7 +43,4 @@ int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, const struct intel_engine_cs *master, const struct intel_engine_cs *sibling); -bool -intel_engine_in_execlists_submission_mode(const struct intel_engine_cs *engine); - #endif /* __INTEL_EXECLISTS_SUBMISSION_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 38742bf33fa3..35069ca5d7de 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -658,7 +658,7 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) goto err_ppgtt; i915_gem_object_lock(ppgtt->vm.scratch[0], NULL); - err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); + err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); i915_gem_object_unlock(ppgtt->vm.scratch[0]); if (err) goto err_stash; @@ -907,9 +907,11 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.insert_entries = gen8_ggtt_insert_entries; - /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ - if (intel_ggtt_update_needs_vtd_wa(i915) || - IS_CHERRYVIEW(i915) /* fails with concurrent use/update */) { + /* + * Serialize GTT updates with aperture access on BXT if VT-d is on, + * and always on CHV. + */ + if (intel_vm_no_concurrent_access_wa(i915)) { ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; ggtt->vm.bind_async_flags = diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index 8a322594210c..7bf84cd21543 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -653,8 +653,8 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt) * banks of memory are paired and unswizzled on the * uneven portion, so leave that as unknown. */ - if (intel_uncore_read16(uncore, C0DRB3) == - intel_uncore_read16(uncore, C1DRB3)) { + if (intel_uncore_read16(uncore, C0DRB3_BW) == + intel_uncore_read16(uncore, C1DRB3_BW)) { swizzle_x = I915_BIT_6_SWIZZLE_9_10; swizzle_y = I915_BIT_6_SWIZZLE_9; } @@ -867,7 +867,7 @@ void intel_ggtt_init_fences(struct i915_ggtt *ggtt) for (i = 0; i < num_fences; i++) { struct i915_fence_reg *fence = &ggtt->fence_regs[i]; - i915_active_init(&fence->active, NULL, NULL); + i915_active_init(&fence->active, NULL, NULL, 0); fence->ggtt = ggtt; fence->id = i; list_add_tail(&fence->link, &ggtt->fence_list); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c index c59468107598..aa0a59c5b614 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c @@ -98,7 +98,6 @@ static void pool_free_work(struct work_struct *wrk) round_jiffies_up_relative(HZ)); } -__i915_active_call static void pool_retire(struct i915_active *ref) { struct intel_gt_buffer_pool_node *node = @@ -154,7 +153,7 @@ node_create(struct intel_gt_buffer_pool *pool, size_t sz, node->age = 0; node->pool = pool; node->pinned = false; - i915_active_init(&node->active, NULL, pool_retire); + i915_active_init(&node->active, NULL, pool_retire, 0); obj = i915_gem_object_create_internal(gt->i915, sz); if (IS_ERR(obj)) { diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index 9fc6c912a4e5..d29126c458ba 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -20,48 +20,6 @@ static void guc_irq_handler(struct intel_guc *guc, u16 iir) intel_guc_to_host_event_handler(guc); } -static void -cs_irq_handler(struct intel_engine_cs *engine, u32 iir) -{ - bool tasklet = false; - - if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) { - u32 eir; - - /* Upper 16b are the enabling mask, rsvd for internal errors */ - eir = ENGINE_READ(engine, RING_EIR) & GENMASK(15, 0); - ENGINE_TRACE(engine, "CS error: %x\n", eir); - - /* Disable the error interrupt until after the reset */ - if (likely(eir)) { - ENGINE_WRITE(engine, RING_EMR, ~0u); - ENGINE_WRITE(engine, RING_EIR, eir); - WRITE_ONCE(engine->execlists.error_interrupt, eir); - tasklet = true; - } - } - - if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) { - WRITE_ONCE(engine->execlists.yield, - ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI)); - ENGINE_TRACE(engine, "semaphore yield: %08x\n", - engine->execlists.yield); - if (del_timer(&engine->execlists.timer)) - tasklet = true; - } - - if (iir & GT_CONTEXT_SWITCH_INTERRUPT) - tasklet = true; - - if (iir & GT_RENDER_USER_INTERRUPT) { - intel_engine_signal_breadcrumbs(engine); - tasklet |= intel_engine_needs_breadcrumb_tasklet(engine); - } - - if (tasklet) - tasklet_hi_schedule(&engine->execlists.tasklet); -} - static u32 gen11_gt_engine_identity(struct intel_gt *gt, const unsigned int bank, const unsigned int bit) @@ -122,7 +80,7 @@ gen11_engine_irq_handler(struct intel_gt *gt, const u8 class, engine = NULL; if (likely(engine)) - return cs_irq_handler(engine, iir); + return intel_engine_cs_irq(engine, iir); WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n", class, instance); @@ -275,9 +233,12 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) { if (gt_iir & GT_RENDER_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], + gt_iir); + if (gt_iir & ILK_BSD_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], + gt_iir); } static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) @@ -301,11 +262,16 @@ static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) { if (gt_iir & GT_RENDER_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], + gt_iir); + if (gt_iir & GT_BSD_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], + gt_iir >> 12); + if (gt_iir & GT_BLT_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine_class[COPY_ENGINE_CLASS][0]); + intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0], + gt_iir >> 22); if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | GT_BSD_CS_ERROR_INTERRUPT | @@ -324,10 +290,10 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl) if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { iir = raw_reg_read(regs, GEN8_GT_IIR(0)); if (likely(iir)) { - cs_irq_handler(gt->engine_class[RENDER_CLASS][0], - iir >> GEN8_RCS_IRQ_SHIFT); - cs_irq_handler(gt->engine_class[COPY_ENGINE_CLASS][0], - iir >> GEN8_BCS_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], + iir >> GEN8_RCS_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0], + iir >> GEN8_BCS_IRQ_SHIFT); raw_reg_write(regs, GEN8_GT_IIR(0), iir); } } @@ -335,10 +301,10 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl) if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) { iir = raw_reg_read(regs, GEN8_GT_IIR(1)); if (likely(iir)) { - cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][0], - iir >> GEN8_VCS0_IRQ_SHIFT); - cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][1], - iir >> GEN8_VCS1_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], + iir >> GEN8_VCS0_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1], + iir >> GEN8_VCS1_IRQ_SHIFT); raw_reg_write(regs, GEN8_GT_IIR(1), iir); } } @@ -346,8 +312,8 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl) if (master_ctl & GEN8_GT_VECS_IRQ) { iir = raw_reg_read(regs, GEN8_GT_IIR(3)); if (likely(iir)) { - cs_irq_handler(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0], - iir >> GEN8_VECS_IRQ_SHIFT); + intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0], + iir >> GEN8_VECS_IRQ_SHIFT); raw_reg_write(regs, GEN8_GT_IIR(3), iir); } } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.h b/drivers/gpu/drm/i915/gt/intel_gt_irq.h index f667e976fb2b..41cad38668c5 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.h @@ -8,6 +8,8 @@ #include <linux/types.h> +#include "intel_engine_types.h" + struct intel_gt; #define GEN8_GT_IRQS (GEN8_GT_RCS_IRQ | \ @@ -39,4 +41,25 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl); void gen8_gt_irq_reset(struct intel_gt *gt); void gen8_gt_irq_postinstall(struct intel_gt *gt); +static inline void intel_engine_cs_irq(struct intel_engine_cs *engine, u16 iir) +{ + if (iir) + engine->irq_handler(engine, iir); +} + +static inline void +intel_engine_set_irq_handler(struct intel_engine_cs *engine, + void (*fn)(struct intel_engine_cs *engine, + u16 iir)) +{ + /* + * As the interrupt is live as allocate and setup the engines, + * err on the side of caution and apply barriers to updating + * the irq handler callback. This assures that when we do use + * the engine, we will receive interrupts only to ourselves, + * and not lose any. + */ + smp_store_mb(engine->irq_handler, fn); +} + #endif /* INTEL_GT_IRQ_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index 0caf6ca0a784..fecfacf551d5 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -31,6 +31,12 @@ struct i915_ggtt; struct intel_engine_cs; struct intel_uncore; +enum intel_submission_method { + INTEL_SUBMISSION_RING, + INTEL_SUBMISSION_ELSP, + INTEL_SUBMISSION_GUC, +}; + struct intel_gt { struct drm_i915_private *i915; struct intel_uncore *uncore; @@ -118,6 +124,7 @@ struct intel_gt { struct intel_engine_cs *engine[I915_NUM_ENGINES]; struct intel_engine_cs *engine_class[MAX_ENGINE_CLASS + 1] [MAX_ENGINE_INSTANCE + 1]; + enum intel_submission_method submission_method; /* * Default address space (either GGTT or ppGTT depending on arch). diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index 941f8af016d6..9b98f9d9faa3 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -7,10 +7,26 @@ #include <linux/fault-inject.h> +#include "gem/i915_gem_lmem.h" #include "i915_trace.h" #include "intel_gt.h" #include "intel_gtt.h" +struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz) +{ + struct drm_i915_gem_object *obj; + + obj = i915_gem_object_create_lmem(vm->i915, sz, 0); + /* + * Ensure all paging structures for this vm share the same dma-resv + * object underneath, with the idea that one object_lock() will lock + * them all at once. + */ + if (!IS_ERR(obj)) + obj->base.resv = &vm->resv; + return obj; +} + struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz) { struct drm_i915_gem_object *obj; @@ -19,33 +35,39 @@ struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz) i915_gem_shrink_all(vm->i915); obj = i915_gem_object_create_internal(vm->i915, sz); - /* ensure all dma objects have the same reservation class */ + /* + * Ensure all paging structures for this vm share the same dma-resv + * object underneath, with the idea that one object_lock() will lock + * them all at once. + */ if (!IS_ERR(obj)) obj->base.resv = &vm->resv; return obj; } -int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj) +int map_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj) { - int err; + enum i915_map_type type; + void *vaddr; - i915_gem_object_lock(obj, NULL); - err = i915_gem_object_pin_pages(obj); - i915_gem_object_unlock(obj); - if (err) - return err; + type = i915_coherent_map_type(vm->i915, obj, true); + vaddr = i915_gem_object_pin_map_unlocked(obj, type); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); i915_gem_object_make_unshrinkable(obj); return 0; } -int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj) +int map_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj) { - int err; + enum i915_map_type type; + void *vaddr; - err = i915_gem_object_pin_pages(obj); - if (err) - return err; + type = i915_coherent_map_type(vm->i915, obj, true); + vaddr = i915_gem_object_pin_map(obj, type); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); i915_gem_object_make_unshrinkable(obj); return 0; @@ -132,7 +154,22 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) */ mutex_init(&vm->mutex); lockdep_set_subclass(&vm->mutex, subclass); - i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); + + if (!intel_vm_no_concurrent_access_wa(vm->i915)) { + i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); + } else { + /* + * CHV + BXT VTD workaround use stop_machine(), + * which is allowed to allocate memory. This means &vm->mutex + * is the outer lock, and in theory we can allocate memory inside + * it through stop_machine(). + * + * Add the annotation for this, we use trylock in shrinker. + */ + mutex_acquire(&vm->mutex.dep_map, 0, 0, _THIS_IP_); + might_alloc(GFP_KERNEL); + mutex_release(&vm->mutex.dep_map, _THIS_IP_); + } dma_resv_init(&vm->resv); GEM_BUG_ON(!vm->total); @@ -155,6 +192,14 @@ void clear_pages(struct i915_vma *vma) memset(&vma->page_sizes, 0, sizeof(vma->page_sizes)); } +void *__px_vaddr(struct drm_i915_gem_object *p) +{ + enum i915_map_type type; + + GEM_BUG_ON(!i915_gem_object_has_pages(p)); + return page_unpack_bits(p->mm.mapping, &type); +} + dma_addr_t __px_dma(struct drm_i915_gem_object *p) { GEM_BUG_ON(!i915_gem_object_has_pages(p)); @@ -170,32 +215,22 @@ struct page *__px_page(struct drm_i915_gem_object *p) void fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count) { - struct page *page = __px_page(p); - void *vaddr; + void *vaddr = __px_vaddr(p); - vaddr = kmap(page); memset64(vaddr, val, count); clflush_cache_range(vaddr, PAGE_SIZE); - kunmap(page); } static void poison_scratch_page(struct drm_i915_gem_object *scratch) { - struct sgt_iter sgt; - struct page *page; + void *vaddr = __px_vaddr(scratch); u8 val; val = 0; if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) val = POISON_FREE; - for_each_sgt_page(page, sgt, scratch->mm.pages) { - void *vaddr; - - vaddr = kmap(page); - memset(vaddr, val, PAGE_SIZE); - kunmap(page); - } + memset(vaddr, val, scratch->base.size); } int setup_scratch_page(struct i915_address_space *vm) @@ -225,7 +260,7 @@ int setup_scratch_page(struct i915_address_space *vm) if (IS_ERR(obj)) goto skip; - if (pin_pt_dma(vm, obj)) + if (map_pt_dma(vm, obj)) goto skip_obj; /* We need a single contiguous page for our scratch */ diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index 79f565aeb8c0..50a98ce39f74 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -180,6 +180,9 @@ struct page *__px_page(struct drm_i915_gem_object *p); dma_addr_t __px_dma(struct drm_i915_gem_object *p); #define px_dma(px) (__px_dma(px_base(px))) +void *__px_vaddr(struct drm_i915_gem_object *p); +#define px_vaddr(px) (__px_vaddr(px_base(px))) + #define px_pt(px) \ __px_choose_expr(px, struct i915_page_table *, __x, \ __px_choose_expr(px, struct i915_page_directory *, &__x->pt, \ @@ -356,6 +359,7 @@ struct i915_ppgtt { #define i915_is_ggtt(vm) ((vm)->is_ggtt) #define i915_is_dpt(vm) ((vm)->is_dpt) +#define i915_is_ggtt_or_dpt(vm) (i915_is_ggtt(vm) || i915_is_dpt(vm)) int __must_check i915_vm_lock_objects(struct i915_address_space *vm, struct i915_gem_ww_ctx *ww); @@ -390,7 +394,7 @@ static inline struct i915_ppgtt * i915_vm_to_ppgtt(struct i915_address_space *vm) { BUILD_BUG_ON(offsetof(struct i915_ppgtt, vm)); - GEM_BUG_ON(i915_is_ggtt(vm)); + GEM_BUG_ON(i915_is_ggtt_or_dpt(vm)); return container_of(vm, struct i915_ppgtt, vm); } @@ -516,8 +520,6 @@ struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt); void i915_ggtt_suspend(struct i915_ggtt *gtt); void i915_ggtt_resume(struct i915_ggtt *ggtt); -#define kmap_atomic_px(px) kmap_atomic(__px_page(px_base(px))) - void fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count); @@ -531,12 +533,13 @@ int setup_scratch_page(struct i915_address_space *vm); void free_scratch(struct i915_address_space *vm); struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz); +struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz); struct i915_page_table *alloc_pt(struct i915_address_space *vm); struct i915_page_directory *alloc_pd(struct i915_address_space *vm); struct i915_page_directory *__alloc_pd(int npde); -int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj); -int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj); +int map_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj); +int map_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj); void free_px(struct i915_address_space *vm, struct i915_page_table *pt, int lvl); @@ -583,7 +586,7 @@ void setup_private_pat(struct intel_uncore *uncore); int i915_vm_alloc_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, u64 size); -int i915_vm_pin_pt_stash(struct i915_address_space *vm, +int i915_vm_map_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash); void i915_vm_free_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash); diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index e86897cde984..aafe2a4df496 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -903,7 +903,9 @@ lrc_pre_pin(struct intel_context *ce, GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); *vaddr = i915_gem_object_pin_map(ce->state->obj, - i915_coherent_map_type(ce->engine->i915) | + i915_coherent_map_type(ce->engine->i915, + ce->state->obj, + false) | I915_MAP_OVERRIDE); return PTR_ERR_OR_ZERO(*vaddr); diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c index 014ae8ac4480..4e3d80c2295c 100644 --- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c @@ -87,11 +87,10 @@ write_dma_entry(struct drm_i915_gem_object * const pdma, const unsigned short idx, const u64 encoded_entry) { - u64 * const vaddr = kmap_atomic(__px_page(pdma)); + u64 * const vaddr = __px_vaddr(pdma); vaddr[idx] = encoded_entry; clflush_cache_range(&vaddr[idx], sizeof(u64)); - kunmap_atomic(vaddr); } void @@ -258,7 +257,7 @@ int i915_vm_alloc_pt_stash(struct i915_address_space *vm, return 0; } -int i915_vm_pin_pt_stash(struct i915_address_space *vm, +int i915_vm_map_pt_stash(struct i915_address_space *vm, struct i915_vm_pt_stash *stash) { struct i915_page_table *pt; @@ -266,7 +265,7 @@ int i915_vm_pin_pt_stash(struct i915_address_space *vm, for (n = 0; n < ARRAY_SIZE(stash->pt); n++) { for (pt = stash->pt[n]; pt; pt = pt->stash) { - err = pin_pt_dma_locked(vm, pt->base); + err = map_pt_dma_locked(vm, pt->base); if (err) return err; } diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index a377c4588aaa..8091846d955b 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -338,15 +338,69 @@ static int gen6_reset_engines(struct intel_gt *gt, return gen6_hw_domain_reset(gt, hw_mask); } -static int gen11_lock_sfc(struct intel_engine_cs *engine, u32 *hw_mask) +static struct intel_engine_cs *find_sfc_paired_vecs_engine(struct intel_engine_cs *engine) +{ + int vecs_id; + + GEM_BUG_ON(engine->class != VIDEO_DECODE_CLASS); + + vecs_id = _VECS((engine->instance) / 2); + + return engine->gt->engine[vecs_id]; +} + +struct sfc_lock_data { + i915_reg_t lock_reg; + i915_reg_t ack_reg; + i915_reg_t usage_reg; + u32 lock_bit; + u32 ack_bit; + u32 usage_bit; + u32 reset_bit; +}; + +static void get_sfc_forced_lock_data(struct intel_engine_cs *engine, + struct sfc_lock_data *sfc_lock) +{ + switch (engine->class) { + default: + MISSING_CASE(engine->class); + fallthrough; + case VIDEO_DECODE_CLASS: + sfc_lock->lock_reg = GEN11_VCS_SFC_FORCED_LOCK(engine); + sfc_lock->lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; + + sfc_lock->ack_reg = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_lock->ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT; + + sfc_lock->usage_reg = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_lock->usage_bit = GEN11_VCS_SFC_USAGE_BIT; + sfc_lock->reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance); + + break; + case VIDEO_ENHANCEMENT_CLASS: + sfc_lock->lock_reg = GEN11_VECS_SFC_FORCED_LOCK(engine); + sfc_lock->lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; + + sfc_lock->ack_reg = GEN11_VECS_SFC_LOCK_ACK(engine); + sfc_lock->ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT; + + sfc_lock->usage_reg = GEN11_VECS_SFC_USAGE(engine); + sfc_lock->usage_bit = GEN11_VECS_SFC_USAGE_BIT; + sfc_lock->reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance); + + break; + } +} + +static int gen11_lock_sfc(struct intel_engine_cs *engine, + u32 *reset_mask, + u32 *unlock_mask) { struct intel_uncore *uncore = engine->uncore; u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access; - i915_reg_t sfc_forced_lock, sfc_forced_lock_ack; - u32 sfc_forced_lock_bit, sfc_forced_lock_ack_bit; - i915_reg_t sfc_usage; - u32 sfc_usage_bit; - u32 sfc_reset_bit; + struct sfc_lock_data sfc_lock; + bool lock_obtained, lock_to_other = false; int ret; switch (engine->class) { @@ -354,53 +408,72 @@ static int gen11_lock_sfc(struct intel_engine_cs *engine, u32 *hw_mask) if ((BIT(engine->instance) & vdbox_sfc_access) == 0) return 0; - sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; - - sfc_forced_lock_ack = GEN11_VCS_SFC_LOCK_STATUS(engine); - sfc_forced_lock_ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT; + fallthrough; + case VIDEO_ENHANCEMENT_CLASS: + get_sfc_forced_lock_data(engine, &sfc_lock); - sfc_usage = GEN11_VCS_SFC_LOCK_STATUS(engine); - sfc_usage_bit = GEN11_VCS_SFC_USAGE_BIT; - sfc_reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance); break; + default: + return 0; + } - case VIDEO_ENHANCEMENT_CLASS: - sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; + if (!(intel_uncore_read_fw(uncore, sfc_lock.usage_reg) & sfc_lock.usage_bit)) { + struct intel_engine_cs *paired_vecs; - sfc_forced_lock_ack = GEN11_VECS_SFC_LOCK_ACK(engine); - sfc_forced_lock_ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT; + if (engine->class != VIDEO_DECODE_CLASS || + !IS_GEN(engine->i915, 12)) + return 0; - sfc_usage = GEN11_VECS_SFC_USAGE(engine); - sfc_usage_bit = GEN11_VECS_SFC_USAGE_BIT; - sfc_reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance); - break; + /* + * Wa_14010733141 + * + * If the VCS-MFX isn't using the SFC, we also need to check + * whether VCS-HCP is using it. If so, we need to issue a *VE* + * forced lock on the VE engine that shares the same SFC. + */ + if (!(intel_uncore_read_fw(uncore, + GEN12_HCP_SFC_LOCK_STATUS(engine)) & + GEN12_HCP_SFC_USAGE_BIT)) + return 0; - default: - return 0; + paired_vecs = find_sfc_paired_vecs_engine(engine); + get_sfc_forced_lock_data(paired_vecs, &sfc_lock); + lock_to_other = true; + *unlock_mask |= paired_vecs->mask; + } else { + *unlock_mask |= engine->mask; } /* - * If the engine is using a SFC, tell the engine that a software reset + * If the engine is using an SFC, tell the engine that a software reset * is going to happen. The engine will then try to force lock the SFC. * If SFC ends up being locked to the engine we want to reset, we have * to reset it as well (we will unlock it once the reset sequence is * completed). */ - if (!(intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit)) - return 0; - - rmw_set_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit); + rmw_set_fw(uncore, sfc_lock.lock_reg, sfc_lock.lock_bit); ret = __intel_wait_for_register_fw(uncore, - sfc_forced_lock_ack, - sfc_forced_lock_ack_bit, - sfc_forced_lock_ack_bit, + sfc_lock.ack_reg, + sfc_lock.ack_bit, + sfc_lock.ack_bit, 1000, 0, NULL); - /* Was the SFC released while we were trying to lock it? */ - if (!(intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit)) + /* + * Was the SFC released while we were trying to lock it? + * + * We should reset both the engine and the SFC if: + * - We were locking the SFC to this engine and the lock succeeded + * OR + * - We were locking the SFC to a different engine (Wa_14010733141) + * but the SFC was released before the lock was obtained. + * + * Otherwise we need only reset the engine by itself and we can + * leave the SFC alone. + */ + lock_obtained = (intel_uncore_read_fw(uncore, sfc_lock.usage_reg) & + sfc_lock.usage_bit) != 0; + if (lock_obtained == lock_to_other) return 0; if (ret) { @@ -408,7 +481,7 @@ static int gen11_lock_sfc(struct intel_engine_cs *engine, u32 *hw_mask) return ret; } - *hw_mask |= sfc_reset_bit; + *reset_mask |= sfc_lock.reset_bit; return 0; } @@ -416,28 +489,19 @@ static void gen11_unlock_sfc(struct intel_engine_cs *engine) { struct intel_uncore *uncore = engine->uncore; u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access; - i915_reg_t sfc_forced_lock; - u32 sfc_forced_lock_bit; - - switch (engine->class) { - case VIDEO_DECODE_CLASS: - if ((BIT(engine->instance) & vdbox_sfc_access) == 0) - return; + struct sfc_lock_data sfc_lock = {}; - sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; - break; - - case VIDEO_ENHANCEMENT_CLASS: - sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); - sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; - break; + if (engine->class != VIDEO_DECODE_CLASS && + engine->class != VIDEO_ENHANCEMENT_CLASS) + return; - default: + if (engine->class == VIDEO_DECODE_CLASS && + (BIT(engine->instance) & vdbox_sfc_access) == 0) return; - } - rmw_clear_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit); + get_sfc_forced_lock_data(engine, &sfc_lock); + + rmw_clear_fw(uncore, sfc_lock.lock_reg, sfc_lock.lock_bit); } static int gen11_reset_engines(struct intel_gt *gt, @@ -456,23 +520,23 @@ static int gen11_reset_engines(struct intel_gt *gt, }; struct intel_engine_cs *engine; intel_engine_mask_t tmp; - u32 hw_mask; + u32 reset_mask, unlock_mask = 0; int ret; if (engine_mask == ALL_ENGINES) { - hw_mask = GEN11_GRDOM_FULL; + reset_mask = GEN11_GRDOM_FULL; } else { - hw_mask = 0; + reset_mask = 0; for_each_engine_masked(engine, gt, engine_mask, tmp) { GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask)); - hw_mask |= hw_engine_mask[engine->id]; - ret = gen11_lock_sfc(engine, &hw_mask); + reset_mask |= hw_engine_mask[engine->id]; + ret = gen11_lock_sfc(engine, &reset_mask, &unlock_mask); if (ret) goto sfc_unlock; } } - ret = gen6_hw_domain_reset(gt, hw_mask); + ret = gen6_hw_domain_reset(gt, reset_mask); sfc_unlock: /* @@ -480,10 +544,14 @@ sfc_unlock: * gen11_lock_sfc to make sure that we clean properly if something * wrong happened during the lock (e.g. lock acquired after timeout * expiration). + * + * Due to Wa_14010733141, we may have locked an SFC to an engine that + * wasn't being reset. So instead of calling gen11_unlock_sfc() + * on engine_mask, we instead call it on the mask of engines that our + * gen11_lock_sfc() calls told us actually had locks attempted. */ - if (engine_mask != ALL_ENGINES) - for_each_engine_masked(engine, gt, engine_mask, tmp) - gen11_unlock_sfc(engine); + for_each_engine_masked(engine, gt, unlock_mask, tmp) + gen11_unlock_sfc(engine); return ret; } @@ -1118,7 +1186,6 @@ static int intel_gt_reset_engine(struct intel_engine_cs *engine) int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg) { struct intel_gt *gt = engine->gt; - bool uses_guc = intel_engine_in_guc_submission_mode(engine); int ret; ENGINE_TRACE(engine, "flags=%lx\n", gt->reset.flags); @@ -1134,10 +1201,10 @@ int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg) "Resetting %s for %s\n", engine->name, msg); atomic_inc(&engine->i915->gpu_error.reset_engine_count[engine->uabi_class]); - if (!uses_guc) - ret = intel_gt_reset_engine(engine); - else + if (intel_engine_uses_guc(engine)) ret = intel_guc_reset_engine(&engine->gt->uc.guc, engine); + else + ret = intel_gt_reset_engine(engine); if (ret) { /* If we fail here, we expect to fallback to a global reset */ ENGINE_TRACE(engine, "Failed to reset, err: %d\n", ret); diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c index aee0a77c77e0..7c4d5158e03b 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.c +++ b/drivers/gpu/drm/i915/gt/intel_ring.c @@ -51,11 +51,14 @@ int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww) if (unlikely(ret)) goto err_unpin; - if (i915_vma_is_map_and_fenceable(vma)) + if (i915_vma_is_map_and_fenceable(vma)) { addr = (void __force *)i915_vma_pin_iomap(vma); - else - addr = i915_gem_object_pin_map(vma->obj, - i915_coherent_map_type(vma->vm->i915)); + } else { + int type = i915_coherent_map_type(vma->vm->i915, vma->obj, false); + + addr = i915_gem_object_pin_map(vma->obj, type); + } + if (IS_ERR(addr)) { ret = PTR_ERR(addr); goto err_ring; diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 9585546556ee..2b6dffcc2262 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -12,6 +12,7 @@ #include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_gt.h" +#include "intel_gt_irq.h" #include "intel_reset.h" #include "intel_ring.h" #include "shmem_utils.h" @@ -989,14 +990,10 @@ static void gen6_bsd_submit_request(struct i915_request *request) static void i9xx_set_default_submission(struct intel_engine_cs *engine) { engine->submit_request = i9xx_submit_request; - - engine->park = NULL; - engine->unpark = NULL; } static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) { - i9xx_set_default_submission(engine); engine->submit_request = gen6_bsd_submit_request; } @@ -1021,10 +1018,17 @@ static void ring_release(struct intel_engine_cs *engine) intel_timeline_put(engine->legacy.timeline); } +static void irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + intel_engine_signal_breadcrumbs(engine); +} + static void setup_irq(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; + intel_engine_set_irq_handler(engine, irq_handler); + if (INTEL_GEN(i915) >= 6) { engine->irq_enable = gen6_irq_enable; engine->irq_disable = gen6_irq_disable; diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 405d814e9040..97cab1b99871 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -1774,7 +1774,7 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) return; if (pm_iir & PM_VEBOX_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(gt->engine[VECS0]); + intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c index f19cf6d2fa85..c4a126c8caef 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.c +++ b/drivers/gpu/drm/i915/gt/intel_timeline.c @@ -32,7 +32,6 @@ static struct i915_vma *hwsp_alloc(struct intel_gt *gt) return vma; } -__i915_active_call static void __timeline_retire(struct i915_active *active) { struct intel_timeline *tl = @@ -104,7 +103,8 @@ static int intel_timeline_init(struct intel_timeline *timeline, INIT_LIST_HEAD(&timeline->requests); i915_syncmap_init(&timeline->sync); - i915_active_init(&timeline->active, __timeline_active, __timeline_retire); + i915_active_init(&timeline->active, __timeline_active, + __timeline_retire, 0); return 0; } diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 2c6f7217469f..62cb9ee5bfc3 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -607,9 +607,38 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); } +/* + * These settings aren't actually workarounds, but general tuning settings that + * need to be programmed on several platforms. + */ +static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine, + struct i915_wa_list *wal) +{ + /* + * Although some platforms refer to it as Wa_1604555607, we need to + * program it even on those that don't explicitly list that + * workaround. + * + * Note that the programming of this register is further modified + * according to the FF_MODE2 guidance given by Wa_1608008084:gen12. + * Wa_1608008084 tells us the FF_MODE2 register will return the wrong + * value when read. The default value for this register is zero for all + * fields and there are no bit masks. So instead of doing a RMW we + * should just write TDS timer value. For the same reason read + * verification is ignored. + */ + wa_add(wal, + FF_MODE2, + FF_MODE2_TDS_TIMER_MASK, + FF_MODE2_TDS_TIMER_128, + 0); +} + static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { + gen12_ctx_gt_tuning_init(engine, wal); + /* * Wa_1409142259:tgl * Wa_1409347922:tgl @@ -628,27 +657,17 @@ static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_field_set(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK, GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); -} - -static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, - struct i915_wa_list *wal) -{ - gen12_ctx_workarounds_init(engine, wal); /* - * Wa_1604555607:tgl,rkl + * Wa_16011163337 * - * Note that the implementation of this workaround is further modified - * according to the FF_MODE2 guidance given by Wa_1608008084:gen12. - * FF_MODE2 register will return the wrong value when read. The default - * value for this register is zero for all fields and there are no bit - * masks. So instead of doing a RMW we should just write the GS Timer - * and TDS timer values for Wa_1604555607 and Wa_16011163337. + * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due + * to Wa_1608008084. */ wa_add(wal, FF_MODE2, - FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK, - FF_MODE2_GS_TIMER_224 | FF_MODE2_TDS_TIMER_128, + FF_MODE2_GS_TIMER_MASK, + FF_MODE2_GS_TIMER_224, 0); } @@ -664,16 +683,6 @@ static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, /* Wa_22010493298 */ wa_masked_en(wal, HIZ_CHICKEN, DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE); - - /* - * Wa_16011163337 - * - * Like in tgl_ctx_workarounds_init(), read verification is ignored due - * to Wa_1608008084. - */ - wa_add(wal, - FF_MODE2, - FF_MODE2_GS_TIMER_MASK, FF_MODE2_GS_TIMER_224, 0); } static void @@ -690,9 +699,6 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, if (IS_DG1(i915)) dg1_ctx_workarounds_init(engine, wal); - else if (IS_ALDERLAKE_S(i915) || IS_ROCKETLAKE(i915) || - IS_TIGERLAKE(i915)) - tgl_ctx_workarounds_init(engine, wal); else if (IS_GEN(i915, 12)) gen12_ctx_workarounds_init(engine, wal); else if (IS_GEN(i915, 11)) @@ -1078,11 +1084,37 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); } +/* + * Though there are per-engine instances of these registers, + * they retain their value through engine resets and should + * only be provided on the GT workaround list rather than + * the engine-specific workaround list. + */ +static void +wa_14011060649(struct drm_i915_private *i915, struct i915_wa_list *wal) +{ + struct intel_engine_cs *engine; + struct intel_gt *gt = &i915->gt; + int id; + + for_each_engine(engine, gt, id) { + if (engine->class != VIDEO_DECODE_CLASS || + (engine->instance % 2)) + continue; + + wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base), + IECPUNIT_CLKGATE_DIS); + } +} + static void gen12_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { wa_init_mcr(i915, wal); + + /* Wa_14011060649:tgl,rkl,dg1,adls */ + wa_14011060649(i915, wal); } static void @@ -1755,11 +1787,10 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN7_FF_THREAD_MODE, GEN12_FF_TESSELATION_DOP_GATE_DISABLE); - /* Wa_22010271021:ehl */ - if (IS_JSL_EHL(i915)) - wa_masked_en(wal, - GEN9_CS_DEBUG_MODE1, - FF_DOP_CLOCK_GATE_DISABLE); + /* Wa_22010271021 */ + wa_masked_en(wal, + GEN9_CS_DEBUG_MODE1, + FF_DOP_CLOCK_GATE_DISABLE); } if (IS_GEN_RANGE(i915, 9, 12)) { @@ -1828,9 +1859,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) CACHE_MODE_0_GEN7, /* enable HiZ Raw Stall Optimization */ HIZ_RAW_STALL_OPT_DISABLE); - - /* WaDisable4x2SubspanOptimization:hsw */ - wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); } if (IS_VALLEYVIEW(i915)) { diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index e1ba03b93ffa..32589c6625e1 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -55,7 +55,7 @@ static struct intel_ring *mock_ring(struct intel_engine_cs *engine) kfree(ring); return NULL; } - i915_active_init(&ring->vma->active, NULL, NULL); + i915_active_init(&ring->vma->active, NULL, NULL, 0); __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(ring->vma)); __set_bit(DRM_MM_NODE_ALLOCATED_BIT, &ring->vma->node.flags); ring->vma->node.size = sz; diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c index b9bdd1d23243..26685b927169 100644 --- a/drivers/gpu/drm/i915/gt/selftest_context.c +++ b/drivers/gpu/drm/i915/gt/selftest_context.c @@ -88,7 +88,8 @@ static int __live_context_size(struct intel_engine_cs *engine) goto err; vaddr = i915_gem_object_pin_map_unlocked(ce->state->obj, - i915_coherent_map_type(engine->i915)); + i915_coherent_map_type(engine->i915, + ce->state->obj, false)); if (IS_ERR(vaddr)) { err = PTR_ERR(vaddr); intel_context_unpin(ce); diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c index b2c369317bf1..4896e4ccad50 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c @@ -77,7 +77,7 @@ static struct pulse *pulse_create(void) return p; kref_init(&p->kref); - i915_active_init(&p->active, pulse_active, pulse_retire); + i915_active_init(&p->active, pulse_active, pulse_retire, 0); return p; } diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c index 1081cd36a2bd..1f93591a8c69 100644 --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c @@ -4716,7 +4716,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) SUBTEST(live_virtual_reset), }; - if (!HAS_EXECLISTS(i915)) + if (i915->gt.submission_method != INTEL_SUBMISSION_ELSP) return 0; if (intel_gt_is_wedged(&i915->gt)) diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c index 746985971c3a..5b63d4df8c93 100644 --- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c @@ -69,7 +69,7 @@ static int hang_init(struct hang *h, struct intel_gt *gt) h->seqno = memset(vaddr, 0xff, PAGE_SIZE); vaddr = i915_gem_object_pin_map_unlocked(h->obj, - i915_coherent_map_type(gt->i915)); + i915_coherent_map_type(gt->i915, h->obj, false)); if (IS_ERR(vaddr)) { err = PTR_ERR(vaddr); goto err_unpin_hws; @@ -130,7 +130,7 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine) return ERR_CAST(obj); } - vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915)); + vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915, obj, false)); if (IS_ERR(vaddr)) { i915_gem_object_put(obj); i915_vm_put(vm); diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 85e7df6a5123..d8f6623524e8 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -1221,7 +1221,9 @@ static int compare_isolation(struct intel_engine_cs *engine, } lrc = i915_gem_object_pin_map_unlocked(ce->state->obj, - i915_coherent_map_type(engine->i915)); + i915_coherent_map_type(engine->i915, + ce->state->obj, + false)); if (IS_ERR(lrc)) { err = PTR_ERR(lrc); goto err_B1; diff --git a/drivers/gpu/drm/i915/gt/selftest_rc6.c b/drivers/gpu/drm/i915/gt/selftest_rc6.c index f097e420ac45..710f825f6e5a 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rc6.c +++ b/drivers/gpu/drm/i915/gt/selftest_rc6.c @@ -34,6 +34,7 @@ int live_rc6_manual(void *arg) struct intel_rc6 *rc6 = >->rc6; u64 rc0_power, rc6_power; intel_wakeref_t wakeref; + bool has_power; ktime_t dt; u64 res[2]; int err = 0; @@ -50,6 +51,7 @@ int live_rc6_manual(void *arg) if (IS_VALLEYVIEW(gt->i915) || IS_CHERRYVIEW(gt->i915)) return 0; + has_power = librapl_supported(gt->i915); wakeref = intel_runtime_pm_get(gt->uncore->rpm); /* Force RC6 off for starters */ @@ -71,11 +73,14 @@ int live_rc6_manual(void *arg) goto out_unlock; } - rc0_power = div64_u64(NSEC_PER_SEC * rc0_power, ktime_to_ns(dt)); - if (!rc0_power) { - pr_err("No power measured while in RC0\n"); - err = -EINVAL; - goto out_unlock; + if (has_power) { + rc0_power = div64_u64(NSEC_PER_SEC * rc0_power, + ktime_to_ns(dt)); + if (!rc0_power) { + pr_err("No power measured while in RC0\n"); + err = -EINVAL; + goto out_unlock; + } } /* Manually enter RC6 */ @@ -97,13 +102,16 @@ int live_rc6_manual(void *arg) err = -EINVAL; } - rc6_power = div64_u64(NSEC_PER_SEC * rc6_power, ktime_to_ns(dt)); - pr_info("GPU consumed %llduW in RC0 and %llduW in RC6\n", - rc0_power, rc6_power); - if (2 * rc6_power > rc0_power) { - pr_err("GPU leaked energy while in RC6!\n"); - err = -EINVAL; - goto out_unlock; + if (has_power) { + rc6_power = div64_u64(NSEC_PER_SEC * rc6_power, + ktime_to_ns(dt)); + pr_info("GPU consumed %llduW in RC0 and %llduW in RC6\n", + rc0_power, rc6_power); + if (2 * rc6_power > rc0_power) { + pr_err("GPU leaked energy while in RC6!\n"); + err = -EINVAL; + goto out_unlock; + } } /* Restore what should have been the original state! */ diff --git a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c index 99609271c3a7..c12e74171b63 100644 --- a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c @@ -291,7 +291,7 @@ int intel_ring_submission_live_selftests(struct drm_i915_private *i915) SUBTEST(live_ctx_switch_wa), }; - if (HAS_EXECLISTS(i915)) + if (i915->gt.submission_method > INTEL_SUBMISSION_RING) return 0; return intel_gt_live_subtests(tests, &i915->gt); diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c index 967641fee42a..3ca1bd5793c3 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rps.c +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -606,7 +606,7 @@ int live_rps_frequency_cs(void *arg) int err = 0; /* - * The premise is that the GPU does change freqency at our behest. + * The premise is that the GPU does change frequency at our behest. * Let's check there is a correspondence between the requested * frequency, the actual frequency, and the observed clock rate. */ @@ -747,7 +747,7 @@ int live_rps_frequency_srm(void *arg) int err = 0; /* - * The premise is that the GPU does change freqency at our behest. + * The premise is that the GPU does change frequency at our behest. * Let's check there is a correspondence between the requested * frequency, the actual frequency, and the observed clock rate. */ @@ -1139,7 +1139,7 @@ int live_rps_power(void *arg) if (!intel_rps_is_enabled(rps) || INTEL_GEN(gt->i915) < 6) return 0; - if (!librapl_energy_uJ()) + if (!librapl_supported(gt->i915)) return 0; if (igt_spinner_init(&spin, gt)) diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index f8f02aab842b..0683b27a3890 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -8,6 +8,7 @@ #include <linux/shmem_fs.h> #include "gem/i915_gem_object.h" +#include "gem/i915_gem_lmem.h" #include "shmem_utils.h" struct file *shmem_create_from_data(const char *name, void *data, size_t len) @@ -39,7 +40,8 @@ struct file *shmem_create_from_object(struct drm_i915_gem_object *obj) return file; } - ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); + ptr = i915_gem_object_pin_map_unlocked(obj, i915_gem_object_is_lmem(obj) ? + I915_MAP_WC : I915_MAP_WB); if (IS_ERR(ptr)) return ERR_CAST(ptr); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index 78305b2ec89d..adae04c47aab 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -682,7 +682,9 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, if (IS_ERR(vma)) return PTR_ERR(vma); - vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB); + vaddr = i915_gem_object_pin_map_unlocked(vma->obj, + i915_coherent_map_type(guc_to_gt(guc)->i915, + vma->obj, true)); if (IS_ERR(vaddr)) { i915_vma_unpin_and_release(&vma, 0); return PTR_ERR(vaddr); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 92688a9b6717..335719f17490 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -11,6 +11,7 @@ #include "gt/intel_context.h" #include "gt/intel_engine_pm.h" #include "gt/intel_gt.h" +#include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm.h" #include "gt/intel_lrc.h" #include "gt/intel_mocs.h" @@ -264,6 +265,14 @@ static void guc_submission_tasklet(struct tasklet_struct *t) spin_unlock_irqrestore(&engine->active.lock, flags); } +static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir) +{ + if (iir & GT_RENDER_USER_INTERRUPT) { + intel_engine_signal_breadcrumbs(engine); + tasklet_hi_schedule(&engine->execlists.tasklet); + } +} + static void guc_reset_prepare(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; @@ -608,35 +617,6 @@ static int guc_resume(struct intel_engine_cs *engine) static void guc_set_default_submission(struct intel_engine_cs *engine) { engine->submit_request = guc_submit_request; - engine->schedule = i915_schedule; - engine->execlists.tasklet.callback = guc_submission_tasklet; - - engine->reset.prepare = guc_reset_prepare; - engine->reset.rewind = guc_reset_rewind; - engine->reset.cancel = guc_reset_cancel; - engine->reset.finish = guc_reset_finish; - - engine->flags |= I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; - engine->flags |= I915_ENGINE_HAS_PREEMPTION; - - /* - * TODO: GuC supports timeslicing and semaphores as well, but they're - * handled by the firmware so some minor tweaks are required before - * enabling. - * - * engine->flags |= I915_ENGINE_HAS_TIMESLICES; - * engine->flags |= I915_ENGINE_HAS_SEMAPHORES; - */ - - engine->emit_bb_start = gen8_emit_bb_start; - - /* - * For the breadcrumb irq to work we need the interrupts to stay - * enabled. However, on all platforms on which we'll have support for - * GuC submission we don't allow disabling the interrupts at runtime, so - * we're always safe with the current flow. - */ - GEM_BUG_ON(engine->irq_enable || engine->irq_disable); } static void guc_release(struct intel_engine_cs *engine) @@ -658,6 +638,13 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->cops = &guc_context_ops; engine->request_alloc = guc_request_alloc; + engine->schedule = i915_schedule; + + engine->reset.prepare = guc_reset_prepare; + engine->reset.rewind = guc_reset_rewind; + engine->reset.cancel = guc_reset_cancel; + engine->reset.finish = guc_reset_finish; + engine->emit_flush = gen8_emit_flush_xcs; engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs; @@ -666,6 +653,19 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->emit_flush = gen12_emit_flush_xcs; } engine->set_default_submission = guc_set_default_submission; + + engine->flags |= I915_ENGINE_HAS_PREEMPTION; + + /* + * TODO: GuC supports timeslicing and semaphores as well, but they're + * handled by the firmware so some minor tweaks are required before + * enabling. + * + * engine->flags |= I915_ENGINE_HAS_TIMESLICES; + * engine->flags |= I915_ENGINE_HAS_SEMAPHORES; + */ + + engine->emit_bb_start = gen8_emit_bb_start; } static void rcs_submission_override(struct intel_engine_cs *engine) @@ -689,6 +689,7 @@ static void rcs_submission_override(struct intel_engine_cs *engine) static inline void guc_default_irqs(struct intel_engine_cs *engine) { engine->irq_keep_mask = GT_RENDER_USER_INTERRUPT; + intel_engine_set_irq_handler(engine, cs_irq_handler); } int intel_guc_submission_setup(struct intel_engine_cs *engine) @@ -753,8 +754,3 @@ void intel_guc_submission_init_early(struct intel_guc *guc) { guc->submission_selected = __guc_submission_selected(guc); } - -bool intel_engine_in_guc_submission_mode(const struct intel_engine_cs *engine) -{ - return engine->set_default_submission == guc_set_default_submission; -} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h index 5f7b9e6347d0..3f7005018939 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h @@ -20,7 +20,6 @@ void intel_guc_submission_fini(struct intel_guc *guc); int intel_guc_preempt_work_create(struct intel_guc *guc); void intel_guc_preempt_work_destroy(struct intel_guc *guc); int intel_guc_submission_setup(struct intel_engine_cs *engine); -bool intel_engine_in_guc_submission_mode(const struct intel_engine_cs *engine); static inline bool intel_guc_submission_is_supported(struct intel_guc *guc) { diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c index 2126dd81ac38..56d2144dc6a0 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c @@ -82,7 +82,9 @@ static int intel_huc_rsa_data_create(struct intel_huc *huc) if (IS_ERR(vma)) return PTR_ERR(vma); - vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB); + vaddr = i915_gem_object_pin_map_unlocked(vma->obj, + i915_coherent_map_type(gt->i915, + vma->obj, true)); if (IS_ERR(vaddr)) { i915_vma_unpin_and_release(&vma, 0); return PTR_ERR(vaddr); diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index ca9c9e27a43d..c4118b808268 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -1006,7 +1006,7 @@ static int cmd_reg_handler(struct parser_exec_state *s, * update reg values in it into vregs, so LRIs in workload with * inhibit context will restore with correct values */ - if (IS_GEN(s->engine->i915, 9) && + if (GRAPHICS_VER(s->engine->i915) == 9 && intel_gvt_mmio_is_sr_in_ctx(gvt, offset) && !strncmp(cmd, "lri", 3)) { intel_gvt_hypervisor_read_gpa(s->vgpu, @@ -1390,7 +1390,7 @@ static int gen8_check_mi_display_flip(struct parser_exec_state *s, if (!info->async_flip) return 0; - if (INTEL_GEN(s->engine->i915) >= 9) { + if (GRAPHICS_VER(s->engine->i915) >= 9) { stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0); tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & GENMASK(12, 10)) >> 10; @@ -1418,7 +1418,7 @@ static int gen8_update_plane_mmio_from_mi_display_flip( set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12), info->surf_val << 12); - if (INTEL_GEN(dev_priv) >= 9) { + if (GRAPHICS_VER(dev_priv) >= 9) { set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0), info->stride_val); set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10), @@ -1446,7 +1446,7 @@ static int decode_mi_display_flip(struct parser_exec_state *s, { if (IS_BROADWELL(s->engine->i915)) return gen8_decode_mi_display_flip(s, info); - if (INTEL_GEN(s->engine->i915) >= 9) + if (GRAPHICS_VER(s->engine->i915) >= 9) return skl_decode_mi_display_flip(s, info); return -ENODEV; diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c index d4f883f35b95..8e65cd8258b9 100644 --- a/drivers/gpu/drm/i915/gvt/dmabuf.c +++ b/drivers/gpu/drm/i915/gvt/dmabuf.c @@ -223,7 +223,7 @@ static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev, obj->read_domains = I915_GEM_DOMAIN_GTT; obj->write_domain = 0; - if (INTEL_GEN(dev_priv) >= 9) { + if (GRAPHICS_VER(dev_priv) >= 9) { unsigned int tiling_mode = 0; unsigned int stride = 0; diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.c b/drivers/gpu/drm/i915/gvt/fb_decoder.c index 0889ad8291b0..11a8baba6822 100644 --- a/drivers/gpu/drm/i915/gvt/fb_decoder.c +++ b/drivers/gpu/drm/i915/gvt/fb_decoder.c @@ -151,7 +151,7 @@ static u32 intel_vgpu_get_stride(struct intel_vgpu *vgpu, int pipe, u32 stride_reg = vgpu_vreg_t(vgpu, DSPSTRIDE(pipe)) & stride_mask; u32 stride = stride_reg; - if (INTEL_GEN(dev_priv) >= 9) { + if (GRAPHICS_VER(dev_priv) >= 9) { switch (tiled) { case PLANE_CTL_TILED_LINEAR: stride = stride_reg * 64; @@ -215,7 +215,7 @@ int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, if (!plane->enabled) return -ENODEV; - if (INTEL_GEN(dev_priv) >= 9) { + if (GRAPHICS_VER(dev_priv) >= 9) { plane->tiled = val & PLANE_CTL_TILED_MASK; fmt = skl_format_to_drm( val & PLANE_CTL_FORMAT_MASK, @@ -256,9 +256,9 @@ int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, } plane->stride = intel_vgpu_get_stride(vgpu, pipe, plane->tiled, - (INTEL_GEN(dev_priv) >= 9) ? - (_PRI_PLANE_STRIDE_MASK >> 6) : - _PRI_PLANE_STRIDE_MASK, plane->bpp); + (GRAPHICS_VER(dev_priv) >= 9) ? + (_PRI_PLANE_STRIDE_MASK >> 6) : + _PRI_PLANE_STRIDE_MASK, plane->bpp); plane->width = (vgpu_vreg_t(vgpu, PIPESRC(pipe)) & _PIPE_H_SRCSZ_MASK) >> _PIPE_H_SRCSZ_SHIFT; diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 9478c132d7b6..cc2c05e18206 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -1055,12 +1055,12 @@ static bool vgpu_ips_enabled(struct intel_vgpu *vgpu) { struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; - if (INTEL_GEN(dev_priv) == 9 || INTEL_GEN(dev_priv) == 10) { + if (GRAPHICS_VER(dev_priv) == 9 || GRAPHICS_VER(dev_priv) == 10) { u32 ips = vgpu_vreg_t(vgpu, GEN8_GAMW_ECO_DEV_RW_IA) & GAMW_ECO_ENABLE_64K_IPS_FIELD; return ips == GAMW_ECO_ENABLE_64K_IPS_FIELD; - } else if (INTEL_GEN(dev_priv) >= 11) { + } else if (GRAPHICS_VER(dev_priv) >= 11) { /* 64K paging only controlled by IPS bit in PTE now. */ return true; } else diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index dda320749c65..98eb48c24c46 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -220,7 +220,7 @@ static int gamw_echo_dev_rw_ia_write(struct intel_vgpu *vgpu, { u32 ips = (*(u32 *)p_data) & GAMW_ECO_ENABLE_64K_IPS_FIELD; - if (INTEL_GEN(vgpu->gvt->gt->i915) <= 10) { + if (GRAPHICS_VER(vgpu->gvt->gt->i915) <= 10) { if (ips == GAMW_ECO_ENABLE_64K_IPS_FIELD) gvt_dbg_core("vgpu%d: ips enabled\n", vgpu->id); else if (!ips) @@ -286,7 +286,7 @@ static int mul_force_wake_write(struct intel_vgpu *vgpu, old = vgpu_vreg(vgpu, offset); new = CALC_MODE_MASK_REG(old, *(u32 *)p_data); - if (INTEL_GEN(vgpu->gvt->gt->i915) >= 9) { + if (GRAPHICS_VER(vgpu->gvt->gt->i915) >= 9) { switch (offset) { case FORCEWAKE_RENDER_GEN9_REG: ack_reg_offset = FORCEWAKE_ACK_RENDER_GEN9_REG; @@ -1174,7 +1174,7 @@ static int dp_aux_ch_ctl_mmio_write(struct intel_vgpu *vgpu, write_vreg(vgpu, offset, p_data, bytes); data = vgpu_vreg(vgpu, offset); - if ((INTEL_GEN(vgpu->gvt->gt->i915) >= 9) + if ((GRAPHICS_VER(vgpu->gvt->gt->i915) >= 9) && offset != _REG_SKL_DP_AUX_CH_CTL(port_index)) { /* SKL DPB/C/D aux ctl register changed */ return 0; @@ -3342,9 +3342,9 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_D(_MMIO(_PLANE_SURF_3_A), D_SKL_PLUS); MMIO_D(_MMIO(_PLANE_SURF_3_B), D_SKL_PLUS); - MMIO_D(CSR_SSP_BASE, D_SKL_PLUS); - MMIO_D(CSR_HTP_SKL, D_SKL_PLUS); - MMIO_D(CSR_LAST_WRITE, D_SKL_PLUS); + MMIO_D(DMC_SSP_BASE, D_SKL_PLUS); + MMIO_D(DMC_HTP_SKL, D_SKL_PLUS); + MMIO_D(DMC_LAST_WRITE, D_SKL_PLUS); MMIO_DFH(BDW_SCRATCH1, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); @@ -3655,7 +3655,7 @@ void intel_gvt_clean_mmio_info(struct intel_gvt *gvt) * otherwise, need to update cmd_reg_handler in cmd_parser.c */ static struct gvt_mmio_block mmio_blocks[] = { - {D_SKL_PLUS, _MMIO(CSR_MMIO_START_RANGE), 0x3000, NULL, NULL}, + {D_SKL_PLUS, _MMIO(DMC_MMIO_START_RANGE), 0x3000, NULL, NULL}, {D_ALL, _MMIO(MCHBAR_MIRROR_BASE_SNB), 0x40000, NULL, NULL}, {D_ALL, _MMIO(VGT_PVINFO_PAGE), VGT_PVINFO_SIZE, pvinfo_mmio_read, pvinfo_mmio_write}, diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c index 497d28ce47df..614b951d919f 100644 --- a/drivers/gpu/drm/i915/gvt/interrupt.c +++ b/drivers/gpu/drm/i915/gvt/interrupt.c @@ -585,7 +585,7 @@ static void gen8_init_irq( SET_BIT_INFO(irq, 4, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); SET_BIT_INFO(irq, 5, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); - } else if (INTEL_GEN(gvt->gt->i915) >= 9) { + } else if (GRAPHICS_VER(gvt->gt->i915) >= 9) { SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_DE_PORT); SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_DE_PORT); SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_DE_PORT); diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c index c9589e26af93..b8ac80765461 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.c +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -373,7 +373,7 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, */ fw = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ | FW_REG_WRITE); - if (engine->id == RCS0 && INTEL_GEN(engine->i915) >= 9) + if (engine->id == RCS0 && GRAPHICS_VER(engine->i915) >= 9) fw |= FORCEWAKE_RENDER; intel_uncore_forcewake_get(uncore, fw); @@ -409,7 +409,7 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next, if (drm_WARN_ON(&engine->i915->drm, engine->id >= ARRAY_SIZE(regs))) return; - if (engine->id == RCS0 && IS_GEN(engine->i915, 9)) + if (engine->id == RCS0 && GRAPHICS_VER(engine->i915) == 9) return; if (!pre && !gen9_render_mocs.initialized) @@ -474,7 +474,7 @@ static void switch_mmio(struct intel_vgpu *pre, struct engine_mmio *mmio; u32 old_v, new_v; - if (INTEL_GEN(engine->i915) >= 9) + if (GRAPHICS_VER(engine->i915) >= 9) switch_mocs(pre, next, engine); for (mmio = engine->i915->gvt->engine_mmio_list.mmio; @@ -486,7 +486,7 @@ static void switch_mmio(struct intel_vgpu *pre, * state image on gen9, it's initialized by lri command and * save or restore with context together. */ - if (IS_GEN(engine->i915, 9) && mmio->in_context) + if (GRAPHICS_VER(engine->i915) == 9 && mmio->in_context) continue; // save @@ -580,7 +580,7 @@ void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt) { struct engine_mmio *mmio; - if (INTEL_GEN(gvt->gt->i915) >= 9) { + if (GRAPHICS_VER(gvt->gt->i915) >= 9) { gvt->engine_mmio_list.mmio = gen9_engine_mmio_list; gvt->engine_mmio_list.tlb_mmio_offset_list = gen8_tlb_mmio_offset_list; gvt->engine_mmio_list.tlb_mmio_offset_list_cnt = ARRAY_SIZE(gen8_tlb_mmio_offset_list); diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index fc735692f21f..734c37c5e347 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -364,7 +364,7 @@ static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) u32 *cs; int err; - if (IS_GEN(req->engine->i915, 9) && is_inhibit_context(req->context)) + if (GRAPHICS_VER(req->engine->i915) == 9 && is_inhibit_context(req->context)) intel_vgpu_restore_inhibit_context(vgpu, req); /* @@ -1148,7 +1148,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id) static int workload_thread(void *arg) { struct intel_engine_cs *engine = arg; - const bool need_force_wake = INTEL_GEN(engine->i915) >= 9; + const bool need_force_wake = GRAPHICS_VER(engine->i915) >= 9; struct intel_gvt *gvt = engine->i915->gvt; struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; struct intel_vgpu_workload *workload = NULL; diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 9039787f123a..fa6b92615799 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -149,10 +149,10 @@ int intel_gvt_init_vgpu_types(struct intel_gvt *gvt) gvt->types[i].avail_instance = min(low_avail / vgpu_types[i].low_mm, high_avail / vgpu_types[i].high_mm); - if (IS_GEN(gvt->gt->i915, 8)) + if (GRAPHICS_VER(gvt->gt->i915) == 8) sprintf(gvt->types[i].name, "GVTg_V4_%s", vgpu_types[i].name); - else if (IS_GEN(gvt->gt->i915, 9)) + else if (GRAPHICS_VER(gvt->gt->i915) == 9) sprintf(gvt->types[i].name, "GVTg_V5_%s", vgpu_types[i].name); diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index aa573b078ae7..b1aa1c482c32 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -343,18 +343,15 @@ out: void __i915_active_init(struct i915_active *ref, int (*active)(struct i915_active *ref), void (*retire)(struct i915_active *ref), + unsigned long flags, struct lock_class_key *mkey, struct lock_class_key *wkey) { - unsigned long bits; - debug_active_init(ref); - ref->flags = 0; + ref->flags = flags; ref->active = active; - ref->retire = ptr_unpack_bits(retire, &bits, 2); - if (bits & I915_ACTIVE_MAY_SLEEP) - ref->flags |= I915_ACTIVE_RETIRE_SLEEPS; + ref->retire = retire; spin_lock_init(&ref->tree_lock); ref->tree = RB_ROOT; @@ -1156,8 +1153,7 @@ static int auto_active(struct i915_active *ref) return 0; } -__i915_active_call static void -auto_retire(struct i915_active *ref) +static void auto_retire(struct i915_active *ref) { i915_active_put(ref); } @@ -1171,7 +1167,7 @@ struct i915_active *i915_active_create(void) return NULL; kref_init(&aa->ref); - i915_active_init(&aa->base, auto_active, auto_retire); + i915_active_init(&aa->base, auto_active, auto_retire, 0); return &aa->base; } diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h index fb165d3f01cf..d0feda68b874 100644 --- a/drivers/gpu/drm/i915/i915_active.h +++ b/drivers/gpu/drm/i915/i915_active.h @@ -152,15 +152,16 @@ i915_active_fence_isset(const struct i915_active_fence *active) void __i915_active_init(struct i915_active *ref, int (*active)(struct i915_active *ref), void (*retire)(struct i915_active *ref), + unsigned long flags, struct lock_class_key *mkey, struct lock_class_key *wkey); /* Specialise each class of i915_active to avoid impossible lockdep cycles. */ -#define i915_active_init(ref, active, retire) do { \ - static struct lock_class_key __mkey; \ - static struct lock_class_key __wkey; \ - \ - __i915_active_init(ref, active, retire, &__mkey, &__wkey); \ +#define i915_active_init(ref, active, retire, flags) do { \ + static struct lock_class_key __mkey; \ + static struct lock_class_key __wkey; \ + \ + __i915_active_init(ref, active, retire, flags, &__mkey, &__wkey); \ } while (0) struct dma_fence * diff --git a/drivers/gpu/drm/i915/i915_active_types.h b/drivers/gpu/drm/i915/i915_active_types.h index 6360c3e4b765..c149f348a972 100644 --- a/drivers/gpu/drm/i915/i915_active_types.h +++ b/drivers/gpu/drm/i915/i915_active_types.h @@ -24,11 +24,6 @@ struct i915_active_fence { struct active_node; -#define I915_ACTIVE_MAY_SLEEP BIT(0) - -#define __i915_active_call __aligned(4) -#define i915_active_may_sleep(fn) ptr_pack_bits(&(fn), I915_ACTIVE_MAY_SLEEP, 2) - struct i915_active { atomic_t count; struct mutex mutex; diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index e6f1e93abbbb..3992c25a191d 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -946,8 +946,8 @@ int intel_engine_init_cmd_parser(struct intel_engine_cs *engine) int cmd_table_count; int ret; - if (!IS_GEN(engine->i915, 7) && !(IS_GEN(engine->i915, 9) && - engine->class == COPY_ENGINE_CLASS)) + if (GRAPHICS_VER(engine->i915) != 7 && !(GRAPHICS_VER(engine->i915) == 9 && + engine->class == COPY_ENGINE_CLASS)) return 0; switch (engine->class) { @@ -977,7 +977,7 @@ int intel_engine_init_cmd_parser(struct intel_engine_cs *engine) break; case COPY_ENGINE_CLASS: engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask; - if (IS_GEN(engine->i915, 9)) { + if (GRAPHICS_VER(engine->i915) == 9) { cmd_tables = gen9_blt_cmd_table; cmd_table_count = ARRAY_SIZE(gen9_blt_cmd_table); engine->get_cmd_length_mask = @@ -993,7 +993,7 @@ int intel_engine_init_cmd_parser(struct intel_engine_cs *engine) cmd_table_count = ARRAY_SIZE(gen7_blt_cmd_table); } - if (IS_GEN(engine->i915, 9)) { + if (GRAPHICS_VER(engine->i915) == 9) { engine->reg_tables = gen9_blt_reg_tables; engine->reg_table_count = ARRAY_SIZE(gen9_blt_reg_tables); @@ -1369,6 +1369,20 @@ static int check_bbstart(u32 *cmd, u32 offset, u32 length, return 0; } +/** + * intel_engine_cmd_parser_alloc_jump_whitelist() - preallocate jump whitelist for intel_engine_cmd_parser() + * @batch_length: length of the commands in batch_obj + * @trampoline: Whether jump trampolines are used. + * + * Preallocates a jump whitelist for parsing the cmd buffer in intel_engine_cmd_parser(). + * This has to be preallocated, because the command parser runs in signaling context, + * and may not allocate any memory. + * + * Return: NULL or pointer to a jump whitelist, or ERR_PTR() on failure. Use + * IS_ERR() to check for errors. Must bre freed() with kfree(). + * + * NULL is a valid value, meaning no allocation was required. + */ unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length, bool trampoline) { @@ -1401,7 +1415,9 @@ unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length, * @batch_offset: byte offset in the batch at which execution starts * @batch_length: length of the commands in batch_obj * @shadow: validated copy of the batch buffer in question - * @trampoline: whether to emit a conditional trampoline at the end of the batch + * @jump_whitelist: buffer preallocated with intel_engine_cmd_parser_alloc_jump_whitelist() + * @shadow_map: mapping to @shadow vma + * @batch_map: mapping to @batch vma * * Parses the specified batch buffer looking for privilege violations as * described in the overview. @@ -1521,7 +1537,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, if (IS_HASWELL(engine->i915)) flags = MI_BATCH_NON_SECURE_HSW; - GEM_BUG_ON(!IS_GEN_RANGE(engine->i915, 6, 7)); + GEM_BUG_ON(!IS_GRAPHICS_VER(engine->i915, 6, 7)); __gen6_emit_bb_start(batch_end, batch_addr, flags); diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index b654b7498bcd..cc745751ac53 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -124,6 +124,17 @@ stringify_page_sizes(unsigned int page_sizes, char *buf, size_t len) } } +static const char *stringify_vma_type(const struct i915_vma *vma) +{ + if (i915_vma_is_ggtt(vma)) + return "ggtt"; + + if (i915_vma_is_dpt(vma)) + return "dpt"; + + return "ppgtt"; +} + void i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) { @@ -156,11 +167,11 @@ i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) if (i915_vma_is_pinned(vma)) pin_count++; - seq_printf(m, " (%sgtt offset: %08llx, size: %08llx, pages: %s", - i915_vma_is_ggtt(vma) ? "g" : "pp", + seq_printf(m, " (%s offset: %08llx, size: %08llx, pages: %s", + stringify_vma_type(vma), vma->node.start, vma->node.size, stringify_page_sizes(vma->page_sizes.gtt, NULL, 0)); - if (i915_vma_is_ggtt(vma)) { + if (i915_vma_is_ggtt(vma) || i915_vma_is_dpt(vma)) { switch (vma->ggtt_view.type) { case I915_GGTT_VIEW_NORMAL: seq_puts(m, ", normal"); @@ -350,7 +361,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); - if (IS_GEN(dev_priv, 5)) { + if (GRAPHICS_VER(dev_priv) == 5) { u16 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); u16 rgvstat = intel_uncore_read16(uncore, MEMSTAT_ILK); @@ -397,7 +408,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) seq_printf(m, "efficient (RPe) frequency: %d MHz\n", intel_gpu_freq(rps, rps->efficient_freq)); - } else if (INTEL_GEN(dev_priv) >= 6) { + } else if (GRAPHICS_VER(dev_priv) >= 6) { u32 rp_state_limits; u32 gt_perf_status; u32 rp_state_cap; @@ -421,7 +432,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); reqf = intel_uncore_read(&dev_priv->uncore, GEN6_RPNSWREQ); - if (INTEL_GEN(dev_priv) >= 9) + if (GRAPHICS_VER(dev_priv) >= 9) reqf >>= 23; else { reqf &= ~GEN6_TURBO_DISABLE; @@ -447,7 +458,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); - if (INTEL_GEN(dev_priv) >= 11) { + if (GRAPHICS_VER(dev_priv) >= 11) { pm_ier = intel_uncore_read(&dev_priv->uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE); pm_imr = intel_uncore_read(&dev_priv->uncore, GEN11_GPM_WGBOXPERF_INTR_MASK); /* @@ -456,7 +467,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) */ pm_isr = 0; pm_iir = 0; - } else if (INTEL_GEN(dev_priv) >= 8) { + } else if (GRAPHICS_VER(dev_priv) >= 8) { pm_ier = intel_uncore_read(&dev_priv->uncore, GEN8_GT_IER(2)); pm_imr = intel_uncore_read(&dev_priv->uncore, GEN8_GT_IMR(2)); pm_isr = intel_uncore_read(&dev_priv->uncore, GEN8_GT_ISR(2)); @@ -479,14 +490,14 @@ static int i915_frequency_info(struct seq_file *m, void *unused) seq_printf(m, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", pm_ier, pm_imr, pm_mask); - if (INTEL_GEN(dev_priv) <= 10) + if (GRAPHICS_VER(dev_priv) <= 10) seq_printf(m, "PM ISR=0x%08x IIR=0x%08x\n", pm_isr, pm_iir); seq_printf(m, "pm_intrmsk_mbz: 0x%08x\n", rps->pm_intrmsk_mbz); seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); seq_printf(m, "Render p-state ratio: %d\n", - (gt_perf_status & (INTEL_GEN(dev_priv) >= 9 ? 0x1ff00 : 0xff00)) >> 8); + (gt_perf_status & (GRAPHICS_VER(dev_priv) >= 9 ? 0x1ff00 : 0xff00)) >> 8); seq_printf(m, "Render p-state VID: %d\n", gt_perf_status & 0xff); seq_printf(m, "Render p-state limit: %d\n", @@ -527,20 +538,20 @@ static int i915_frequency_info(struct seq_file *m, void *unused) max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 : rp_state_cap >> 16) & 0xff; max_freq *= (IS_GEN9_BC(dev_priv) || - INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (rp_state_cap & 0xff00) >> 8; max_freq *= (IS_GEN9_BC(dev_priv) || - INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 : rp_state_cap >> 0) & 0xff; max_freq *= (IS_GEN9_BC(dev_priv) || - INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); + GRAPHICS_VER(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); seq_printf(m, "Max overclocked frequency: %dMHz\n", @@ -611,20 +622,20 @@ static int i915_swizzle_info(struct seq_file *m, void *data) seq_puts(m, "L-shaped memory detected\n"); /* On BDW+, swizzling is not used. See detect_bit_6_swizzle() */ - if (INTEL_GEN(dev_priv) >= 8 || IS_VALLEYVIEW(dev_priv)) + if (GRAPHICS_VER(dev_priv) >= 8 || IS_VALLEYVIEW(dev_priv)) return 0; wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); - if (IS_GEN_RANGE(dev_priv, 3, 4)) { + if (IS_GRAPHICS_VER(dev_priv, 3, 4)) { seq_printf(m, "DDC = 0x%08x\n", intel_uncore_read(uncore, DCC)); seq_printf(m, "DDC2 = 0x%08x\n", intel_uncore_read(uncore, DCC2)); seq_printf(m, "C0DRB3 = 0x%04x\n", - intel_uncore_read16(uncore, C0DRB3)); + intel_uncore_read16(uncore, C0DRB3_BW)); seq_printf(m, "C1DRB3 = 0x%04x\n", - intel_uncore_read16(uncore, C1DRB3)); + intel_uncore_read16(uncore, C1DRB3_BW)); } else if (INTEL_GEN(dev_priv) >= 6) { seq_printf(m, "MAD_DIMM_C0 = 0x%08x\n", intel_uncore_read(uncore, MAD_DIMM_C0)); @@ -634,7 +645,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data) intel_uncore_read(uncore, MAD_DIMM_C2)); seq_printf(m, "TILECTL = 0x%08x\n", intel_uncore_read(uncore, TILECTL)); - if (INTEL_GEN(dev_priv) >= 8) + if (GRAPHICS_VER(dev_priv) >= 8) seq_printf(m, "GAMTARBMODE = 0x%08x\n", intel_uncore_read(uncore, GAMTARBMODE)); else @@ -945,7 +956,7 @@ static int i915_forcewake_open(struct inode *inode, struct file *file) atomic_inc(>->user_wakeref); intel_gt_pm_get(gt); - if (INTEL_GEN(i915) >= 6) + if (GRAPHICS_VER(i915) >= 6) intel_uncore_forcewake_user_get(gt->uncore); return 0; @@ -956,7 +967,7 @@ static int i915_forcewake_release(struct inode *inode, struct file *file) struct drm_i915_private *i915 = inode->i_private; struct intel_gt *gt = &i915->gt; - if (INTEL_GEN(i915) >= 6) + if (GRAPHICS_VER(i915) >= 6) intel_uncore_forcewake_user_put(&i915->uncore); intel_gt_pm_put(gt); atomic_dec(>->user_wakeref); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index db513f93f0f5..d5b3c5ba6bd2 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -49,7 +49,7 @@ #include "display/intel_acpi.h" #include "display/intel_bw.h" #include "display/intel_cdclk.h" -#include "display/intel_csr.h" +#include "display/intel_dmc.h" #include "display/intel_display_types.h" #include "display/intel_dp.h" #include "display/intel_fbdev.h" @@ -106,12 +106,12 @@ static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) static int intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv) { - int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; + int reg = GRAPHICS_VER(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; u32 temp_lo, temp_hi = 0; u64 mchbar_addr; int ret; - if (INTEL_GEN(dev_priv) >= 4) + if (GRAPHICS_VER(dev_priv) >= 4) pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); mchbar_addr = ((u64)temp_hi << 32) | temp_lo; @@ -138,7 +138,7 @@ intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv) return ret; } - if (INTEL_GEN(dev_priv) >= 4) + if (GRAPHICS_VER(dev_priv) >= 4) pci_write_config_dword(dev_priv->bridge_dev, reg + 4, upper_32_bits(dev_priv->mch_res.start)); @@ -151,7 +151,7 @@ intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv) static void intel_setup_mchbar(struct drm_i915_private *dev_priv) { - int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; + int mchbar_reg = GRAPHICS_VER(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; u32 temp; bool enabled; @@ -190,7 +190,7 @@ intel_setup_mchbar(struct drm_i915_private *dev_priv) static void intel_teardown_mchbar(struct drm_i915_private *dev_priv) { - int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; + int mchbar_reg = GRAPHICS_VER(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; if (dev_priv->mchbar_need_disable) { if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { @@ -475,7 +475,7 @@ static int i915_set_dma_info(struct drm_i915_private *i915) goto mask_err; /* overlay on gen2 is broken and can't address above 1G */ - if (IS_GEN(i915, 2)) + if (GRAPHICS_VER(i915) == 2) mask_size = 30; /* @@ -601,7 +601,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) * device. The kernel then disables that interrupt source and so * prevents the other device from working properly. */ - if (INTEL_GEN(dev_priv) >= 5) { + if (GRAPHICS_VER(dev_priv) >= 5) { if (pci_enable_msi(pdev) < 0) drm_dbg(&dev_priv->drm, "can't enable MSI"); } @@ -729,7 +729,7 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv) intel_platform_name(INTEL_INFO(dev_priv)->platform), intel_subplatform(RUNTIME_INFO(dev_priv), INTEL_INFO(dev_priv)->platform), - INTEL_GEN(dev_priv)); + GRAPHICS_VER(dev_priv)); intel_device_info_print_static(INTEL_INFO(dev_priv), &p); intel_device_info_print_runtime(RUNTIME_INFO(dev_priv), &p); @@ -803,7 +803,7 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent) */ #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) if (IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) { - if (INTEL_GEN(i915) >= 9 && i915_selftest.live < 0 && + if (GRAPHICS_VER(i915) >= 9 && i915_selftest.live < 0 && i915->params.fake_lmem_start) { mkwrite_device_info(i915)->memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_SMEM; @@ -1043,7 +1043,7 @@ void i915_driver_shutdown(struct drm_i915_private *i915) intel_suspend_encoders(i915); intel_shutdown_encoders(i915); - intel_csr_ucode_suspend(i915); + intel_dmc_ucode_suspend(i915); /* * The only requirement is to reboot with display DC states disabled, @@ -1124,7 +1124,7 @@ static int i915_drm_suspend(struct drm_device *dev) dev_priv->suspend_count++; - intel_csr_ucode_suspend(dev_priv); + intel_dmc_ucode_suspend(dev_priv); enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); @@ -1182,7 +1182,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) * Fujitsu FSC S7110 * Acer Aspire 1830T */ - if (!(hibernation && INTEL_GEN(dev_priv) < 6)) + if (!(hibernation && GRAPHICS_VER(dev_priv) < 6)) pci_set_power_state(pdev, PCI_D3hot); out: @@ -1226,7 +1226,7 @@ static int i915_drm_resume(struct drm_device *dev) i915_ggtt_resume(&dev_priv->ggtt); - intel_csr_ucode_resume(dev_priv); + intel_dmc_ucode_resume(dev_priv); i915_restore_display(dev_priv); intel_pps_unlock_regs_wa(dev_priv); @@ -1727,6 +1727,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CREATE_EXT, i915_gem_create_ext_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 501a75cd306c..7b7918f72c41 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -67,6 +67,7 @@ #include "display/intel_bios.h" #include "display/intel_display.h" #include "display/intel_display_power.h" +#include "display/intel_dmc.h" #include "display/intel_dpll_mgr.h" #include "display/intel_dsb.h" #include "display/intel_frontbuffer.h" @@ -77,6 +78,7 @@ #include "gem/i915_gem_context_types.h" #include "gem/i915_gem_shrinker.h" #include "gem/i915_gem_stolen.h" +#include "gem/i915_gem_lmem.h" #include "gt/intel_engine.h" #include "gt/intel_gt_types.h" @@ -327,23 +329,6 @@ struct drm_i915_display_funcs { void (*read_luts)(struct intel_crtc_state *crtc_state); }; -struct intel_csr { - struct work_struct work; - const char *fw_path; - u32 required_version; - u32 max_fw_size; /* bytes */ - u32 *dmc_payload; - u32 dmc_fw_size; /* dwords */ - u32 version; - u32 mmio_count; - i915_reg_t mmioaddr[20]; - u32 mmiodata[20]; - u32 dc_state; - u32 target_dc_state; - u32 allowed_dc_mask; - intel_wakeref_t wakeref; -}; - enum i915_cache_level { I915_CACHE_NONE = 0, I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */ @@ -513,6 +498,13 @@ struct intel_l3_parity { }; struct i915_gem_mm { + /* + * Shortcut for the stolen region. This points to either + * INTEL_REGION_STOLEN_SMEM for integrated platforms, or + * INTEL_REGION_STOLEN_LMEM for discrete, or NULL if the device doesn't + * support stolen. + */ + struct intel_memory_region *stolen_region; /** Memory allocator for GTT stolen memory */ struct drm_mm stolen; /** Protects the usage of the GTT stolen memory allocator. This is @@ -581,6 +573,8 @@ i915_fence_timeout(const struct drm_i915_private *i915) /* Amount of SAGV/QGV points, BSpec precisely defines this */ #define I915_NUM_QGV_POINTS 8 +#define HAS_HW_SAGV_WM(i915) (DISPLAY_VER(i915) >= 13 && !IS_DGFX(i915)) + struct ddi_vbt_port_info { /* Non-NULL if port present. */ struct intel_bios_encoder_data *devdata; @@ -816,7 +810,7 @@ struct drm_i915_private { struct intel_wopcm wopcm; - struct intel_csr csr; + struct intel_dmc dmc; struct intel_gmbus gmbus[GMBUS_NUM_PINS]; @@ -1130,6 +1124,9 @@ struct drm_i915_private { u8 framestart_delay; + /* Window2 specifies time required to program DSB (Window2) in number of scan lines */ + u8 window2_delay; + u8 pch_ssc_use; /* For i915gm/i945gm vblank irq workaround */ @@ -1550,9 +1547,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, (IS_ALDERLAKE_P(__i915) && \ IS_GT_STEP(__i915, since, until)) -#define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp) -#define IS_GEN9_LP(dev_priv) (IS_GEN(dev_priv, 9) && IS_LP(dev_priv)) -#define IS_GEN9_BC(dev_priv) (IS_GEN(dev_priv, 9) && !IS_LP(dev_priv)) +#define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp) +#define IS_GEN9_LP(dev_priv) (GRAPHICS_VER(dev_priv) == 9 && IS_LP(dev_priv)) +#define IS_GEN9_BC(dev_priv) (GRAPHICS_VER(dev_priv) == 9 && !IS_LP(dev_priv)) #define __HAS_ENGINE(engine_mask, id) ((engine_mask) & BIT(id)) #define HAS_ENGINE(gt, id) __HAS_ENGINE((gt)->info.engine_mask, id) @@ -1572,12 +1569,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, * The Gen7 cmdparser copies the scanned buffer to the ggtt for execution * All later gens can run the final buffer from the ppgtt */ -#define CMDPARSER_USES_GGTT(dev_priv) IS_GEN(dev_priv, 7) +#define CMDPARSER_USES_GGTT(dev_priv) (GRAPHICS_VER(dev_priv) == 7) #define HAS_LLC(dev_priv) (INTEL_INFO(dev_priv)->has_llc) #define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop) #define HAS_EDRAM(dev_priv) ((dev_priv)->edram_size_mb) -#define HAS_SECURE_BATCHES(dev_priv) (INTEL_GEN(dev_priv) < 6) +#define HAS_SECURE_BATCHES(dev_priv) (GRAPHICS_VER(dev_priv) < 6) #define HAS_WT(dev_priv) HAS_EDRAM(dev_priv) #define HWS_NEEDS_PHYSICAL(dev_priv) (INTEL_INFO(dev_priv)->hws_needs_physical) @@ -1610,7 +1607,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) #define NEEDS_RC6_CTX_CORRUPTION_WA(dev_priv) \ - (IS_BROADWELL(dev_priv) || IS_GEN(dev_priv, 9)) + (IS_BROADWELL(dev_priv) || GRAPHICS_VER(dev_priv) == 9) /* WaRsDisableCoarsePowerGating:skl,cnl */ #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ @@ -1618,23 +1615,22 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, IS_SKL_GT3(dev_priv) || \ IS_SKL_GT4(dev_priv)) -#define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4) -#define HAS_GMBUS_BURST_READ(dev_priv) (INTEL_GEN(dev_priv) >= 10 || \ +#define HAS_GMBUS_IRQ(dev_priv) (GRAPHICS_VER(dev_priv) >= 4) +#define HAS_GMBUS_BURST_READ(dev_priv) (GRAPHICS_VER(dev_priv) >= 10 || \ IS_GEMINILAKE(dev_priv) || \ IS_KABYLAKE(dev_priv)) /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte * rows, which changed the alignment requirements and fence programming. */ -#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN(dev_priv, 2) && \ - !(IS_I915G(dev_priv) || \ - IS_I915GM(dev_priv))) +#define HAS_128_BYTE_Y_TILING(dev_priv) (GRAPHICS_VER(dev_priv) != 2 && \ + !(IS_I915G(dev_priv) || IS_I915GM(dev_priv))) #define SUPPORTS_TV(dev_priv) (INTEL_INFO(dev_priv)->display.supports_tv) #define I915_HAS_HOTPLUG(dev_priv) (INTEL_INFO(dev_priv)->display.has_hotplug) -#define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) +#define HAS_FW_BLC(dev_priv) (GRAPHICS_VER(dev_priv) > 2) #define HAS_FBC(dev_priv) (INTEL_INFO(dev_priv)->display.has_fbc) -#define HAS_CUR_FBC(dev_priv) (!HAS_GMCH(dev_priv) && INTEL_GEN(dev_priv) >= 7) +#define HAS_CUR_FBC(dev_priv) (!HAS_GMCH(dev_priv) && GRAPHICS_VER(dev_priv) >= 7) #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) @@ -1645,7 +1641,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_PSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_psr) #define HAS_PSR_HW_TRACKING(dev_priv) \ (INTEL_INFO(dev_priv)->display.has_psr_hw_tracking) -#define HAS_PSR2_SEL_FETCH(dev_priv) (INTEL_GEN(dev_priv) >= 12) +#define HAS_PSR2_SEL_FETCH(dev_priv) (GRAPHICS_VER(dev_priv) >= 12) #define HAS_TRANSCODER(dev_priv, trans) ((INTEL_INFO(dev_priv)->cpu_transcoder_mask & BIT(trans)) != 0) #define HAS_RC6(dev_priv) (INTEL_INFO(dev_priv)->has_rc6) @@ -1654,9 +1650,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_RPS(dev_priv) (INTEL_INFO(dev_priv)->has_rps) -#define HAS_CSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_csr) +#define HAS_DMC(dev_priv) (INTEL_INFO(dev_priv)->display.has_dmc) -#define HAS_MSO(i915) (INTEL_GEN(i915) >= 12) +#define HAS_MSO(i915) (GRAPHICS_VER(i915) >= 12) #define HAS_RUNTIME_PM(dev_priv) (INTEL_INFO(dev_priv)->has_runtime_pm) #define HAS_64BIT_RELOC(dev_priv) (INTEL_INFO(dev_priv)->has_64bit_reloc) @@ -1675,7 +1671,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_GMCH(dev_priv) (INTEL_INFO(dev_priv)->display.has_gmch) -#define HAS_LSPCON(dev_priv) (IS_GEN_RANGE(dev_priv, 9, 10)) +#define HAS_LSPCON(dev_priv) (IS_GRAPHICS_VER(dev_priv, 9, 10)) /* DPF == dynamic parity feature */ #define HAS_L3_DPF(dev_priv) (INTEL_INFO(dev_priv)->has_l3_dpf) @@ -1689,7 +1685,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_DISPLAY(dev_priv) (INTEL_INFO(dev_priv)->pipe_mask != 0) -#define HAS_VRR(i915) (INTEL_GEN(i915) >= 12) +#define HAS_VRR(i915) (GRAPHICS_VER(i915) >= 12) /* Only valid when HAS_DISPLAY() is true */ #define INTEL_DISPLAY_ENABLED(dev_priv) \ @@ -1716,13 +1712,19 @@ static inline bool intel_vtd_active(void) static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) { - return INTEL_GEN(dev_priv) >= 6 && intel_vtd_active(); + return GRAPHICS_VER(dev_priv) >= 6 && intel_vtd_active(); } static inline bool -intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv) +intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *i915) { - return IS_BROXTON(dev_priv) && intel_vtd_active(); + return IS_BROXTON(i915) && intel_vtd_active(); +} + +static inline bool +intel_vm_no_concurrent_access_wa(struct drm_i915_private *i915) +{ + return IS_CHERRYVIEW(i915) || intel_ggtt_update_needs_vtd_wa(i915); } /* i915_drv.c */ @@ -1802,6 +1804,7 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj, #define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0) #define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1) #define I915_GEM_OBJECT_UNBIND_TEST BIT(2) +#define I915_GEM_OBJECT_UNBIND_VM_TRYLOCK BIT(3) void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); @@ -1927,16 +1930,22 @@ int remap_io_sg(struct vm_area_struct *vma, static inline int intel_hws_csb_write_index(struct drm_i915_private *i915) { - if (INTEL_GEN(i915) >= 10) + if (GRAPHICS_VER(i915) >= 10) return CNL_HWS_CSB_WRITE_INDEX; else return I915_HWS_CSB_WRITE_INDEX; } static inline enum i915_map_type -i915_coherent_map_type(struct drm_i915_private *i915) +i915_coherent_map_type(struct drm_i915_private *i915, + struct drm_i915_gem_object *obj, bool always_coherent) { - return HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC; + if (i915_gem_object_is_lmem(obj)) + return I915_MAP_WC; + if (HAS_LLC(i915) || always_coherent) + return I915_MAP_WB; + else + return I915_MAP_WC; } #endif diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index b3cedd20f365..1aadc021d92e 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -157,8 +157,18 @@ try_again: if (vma) { ret = -EBUSY; if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE || - !i915_vma_is_active(vma)) - ret = i915_vma_unbind(vma); + !i915_vma_is_active(vma)) { + if (flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK) { + if (mutex_trylock(&vma->vm->mutex)) { + ret = __i915_vma_unbind(vma); + mutex_unlock(&vma->vm->mutex); + } else { + ret = -EBUSY; + } + } else { + ret = i915_vma_unbind(vma); + } + } __i915_vma_put(vma); } @@ -432,7 +442,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, /* PREAD is disallowed for all platforms after TGL-LP. This also * covers all platforms with local memory. */ - if (INTEL_GEN(i915) >= 12 && !IS_TIGERLAKE(i915)) + if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915)) return -EOPNOTSUPP; if (args->size == 0) @@ -712,7 +722,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, /* PWRITE is disallowed for all platforms after TGL-LP. This also * covers all platforms with local memory. */ - if (INTEL_GEN(i915) >= 12 && !IS_TIGERLAKE(i915)) + if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915)) return -EOPNOTSUPP; if (args->size == 0) diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 99ca242ec13b..35c97c39f125 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -36,7 +36,7 @@ #include <drm/drm_print.h> -#include "display/intel_csr.h" +#include "display/intel_dmc.h" #include "display/intel_overlay.h" #include "gem/i915_gem_context.h" @@ -435,13 +435,13 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m, err_printf(m, " INSTDONE: 0x%08x\n", ee->instdone.instdone); - if (ee->engine->class != RENDER_CLASS || INTEL_GEN(m->i915) <= 3) + if (ee->engine->class != RENDER_CLASS || GRAPHICS_VER(m->i915) <= 3) return; err_printf(m, " SC_INSTDONE: 0x%08x\n", ee->instdone.slice_common); - if (INTEL_GEN(m->i915) <= 6) + if (GRAPHICS_VER(m->i915) <= 6) return; for_each_instdone_slice_subslice(m->i915, sseu, slice, subslice) @@ -454,7 +454,7 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m, slice, subslice, ee->instdone.row[slice][subslice]); - if (INTEL_GEN(m->i915) < 12) + if (GRAPHICS_VER(m->i915) < 12) return; err_printf(m, " SC_INSTDONE_EXTRA: 0x%08x\n", @@ -543,7 +543,7 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, upper_32_bits(start), lower_32_bits(start), upper_32_bits(end), lower_32_bits(end)); } - if (INTEL_GEN(m->i915) >= 4) { + if (GRAPHICS_VER(m->i915) >= 4) { err_printf(m, " BBADDR: 0x%08x_%08x\n", (u32)(ee->bbaddr>>32), (u32)ee->bbaddr); err_printf(m, " BB_STATE: 0x%08x\n", ee->bbstate); @@ -552,14 +552,14 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, err_printf(m, " INSTPM: 0x%08x\n", ee->instpm); err_printf(m, " FADDR: 0x%08x %08x\n", upper_32_bits(ee->faddr), lower_32_bits(ee->faddr)); - if (INTEL_GEN(m->i915) >= 6) { + if (GRAPHICS_VER(m->i915) >= 6) { err_printf(m, " RC PSMI: 0x%08x\n", ee->rc_psmi); err_printf(m, " FAULT_REG: 0x%08x\n", ee->fault_reg); } if (HAS_PPGTT(m->i915)) { err_printf(m, " GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode); - if (INTEL_GEN(m->i915) >= 8) { + if (GRAPHICS_VER(m->i915) >= 8) { int i; for (i = 0; i < 4; i++) err_printf(m, " PDP%d: 0x%016llx\n", @@ -706,25 +706,25 @@ static void err_print_gt(struct drm_i915_error_state_buf *m, for (i = 0; i < gt->nfence; i++) err_printf(m, " fence[%d] = %08llx\n", i, gt->fence[i]); - if (IS_GEN_RANGE(m->i915, 6, 11)) { + if (IS_GRAPHICS_VER(m->i915, 6, 11)) { err_printf(m, "ERROR: 0x%08x\n", gt->error); err_printf(m, "DONE_REG: 0x%08x\n", gt->done_reg); } - if (INTEL_GEN(m->i915) >= 8) + if (GRAPHICS_VER(m->i915) >= 8) err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n", gt->fault_data1, gt->fault_data0); - if (IS_GEN(m->i915, 7)) + if (GRAPHICS_VER(m->i915) == 7) err_printf(m, "ERR_INT: 0x%08x\n", gt->err_int); - if (IS_GEN_RANGE(m->i915, 8, 11)) + if (IS_GRAPHICS_VER(m->i915, 8, 11)) err_printf(m, "GTT_CACHE_EN: 0x%08x\n", gt->gtt_cache); - if (IS_GEN(m->i915, 12)) + if (GRAPHICS_VER(m->i915) == 12) err_printf(m, "AUX_ERR_DBG: 0x%08x\n", gt->aux_err); - if (INTEL_GEN(m->i915) >= 12) { + if (GRAPHICS_VER(m->i915) >= 12) { int i; for (i = 0; i < GEN12_SFC_DONE_MAX; i++) @@ -788,14 +788,14 @@ static void __err_print_to_sgl(struct drm_i915_error_state_buf *m, err_printf(m, "IOMMU enabled?: %d\n", error->iommu); - if (HAS_CSR(m->i915)) { - struct intel_csr *csr = &m->i915->csr; + if (HAS_DMC(m->i915)) { + struct intel_dmc *dmc = &m->i915->dmc; err_printf(m, "DMC loaded: %s\n", - yesno(csr->dmc_payload != NULL)); + yesno(intel_dmc_has_payload(m->i915) != 0)); err_printf(m, "DMC fw version: %d.%d\n", - CSR_VERSION_MAJOR(csr->version), - CSR_VERSION_MINOR(csr->version)); + DMC_VERSION_MAJOR(dmc->version), + DMC_VERSION_MINOR(dmc->version)); } err_printf(m, "RPM wakelock: %s\n", yesno(error->wakelock)); @@ -1092,12 +1092,12 @@ static void gt_record_fences(struct intel_gt_coredump *gt) struct intel_uncore *uncore = gt->_gt->uncore; int i; - if (INTEL_GEN(uncore->i915) >= 6) { + if (GRAPHICS_VER(uncore->i915) >= 6) { for (i = 0; i < ggtt->num_fences; i++) gt->fence[i] = intel_uncore_read64(uncore, FENCE_REG_GEN6_LO(i)); - } else if (INTEL_GEN(uncore->i915) >= 4) { + } else if (GRAPHICS_VER(uncore->i915) >= 4) { for (i = 0; i < ggtt->num_fences; i++) gt->fence[i] = intel_uncore_read64(uncore, @@ -1115,20 +1115,20 @@ static void engine_record_registers(struct intel_engine_coredump *ee) const struct intel_engine_cs *engine = ee->engine; struct drm_i915_private *i915 = engine->i915; - if (INTEL_GEN(i915) >= 6) { + if (GRAPHICS_VER(i915) >= 6) { ee->rc_psmi = ENGINE_READ(engine, RING_PSMI_CTL); - if (INTEL_GEN(i915) >= 12) + if (GRAPHICS_VER(i915) >= 12) ee->fault_reg = intel_uncore_read(engine->uncore, GEN12_RING_FAULT_REG); - else if (INTEL_GEN(i915) >= 8) + else if (GRAPHICS_VER(i915) >= 8) ee->fault_reg = intel_uncore_read(engine->uncore, GEN8_RING_FAULT_REG); else ee->fault_reg = GEN6_RING_FAULT_REG_READ(engine); } - if (INTEL_GEN(i915) >= 4) { + if (GRAPHICS_VER(i915) >= 4) { ee->esr = ENGINE_READ(engine, RING_ESR); ee->faddr = ENGINE_READ(engine, RING_DMA_FADD); ee->ipeir = ENGINE_READ(engine, RING_IPEIR); @@ -1136,7 +1136,7 @@ static void engine_record_registers(struct intel_engine_coredump *ee) ee->instps = ENGINE_READ(engine, RING_INSTPS); ee->bbaddr = ENGINE_READ(engine, RING_BBADDR); ee->ccid = ENGINE_READ(engine, CCID); - if (INTEL_GEN(i915) >= 8) { + if (GRAPHICS_VER(i915) >= 8) { ee->faddr |= (u64)ENGINE_READ(engine, RING_DMA_FADD_UDW) << 32; ee->bbaddr |= (u64)ENGINE_READ(engine, RING_BBADDR_UDW) << 32; } @@ -1155,13 +1155,13 @@ static void engine_record_registers(struct intel_engine_coredump *ee) ee->head = ENGINE_READ(engine, RING_HEAD); ee->tail = ENGINE_READ(engine, RING_TAIL); ee->ctl = ENGINE_READ(engine, RING_CTL); - if (INTEL_GEN(i915) > 2) + if (GRAPHICS_VER(i915) > 2) ee->mode = ENGINE_READ(engine, RING_MI_MODE); if (!HWS_NEEDS_PHYSICAL(i915)) { i915_reg_t mmio; - if (IS_GEN(i915, 7)) { + if (GRAPHICS_VER(i915) == 7) { switch (engine->id) { default: MISSING_CASE(engine->id); @@ -1179,7 +1179,7 @@ static void engine_record_registers(struct intel_engine_coredump *ee) mmio = VEBOX_HWS_PGA_GEN7; break; } - } else if (IS_GEN(engine->i915, 6)) { + } else if (GRAPHICS_VER(engine->i915) == 6) { mmio = RING_HWS_PGA_GEN6(engine->mmio_base); } else { /* XXX: gen8 returns to sanity */ @@ -1196,13 +1196,13 @@ static void engine_record_registers(struct intel_engine_coredump *ee) ee->vm_info.gfx_mode = ENGINE_READ(engine, RING_MODE_GEN7); - if (IS_GEN(i915, 6)) { + if (GRAPHICS_VER(i915) == 6) { ee->vm_info.pp_dir_base = ENGINE_READ(engine, RING_PP_DIR_BASE_READ); - } else if (IS_GEN(i915, 7)) { + } else if (GRAPHICS_VER(i915) == 7) { ee->vm_info.pp_dir_base = ENGINE_READ(engine, RING_PP_DIR_BASE); - } else if (INTEL_GEN(i915) >= 8) { + } else if (GRAPHICS_VER(i915) >= 8) { u32 base = engine->mmio_base; for (i = 0; i < 4; i++) { @@ -1534,52 +1534,52 @@ static void gt_record_regs(struct intel_gt_coredump *gt) gt->forcewake = intel_uncore_read_fw(uncore, FORCEWAKE_VLV); } - if (IS_GEN(i915, 7)) + if (GRAPHICS_VER(i915) == 7) gt->err_int = intel_uncore_read(uncore, GEN7_ERR_INT); - if (INTEL_GEN(i915) >= 12) { + if (GRAPHICS_VER(i915) >= 12) { gt->fault_data0 = intel_uncore_read(uncore, GEN12_FAULT_TLB_DATA0); gt->fault_data1 = intel_uncore_read(uncore, GEN12_FAULT_TLB_DATA1); - } else if (INTEL_GEN(i915) >= 8) { + } else if (GRAPHICS_VER(i915) >= 8) { gt->fault_data0 = intel_uncore_read(uncore, GEN8_FAULT_TLB_DATA0); gt->fault_data1 = intel_uncore_read(uncore, GEN8_FAULT_TLB_DATA1); } - if (IS_GEN(i915, 6)) { + if (GRAPHICS_VER(i915) == 6) { gt->forcewake = intel_uncore_read_fw(uncore, FORCEWAKE); gt->gab_ctl = intel_uncore_read(uncore, GAB_CTL); gt->gfx_mode = intel_uncore_read(uncore, GFX_MODE); } /* 2: Registers which belong to multiple generations */ - if (INTEL_GEN(i915) >= 7) + if (GRAPHICS_VER(i915) >= 7) gt->forcewake = intel_uncore_read_fw(uncore, FORCEWAKE_MT); - if (INTEL_GEN(i915) >= 6) { + if (GRAPHICS_VER(i915) >= 6) { gt->derrmr = intel_uncore_read(uncore, DERRMR); - if (INTEL_GEN(i915) < 12) { + if (GRAPHICS_VER(i915) < 12) { gt->error = intel_uncore_read(uncore, ERROR_GEN6); gt->done_reg = intel_uncore_read(uncore, DONE_REG); } } /* 3: Feature specific registers */ - if (IS_GEN_RANGE(i915, 6, 7)) { + if (IS_GRAPHICS_VER(i915, 6, 7)) { gt->gam_ecochk = intel_uncore_read(uncore, GAM_ECOCHK); gt->gac_eco = intel_uncore_read(uncore, GAC_ECO_BITS); } - if (IS_GEN_RANGE(i915, 8, 11)) + if (IS_GRAPHICS_VER(i915, 8, 11)) gt->gtt_cache = intel_uncore_read(uncore, HSW_GTT_CACHE_EN); - if (IS_GEN(i915, 12)) + if (GRAPHICS_VER(i915) == 12) gt->aux_err = intel_uncore_read(uncore, GEN12_AUX_ERR_DBG); - if (INTEL_GEN(i915) >= 12) { + if (GRAPHICS_VER(i915) >= 12) { for (i = 0; i < GEN12_SFC_DONE_MAX; i++) { gt->sfc_done[i] = intel_uncore_read(uncore, GEN12_SFC_DONE(i)); @@ -1589,7 +1589,7 @@ static void gt_record_regs(struct intel_gt_coredump *gt) } /* 4: Everything else */ - if (INTEL_GEN(i915) >= 11) { + if (GRAPHICS_VER(i915) >= 11) { gt->ier = intel_uncore_read(uncore, GEN8_DE_MISC_IER); gt->gtier[0] = intel_uncore_read(uncore, @@ -1608,7 +1608,7 @@ static void gt_record_regs(struct intel_gt_coredump *gt) intel_uncore_read(uncore, GEN11_GUNIT_CSME_INTR_ENABLE); gt->ngtier = 6; - } else if (INTEL_GEN(i915) >= 8) { + } else if (GRAPHICS_VER(i915) >= 8) { gt->ier = intel_uncore_read(uncore, GEN8_DE_MISC_IER); for (i = 0; i < 4; i++) gt->gtier[i] = @@ -1618,7 +1618,7 @@ static void gt_record_regs(struct intel_gt_coredump *gt) gt->ier = intel_uncore_read(uncore, DEIER); gt->gtier[0] = intel_uncore_read(uncore, GTIER); gt->ngtier = 1; - } else if (IS_GEN(i915, 2)) { + } else if (GRAPHICS_VER(i915) == 2) { gt->ier = intel_uncore_read16(uncore, GEN2_IER); } else if (!IS_VALLEYVIEW(i915)) { gt->ier = intel_uncore_read(uncore, GEN2_IER); @@ -1674,7 +1674,7 @@ static const char *error_msg(struct i915_gpu_coredump *error) len = scnprintf(error->error_msg, sizeof(error->error_msg), "GPU HANG: ecode %d:%x:%08x", - INTEL_GEN(error->i915), hung_classes, + GRAPHICS_VER(error->i915), hung_classes, generate_ecode(first)); if (first && first->context.pid) { /* Just show the first executing process, more is confusing */ diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index d4611c643446..a11bdb667241 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -2175,7 +2175,7 @@ static irqreturn_t ilk_irq_handler(int irq, void *arg) gt_iir = raw_reg_read(regs, GTIIR); if (gt_iir) { raw_reg_write(regs, GTIIR, gt_iir); - if (INTEL_GEN(i915) >= 6) + if (GRAPHICS_VER(i915) >= 6) gen6_gt_irq_handler(&i915->gt, gt_iir); else gen5_gt_irq_handler(&i915->gt, gt_iir); @@ -2192,7 +2192,7 @@ static irqreturn_t ilk_irq_handler(int irq, void *arg) ret = IRQ_HANDLED; } - if (INTEL_GEN(i915) >= 6) { + if (GRAPHICS_VER(i915) >= 6) { u32 pm_iir = raw_reg_read(regs, GEN6_PMIIR); if (pm_iir) { raw_reg_write(regs, GEN6_PMIIR, pm_iir); @@ -2425,6 +2425,17 @@ static u32 gen8_de_pipe_flip_done_mask(struct drm_i915_private *i915) return GEN8_PIPE_PRIMARY_FLIP_DONE; } +u32 gen8_de_pipe_underrun_mask(struct drm_i915_private *dev_priv) +{ + u32 mask = GEN8_PIPE_FIFO_UNDERRUN; + + if (DISPLAY_VER(dev_priv) >= 13) + mask |= XELPD_PIPE_SOFT_UNDERRUN | + XELPD_PIPE_HARD_UNDERRUN; + + return mask; +} + static irqreturn_t gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) { @@ -2536,7 +2547,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) if (iir & GEN8_PIPE_CDCLK_CRC_DONE) hsw_pipe_crc_irq_handler(dev_priv, pipe); - if (iir & GEN8_PIPE_FIFO_UNDERRUN) + if (iir & gen8_de_pipe_underrun_mask(dev_priv)) intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv); @@ -3028,7 +3039,7 @@ static void ilk_irq_reset(struct drm_i915_private *dev_priv) GEN3_IRQ_RESET(uncore, DE); dev_priv->irq_mask = ~0u; - if (IS_GEN(dev_priv, 7)) + if (GRAPHICS_VER(dev_priv) == 7) intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff); if (IS_HASWELL(dev_priv)) { @@ -3173,7 +3184,8 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, u8 pipe_mask) { struct intel_uncore *uncore = &dev_priv->uncore; - u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN | + u32 extra_ier = GEN8_PIPE_VBLANK | + gen8_de_pipe_underrun_mask(dev_priv) | gen8_de_pipe_flip_done_mask(dev_priv); enum pipe pipe; @@ -3646,7 +3658,7 @@ static void ilk_irq_postinstall(struct drm_i915_private *dev_priv) struct intel_uncore *uncore = &dev_priv->uncore; u32 display_mask, extra_mask; - if (INTEL_GEN(dev_priv) >= 7) { + if (GRAPHICS_VER(dev_priv) >= 7) { display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB); extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB | @@ -3757,7 +3769,8 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) } de_pipe_enables = de_pipe_masked | - GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN | + GEN8_PIPE_VBLANK | + gen8_de_pipe_underrun_mask(dev_priv) | gen8_de_pipe_flip_done_mask(dev_priv); de_port_enables = de_port_masked; @@ -4024,7 +4037,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[RCS0], iir); if (iir & I915_MASTER_ERROR_INTERRUPT) i8xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4132,7 +4145,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[RCS0], iir); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4277,10 +4290,12 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[RCS0], + iir); if (iir & I915_BSD_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->gt.engine[VCS0]); + intel_engine_cs_irq(dev_priv->gt.engine[VCS0], + iir >> 25); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4315,7 +4330,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev_priv->l3_parity.remap_info[i] = NULL; /* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */ - if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11) + if (HAS_GT_UC(dev_priv) && GRAPHICS_VER(dev_priv) < 11) dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16; if (!HAS_DISPLAY(dev_priv)) @@ -4386,18 +4401,18 @@ static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv) return cherryview_irq_handler; else if (IS_VALLEYVIEW(dev_priv)) return valleyview_irq_handler; - else if (IS_GEN(dev_priv, 4)) + else if (GRAPHICS_VER(dev_priv) == 4) return i965_irq_handler; - else if (IS_GEN(dev_priv, 3)) + else if (GRAPHICS_VER(dev_priv) == 3) return i915_irq_handler; else return i8xx_irq_handler; } else { if (HAS_MASTER_UNIT_IRQ(dev_priv)) return dg1_irq_handler; - if (INTEL_GEN(dev_priv) >= 11) + if (GRAPHICS_VER(dev_priv) >= 11) return gen11_irq_handler; - else if (INTEL_GEN(dev_priv) >= 8) + else if (GRAPHICS_VER(dev_priv) >= 8) return gen8_irq_handler; else return ilk_irq_handler; @@ -4411,16 +4426,16 @@ static void intel_irq_reset(struct drm_i915_private *dev_priv) cherryview_irq_reset(dev_priv); else if (IS_VALLEYVIEW(dev_priv)) valleyview_irq_reset(dev_priv); - else if (IS_GEN(dev_priv, 4)) + else if (GRAPHICS_VER(dev_priv) == 4) i965_irq_reset(dev_priv); - else if (IS_GEN(dev_priv, 3)) + else if (GRAPHICS_VER(dev_priv) == 3) i915_irq_reset(dev_priv); else i8xx_irq_reset(dev_priv); } else { - if (INTEL_GEN(dev_priv) >= 11) + if (GRAPHICS_VER(dev_priv) >= 11) gen11_irq_reset(dev_priv); - else if (INTEL_GEN(dev_priv) >= 8) + else if (GRAPHICS_VER(dev_priv) >= 8) gen8_irq_reset(dev_priv); else ilk_irq_reset(dev_priv); @@ -4434,16 +4449,16 @@ static void intel_irq_postinstall(struct drm_i915_private *dev_priv) cherryview_irq_postinstall(dev_priv); else if (IS_VALLEYVIEW(dev_priv)) valleyview_irq_postinstall(dev_priv); - else if (IS_GEN(dev_priv, 4)) + else if (GRAPHICS_VER(dev_priv) == 4) i965_irq_postinstall(dev_priv); - else if (IS_GEN(dev_priv, 3)) + else if (GRAPHICS_VER(dev_priv) == 3) i915_irq_postinstall(dev_priv); else i8xx_irq_postinstall(dev_priv); } else { - if (INTEL_GEN(dev_priv) >= 11) + if (GRAPHICS_VER(dev_priv) >= 11) gen11_irq_postinstall(dev_priv); - else if (INTEL_GEN(dev_priv) >= 8) + else if (GRAPHICS_VER(dev_priv) >= 8) gen8_irq_postinstall(dev_priv); else ilk_irq_postinstall(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h index 25f25cd95818..db34d5dbe402 100644 --- a/drivers/gpu/drm/i915/i915_irq.h +++ b/drivers/gpu/drm/i915/i915_irq.h @@ -100,6 +100,7 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, u8 pipe_mask); void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, u8 pipe_mask); +u32 gen8_de_pipe_underrun_mask(struct drm_i915_private *dev_priv); bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time, bool in_vblank_irq); diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index 14cd64cc61d0..4a114a5ad000 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -71,18 +71,18 @@ struct drm_printer; param(int, fastboot, -1, 0600) \ param(int, enable_dpcd_backlight, -1, 0600) \ param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \ - param(unsigned long, fake_lmem_start, 0, 0400) \ - param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, 0600) \ + param(unsigned long, fake_lmem_start, 0, IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM) ? 0400 : 0) \ + param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, CONFIG_DRM_I915_REQUEST_TIMEOUT ? 0600 : 0) \ /* leave bools at the end to not create holes */ \ param(bool, enable_hangcheck, true, 0600) \ param(bool, load_detect_test, false, 0600) \ param(bool, force_reset_modeset_test, false, 0600) \ - param(bool, error_capture, true, 0600) \ + param(bool, error_capture, true, IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ? 0600 : 0) \ param(bool, disable_display, false, 0400) \ param(bool, verbose_state_checks, true, 0) \ param(bool, nuclear_pageflip, false, 0400) \ param(bool, enable_dp_mst, true, 0600) \ - param(bool, enable_gvt, false, 0400) + param(bool, enable_gvt, false, IS_ENABLED(CONFIG_DRM_I915_GVT) ? 0400 : 0) #define MEMBER(T, member, ...) T member; struct i915_params { diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 1680062a2149..83b500bb170c 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -643,7 +643,7 @@ static const struct intel_device_info chv_info = { GEN8_FEATURES, \ GEN(9), \ GEN9_DEFAULT_PAGE_SIZES, \ - .display.has_csr = 1, \ + .display.has_dmc = 1, \ .has_gt_uc = 1, \ .display.has_hdcp = 1, \ .display.has_ipc = 1, \ @@ -698,7 +698,7 @@ static const struct intel_device_info skl_gt4_info = { .display.has_psr = 1, \ .display.has_psr_hw_tracking = 1, \ .has_runtime_pm = 1, \ - .display.has_csr = 1, \ + .display.has_dmc = 1, \ .has_rc6 = 1, \ .has_rps = true, \ .display.has_dp_mst = 1, \ @@ -908,7 +908,7 @@ static const struct intel_device_info rkl_info = { }; #define DGFX_FEATURES \ - .memory_regions = REGION_SMEM | REGION_LMEM, \ + .memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \ .has_master_unit_irq = 1, \ .has_llc = 0, \ .has_snoop = 1, \ @@ -953,6 +953,7 @@ static const struct intel_device_info adl_p_info = { GEN12_FEATURES, XE_LPD_FEATURES, PLATFORM(INTEL_ALDERLAKE_P), + .has_cdclk_crawl = 1, .require_force_probe = 1, .display.has_modular_fia = 1, .platform_engine_mask = diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index de8ebc34af0f..9f94914958c3 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -719,7 +719,7 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream, * it to userspace... */ reason = ((report32[0] >> OAREPORT_REASON_SHIFT) & - (IS_GEN(stream->perf->i915, 12) ? + (GRAPHICS_VER(stream->perf->i915) == 12 ? OAREPORT_REASON_MASK_EXTENDED : OAREPORT_REASON_MASK)); @@ -734,7 +734,7 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream, * understand that the ID has been squashed by the kernel. */ if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) && - INTEL_GEN(stream->perf->i915) <= 11) + GRAPHICS_VER(stream->perf->i915) <= 11) ctx_id = report32[2] = INVALID_CTX_ID; /* @@ -801,7 +801,7 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream, if (start_offset != *offset) { i915_reg_t oaheadptr; - oaheadptr = IS_GEN(stream->perf->i915, 12) ? + oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ? GEN12_OAG_OAHEADPTR : GEN8_OAHEADPTR; spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); @@ -854,7 +854,7 @@ static int gen8_oa_read(struct i915_perf_stream *stream, if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) return -EIO; - oastatus_reg = IS_GEN(stream->perf->i915, 12) ? + oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ? GEN12_OAG_OASTATUS : GEN8_OASTATUS; oastatus = intel_uncore_read(uncore, oastatus_reg); @@ -901,7 +901,7 @@ static int gen8_oa_read(struct i915_perf_stream *stream, intel_uncore_rmw(uncore, oastatus_reg, GEN8_OASTATUS_COUNTER_OVERFLOW | GEN8_OASTATUS_REPORT_LOST, - IS_GEN_RANGE(uncore->i915, 8, 11) ? + IS_GRAPHICS_VER(uncore->i915, 8, 11) ? (GEN8_OASTATUS_HEAD_POINTER_WRAP | GEN8_OASTATUS_TAIL_POINTER_WRAP) : 0); } @@ -1243,7 +1243,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) if (IS_ERR(ce)) return PTR_ERR(ce); - switch (INTEL_GEN(ce->engine->i915)) { + switch (GRAPHICS_VER(ce->engine->i915)) { case 7: { /* * On Haswell we don't do any post processing of the reports @@ -1257,11 +1257,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) case 8: case 9: case 10: - if (intel_engine_in_execlists_submission_mode(ce->engine)) { - stream->specific_ctx_id_mask = - (1U << GEN8_CTX_ID_WIDTH) - 1; - stream->specific_ctx_id = stream->specific_ctx_id_mask; - } else { + if (intel_engine_uses_guc(ce->engine)) { /* * When using GuC, the context descriptor we write in * i915 is read by GuC and rewritten before it's @@ -1280,6 +1276,10 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) */ stream->specific_ctx_id_mask = (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1; + } else { + stream->specific_ctx_id_mask = + (1U << GEN8_CTX_ID_WIDTH) - 1; + stream->specific_ctx_id = stream->specific_ctx_id_mask; } break; @@ -1297,7 +1297,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) } default: - MISSING_CASE(INTEL_GEN(ce->engine->i915)); + MISSING_CASE(GRAPHICS_VER(ce->engine->i915)); } ce->tag = stream->specific_ctx_id; @@ -1602,7 +1602,7 @@ static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs, cmd = save ? MI_STORE_REGISTER_MEM : MI_LOAD_REGISTER_MEM; cmd |= MI_SRM_LRM_GLOBAL_GTT; - if (INTEL_GEN(stream->perf->i915) >= 8) + if (GRAPHICS_VER(stream->perf->i915) >= 8) cmd++; for (d = 0; d < dword_count; d++) { @@ -1731,7 +1731,7 @@ retry: *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1); /* Restart from the beginning if we had timestamps roll over. */ - *cs++ = (INTEL_GEN(i915) < 8 ? + *cs++ = (GRAPHICS_VER(i915) < 8 ? MI_BATCH_BUFFER_START : MI_BATCH_BUFFER_START_GEN8) | MI_BATCH_PREDICATE; @@ -1768,7 +1768,7 @@ retry: *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1); /* Predicate the jump. */ - *cs++ = (INTEL_GEN(i915) < 8 ? + *cs++ = (GRAPHICS_VER(i915) < 8 ? MI_BATCH_BUFFER_START : MI_BATCH_BUFFER_START_GEN8) | MI_BATCH_PREDICATE; @@ -1892,7 +1892,7 @@ retry: oa_config->flex_regs_len); /* Jump into the active wait. */ - *cs++ = (INTEL_GEN(stream->perf->i915) < 8 ? + *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ? MI_BATCH_BUFFER_START : MI_BATCH_BUFFER_START_GEN8); *cs++ = i915_ggtt_offset(stream->noa_wait); @@ -2492,7 +2492,7 @@ gen8_enable_metric_set(struct i915_perf_stream *stream, * be read back from automatically triggered reports, as part of the * RPT_ID field. */ - if (IS_GEN_RANGE(stream->perf->i915, 9, 11)) { + if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) { intel_uncore_write(uncore, GEN8_OA_DEBUG, _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS | GEN9_OA_DEBUG_INCLUDE_CLK_RATIO)); @@ -2797,7 +2797,7 @@ get_default_sseu_config(struct intel_sseu *out_sseu, *out_sseu = intel_sseu_from_device_info(devinfo_sseu); - if (IS_GEN(engine->i915, 11)) { + if (GRAPHICS_VER(engine->i915) == 11) { /* * We only need subslice count so it doesn't matter which ones * we select - just turn off low bits in the amount of half of @@ -2864,7 +2864,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream, } if (!(props->sample_flags & SAMPLE_OA_REPORT) && - (INTEL_GEN(perf->i915) < 12 || !stream->ctx)) { + (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) { DRM_DEBUG("Only OA report sampling supported\n"); return -EINVAL; } @@ -3006,7 +3006,7 @@ void i915_oa_init_reg_state(const struct intel_context *ce, /* perf.exclusive_stream serialised by lrc_configure_all_contexts() */ stream = READ_ONCE(engine->i915->perf.exclusive_stream); - if (stream && INTEL_GEN(stream->perf->i915) < 12) + if (stream && GRAPHICS_VER(stream->perf->i915) < 12) gen8_update_reg_state_unlocked(ce, stream); } @@ -3443,7 +3443,7 @@ i915_perf_open_ioctl_locked(struct i915_perf *perf, */ if (IS_HASWELL(perf->i915) && specific_ctx) privileged_op = false; - else if (IS_GEN(perf->i915, 12) && specific_ctx && + else if (GRAPHICS_VER(perf->i915) == 12 && specific_ctx && (props->sample_flags & SAMPLE_OA_REPORT) == 0) privileged_op = false; @@ -4119,7 +4119,7 @@ int i915_perf_add_config_ioctl(struct drm_device *dev, void *data, } oa_config->b_counter_regs = regs; - if (INTEL_GEN(perf->i915) < 8) { + if (GRAPHICS_VER(perf->i915) < 8) { if (args->n_flex_regs != 0) { err = -EINVAL; goto reg_err; @@ -4365,7 +4365,7 @@ void i915_perf_init(struct drm_i915_private *i915) */ perf->ops.read = gen8_oa_read; - if (IS_GEN_RANGE(i915, 8, 9)) { + if (IS_GRAPHICS_VER(i915, 8, 9)) { perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; perf->ops.is_valid_mux_reg = @@ -4384,7 +4384,7 @@ void i915_perf_init(struct drm_i915_private *i915) perf->ops.disable_metric_set = gen8_disable_metric_set; perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read; - if (IS_GEN(i915, 8)) { + if (GRAPHICS_VER(i915) == 8) { perf->ctx_oactxctrl_offset = 0x120; perf->ctx_flexeu0_offset = 0x2ce; @@ -4395,7 +4395,7 @@ void i915_perf_init(struct drm_i915_private *i915) perf->gen8_valid_ctx_bit = BIT(16); } - } else if (IS_GEN_RANGE(i915, 10, 11)) { + } else if (IS_GRAPHICS_VER(i915, 10, 11)) { perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; perf->ops.is_valid_mux_reg = @@ -4409,7 +4409,7 @@ void i915_perf_init(struct drm_i915_private *i915) perf->ops.disable_metric_set = gen10_disable_metric_set; perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read; - if (IS_GEN(i915, 10)) { + if (GRAPHICS_VER(i915) == 10) { perf->ctx_oactxctrl_offset = 0x128; perf->ctx_flexeu0_offset = 0x3de; } else { @@ -4417,7 +4417,7 @@ void i915_perf_init(struct drm_i915_private *i915) perf->ctx_flexeu0_offset = 0x78e; } perf->gen8_valid_ctx_bit = BIT(16); - } else if (IS_GEN(i915, 12)) { + } else if (GRAPHICS_VER(i915) == 12) { perf->ops.is_valid_b_counter_reg = gen12_is_valid_b_counter_addr; perf->ops.is_valid_mux_reg = diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index 41651ac255fa..22c4d4178766 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -287,7 +287,7 @@ static bool exclusive_mmio_access(const struct drm_i915_private *i915) * risk a machine hang. For a fun history lesson dig out the old * userspace intel_gpu_top and run it on Ivybridge or Haswell! */ - return IS_GEN(i915, 7); + return GRAPHICS_VER(i915) == 7; } static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns) @@ -463,7 +463,7 @@ engine_event_status(struct intel_engine_cs *engine, case I915_SAMPLE_WAIT: break; case I915_SAMPLE_SEMA: - if (INTEL_GEN(engine->i915) < 6) + if (GRAPHICS_VER(engine->i915) < 6) return -ENODEV; break; default: @@ -476,6 +476,8 @@ engine_event_status(struct intel_engine_cs *engine, static int config_status(struct drm_i915_private *i915, u64 config) { + struct intel_gt *gt = &i915->gt; + switch (config) { case I915_PMU_ACTUAL_FREQUENCY: if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) @@ -483,13 +485,13 @@ config_status(struct drm_i915_private *i915, u64 config) return -ENODEV; fallthrough; case I915_PMU_REQUESTED_FREQUENCY: - if (INTEL_GEN(i915) < 6) + if (GRAPHICS_VER(i915) < 6) return -ENODEV; break; case I915_PMU_INTERRUPTS: break; case I915_PMU_RC6_RESIDENCY: - if (!HAS_RC6(i915)) + if (!gt->rc6.supported) return -ENODEV; break; case I915_PMU_SOFTWARE_GT_AWAKE_TIME: @@ -1145,7 +1147,7 @@ void i915_pmu_register(struct drm_i915_private *i915) int ret = -ENOMEM; - if (INTEL_GEN(i915) <= 2) { + if (GRAPHICS_VER(i915) <= 2) { drm_info(&i915->drm, "PMU not supported for this GPU."); return; } diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index fed337ad7b68..e49da36c62fb 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -419,11 +419,73 @@ static int query_perf_config(struct drm_i915_private *i915, } } +static int query_memregion_info(struct drm_i915_private *i915, + struct drm_i915_query_item *query_item) +{ + struct drm_i915_query_memory_regions __user *query_ptr = + u64_to_user_ptr(query_item->data_ptr); + struct drm_i915_memory_region_info __user *info_ptr = + &query_ptr->regions[0]; + struct drm_i915_memory_region_info info = { }; + struct drm_i915_query_memory_regions query; + struct intel_memory_region *mr; + u32 total_length; + int ret, id, i; + + if (!IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) + return -ENODEV; + + if (query_item->flags != 0) + return -EINVAL; + + total_length = sizeof(query); + for_each_memory_region(mr, i915, id) { + if (mr->private) + continue; + + total_length += sizeof(info); + } + + ret = copy_query_item(&query, sizeof(query), total_length, query_item); + if (ret != 0) + return ret; + + if (query.num_regions) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(query.rsvd); i++) { + if (query.rsvd[i]) + return -EINVAL; + } + + for_each_memory_region(mr, i915, id) { + if (mr->private) + continue; + + info.region.memory_class = mr->type; + info.region.memory_instance = mr->instance; + info.probed_size = mr->total; + info.unallocated_size = mr->avail; + + if (__copy_to_user(info_ptr, &info, sizeof(info))) + return -EFAULT; + + query.num_regions++; + info_ptr++; + } + + if (__copy_to_user(query_ptr, &query, sizeof(query))) + return -EFAULT; + + return total_length; +} + static int (* const i915_query_funcs[])(struct drm_i915_private *dev_priv, struct drm_i915_query_item *query_item) = { query_topology_info, query_engine_info, query_perf_config, + query_memregion_info, }; int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 07cca3b423bc..e915ec034c98 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -416,6 +416,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN11_VECS_SFC_USAGE(engine) _MMIO((engine)->mmio_base + 0x2014) #define GEN11_VECS_SFC_USAGE_BIT (1 << 0) +#define GEN12_HCP_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x2910) +#define GEN12_HCP_SFC_FORCED_LOCK_BIT REG_BIT(0) +#define GEN12_HCP_SFC_LOCK_STATUS(engine) _MMIO((engine)->mmio_base + 0x2914) +#define GEN12_HCP_SFC_LOCK_ACK_BIT REG_BIT(1) +#define GEN12_HCP_SFC_USAGE_BIT REG_BIT(0) + #define GEN12_SFC_DONE(n) _MMIO(0x1cc00 + (n) * 0x100) #define GEN12_SFC_DONE_MAX 4 @@ -487,6 +493,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GAB_CTL _MMIO(0x24000) #define GAB_CTL_CONT_AFTER_PAGEFAULT (1 << 8) +#define GU_CNTL _MMIO(0x101010) +#define LMEM_INIT REG_BIT(7) + #define GEN6_STOLEN_RESERVED _MMIO(0x1082C0) #define GEN6_STOLEN_RESERVED_ADDR_MASK (0xFFF << 20) #define GEN7_STOLEN_RESERVED_ADDR_MASK (0x3FFF << 18) @@ -2715,6 +2724,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define RING_INDIRECT_CTX_OFFSET(base) _MMIO((base) + 0x1c8) /* gen8+ */ #define RING_CTX_TIMESTAMP(base) _MMIO((base) + 0x3a8) /* gen8+ */ +#define VDBOX_CGCTL3F10(base) _MMIO((base) + 0x3f10) +#define IECPUNIT_CLKGATE_DIS REG_BIT(22) + #define ERROR_GEN6 _MMIO(0x40a0) #define GEN7_ERR_INT _MMIO(0x44040) #define ERR_INT_POISON (1 << 31) @@ -2929,6 +2941,15 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define MBUS_BBOX_CTL_S1 _MMIO(0x45040) #define MBUS_BBOX_CTL_S2 _MMIO(0x45044) +#define MBUS_CTL _MMIO(0x4438C) +#define MBUS_JOIN REG_BIT(31) +#define MBUS_HASHING_MODE_MASK REG_BIT(30) +#define MBUS_HASHING_MODE_2x2 REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 0) +#define MBUS_HASHING_MODE_1x4 REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 1) +#define MBUS_JOIN_PIPE_SELECT_MASK REG_GENMASK(28, 26) +#define MBUS_JOIN_PIPE_SELECT(pipe) REG_FIELD_PREP(MBUS_JOIN_PIPE_SELECT_MASK, pipe) +#define MBUS_JOIN_PIPE_SELECT_NONE MBUS_JOIN_PIPE_SELECT(7) + #define HDPORT_STATE _MMIO(0x45050) #define HDPORT_DPLL_USED_MASK REG_GENMASK(15, 12) #define HDPORT_DDI_USED(phy) REG_BIT(2 * (phy) + 1) @@ -3781,8 +3802,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define CSHRDDR3CTL_DDR3 (1 << 2) /* 965 MCH register controlling DRAM channel configuration */ -#define C0DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x206) -#define C1DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x606) +#define C0DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x206) +#define C1DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x606) /* snb MCH registers for reading the DRAM channel configuration */ #define MAD_DIMM_C0 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004) @@ -4370,6 +4391,8 @@ enum { #define VRR_CTL_PIPELINE_FULL_MASK REG_GENMASK(10, 3) #define VRR_CTL_PIPELINE_FULL(x) REG_FIELD_PREP(VRR_CTL_PIPELINE_FULL_MASK, (x)) #define VRR_CTL_PIPELINE_FULL_OVERRIDE REG_BIT(0) +#define XELPD_VRR_CTL_VRR_GUARDBAND_MASK REG_GENMASK(15, 0) +#define XELPD_VRR_CTL_VRR_GUARDBAND(x) REG_FIELD_PREP(XELPD_VRR_CTL_VRR_GUARDBAND_MASK, (x)) #define _TRANS_VRR_VMAX_A 0x60424 #define _TRANS_VRR_VMAX_B 0x61424 @@ -6129,6 +6152,10 @@ enum { #define _PIPEBGCMAX 0x71010 #define PIPEGCMAX(pipe, i) _MMIO_PIPE2(pipe, _PIPEAGCMAX + (i) * 4) +#define _PIPE_ARB_CTL_A 0x70028 /* icl+ */ +#define PIPE_ARB_CTL(pipe) _MMIO_PIPE2(pipe, _PIPE_ARB_CTL_A) +#define PIPE_ARB_USE_PROG_SLOTS REG_BIT(13) + #define _PIPE_MISC_A 0x70030 #define _PIPE_MISC_B 0x71030 #define PIPEMISC_YUV420_ENABLE (1 << 27) /* glk+ */ @@ -6146,12 +6173,26 @@ enum { #define PIPEMISC_DITHER_TYPE_SP (0 << 2) #define PIPEMISC(pipe) _MMIO_PIPE2(pipe, _PIPE_MISC_A) +#define _PIPE_MISC2_A 0x7002C +#define _PIPE_MISC2_B 0x7102C +#define PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN (0x50 << 24) +#define PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS (0x14 << 24) +#define PIPE_MISC2_UNDERRUN_BUBBLE_COUNTER_MASK (0xff << 24) +#define PIPE_MISC2(pipe) _MMIO_PIPE2(pipe, _PIPE_MISC2_A) + /* Skylake+ pipe bottom (background) color */ #define _SKL_BOTTOM_COLOR_A 0x70034 #define SKL_BOTTOM_COLOR_GAMMA_ENABLE (1 << 31) #define SKL_BOTTOM_COLOR_CSC_ENABLE (1 << 30) #define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE2(pipe, _SKL_BOTTOM_COLOR_A) +#define _ICL_PIPE_A_STATUS 0x70058 +#define ICL_PIPESTATUS(pipe) _MMIO_PIPE2(pipe, _ICL_PIPE_A_STATUS) +#define PIPE_STATUS_UNDERRUN REG_BIT(31) +#define PIPE_STATUS_SOFT_UNDERRUN_XELPD REG_BIT(28) +#define PIPE_STATUS_HARD_UNDERRUN_XELPD REG_BIT(27) +#define PIPE_STATUS_PORT_UNDERRUN_XELPD REG_BIT(26) + #define VLV_DPFLIPSTAT _MMIO(VLV_DISPLAY_BASE + 0x70028) #define PIPEB_LINE_COMPARE_INT_EN (1 << 29) #define PIPEB_HLINE_INT_EN (1 << 28) @@ -6420,16 +6461,28 @@ enum { /* Watermark register definitions for SKL */ #define _CUR_WM_A_0 0x70140 #define _CUR_WM_B_0 0x71140 +#define _CUR_WM_SAGV_A 0x70158 +#define _CUR_WM_SAGV_B 0x71158 +#define _CUR_WM_SAGV_TRANS_A 0x7015C +#define _CUR_WM_SAGV_TRANS_B 0x7115C +#define _CUR_WM_TRANS_A 0x70168 +#define _CUR_WM_TRANS_B 0x71168 #define _PLANE_WM_1_A_0 0x70240 #define _PLANE_WM_1_B_0 0x71240 #define _PLANE_WM_2_A_0 0x70340 #define _PLANE_WM_2_B_0 0x71340 -#define _PLANE_WM_TRANS_1_A_0 0x70268 -#define _PLANE_WM_TRANS_1_B_0 0x71268 -#define _PLANE_WM_TRANS_2_A_0 0x70368 -#define _PLANE_WM_TRANS_2_B_0 0x71368 -#define _CUR_WM_TRANS_A_0 0x70168 -#define _CUR_WM_TRANS_B_0 0x71168 +#define _PLANE_WM_SAGV_1_A 0x70258 +#define _PLANE_WM_SAGV_1_B 0x71258 +#define _PLANE_WM_SAGV_2_A 0x70358 +#define _PLANE_WM_SAGV_2_B 0x71358 +#define _PLANE_WM_SAGV_TRANS_1_A 0x7025C +#define _PLANE_WM_SAGV_TRANS_1_B 0x7125C +#define _PLANE_WM_SAGV_TRANS_2_A 0x7035C +#define _PLANE_WM_SAGV_TRANS_2_B 0x7135C +#define _PLANE_WM_TRANS_1_A 0x70268 +#define _PLANE_WM_TRANS_1_B 0x71268 +#define _PLANE_WM_TRANS_2_A 0x70368 +#define _PLANE_WM_TRANS_2_B 0x71368 #define PLANE_WM_EN (1 << 31) #define PLANE_WM_IGNORE_LINES (1 << 30) #define PLANE_WM_LINES_MASK REG_GENMASK(26, 14) @@ -6437,19 +6490,32 @@ enum { #define _CUR_WM_0(pipe) _PIPE(pipe, _CUR_WM_A_0, _CUR_WM_B_0) #define CUR_WM(pipe, level) _MMIO(_CUR_WM_0(pipe) + ((4) * (level))) -#define CUR_WM_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_TRANS_A_0, _CUR_WM_TRANS_B_0) - +#define CUR_WM_SAGV(pipe) _MMIO_PIPE(pipe, _CUR_WM_SAGV_A, _CUR_WM_SAGV_B) +#define CUR_WM_SAGV_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_SAGV_TRANS_A, _CUR_WM_SAGV_TRANS_B) +#define CUR_WM_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_TRANS_A, _CUR_WM_TRANS_B) #define _PLANE_WM_1(pipe) _PIPE(pipe, _PLANE_WM_1_A_0, _PLANE_WM_1_B_0) #define _PLANE_WM_2(pipe) _PIPE(pipe, _PLANE_WM_2_A_0, _PLANE_WM_2_B_0) -#define _PLANE_WM_BASE(pipe, plane) \ - _PLANE(plane, _PLANE_WM_1(pipe), _PLANE_WM_2(pipe)) -#define PLANE_WM(pipe, plane, level) \ - _MMIO(_PLANE_WM_BASE(pipe, plane) + ((4) * (level))) -#define _PLANE_WM_TRANS_1(pipe) \ - _PIPE(pipe, _PLANE_WM_TRANS_1_A_0, _PLANE_WM_TRANS_1_B_0) -#define _PLANE_WM_TRANS_2(pipe) \ - _PIPE(pipe, _PLANE_WM_TRANS_2_A_0, _PLANE_WM_TRANS_2_B_0) -#define PLANE_WM_TRANS(pipe, plane) \ +#define _PLANE_WM_BASE(pipe, plane) \ + _PLANE(plane, _PLANE_WM_1(pipe), _PLANE_WM_2(pipe)) +#define PLANE_WM(pipe, plane, level) \ + _MMIO(_PLANE_WM_BASE(pipe, plane) + ((4) * (level))) +#define _PLANE_WM_SAGV_1(pipe) \ + _PIPE(pipe, _PLANE_WM_SAGV_1_A, _PLANE_WM_SAGV_1_B) +#define _PLANE_WM_SAGV_2(pipe) \ + _PIPE(pipe, _PLANE_WM_SAGV_2_A, _PLANE_WM_SAGV_2_B) +#define PLANE_WM_SAGV(pipe, plane) \ + _MMIO(_PLANE(plane, _PLANE_WM_SAGV_1(pipe), _PLANE_WM_SAGV_2(pipe))) +#define _PLANE_WM_SAGV_TRANS_1(pipe) \ + _PIPE(pipe, _PLANE_WM_SAGV_TRANS_1_A, _PLANE_WM_SAGV_TRANS_1_B) +#define _PLANE_WM_SAGV_TRANS_2(pipe) \ + _PIPE(pipe, _PLANE_WM_SAGV_TRANS_2_A, _PLANE_WM_SAGV_TRANS_2_B) +#define PLANE_WM_SAGV_TRANS(pipe, plane) \ + _MMIO(_PLANE(plane, _PLANE_WM_SAGV_TRANS_1(pipe), _PLANE_WM_SAGV_TRANS_2(pipe))) +#define _PLANE_WM_TRANS_1(pipe) \ + _PIPE(pipe, _PLANE_WM_TRANS_1_A, _PLANE_WM_TRANS_1_B) +#define _PLANE_WM_TRANS_2(pipe) \ + _PIPE(pipe, _PLANE_WM_TRANS_2_A, _PLANE_WM_TRANS_2_B) +#define PLANE_WM_TRANS(pipe, plane) \ _MMIO(_PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe))) /* define the Watermark register on Ironlake */ @@ -6554,6 +6620,8 @@ enum { #define MCURSOR_MODE_128_ARGB_AX ((1 << 5) | MCURSOR_MODE_128_32B_AX) #define MCURSOR_MODE_256_ARGB_AX ((1 << 5) | MCURSOR_MODE_256_32B_AX) #define MCURSOR_MODE_64_ARGB_AX ((1 << 5) | MCURSOR_MODE_64_32B_AX) +#define MCURSOR_ARB_SLOTS_MASK REG_GENMASK(30, 28) /* icl+ */ +#define MCURSOR_ARB_SLOTS(x) REG_FIELD_PREP(MCURSOR_ARB_SLOTS_MASK, (x)) /* icl+ */ #define MCURSOR_PIPE_SELECT_MASK (0x3 << 28) #define MCURSOR_PIPE_SELECT_SHIFT 28 #define MCURSOR_PIPE_SELECT(pipe) ((pipe) << 28) @@ -7005,6 +7073,8 @@ enum { #define _PLANE_CTL_2_A 0x70280 #define _PLANE_CTL_3_A 0x70380 #define PLANE_CTL_ENABLE (1 << 31) +#define PLANE_CTL_ARB_SLOTS_MASK REG_GENMASK(30, 28) /* icl+ */ +#define PLANE_CTL_ARB_SLOTS(x) REG_FIELD_PREP(PLANE_CTL_ARB_SLOTS_MASK, (x)) /* icl+ */ #define PLANE_CTL_PIPE_GAMMA_ENABLE (1 << 30) /* Pre-GLK */ #define PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE (1 << 28) /* @@ -7268,7 +7338,7 @@ enum { #define _PLANE_BUF_CFG_1_B 0x7127c #define _PLANE_BUF_CFG_2_B 0x7137c -#define DDB_ENTRY_MASK 0x7FF /* skl+: 10 bits, icl+ 11 bits */ +#define DDB_ENTRY_MASK 0xFFF /* skl+: 10 bits, icl+ 11 bits, adlp+ 12 bits */ #define DDB_ENTRY_END_SHIFT 16 #define _PLANE_BUF_CFG_1(pipe) \ _PIPE(pipe, _PLANE_BUF_CFG_1_A, _PLANE_BUF_CFG_1_B) @@ -7680,20 +7750,20 @@ enum { #define GAMMA_MODE_MODE_SPLIT (3 << 0) /* ivb-bdw */ #define GAMMA_MODE_MODE_12BIT_MULTI_SEGMENTED (3 << 0) /* icl + */ -/* DMC/CSR */ -#define CSR_PROGRAM(i) _MMIO(0x80000 + (i) * 4) -#define CSR_SSP_BASE_ADDR_GEN9 0x00002FC0 -#define CSR_HTP_ADDR_SKL 0x00500034 -#define CSR_SSP_BASE _MMIO(0x8F074) -#define CSR_HTP_SKL _MMIO(0x8F004) -#define CSR_LAST_WRITE _MMIO(0x8F034) -#define CSR_LAST_WRITE_VALUE 0xc003b400 -/* MMIO address range for CSR program (0x80000 - 0x82FFF) */ -#define CSR_MMIO_START_RANGE 0x80000 -#define CSR_MMIO_END_RANGE 0x8FFFF -#define SKL_CSR_DC3_DC5_COUNT _MMIO(0x80030) -#define SKL_CSR_DC5_DC6_COUNT _MMIO(0x8002C) -#define BXT_CSR_DC3_DC5_COUNT _MMIO(0x80038) +/* DMC */ +#define DMC_PROGRAM(i) _MMIO(0x80000 + (i) * 4) +#define DMC_SSP_BASE_ADDR_GEN9 0x00002FC0 +#define DMC_HTP_ADDR_SKL 0x00500034 +#define DMC_SSP_BASE _MMIO(0x8F074) +#define DMC_HTP_SKL _MMIO(0x8F004) +#define DMC_LAST_WRITE _MMIO(0x8F034) +#define DMC_LAST_WRITE_VALUE 0xc003b400 +/* MMIO address range for DMC program (0x80000 - 0x82FFF) */ +#define DMC_MMIO_START_RANGE 0x80000 +#define DMC_MMIO_END_RANGE 0x8FFFF +#define SKL_DMC_DC3_DC5_COUNT _MMIO(0x80030) +#define SKL_DMC_DC5_DC6_COUNT _MMIO(0x8002C) +#define BXT_DMC_DC3_DC5_COUNT _MMIO(0x80038) #define TGL_DMC_DEBUG_DC5_COUNT _MMIO(0x101084) #define TGL_DMC_DEBUG_DC6_COUNT _MMIO(0x101088) #define DG1_DMC_DEBUG_DC5_COUNT _MMIO(0x134154) @@ -7808,6 +7878,8 @@ enum { #define GEN8_PIPE_FIFO_UNDERRUN (1 << 31) #define GEN8_PIPE_CDCLK_CRC_ERROR (1 << 29) #define GEN8_PIPE_CDCLK_CRC_DONE (1 << 28) +#define XELPD_PIPE_SOFT_UNDERRUN (1 << 22) +#define XELPD_PIPE_HARD_UNDERRUN (1 << 21) #define GEN8_PIPE_CURSOR_FAULT (1 << 10) #define GEN8_PIPE_SPRITE_FAULT (1 << 9) #define GEN8_PIPE_PRIMARY_FAULT (1 << 8) @@ -8103,13 +8175,29 @@ enum { #define DISP_DATA_PARTITION_5_6 (1 << 6) #define DISP_IPC_ENABLE (1 << 3) -#define _DBUF_CTL_S1 0x45008 -#define _DBUF_CTL_S2 0x44FE8 -#define DBUF_CTL_S(slice) _MMIO(_PICK_EVEN(slice, _DBUF_CTL_S1, _DBUF_CTL_S2)) +/* + * The below are numbered starting from "S1" on gen11/gen12, but starting + * with gen13 display, the bspec switches to a 0-based numbering scheme + * (although the addresses stay the same so new S0 = old S1, new S1 = old S2). + * We'll just use the 0-based numbering here for all platforms since it's the + * way things will be named by the hardware team going forward, plus it's more + * consistent with how most of the rest of our registers are named. + */ +#define _DBUF_CTL_S0 0x45008 +#define _DBUF_CTL_S1 0x44FE8 +#define _DBUF_CTL_S2 0x44300 +#define _DBUF_CTL_S3 0x44304 +#define DBUF_CTL_S(slice) _MMIO(_PICK(slice, \ + _DBUF_CTL_S0, \ + _DBUF_CTL_S1, \ + _DBUF_CTL_S2, \ + _DBUF_CTL_S3)) #define DBUF_POWER_REQUEST REG_BIT(31) #define DBUF_POWER_STATE REG_BIT(30) #define DBUF_TRACKER_STATE_SERVICE_MASK REG_GENMASK(23, 19) #define DBUF_TRACKER_STATE_SERVICE(x) REG_FIELD_PREP(DBUF_TRACKER_STATE_SERVICE_MASK, x) +#define DBUF_MIN_TRACKER_STATE_SERVICE_MASK REG_GENMASK(18, 16) /* ADL-P+ */ +#define DBUF_MIN_TRACKER_STATE_SERVICE(x) REG_FIELD_PREP(DBUF_MIN_TRACKER_STATE_SERVICE_MASK, x) /* ADL-P+ */ #define GEN7_MSG_CTL _MMIO(0x45010) #define WAIT_FOR_PCH_RESET_ACK (1 << 1) @@ -8294,6 +8382,7 @@ enum { #define _PIPEC_CHICKEN 0x72038 #define PIPE_CHICKEN(pipe) _MMIO_PIPE(pipe, _PIPEA_CHICKEN,\ _PIPEB_CHICKEN) +#define UNDERRUN_RECOVERY_DISABLE REG_BIT(30) #define PIXEL_ROUNDING_TRUNC_FB_PASSTHRU (1 << 15) #define PER_PIXEL_ALPHA_BYPASS_EN (1 << 7) @@ -9819,7 +9908,7 @@ enum skl_power_gate { #define TRANS_HDCP_CONF(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_CONF, \ _TRANSB_HDCP_CONF) #define HDCP_CONF(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_CONF(trans) : \ PORT_HDCP_CONF(port)) @@ -9832,7 +9921,7 @@ enum skl_power_gate { _TRANSA_HDCP_ANINIT, \ _TRANSB_HDCP_ANINIT) #define HDCP_ANINIT(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_ANINIT(trans) : \ PORT_HDCP_ANINIT(port)) @@ -9842,7 +9931,7 @@ enum skl_power_gate { #define TRANS_HDCP_ANLO(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_ANLO, \ _TRANSB_HDCP_ANLO) #define HDCP_ANLO(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_ANLO(trans) : \ PORT_HDCP_ANLO(port)) @@ -9852,7 +9941,7 @@ enum skl_power_gate { #define TRANS_HDCP_ANHI(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_ANHI, \ _TRANSB_HDCP_ANHI) #define HDCP_ANHI(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_ANHI(trans) : \ PORT_HDCP_ANHI(port)) @@ -9863,7 +9952,7 @@ enum skl_power_gate { _TRANSA_HDCP_BKSVLO, \ _TRANSB_HDCP_BKSVLO) #define HDCP_BKSVLO(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_BKSVLO(trans) : \ PORT_HDCP_BKSVLO(port)) @@ -9874,7 +9963,7 @@ enum skl_power_gate { _TRANSA_HDCP_BKSVHI, \ _TRANSB_HDCP_BKSVHI) #define HDCP_BKSVHI(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_BKSVHI(trans) : \ PORT_HDCP_BKSVHI(port)) @@ -9885,7 +9974,7 @@ enum skl_power_gate { _TRANSA_HDCP_RPRIME, \ _TRANSB_HDCP_RPRIME) #define HDCP_RPRIME(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_RPRIME(trans) : \ PORT_HDCP_RPRIME(port)) @@ -9896,7 +9985,7 @@ enum skl_power_gate { _TRANSA_HDCP_STATUS, \ _TRANSB_HDCP_STATUS) #define HDCP_STATUS(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP_STATUS(trans) : \ PORT_HDCP_STATUS(port)) @@ -9937,7 +10026,7 @@ enum skl_power_gate { #define AUTH_FORCE_CLR_INPUTCTR BIT(19) #define AUTH_CLR_KEYS BIT(18) #define HDCP2_AUTH(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP2_AUTH(trans) : \ PORT_HDCP2_AUTH(port)) @@ -9948,7 +10037,7 @@ enum skl_power_gate { _TRANSB_HDCP2_CTL) #define CTL_LINK_ENCRYPTION_REQ BIT(31) #define HDCP2_CTL(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP2_CTL(trans) : \ PORT_HDCP2_CTL(port)) @@ -9962,7 +10051,7 @@ enum skl_power_gate { #define LINK_AUTH_STATUS BIT(21) #define LINK_ENCRYPTION_STATUS BIT(20) #define HDCP2_STATUS(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP2_STATUS(trans) : \ PORT_HDCP2_STATUS(port)) @@ -9984,7 +10073,7 @@ enum skl_power_gate { #define STREAM_ENCRYPTION_STATUS BIT(31) #define STREAM_TYPE_STATUS BIT(30) #define HDCP2_STREAM_STATUS(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP2_STREAM_STATUS(trans) : \ PIPE_HDCP2_STREAM_STATUS(pipe)) @@ -10000,7 +10089,7 @@ enum skl_power_gate { _TRANSB_HDCP2_AUTH_STREAM) #define AUTH_STREAM_TYPE BIT(31) #define HDCP2_AUTH_STREAM(dev_priv, trans, port) \ - (INTEL_GEN(dev_priv) >= 12 ? \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ TRANS_HDCP2_AUTH_STREAM(trans) : \ PORT_HDCP2_AUTH_STREAM(port)) @@ -10116,8 +10205,10 @@ enum skl_power_gate { #define DDI_BUF_CTL_ENABLE (1 << 31) #define DDI_BUF_TRANS_SELECT(n) ((n) << 24) #define DDI_BUF_EMP_MASK (0xf << 24) +#define DDI_BUF_PHY_LINK_RATE(r) ((r) << 20) #define DDI_BUF_PORT_REVERSAL (1 << 16) #define DDI_BUF_IS_IDLE (1 << 7) +#define DDI_BUF_CTL_TC_PHY_OWNERSHIP REG_BIT(6) #define DDI_A_4_LANES (1 << 4) #define DDI_PORT_WIDTH(width) (((width) - 1) << 1) #define DDI_PORT_WIDTH_MASK (7 << 1) @@ -10480,6 +10571,14 @@ enum skl_power_gate { #define DG1_DPLL_ENABLE(pll) _MMIO_PLL3(pll, DPLL0_ENABLE, DPLL1_ENABLE, \ _MG_PLL1_ENABLE, _MG_PLL2_ENABLE) +/* ADL-P Type C PLL */ +#define PORTTC1_PLL_ENABLE 0x46038 +#define PORTTC2_PLL_ENABLE 0x46040 + +#define ADLP_PORTTC_PLL_ENABLE(tc_port) _MMIO_PORT((tc_port), \ + PORTTC1_PLL_ENABLE, \ + PORTTC2_PLL_ENABLE) + #define _MG_REFCLKIN_CTL_PORT1 0x16892C #define _MG_REFCLKIN_CTL_PORT2 0x16992C #define _MG_REFCLKIN_CTL_PORT3 0x16A92C @@ -10894,6 +10993,8 @@ enum skl_power_gate { #define BXT_DE_PLL_ENABLE _MMIO(0x46070) #define BXT_DE_PLL_PLL_ENABLE (1 << 31) #define BXT_DE_PLL_LOCK (1 << 30) +#define BXT_DE_PLL_FREQ_REQ (1 << 23) +#define BXT_DE_PLL_FREQ_REQ_ACK (1 << 22) #define CNL_CDCLK_PLL_RATIO(x) (x) #define CNL_CDCLK_PLL_RATIO_MASK 0xff @@ -11268,6 +11369,12 @@ enum skl_power_gate { #define ICL_ESC_CLK_DIV_SHIFT 0 #define DSI_MAX_ESC_CLK 20000 /* in KHz */ +#define _ADL_MIPIO_REG 0x180 +#define ADL_MIPIO_DW(port, dw) _MMIO(_ICL_COMBOPHY(port) + _ADL_MIPIO_REG + 4 * (dw)) +#define TX_ESC_CLK_DIV_PHY_SEL REGBIT(16) +#define TX_ESC_CLK_DIV_PHY_MASK REG_GENMASK(23, 16) +#define TX_ESC_CLK_DIV_PHY REG_FIELD_PREP(TX_ESC_CLK_DIV_PHY_MASK, 0x7f) + #define _DSI_CMD_FRMCTL_0 0x6b034 #define _DSI_CMD_FRMCTL_1 0x6b834 #define DSI_CMD_FRMCTL(port) _MMIO_PORT(port, \ @@ -12208,6 +12315,7 @@ enum skl_power_gate { #define GEN12_GLOBAL_MOCS(i) _MMIO(0x4000 + (i) * 4) /* Global MOCS regs */ #define GEN12_GSMBASE _MMIO(0x108100) +#define GEN12_DSMBASE _MMIO(0x1080C0) /* gamt regs */ #define GEN8_L3_LRA_1_GPGPU _MMIO(0x4dd4) @@ -12553,6 +12661,15 @@ enum skl_power_gate { #define DP_PIN_ASSIGNMENT_MASK(idx) (0xf << ((idx) * 4)) #define DP_PIN_ASSIGNMENT(idx, x) ((x) << ((idx) * 4)) +#define _TCSS_DDI_STATUS_1 0x161500 +#define _TCSS_DDI_STATUS_2 0x161504 +#define TCSS_DDI_STATUS(tc) _MMIO(_PICK_EVEN(tc, \ + _TCSS_DDI_STATUS_1, \ + _TCSS_DDI_STATUS_2)) +#define TCSS_DDI_STATUS_READY REG_BIT(2) +#define TCSS_DDI_STATUS_HPD_LIVE_STATUS_TBT REG_BIT(1) +#define TCSS_DDI_STATUS_HPD_LIVE_STATUS_ALT REG_BIT(0) + /* This register controls the Display State Buffer (DSB) engines. */ #define _DSBSL_INSTANCE_BASE 0x70B00 #define DSBSL_INSTANCE(pipe, id) (_DSBSL_INSTANCE_BASE + \ diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 6cb91f042642..1014c71cf7f5 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -929,7 +929,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) u32 seqno; int ret; - might_sleep_if(gfpflags_allow_blocking(gfp)); + might_alloc(gfp); /* Check that the caller provided an already pinned context */ __intel_context_pin(ce); @@ -1176,12 +1176,12 @@ __emit_semaphore_wait(struct i915_request *to, struct i915_request *from, u32 seqno) { - const int has_token = INTEL_GEN(to->engine->i915) >= 12; + const int has_token = GRAPHICS_VER(to->engine->i915) >= 12; u32 hwsp_offset; int len, err; u32 *cs; - GEM_BUG_ON(INTEL_GEN(to->engine->i915) < 8); + GEM_BUG_ON(GRAPHICS_VER(to->engine->i915) < 8); GEM_BUG_ON(i915_request_has_initial_breadcrumb(to)); /* We need to pin the signaler's HWSP until we are finished reading. */ diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index 5fcc32821e18..f7b55f34dba8 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -38,14 +38,14 @@ static void intel_save_swf(struct drm_i915_private *dev_priv) int i; /* Scratch space */ - if (IS_GEN(dev_priv, 2) && IS_MOBILE(dev_priv)) { + if (GRAPHICS_VER(dev_priv) == 2 && IS_MOBILE(dev_priv)) { for (i = 0; i < 7; i++) { dev_priv->regfile.saveSWF0[i] = intel_de_read(dev_priv, SWF0(i)); dev_priv->regfile.saveSWF1[i] = intel_de_read(dev_priv, SWF1(i)); } for (i = 0; i < 3; i++) dev_priv->regfile.saveSWF3[i] = intel_de_read(dev_priv, SWF3(i)); - } else if (IS_GEN(dev_priv, 2)) { + } else if (GRAPHICS_VER(dev_priv) == 2) { for (i = 0; i < 7; i++) dev_priv->regfile.saveSWF1[i] = intel_de_read(dev_priv, SWF1(i)); } else if (HAS_GMCH(dev_priv)) { @@ -63,14 +63,14 @@ static void intel_restore_swf(struct drm_i915_private *dev_priv) int i; /* Scratch space */ - if (IS_GEN(dev_priv, 2) && IS_MOBILE(dev_priv)) { + if (GRAPHICS_VER(dev_priv) == 2 && IS_MOBILE(dev_priv)) { for (i = 0; i < 7; i++) { intel_de_write(dev_priv, SWF0(i), dev_priv->regfile.saveSWF0[i]); intel_de_write(dev_priv, SWF1(i), dev_priv->regfile.saveSWF1[i]); } for (i = 0; i < 3; i++) intel_de_write(dev_priv, SWF3(i), dev_priv->regfile.saveSWF3[i]); - } else if (IS_GEN(dev_priv, 2)) { + } else if (GRAPHICS_VER(dev_priv) == 2) { for (i = 0; i < 7; i++) intel_de_write(dev_priv, SWF1(i), dev_priv->regfile.saveSWF1[i]); } else if (HAS_GMCH(dev_priv)) { @@ -91,10 +91,10 @@ void i915_save_display(struct drm_i915_private *dev_priv) return; /* Display arbitration control */ - if (INTEL_GEN(dev_priv) <= 4) + if (GRAPHICS_VER(dev_priv) <= 4) dev_priv->regfile.saveDSPARB = intel_de_read(dev_priv, DSPARB); - if (IS_GEN(dev_priv, 4)) + if (GRAPHICS_VER(dev_priv) == 4) pci_read_config_word(pdev, GCDGMBUS, &dev_priv->regfile.saveGCDGMBUS); @@ -110,12 +110,12 @@ void i915_restore_display(struct drm_i915_private *dev_priv) intel_restore_swf(dev_priv); - if (IS_GEN(dev_priv, 4)) + if (GRAPHICS_VER(dev_priv) == 4) pci_write_config_word(pdev, GCDGMBUS, dev_priv->regfile.saveGCDGMBUS); /* Display arbitration */ - if (INTEL_GEN(dev_priv) <= 4) + if (GRAPHICS_VER(dev_priv) <= 4) intel_de_write(dev_priv, DSPARB, dev_priv->regfile.saveDSPARB); /* only restore FBC info on the platform that supports FBC*/ diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index 4c6b5d52b5ca..b099e09ccc32 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -595,7 +595,7 @@ void i915_setup_sysfs(struct drm_i915_private *dev_priv) ret = 0; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ret = sysfs_create_files(&kdev->kobj, vlv_attrs); - else if (INTEL_GEN(dev_priv) >= 6) + else if (GRAPHICS_VER(dev_priv) >= 6) ret = sysfs_create_files(&kdev->kobj, gen6_attrs); if (ret) drm_err(&dev_priv->drm, "RPS sysfs setup failed\n"); diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c index 172799277dd5..31a105bc1792 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.c +++ b/drivers/gpu/drm/i915/i915_vgpu.c @@ -74,7 +74,7 @@ void intel_vgpu_detect(struct drm_i915_private *dev_priv) * we do not support VGT on older gens, return early so we don't have * to consider differently numbered or sized MMIO bars */ - if (INTEL_GEN(dev_priv) < 6) + if (GRAPHICS_VER(dev_priv) < 6) return; shared_area = pci_iomap_range(pdev, 0, VGT_PVINFO_PAGE, VGT_PVINFO_SIZE); diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 07490db51cdc..b319fd3f91cc 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -27,6 +27,7 @@ #include "display/intel_frontbuffer.h" +#include "gem/i915_gem_lmem.h" #include "gt/intel_engine.h" #include "gt/intel_engine_heartbeat.h" #include "gt/intel_gt.h" @@ -93,7 +94,6 @@ static int __i915_vma_active(struct i915_active *ref) return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT; } -__i915_active_call static void __i915_vma_retire(struct i915_active *ref) { i915_vma_put(active_to_vma(ref)); @@ -124,7 +124,7 @@ vma_create(struct drm_i915_gem_object *obj, vma->size = obj->base.size; vma->display_alignment = I915_GTT_MIN_ALIGNMENT; - i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire); + i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0); /* Declare ourselves safe for use inside shrinkers */ if (IS_ENABLED(CONFIG_LOCKDEP)) { @@ -274,7 +274,7 @@ i915_vma_instance(struct drm_i915_gem_object *obj, { struct i915_vma *vma; - GEM_BUG_ON(view && !i915_is_ggtt(vm)); + GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm)); GEM_BUG_ON(!atomic_read(&vm->open)); spin_lock(&obj->vma.lock); @@ -448,9 +448,11 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) void __iomem *ptr; int err; - if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { - err = -ENODEV; - goto err; + if (!i915_gem_object_is_lmem(vma->obj)) { + if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { + err = -ENODEV; + goto err; + } } GEM_BUG_ON(!i915_vma_is_ggtt(vma)); @@ -458,9 +460,19 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) ptr = READ_ONCE(vma->iomap); if (ptr == NULL) { - ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, - vma->node.start, - vma->node.size); + /* + * TODO: consider just using i915_gem_object_pin_map() for lmem + * instead, which already supports mapping non-contiguous chunks + * of pages, that way we can also drop the + * I915_BO_ALLOC_CONTIGUOUS when allocating the object. + */ + if (i915_gem_object_is_lmem(vma->obj)) + ptr = i915_gem_object_lmem_io_map(vma->obj, 0, + vma->obj->base.size); + else + ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, + vma->node.start, + vma->node.size); if (ptr == NULL) { err = -ENOMEM; goto err; @@ -905,8 +917,7 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, if (err) goto err_fence; - err = i915_vm_pin_pt_stash(vma->vm, - &work->stash); + err = i915_vm_map_pt_stash(vma->vm, &work->stash); if (err) goto err_fence; } diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h index 8df784a026d2..dc6926d89626 100644 --- a/drivers/gpu/drm/i915/i915_vma.h +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -68,6 +68,11 @@ static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) return test_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma)); } +static inline bool i915_vma_is_dpt(const struct i915_vma *vma) +{ + return i915_is_dpt(vma->vm); +} + static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma) { return test_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma)); @@ -158,7 +163,7 @@ i915_vma_compare(struct i915_vma *vma, { ptrdiff_t cmp; - GEM_BUG_ON(view && !i915_is_ggtt(vm)); + GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm)); cmp = ptrdiff(vma->vm, vm); if (cmp) diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index 8cb58a238c68..7eaa92fee421 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -257,10 +257,10 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) if (IS_ADLS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0)) for_each_pipe(dev_priv, pipe) runtime->num_scalers[pipe] = 0; - else if (INTEL_GEN(dev_priv) >= 10) { + else if (GRAPHICS_VER(dev_priv) >= 10) { for_each_pipe(dev_priv, pipe) runtime->num_scalers[pipe] = 2; - } else if (IS_GEN(dev_priv, 9)) { + } else if (GRAPHICS_VER(dev_priv) == 9) { runtime->num_scalers[PIPE_A] = 2; runtime->num_scalers[PIPE_B] = 2; runtime->num_scalers[PIPE_C] = 1; @@ -271,10 +271,10 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) if (DISPLAY_VER(dev_priv) >= 13 || HAS_D12_PLANE_MINIMIZATION(dev_priv)) for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 4; - else if (INTEL_GEN(dev_priv) >= 11) + else if (GRAPHICS_VER(dev_priv) >= 11) for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 6; - else if (IS_GEN(dev_priv, 10) || IS_GEMINILAKE(dev_priv)) + else if (GRAPHICS_VER(dev_priv) == 10 || IS_GEMINILAKE(dev_priv)) for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 3; else if (IS_BROXTON(dev_priv)) { @@ -293,12 +293,12 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 2; - } else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) { + } else if (GRAPHICS_VER(dev_priv) >= 5 || IS_G4X(dev_priv)) { for_each_pipe(dev_priv, pipe) runtime->num_sprites[pipe] = 1; } - if (HAS_DISPLAY(dev_priv) && IS_GEN_RANGE(dev_priv, 7, 8) && + if (HAS_DISPLAY(dev_priv) && IS_GRAPHICS_VER(dev_priv, 7, 8) && HAS_PCH_SPLIT(dev_priv)) { u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP); u32 sfuse_strap = intel_de_read(dev_priv, SFUSE_STRAP); @@ -325,7 +325,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) info->pipe_mask &= ~BIT(PIPE_C); info->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); } - } else if (HAS_DISPLAY(dev_priv) && INTEL_GEN(dev_priv) >= 9) { + } else if (HAS_DISPLAY(dev_priv) && GRAPHICS_VER(dev_priv) >= 9) { u32 dfsm = intel_de_read(dev_priv, SKL_DFSM); if (dfsm & SKL_DFSM_PIPE_A_DISABLE) { @@ -340,7 +340,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) info->pipe_mask &= ~BIT(PIPE_C); info->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); } - if (INTEL_GEN(dev_priv) >= 12 && + if (GRAPHICS_VER(dev_priv) >= 12 && (dfsm & TGL_DFSM_PIPE_D_DISABLE)) { info->pipe_mask &= ~BIT(PIPE_D); info->cpu_transcoder_mask &= ~BIT(TRANSCODER_D); @@ -352,15 +352,15 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) if (dfsm & SKL_DFSM_DISPLAY_PM_DISABLE) info->display.has_fbc = 0; - if (INTEL_GEN(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE)) - info->display.has_csr = 0; + if (GRAPHICS_VER(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE)) + info->display.has_dmc = 0; - if (INTEL_GEN(dev_priv) >= 10 && + if (GRAPHICS_VER(dev_priv) >= 10 && (dfsm & CNL_DFSM_DISPLAY_DSC_DISABLE)) info->display.has_dsc = 0; } - if (IS_GEN(dev_priv, 6) && intel_vtd_active()) { + if (GRAPHICS_VER(dev_priv) == 6 && intel_vtd_active()) { drm_info(&dev_priv->drm, "Disabling ppGTT for VT-d support\n"); info->ppgtt_type = INTEL_PPGTT_NONE; diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index e98b36959736..b326aff65cd6 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -141,7 +141,7 @@ enum intel_ppgtt_type { #define DEV_INFO_DISPLAY_FOR_EACH_FLAG(func) \ /* Keep in alphabetical order */ \ func(cursor_needs_physical); \ - func(has_csr); \ + func(has_dmc); \ func(has_ddi); \ func(has_dp_mst); \ func(has_dsb); \ @@ -185,6 +185,8 @@ struct intel_device_info { u8 abox_mask; + u8 has_cdclk_crawl; /* does support CDCLK crawling */ + #define DEFINE_FLAG(name) u8 name:1 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); #undef DEFINE_FLAG diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c index 1e53c017c30d..50fdea84ba70 100644 --- a/drivers/gpu/drm/i915/intel_dram.c +++ b/drivers/gpu/drm/i915/intel_dram.c @@ -121,7 +121,7 @@ skl_dram_get_dimm_info(struct drm_i915_private *i915, struct dram_dimm_info *dimm, int channel, char dimm_name, u16 val) { - if (INTEL_GEN(i915) >= 10) { + if (GRAPHICS_VER(i915) >= 10) { dimm->size = cnl_get_dimm_size(val); dimm->width = cnl_get_dimm_width(val); dimm->ranks = cnl_get_dimm_ranks(val); @@ -422,7 +422,7 @@ static int icl_pcode_read_mem_global_info(struct drm_i915_private *dev_priv) if (ret) return ret; - if (IS_GEN(dev_priv, 12)) { + if (GRAPHICS_VER(dev_priv) == 12) { switch (val & 0xf) { case 0: dram_info->type = INTEL_DRAM_DDR4; @@ -501,12 +501,12 @@ void intel_dram_detect(struct drm_i915_private *i915) */ dram_info->wm_lv_0_adjust_needed = !IS_GEN9_LP(i915); - if (INTEL_GEN(i915) < 9 || !HAS_DISPLAY(i915)) + if (GRAPHICS_VER(i915) < 9 || !HAS_DISPLAY(i915)) return; - if (INTEL_GEN(i915) >= 12) + if (GRAPHICS_VER(i915) >= 12) ret = gen12_get_dram_info(i915); - else if (INTEL_GEN(i915) >= 11) + else if (GRAPHICS_VER(i915) >= 11) ret = gen11_get_dram_info(i915); else if (IS_GEN9_LP(i915)) ret = bxt_get_dram_info(i915); @@ -535,7 +535,7 @@ void intel_dram_edram_detect(struct drm_i915_private *i915) { u32 edram_cap = 0; - if (!(IS_HASWELL(i915) || IS_BROADWELL(i915) || INTEL_GEN(i915) >= 9)) + if (!(IS_HASWELL(i915) || IS_BROADWELL(i915) || GRAPHICS_VER(i915) >= 9)) return; edram_cap = __raw_uncore_read32(&i915->uncore, HSW_EDRAM_CAP); @@ -549,7 +549,7 @@ void intel_dram_edram_detect(struct drm_i915_private *i915) * The needed capability bits for size calculation are not there with * pre gen9 so return 128MB always. */ - if (INTEL_GEN(i915) < 9) + if (GRAPHICS_VER(i915) < 9) i915->edram_size_mb = 128; else i915->edram_size_mb = gen9_edram_size_mb(i915, edram_cap); diff --git a/drivers/gpu/drm/i915/intel_memory_region.c b/drivers/gpu/drm/i915/intel_memory_region.c index bf837b6bb185..d98e8b81d322 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.c +++ b/drivers/gpu/drm/i915/intel_memory_region.c @@ -22,9 +22,29 @@ static const struct { .class = INTEL_MEMORY_STOLEN_SYSTEM, .instance = 0, }, + [INTEL_REGION_STOLEN_LMEM] = { + .class = INTEL_MEMORY_STOLEN_LOCAL, + .instance = 0, + }, }; struct intel_memory_region * +intel_memory_region_lookup(struct drm_i915_private *i915, + u16 class, u16 instance) +{ + struct intel_memory_region *mr; + int id; + + /* XXX: consider maybe converting to an rb tree at some point */ + for_each_memory_region(mr, i915, id) { + if (mr->type == class && mr->instance == instance) + return mr; + } + + return NULL; +} + +struct intel_memory_region * intel_memory_region_by_type(struct drm_i915_private *i915, enum intel_memory_type mem_type) { @@ -278,8 +298,15 @@ int intel_memory_regions_hw_probe(struct drm_i915_private *i915) case INTEL_MEMORY_SYSTEM: mem = i915_gem_shmem_setup(i915); break; + case INTEL_MEMORY_STOLEN_LOCAL: + mem = i915_gem_stolen_lmem_setup(i915); + if (!IS_ERR(mem)) + i915->mm.stolen_region = mem; + break; case INTEL_MEMORY_STOLEN_SYSTEM: - mem = i915_gem_stolen_setup(i915); + mem = i915_gem_stolen_smem_setup(i915); + if (!IS_ERR(mem)) + i915->mm.stolen_region = mem; break; default: continue; diff --git a/drivers/gpu/drm/i915/intel_memory_region.h b/drivers/gpu/drm/i915/intel_memory_region.h index edd49067c8ca..d24ce5a0b30b 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.h +++ b/drivers/gpu/drm/i915/intel_memory_region.h @@ -11,6 +11,7 @@ #include <linux/mutex.h> #include <linux/io-mapping.h> #include <drm/drm_mm.h> +#include <drm/i915_drm.h> #include "i915_buddy.h" @@ -19,25 +20,25 @@ struct drm_i915_gem_object; struct intel_memory_region; struct sg_table; -/** - * Base memory type - */ enum intel_memory_type { - INTEL_MEMORY_SYSTEM = 0, - INTEL_MEMORY_LOCAL, + INTEL_MEMORY_SYSTEM = I915_MEMORY_CLASS_SYSTEM, + INTEL_MEMORY_LOCAL = I915_MEMORY_CLASS_DEVICE, INTEL_MEMORY_STOLEN_SYSTEM, + INTEL_MEMORY_STOLEN_LOCAL, }; enum intel_region_id { INTEL_REGION_SMEM = 0, INTEL_REGION_LMEM, INTEL_REGION_STOLEN_SMEM, + INTEL_REGION_STOLEN_LMEM, INTEL_REGION_UNKNOWN, /* Should be last */ }; #define REGION_SMEM BIT(INTEL_REGION_SMEM) #define REGION_LMEM BIT(INTEL_REGION_LMEM) #define REGION_STOLEN_SMEM BIT(INTEL_REGION_STOLEN_SMEM) +#define REGION_STOLEN_LMEM BIT(INTEL_REGION_STOLEN_LMEM) #define I915_ALLOC_MIN_PAGE_SIZE BIT(0) #define I915_ALLOC_CONTIGUOUS BIT(1) @@ -82,7 +83,8 @@ struct intel_memory_region { u16 type; u16 instance; enum intel_region_id id; - char name[8]; + char name[16]; + bool private; /* not for userspace */ struct list_head reserved; @@ -95,6 +97,10 @@ struct intel_memory_region { } objects; }; +struct intel_memory_region * +intel_memory_region_lookup(struct drm_i915_private *i915, + u16 class, u16 instance); + int intel_memory_region_init_buddy(struct intel_memory_region *mem); void intel_memory_region_release_buddy(struct intel_memory_region *mem); diff --git a/drivers/gpu/drm/i915/intel_pch.c b/drivers/gpu/drm/i915/intel_pch.c index 98a17dd1bda4..4e92ae19189e 100644 --- a/drivers/gpu/drm/i915/intel_pch.c +++ b/drivers/gpu/drm/i915/intel_pch.c @@ -13,17 +13,17 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) switch (id) { case INTEL_PCH_IBX_DEVICE_ID_TYPE: drm_dbg_kms(&dev_priv->drm, "Found Ibex Peak PCH\n"); - drm_WARN_ON(&dev_priv->drm, !IS_GEN(dev_priv, 5)); + drm_WARN_ON(&dev_priv->drm, GRAPHICS_VER(dev_priv) != 5); return PCH_IBX; case INTEL_PCH_CPT_DEVICE_ID_TYPE: drm_dbg_kms(&dev_priv->drm, "Found CougarPoint PCH\n"); drm_WARN_ON(&dev_priv->drm, - !IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); + GRAPHICS_VER(dev_priv) != 6 && !IS_IVYBRIDGE(dev_priv)); return PCH_CPT; case INTEL_PCH_PPT_DEVICE_ID_TYPE: drm_dbg_kms(&dev_priv->drm, "Found PantherPoint PCH\n"); drm_WARN_ON(&dev_priv->drm, - !IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); + GRAPHICS_VER(dev_priv) != 6 && !IS_IVYBRIDGE(dev_priv)); /* PantherPoint is CPT compatible */ return PCH_CPT; case INTEL_PCH_LPT_DEVICE_ID_TYPE: @@ -181,9 +181,9 @@ intel_virt_detect_pch(const struct drm_i915_private *dev_priv, id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE; else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) id = INTEL_PCH_LPT_DEVICE_ID_TYPE; - else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) + else if (GRAPHICS_VER(dev_priv) == 6 || IS_IVYBRIDGE(dev_priv)) id = INTEL_PCH_CPT_DEVICE_ID_TYPE; - else if (IS_GEN(dev_priv, 5)) + else if (GRAPHICS_VER(dev_priv) == 5) id = INTEL_PCH_IBX_DEVICE_ID_TYPE; if (id) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 15d9a64e7b4c..45fefa0ed160 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -2983,7 +2983,9 @@ static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv, int ilk_wm_max_level(const struct drm_i915_private *dev_priv) { /* how many WM levels are we expecting */ - if (DISPLAY_VER(dev_priv) >= 9) + if (HAS_HW_SAGV_WM(dev_priv)) + return 5; + else if (DISPLAY_VER(dev_priv) >= 9) return 7; else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) return 4; @@ -4011,8 +4013,9 @@ static int intel_compute_sagv_mask(struct intel_atomic_state *state) * latter from the plane commit hooks (especially in the legacy * cursor case) */ - pipe_wm->use_sagv_wm = DISPLAY_VER(dev_priv) >= 12 && - intel_can_enable_sagv(dev_priv, new_bw_state); + pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) && + DISPLAY_VER(dev_priv) >= 12 && + intel_can_enable_sagv(dev_priv, new_bw_state); } if (intel_can_enable_sagv(dev_priv, new_bw_state) != @@ -4054,6 +4057,20 @@ skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask, WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size); } +static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask) +{ + struct skl_ddb_entry ddb; + + if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2))) + slice_mask = BIT(DBUF_S1); + else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4))) + slice_mask = BIT(DBUF_S3); + + skl_ddb_entry_for_slices(i915, slice_mask, &ddb); + + return ddb.start; +} + u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv, const struct skl_ddb_entry *entry) { @@ -4146,6 +4163,7 @@ skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) struct intel_crtc_state *crtc_state; struct skl_ddb_entry ddb_slices; enum pipe pipe = crtc->pipe; + unsigned int mbus_offset = 0; u32 ddb_range_size; u32 dbuf_slice_mask; u32 start, end; @@ -4160,6 +4178,7 @@ skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) dbuf_slice_mask = new_dbuf_state->slices[pipe]; skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices); + mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask); ddb_range_size = skl_ddb_entry_size(&ddb_slices); intel_crtc_dbuf_weights(new_dbuf_state, pipe, @@ -4168,11 +4187,11 @@ skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) start = ddb_range_size * weight_start / weight_total; end = ddb_range_size * weight_end / weight_total; - new_dbuf_state->ddb[pipe].start = ddb_slices.start + start; - new_dbuf_state->ddb[pipe].end = ddb_slices.start + end; - + new_dbuf_state->ddb[pipe].start = ddb_slices.start - mbus_offset + start; + new_dbuf_state->ddb[pipe].end = ddb_slices.start - mbus_offset + end; out: - if (skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe], + if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] && + skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe], &new_dbuf_state->ddb[pipe])) return 0; @@ -4184,7 +4203,12 @@ out: if (IS_ERR(crtc_state)) return PTR_ERR(crtc_state); - crtc_state->wm.skl.ddb = new_dbuf_state->ddb[pipe]; + /* + * Used for checking overlaps, so we need absolute + * offsets instead of MBUS relative offsets. + */ + crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start; + crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end; drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n", @@ -4242,7 +4266,6 @@ skl_cursor_allocation(const struct intel_crtc_state *crtc_state, static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv, struct skl_ddb_entry *entry, u32 reg) { - entry->start = reg & DDB_ENTRY_MASK; entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK; @@ -4367,6 +4390,7 @@ skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state, struct dbuf_slice_conf_entry { u8 active_pipes; u8 dbuf_mask[I915_MAX_PIPES]; + bool join_mbus; }; /* @@ -4555,6 +4579,137 @@ static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = {} }; +static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = { + { + .active_pipes = BIT(PIPE_A), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), + }, + .join_mbus = true, + }, + { + .active_pipes = BIT(PIPE_B), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), + }, + .join_mbus = true, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_C), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_D), + .dbuf_mask = { + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + {} + +}; + +static bool check_mbus_joined(u8 active_pipes, + const struct dbuf_slice_conf_entry *dbuf_slices) +{ + int i; + + for (i = 0; i < dbuf_slices[i].active_pipes; i++) { + if (dbuf_slices[i].active_pipes == active_pipes) + return dbuf_slices[i].join_mbus; + } + return false; +} + +static bool adlp_check_mbus_joined(u8 active_pipes) +{ + return check_mbus_joined(active_pipes, adlp_allowed_dbufs); +} + static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, const struct dbuf_slice_conf_entry *dbuf_slices) { @@ -4594,12 +4749,19 @@ static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes) return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs); } +static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes) +{ + return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs); +} + static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - if (DISPLAY_VER(dev_priv) == 12) + if (IS_ALDERLAKE_P(dev_priv)) + return adlp_compute_dbuf_slices(pipe, active_pipes); + else if (DISPLAY_VER(dev_priv) == 12) return tgl_compute_dbuf_slices(pipe, active_pipes); else if (DISPLAY_VER(dev_priv) == 11) return icl_compute_dbuf_slices(pipe, active_pipes); @@ -5619,6 +5781,13 @@ void skl_write_plane_wm(struct intel_plane *plane, skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id), skl_plane_trans_wm(pipe_wm, plane_id)); + if (HAS_HW_SAGV_WM(dev_priv)) { + skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id), + &wm->sagv.wm0); + skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id), + &wm->sagv.trans_wm); + } + if (DISPLAY_VER(dev_priv) >= 11) { skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id), ddb_y); @@ -5652,6 +5821,15 @@ void skl_write_cursor_wm(struct intel_plane *plane, skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), skl_plane_trans_wm(pipe_wm, plane_id)); + if (HAS_HW_SAGV_WM(dev_priv)) { + const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; + + skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe), + &wm->sagv.wm0); + skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe), + &wm->sagv.trans_wm); + } + skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb); } @@ -5813,16 +5991,29 @@ skl_compute_ddb(struct intel_atomic_state *state) new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state); - if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) { + if (IS_ALDERLAKE_P(dev_priv)) + new_dbuf_state->joined_mbus = adlp_check_mbus_joined(new_dbuf_state->active_pipes); + + if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices || + old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { ret = intel_atomic_serialize_global_state(&new_dbuf_state->base); if (ret) return ret; + if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { + /* TODO: Implement vblank synchronized MBUS joining changes */ + ret = intel_modeset_all_pipes(state); + if (ret) + return ret; + } + drm_dbg_kms(&dev_priv->drm, - "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x)\n", + "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n", old_dbuf_state->enabled_slices, new_dbuf_state->enabled_slices, - INTEL_INFO(dev_priv)->dbuf.slice_mask); + INTEL_INFO(dev_priv)->dbuf.slice_mask, + yesno(old_dbuf_state->joined_mbus), + yesno(new_dbuf_state->joined_mbus)); } for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { @@ -6016,6 +6207,15 @@ static bool skl_plane_selected_wm_equals(struct intel_plane *plane, return false; } + if (HAS_HW_SAGV_WM(i915)) { + const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id]; + const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id]; + + if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) || + !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm)) + return false; + } + return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id), skl_plane_trans_wm(new_pipe_wm, plane->id)); } @@ -6234,7 +6434,25 @@ void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, skl_wm_level_from_reg_val(val, &wm->trans_wm); - if (DISPLAY_VER(dev_priv) >= 12) { + if (HAS_HW_SAGV_WM(dev_priv)) { + if (plane_id != PLANE_CURSOR) + val = intel_uncore_read(&dev_priv->uncore, + PLANE_WM_SAGV(pipe, plane_id)); + else + val = intel_uncore_read(&dev_priv->uncore, + CUR_WM_SAGV(pipe)); + + skl_wm_level_from_reg_val(val, &wm->sagv.wm0); + + if (plane_id != PLANE_CURSOR) + val = intel_uncore_read(&dev_priv->uncore, + PLANE_WM_SAGV_TRANS(pipe, plane_id)); + else + val = intel_uncore_read(&dev_priv->uncore, + CUR_WM_SAGV_TRANS(pipe)); + + skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm); + } else if (DISPLAY_VER(dev_priv) >= 12) { wm->sagv.wm0 = wm->wm[0]; wm->sagv.trans_wm = wm->trans_wm; } @@ -6247,10 +6465,14 @@ void skl_wm_get_hw_state(struct drm_i915_private *dev_priv) to_intel_dbuf_state(dev_priv->dbuf.obj.state); struct intel_crtc *crtc; + if (IS_ALDERLAKE_P(dev_priv)) + dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN; + for_each_intel_crtc(&dev_priv->drm, crtc) { struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); enum pipe pipe = crtc->pipe; + unsigned int mbus_offset; enum plane_id plane_id; skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal); @@ -6276,13 +6498,20 @@ void skl_wm_get_hw_state(struct drm_i915_private *dev_priv) dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state); - crtc_state->wm.skl.ddb = dbuf_state->ddb[pipe]; + /* + * Used for checking overlaps, so we need absolute + * offsets instead of MBUS relative offsets. + */ + mbus_offset = mbus_ddb_offset(dev_priv, dbuf_state->slices[pipe]); + crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start; + crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end; drm_dbg_kms(&dev_priv->drm, - "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n", + "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n", crtc->base.base.id, crtc->base.name, dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start, - dbuf_state->ddb[pipe].end, dbuf_state->active_pipes); + dbuf_state->ddb[pipe].end, dbuf_state->active_pipes, + yesno(dbuf_state->joined_mbus)); } dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices; @@ -7630,9 +7859,9 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) dev_priv->display.init_clock_gating = adlp_init_clock_gating; else if (IS_DG1(dev_priv)) dev_priv->display.init_clock_gating = dg1_init_clock_gating; - else if (IS_GEN(dev_priv, 12)) + else if (GRAPHICS_VER(dev_priv) == 12) dev_priv->display.init_clock_gating = gen12lp_init_clock_gating; - else if (IS_GEN(dev_priv, 11)) + else if (GRAPHICS_VER(dev_priv) == 11) dev_priv->display.init_clock_gating = icl_init_clock_gating; else if (IS_CANNONLAKE(dev_priv)) dev_priv->display.init_clock_gating = cnl_init_clock_gating; @@ -7656,9 +7885,9 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) dev_priv->display.init_clock_gating = ivb_init_clock_gating; else if (IS_VALLEYVIEW(dev_priv)) dev_priv->display.init_clock_gating = vlv_init_clock_gating; - else if (IS_GEN(dev_priv, 6)) + else if (GRAPHICS_VER(dev_priv) == 6) dev_priv->display.init_clock_gating = gen6_init_clock_gating; - else if (IS_GEN(dev_priv, 5)) + else if (GRAPHICS_VER(dev_priv) == 5) dev_priv->display.init_clock_gating = ilk_init_clock_gating; else if (IS_G4X(dev_priv)) dev_priv->display.init_clock_gating = g4x_init_clock_gating; @@ -7666,11 +7895,11 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) dev_priv->display.init_clock_gating = i965gm_init_clock_gating; else if (IS_I965G(dev_priv)) dev_priv->display.init_clock_gating = i965g_init_clock_gating; - else if (IS_GEN(dev_priv, 3)) + else if (GRAPHICS_VER(dev_priv) == 3) dev_priv->display.init_clock_gating = gen3_init_clock_gating; else if (IS_I85X(dev_priv) || IS_I865G(dev_priv)) dev_priv->display.init_clock_gating = i85x_init_clock_gating; - else if (IS_GEN(dev_priv, 2)) + else if (GRAPHICS_VER(dev_priv) == 2) dev_priv->display.init_clock_gating = i830_init_clock_gating; else { MISSING_CASE(INTEL_DEVID(dev_priv)); @@ -7684,7 +7913,7 @@ void intel_init_pm(struct drm_i915_private *dev_priv) /* For cxsr */ if (IS_PINEVIEW(dev_priv)) pnv_get_mem_freq(dev_priv); - else if (IS_GEN(dev_priv, 5)) + else if (GRAPHICS_VER(dev_priv) == 5) ilk_get_mem_freq(dev_priv); if (intel_has_sagv(dev_priv)) @@ -7816,6 +8045,45 @@ int intel_dbuf_init(struct drm_i915_private *dev_priv) return 0; } +/* + * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before + * update the request state of all DBUS slices. + */ +static void update_mbus_pre_enable(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + u32 mbus_ctl, dbuf_min_tracker_val; + enum dbuf_slice slice; + const struct intel_dbuf_state *dbuf_state = + intel_atomic_get_new_dbuf_state(state); + + if (!IS_ALDERLAKE_P(dev_priv)) + return; + + /* + * TODO: Implement vblank synchronized MBUS joining changes. + * Must be properly coordinated with dbuf reprogramming. + */ + if (dbuf_state->joined_mbus) { + mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN | + MBUS_JOIN_PIPE_SELECT_NONE; + dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3); + } else { + mbus_ctl = MBUS_HASHING_MODE_2x2 | + MBUS_JOIN_PIPE_SELECT_NONE; + dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1); + } + + intel_de_rmw(dev_priv, MBUS_CTL, + MBUS_HASHING_MODE_MASK | MBUS_JOIN | + MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl); + + for_each_dbuf_slice(dev_priv, slice) + intel_de_rmw(dev_priv, DBUF_CTL_S(slice), + DBUF_MIN_TRACKER_STATE_SERVICE_MASK, + dbuf_min_tracker_val); +} + void intel_dbuf_pre_plane_update(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); @@ -7825,11 +8093,13 @@ void intel_dbuf_pre_plane_update(struct intel_atomic_state *state) intel_atomic_get_old_dbuf_state(state); if (!new_dbuf_state || - new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices) + ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices) + && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))) return; WARN_ON(!new_dbuf_state->base.changed); + update_mbus_pre_enable(state); gen9_dbuf_slices_update(dev_priv, old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices); @@ -7844,7 +8114,8 @@ void intel_dbuf_post_plane_update(struct intel_atomic_state *state) intel_atomic_get_old_dbuf_state(state); if (!new_dbuf_state || - new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices) + ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices) + && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))) return; WARN_ON(!new_dbuf_state->base.changed); diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h index 669c8d505677..91f23b7f0af2 100644 --- a/drivers/gpu/drm/i915/intel_pm.h +++ b/drivers/gpu/drm/i915/intel_pm.h @@ -78,13 +78,11 @@ struct intel_dbuf_state { struct skl_ddb_entry ddb[I915_MAX_PIPES]; unsigned int weight[I915_MAX_PIPES]; u8 slices[I915_MAX_PIPES]; - u8 enabled_slices; u8 active_pipes; + bool joined_mbus; }; -int intel_dbuf_init(struct drm_i915_private *dev_priv); - struct intel_dbuf_state * intel_atomic_get_dbuf_state(struct intel_atomic_state *state); diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c index 0ec0cf191955..f0a82b37bd1a 100644 --- a/drivers/gpu/drm/i915/intel_sideband.c +++ b/drivers/gpu/drm/i915/intel_sideband.c @@ -430,7 +430,7 @@ static int __sandybridge_pcode_rw(struct drm_i915_private *i915, if (is_read && val1) *val1 = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA1); - if (INTEL_GEN(i915) > 6) + if (GRAPHICS_VER(i915) > 6) return gen7_check_mailbox_status(mbox); else return gen6_check_mailbox_status(mbox); diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index ed5abe7be498..1bed8f666048 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -1635,7 +1635,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) #define fw_domain_init(uncore__, id__, set__, ack__) \ (ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__)))) - if (INTEL_GEN(i915) >= 11) { + if (GRAPHICS_VER(i915) >= 11) { /* we'll prune the domains of missing engines later */ intel_engine_mask_t emask = INTEL_INFO(i915)->platform_engine_mask; int i; @@ -1665,7 +1665,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) FORCEWAKE_MEDIA_VEBOX_GEN11(i), FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)); } - } else if (IS_GEN_RANGE(i915, 9, 10)) { + } else if (IS_GRAPHICS_VER(i915, 9, 10)) { uncore->funcs.force_wake_get = fw_domains_get_with_fallback; uncore->funcs.force_wake_put = fw_domains_put; fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, @@ -1733,7 +1733,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, FORCEWAKE, FORCEWAKE_ACK); } - } else if (IS_GEN(i915, 6)) { + } else if (GRAPHICS_VER(i915) == 6) { uncore->funcs.force_wake_get = fw_domains_get_with_thread_status; uncore->funcs.force_wake_put = fw_domains_put; @@ -1800,7 +1800,7 @@ static int uncore_mmio_setup(struct intel_uncore *uncore) int mmio_bar; int mmio_size; - mmio_bar = IS_GEN(i915, 2) ? 1 : 0; + mmio_bar = GRAPHICS_VER(i915) == 2 ? 1 : 0; /* * Before gen4, the registers and the GTT are behind different BARs. * However, from gen4 onwards, the registers and the GTT are shared @@ -1810,7 +1810,7 @@ static int uncore_mmio_setup(struct intel_uncore *uncore) * generations up to Ironlake. * For dgfx chips register range is expanded to 4MB. */ - if (INTEL_GEN(i915) < 5) + if (GRAPHICS_VER(i915) < 5) mmio_size = 512 * 1024; else if (IS_DGFX(i915)) mmio_size = 4 * 1024 * 1024; @@ -1849,7 +1849,7 @@ static void uncore_raw_init(struct intel_uncore *uncore) if (intel_vgpu_active(uncore->i915)) { ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu); ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu); - } else if (IS_GEN(uncore->i915, 5)) { + } else if (GRAPHICS_VER(uncore->i915) == 5) { ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5); ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5); } else { @@ -1870,7 +1870,7 @@ static int uncore_forcewake_init(struct intel_uncore *uncore) return ret; forcewake_early_sanitize(uncore, 0); - if (IS_GEN_RANGE(i915, 6, 7)) { + if (IS_GRAPHICS_VER(i915, 6, 7)) { ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); if (IS_VALLEYVIEW(i915)) { @@ -1879,7 +1879,7 @@ static int uncore_forcewake_init(struct intel_uncore *uncore) } else { ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); } - } else if (IS_GEN(i915, 8)) { + } else if (GRAPHICS_VER(i915) == 8) { if (IS_CHERRYVIEW(i915)) { ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); @@ -1888,11 +1888,11 @@ static int uncore_forcewake_init(struct intel_uncore *uncore) ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen8); ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); } - } else if (IS_GEN_RANGE(i915, 9, 10)) { + } else if (IS_GRAPHICS_VER(i915, 9, 10)) { ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); - } else if (IS_GEN(i915, 11)) { + } else if (GRAPHICS_VER(i915) == 11) { ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen11_fwtable); ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable); @@ -1917,6 +1917,18 @@ int intel_uncore_init_mmio(struct intel_uncore *uncore) if (ret) return ret; + /* + * The boot firmware initializes local memory and assesses its health. + * If memory training fails, the punit will have been instructed to + * keep the GT powered down; we won't be able to communicate with it + * and we should not continue with driver initialization. + */ + if (IS_DGFX(i915) && + !(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) { + drm_err(&i915->drm, "LMEM not initialized by firmware\n"); + return -ENODEV; + } + if (INTEL_GEN(i915) > 5 && !intel_vgpu_active(i915)) uncore->flags |= UNCORE_HAS_FORCEWAKE; @@ -1940,7 +1952,7 @@ int intel_uncore_init_mmio(struct intel_uncore *uncore) if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED; - if (IS_GEN_RANGE(i915, 6, 7)) + if (IS_GRAPHICS_VER(i915, 6, 7)) uncore->flags |= UNCORE_HAS_FIFO; /* clear out unclaimed reg detection bit */ @@ -1967,7 +1979,7 @@ void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore, enum forcewake_domain_id domain_id; int i; - if (!intel_uncore_has_forcewake(uncore) || INTEL_GEN(uncore->i915) < 11) + if (!intel_uncore_has_forcewake(uncore) || GRAPHICS_VER(uncore->i915) < 11) return; for (i = 0; i < I915_MAX_VCS; i++) { diff --git a/drivers/gpu/drm/i915/intel_wopcm.c b/drivers/gpu/drm/i915/intel_wopcm.c index ec776591e1cf..8309455f13ea 100644 --- a/drivers/gpu/drm/i915/intel_wopcm.c +++ b/drivers/gpu/drm/i915/intel_wopcm.c @@ -81,7 +81,7 @@ void intel_wopcm_init_early(struct intel_wopcm *wopcm) if (!HAS_GT_UC(i915)) return; - if (INTEL_GEN(i915) >= 11) + if (GRAPHICS_VER(i915) >= 11) wopcm->size = GEN11_WOPCM_SIZE; else wopcm->size = GEN9_WOPCM_SIZE; @@ -93,7 +93,7 @@ static u32 context_reserved_size(struct drm_i915_private *i915) { if (IS_GEN9_LP(i915)) return BXT_WOPCM_RC6_CTX_RESERVED; - else if (INTEL_GEN(i915) >= 10) + else if (GRAPHICS_VER(i915) >= 10) return CNL_WOPCM_HW_CTX_RESERVED; else return 0; @@ -145,11 +145,11 @@ static bool check_hw_restrictions(struct drm_i915_private *i915, u32 guc_wopcm_base, u32 guc_wopcm_size, u32 huc_fw_size) { - if (IS_GEN(i915, 9) && !gen9_check_dword_gap(i915, guc_wopcm_base, - guc_wopcm_size)) + if (GRAPHICS_VER(i915) == 9 && !gen9_check_dword_gap(i915, guc_wopcm_base, + guc_wopcm_size)) return false; - if (IS_GEN(i915, 9) && + if (GRAPHICS_VER(i915) == 9 && !gen9_check_huc_fw_fits(i915, guc_wopcm_size, huc_fw_size)) return false; diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c index 4002c984c2e0..61bf4560d8af 100644 --- a/drivers/gpu/drm/i915/selftests/i915_active.c +++ b/drivers/gpu/drm/i915/selftests/i915_active.c @@ -68,7 +68,7 @@ static struct live_active *__live_alloc(struct drm_i915_private *i915) return NULL; kref_init(&active->ref); - i915_active_init(&active->base, __live_active, __live_retire); + i915_active_init(&active->base, __live_active, __live_retire, 0); return active; } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index dc394fb7ccfa..152d9ab135b1 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -87,14 +87,14 @@ static void simulate_hibernate(struct drm_i915_private *i915) intel_runtime_pm_put(&i915->runtime_pm, wakeref); } -static int pm_prepare(struct drm_i915_private *i915) +static int igt_pm_prepare(struct drm_i915_private *i915) { i915_gem_suspend(i915); return 0; } -static void pm_suspend(struct drm_i915_private *i915) +static void igt_pm_suspend(struct drm_i915_private *i915) { intel_wakeref_t wakeref; @@ -104,7 +104,7 @@ static void pm_suspend(struct drm_i915_private *i915) } } -static void pm_hibernate(struct drm_i915_private *i915) +static void igt_pm_hibernate(struct drm_i915_private *i915) { intel_wakeref_t wakeref; @@ -116,7 +116,7 @@ static void pm_hibernate(struct drm_i915_private *i915) } } -static void pm_resume(struct drm_i915_private *i915) +static void igt_pm_resume(struct drm_i915_private *i915) { intel_wakeref_t wakeref; @@ -148,16 +148,16 @@ static int igt_gem_suspend(void *arg) if (err) goto out; - err = pm_prepare(i915); + err = igt_pm_prepare(i915); if (err) goto out; - pm_suspend(i915); + igt_pm_suspend(i915); /* Here be dragons! Note that with S3RST any S3 may become S4! */ simulate_hibernate(i915); - pm_resume(i915); + igt_pm_resume(i915); err = switch_to_context(ctx); out: @@ -183,16 +183,16 @@ static int igt_gem_hibernate(void *arg) if (err) goto out; - err = pm_prepare(i915); + err = igt_pm_prepare(i915); if (err) goto out; - pm_hibernate(i915); + igt_pm_hibernate(i915); /* Here be dragons! */ simulate_hibernate(i915); - pm_resume(i915); + igt_pm_resume(i915); err = switch_to_context(ctx); out: diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 45c6c0107c7c..f843a5040706 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -186,7 +186,7 @@ retry: if (err) goto err_ppgtt_cleanup; - err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); + err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); if (err) { i915_vm_free_pt_stash(&ppgtt->vm, &stash); goto err_ppgtt_cleanup; @@ -208,7 +208,7 @@ retry: if (err) goto err_ppgtt_cleanup; - err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); + err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); if (err) { i915_vm_free_pt_stash(&ppgtt->vm, &stash); goto err_ppgtt_cleanup; @@ -325,11 +325,10 @@ retry: BIT_ULL(size))) goto alloc_vm_end; - err = i915_vm_pin_pt_stash(vm, &stash); + err = i915_vm_map_pt_stash(vm, &stash); if (!err) vm->allocate_va_range(vm, &stash, addr, BIT_ULL(size)); - i915_vm_free_pt_stash(vm, &stash); alloc_vm_end: if (err == -EDEADLK) { @@ -1885,9 +1884,9 @@ static int igt_cs_tlb(void *arg) u32 *cs = batch + i * 64 / sizeof(*cs); u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32); - GEM_BUG_ON(INTEL_GEN(i915) < 6); + GEM_BUG_ON(GRAPHICS_VER(i915) < 6); cs[0] = MI_STORE_DWORD_IMM_GEN4; - if (INTEL_GEN(i915) >= 8) { + if (GRAPHICS_VER(i915) >= 8) { cs[1] = lower_32_bits(addr); cs[2] = upper_32_bits(addr); cs[3] = i; @@ -1968,10 +1967,9 @@ retry: if (err) goto end_ww; - err = i915_vm_pin_pt_stash(vm, &stash); + err = i915_vm_map_pt_stash(vm, &stash); if (!err) vm->allocate_va_range(vm, &stash, offset, chunk_size); - i915_vm_free_pt_stash(vm, &stash); end_ww: if (err == -EDEADLK) { diff --git a/drivers/gpu/drm/i915/selftests/i915_perf.c b/drivers/gpu/drm/i915/selftests/i915_perf.c index e9d86dab8677..9e9a6cb1d9e5 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf.c +++ b/drivers/gpu/drm/i915/selftests/i915_perf.c @@ -98,7 +98,7 @@ test_stream(struct i915_perf *perf) I915_ENGINE_CLASS_RENDER, 0), .sample_flags = SAMPLE_OA_REPORT, - .oa_format = IS_GEN(perf->i915, 12) ? + .oa_format = GRAPHICS_VER(perf->i915) == 12 ? I915_OA_FORMAT_A32u40_A4u32_B8_C8 : I915_OA_FORMAT_C4_B8, }; struct i915_perf_stream *stream; @@ -162,7 +162,7 @@ static int write_timestamp(struct i915_request *rq, int slot) return PTR_ERR(cs); len = 5; - if (INTEL_GEN(rq->engine->i915) >= 8) + if (GRAPHICS_VER(rq->engine->i915) >= 8) len++; *cs++ = GFX_OP_PIPE_CONTROL(len); @@ -307,7 +307,7 @@ static int live_noa_gpr(void *arg) } /* Poison the ce->vm so we detect writes not to the GGTT gt->scratch */ - scratch = kmap(__px_page(ce->vm->scratch[0])); + scratch = __px_vaddr(ce->vm->scratch[0]); memset(scratch, POISON_FREE, PAGE_SIZE); rq = intel_context_create_request(ce); @@ -363,7 +363,7 @@ static int live_noa_gpr(void *arg) } cmd = MI_STORE_REGISTER_MEM; - if (INTEL_GEN(i915) >= 8) + if (GRAPHICS_VER(i915) >= 8) cmd++; cmd |= MI_USE_GGTT; @@ -405,7 +405,6 @@ static int live_noa_gpr(void *arg) out_rq: i915_request_put(rq); out_ce: - kunmap(__px_page(ce->vm->scratch[0])); intel_context_put(ce); out: stream_destroy(stream); diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index ee8e753d98ce..db367a6721c5 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -963,7 +963,7 @@ out_batch: static struct i915_vma *recursive_batch(struct drm_i915_private *i915) { struct drm_i915_gem_object *obj; - const int gen = INTEL_GEN(i915); + const int ver = GRAPHICS_VER(i915); struct i915_vma *vma; u32 *cmd; int err; @@ -988,11 +988,11 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915) goto err; } - if (gen >= 8) { + if (ver >= 8) { *cmd++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; *cmd++ = lower_32_bits(vma->node.start); *cmd++ = upper_32_bits(vma->node.start); - } else if (gen >= 6) { + } else if (ver >= 6) { *cmd++ = MI_BATCH_BUFFER_START | 1 << 8; *cmd++ = lower_32_bits(vma->node.start); } else { @@ -2482,7 +2482,7 @@ static int perf_request_latency(void *arg) struct pm_qos_request qos; int err = 0; - if (INTEL_GEN(i915) < 8) /* per-engine CS timestamp, semaphores */ + if (GRAPHICS_VER(i915) < 8) /* per-engine CS timestamp, semaphores */ return 0; cpu_latency_qos_add_request(&qos, 0); /* disable cstates */ diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index 5fe7b80ca0bd..dd0607254a95 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -967,6 +967,9 @@ static int igt_vma_remapped_gtt(void *arg) intel_wakeref_t wakeref; int err = 0; + if (!i915_ggtt_has_aperture(&i915->ggtt)) + return 0; + obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c index cfbbe415b57c..24d87d0fc747 100644 --- a/drivers/gpu/drm/i915/selftests/igt_spinner.c +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c @@ -94,9 +94,9 @@ int igt_spinner_pin(struct igt_spinner *spin, } if (!spin->batch) { - unsigned int mode = - i915_coherent_map_type(spin->gt->i915); + unsigned int mode; + mode = i915_coherent_map_type(spin->gt->i915, spin->obj, false); vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma); if (IS_ERR(vaddr)) return PTR_ERR(vaddr); @@ -174,15 +174,15 @@ igt_spinner_create_request(struct igt_spinner *spin, batch = spin->batch; - if (INTEL_GEN(rq->engine->i915) >= 8) { + if (GRAPHICS_VER(rq->engine->i915) >= 8) { *batch++ = MI_STORE_DWORD_IMM_GEN4; *batch++ = lower_32_bits(hws_address(hws, rq)); *batch++ = upper_32_bits(hws_address(hws, rq)); - } else if (INTEL_GEN(rq->engine->i915) >= 6) { + } else if (GRAPHICS_VER(rq->engine->i915) >= 6) { *batch++ = MI_STORE_DWORD_IMM_GEN4; *batch++ = 0; *batch++ = hws_address(hws, rq); - } else if (INTEL_GEN(rq->engine->i915) >= 4) { + } else if (GRAPHICS_VER(rq->engine->i915) >= 4) { *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *batch++ = 0; *batch++ = hws_address(hws, rq); @@ -194,11 +194,11 @@ igt_spinner_create_request(struct igt_spinner *spin, *batch++ = arbitration_command; - if (INTEL_GEN(rq->engine->i915) >= 8) + if (GRAPHICS_VER(rq->engine->i915) >= 8) *batch++ = MI_BATCH_BUFFER_START | BIT(8) | 1; else if (IS_HASWELL(rq->engine->i915)) *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW; - else if (INTEL_GEN(rq->engine->i915) >= 6) + else if (GRAPHICS_VER(rq->engine->i915) >= 6) *batch++ = MI_BATCH_BUFFER_START; else *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; @@ -216,7 +216,7 @@ igt_spinner_create_request(struct igt_spinner *spin, } flags = 0; - if (INTEL_GEN(rq->engine->i915) <= 5) + if (GRAPHICS_VER(rq->engine->i915) <= 5) flags |= I915_DISPATCH_SECURE; err = engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index a5fc0bf3feb9..f85fd8cbfbf5 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -513,7 +513,7 @@ static int igt_cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) if (err) return err; - ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); + ptr = i915_gem_object_pin_map(obj, I915_MAP_WC); if (IS_ERR(ptr)) return PTR_ERR(ptr); @@ -593,7 +593,9 @@ static int igt_gpu_write(struct i915_gem_context *ctx, if (err) break; + i915_gem_object_lock(obj, NULL); err = igt_cpu_check(obj, dword, rng); + i915_gem_object_unlock(obj); if (err) break; } while (!__igt_timeout(end_time, NULL)); @@ -629,6 +631,88 @@ out_put: return err; } +static int igt_lmem_create_cleared_cpu(void *arg) +{ + struct drm_i915_private *i915 = arg; + I915_RND_STATE(prng); + IGT_TIMEOUT(end_time); + u32 size, i; + int err; + + i915_gem_drain_freed_objects(i915); + + size = max_t(u32, PAGE_SIZE, i915_prandom_u32_max_state(SZ_32M, &prng)); + size = round_up(size, PAGE_SIZE); + i = 0; + + do { + struct drm_i915_gem_object *obj; + unsigned int flags; + u32 dword, val; + void *vaddr; + + /* + * Alternate between cleared and uncleared allocations, while + * also dirtying the pages each time to check that the pages are + * always cleared if requested, since we should get some overlap + * of the underlying pages, if not all, since we are the only + * user. + */ + + flags = I915_BO_ALLOC_CPU_CLEAR; + if (i & 1) + flags = 0; + + obj = i915_gem_object_create_lmem(i915, size, flags); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + i915_gem_object_lock(obj, NULL); + err = i915_gem_object_pin_pages(obj); + if (err) + goto out_put; + + dword = i915_prandom_u32_max_state(PAGE_SIZE / sizeof(u32), + &prng); + + if (flags & I915_BO_ALLOC_CPU_CLEAR) { + err = igt_cpu_check(obj, dword, 0); + if (err) { + pr_err("%s failed with size=%u, flags=%u\n", + __func__, size, flags); + goto out_unpin; + } + } + + vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); + if (IS_ERR(vaddr)) { + err = PTR_ERR(vaddr); + goto out_unpin; + } + + val = prandom_u32_state(&prng); + + memset32(vaddr, val, obj->base.size / sizeof(u32)); + + i915_gem_object_flush_map(obj); + i915_gem_object_unpin_map(obj); +out_unpin: + i915_gem_object_unpin_pages(obj); + __i915_gem_object_put_pages(obj); +out_put: + i915_gem_object_unlock(obj); + i915_gem_object_put(obj); + + if (err) + break; + ++i; + } while (!__igt_timeout(end_time, NULL)); + + pr_info("%s completed (%u) iterations\n", __func__, i); + + return err; +} + static int igt_lmem_write_gpu(void *arg) { struct drm_i915_private *i915 = arg; @@ -1043,6 +1127,7 @@ int intel_memory_region_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_lmem_create), + SUBTEST(igt_lmem_create_cleared_cpu), SUBTEST(igt_lmem_write_cpu), SUBTEST(igt_lmem_write_gpu), }; diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c index f76c9bcec735..8ef9e6a4ad05 100644 --- a/drivers/gpu/drm/i915/selftests/intel_uncore.c +++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c @@ -321,7 +321,7 @@ static int live_fw_table(void *arg) /* Confirm the table we load is still valid */ return intel_fw_table_check(gt->uncore->fw_domains_table, gt->uncore->fw_domains_table_entries, - INTEL_GEN(gt->i915) >= 9); + GRAPHICS_VER(gt->i915) >= 9); } int intel_uncore_live_selftests(struct drm_i915_private *i915) diff --git a/drivers/gpu/drm/i915/selftests/librapl.c b/drivers/gpu/drm/i915/selftests/librapl.c index 58710ac3f979..eb03b5b28bad 100644 --- a/drivers/gpu/drm/i915/selftests/librapl.c +++ b/drivers/gpu/drm/i915/selftests/librapl.c @@ -5,8 +5,18 @@ #include <asm/msr.h> +#include "i915_drv.h" #include "librapl.h" +bool librapl_supported(const struct drm_i915_private *i915) +{ + /* Discrete cards require hwmon integration */ + if (IS_DGFX(i915)) + return false; + + return librapl_energy_uJ(); +} + u64 librapl_energy_uJ(void) { unsigned long long power; diff --git a/drivers/gpu/drm/i915/selftests/librapl.h b/drivers/gpu/drm/i915/selftests/librapl.h index 887f3e91dd05..e3b24fad0a7a 100644 --- a/drivers/gpu/drm/i915/selftests/librapl.h +++ b/drivers/gpu/drm/i915/selftests/librapl.h @@ -8,6 +8,10 @@ #include <linux/types.h> +struct drm_i915_private; + +bool librapl_supported(const struct drm_i915_private *i915); + u64 librapl_energy_uJ(void); #endif /* SELFTEST_LIBRAPL_H */ |