/*
 * Copyright (C) 2020-2021 Collabora, Ltd.
 * SPDX-License-Identifier: MIT
 */

#include <math.h>
#include <stdio.h>
#include "compiler/nir/nir_builder.h"
#include "util/u_math.h"
#include "pan_afbc.h"
#include "pan_blend_cso.h"
#include "pan_desc.h"
#include "pan_encoder.h"
#include "pan_fb_preload.h"
#include "pan_jc.h"
#include "pan_pool.h"
#include "pan_shader.h"
#include "pan_texture.h"
#include "pan_trace.h"
#include "compiler/pan_compiler.h"
#include "compiler/pan_nir.h"

#if PAN_ARCH >= 6
/* On Midgard, the native preload infrastructure (via MFBD preloads) is broken
 * or missing in many cases. We instead use software paths as fallbacks, which
 * are done as TILER jobs. No vertex shader is necessary since we can supply
 * screen-space coordinates directly.
 *
 * This is primarily designed as a fallback for preloads but could be extended
 * for other clears/blits if needed in the future. */

static enum mali_register_file_format
nir_type_to_reg_fmt(nir_alu_type in)
{
   switch (in) {
   case nir_type_float32:
      return MALI_REGISTER_FILE_FORMAT_F32;
   case nir_type_int32:
      return MALI_REGISTER_FILE_FORMAT_I32;
   case nir_type_uint32:
      return MALI_REGISTER_FILE_FORMAT_U32;
   default:
      UNREACHABLE("Invalid type");
   }
}
#endif

/* On Valhall, the driver gives the hardware a table of resource tables.
 * Resources are addressed as the index of the table together with the index of
 * the resource within the table. For simplicity, we put one type of resource
 * in each table and fix the numbering of the tables.
 *
 * This numbering is arbitrary.
 */
enum pan_preload_resource_table {
   PAN_BLIT_TABLE_ATTRIBUTE = 0,
   PAN_BLIT_TABLE_ATTRIBUTE_BUFFER,
   PAN_BLIT_TABLE_SAMPLER,
   PAN_BLIT_TABLE_TEXTURE,

   PAN_BLIT_NUM_RESOURCE_TABLES
};

struct pan_preload_surface {
   gl_frag_result loc              : 4;
   nir_alu_type type               : 8;
   enum mali_texture_dimension dim : 3;
   bool array                      : 1;
   unsigned samples                : 5;
};

struct pan_preload_shader_key {
   struct pan_preload_surface surfaces[8];
};

struct pan_preload_shader_data {
   struct pan_preload_shader_key key;
   struct pan_shader_info info;
   uint64_t address;
   unsigned blend_ret_offsets[8];
   nir_alu_type blend_types[8];
};

struct pan_preload_rsd_key {
   struct {
      enum pipe_format format;
      nir_alu_type type               : 8;
      unsigned samples                : 5;
      enum mali_texture_dimension dim : 3;
      bool array                      : 1;
   } rts[8], z, s;
};

struct pan_preload_rsd_data {
   struct pan_preload_rsd_key key;
   uint64_t address;
};

#if PAN_ARCH >= 5
static void
pan_preload_emit_blend(unsigned rt,
                       const struct pan_image_view *iview,
                       const struct pan_preload_shader_data *preload_shader,
                       uint64_t blend_shader, struct mali_blend_packed *out)
{
   assert(blend_shader == 0 || PAN_ARCH <= 5);

   pan_pack(out, BLEND, cfg) {
      if (!iview) {
         cfg.enable = false;
#if PAN_ARCH >= 6
         cfg.internal.mode = MALI_BLEND_MODE_OFF;
#endif
         continue;
      }

      cfg.round_to_fb_precision = true;
      cfg.srgb = util_format_is_srgb(iview->format);

#if PAN_ARCH >= 6
      cfg.internal.mode = MALI_BLEND_MODE_OPAQUE;
#endif

      if (!blend_shader) {
         cfg.equation.rgb.a = MALI_BLEND_OPERAND_A_SRC;
         cfg.equation.rgb.b = MALI_BLEND_OPERAND_B_SRC;
         cfg.equation.rgb.c = MALI_BLEND_OPERAND_C_ZERO;
         cfg.equation.alpha.a = MALI_BLEND_OPERAND_A_SRC;
         cfg.equation.alpha.b = MALI_BLEND_OPERAND_B_SRC;
         cfg.equation.alpha.c = MALI_BLEND_OPERAND_C_ZERO;
         cfg.equation.color_mask = 0xf;

#if PAN_ARCH >= 6
         cfg.internal.fixed_function.num_comps = 4;
         cfg.internal.fixed_function.conversion.memory_format =
            GENX(pan_dithered_format_from_pipe_format)(iview->format, false);
#if PAN_ARCH < 9
         nir_alu_type type = preload_shader->key.surfaces[rt].type;
         cfg.internal.fixed_function.conversion.register_format =
            nir_type_to_reg_fmt(type);
#endif

         cfg.internal.fixed_function.rt = rt;
#endif
      } else {
#if PAN_ARCH <= 5
         cfg.blend_shader = true;
         cfg.shader_pc = blend_shader;
#endif
      }
   }
}
#endif

struct pan_preload_views {
   unsigned rt_count;
   const struct pan_image_view *rts[8];
   const struct pan_image_view *z;
   const struct pan_image_view *s;
};

static bool
pan_preload_is_ms(struct pan_preload_views *views)
{
   for (unsigned i = 0; i < views->rt_count; i++) {
      if (views->rts[i]) {
         if (pan_image_view_get_nr_samples(views->rts[i]) > 1)
            return true;
      }
   }

   if (views->z && pan_image_view_get_nr_samples(views->z) > 1)
      return true;

   if (views->s && pan_image_view_get_nr_samples(views->s) > 1)
      return true;

   return false;
}

#if PAN_ARCH >= 5
static void
pan_preload_emit_blends(const struct pan_preload_shader_data *preload_shader,
                        struct pan_preload_views *views,
                        uint64_t *blend_shaders, struct mali_blend_packed *out)
{
   for (unsigned i = 0; i < MAX2(views->rt_count, 1); ++i) {
      const struct pan_image_view *rt_view = views->rts[i];
      uint64_t blend_shader = blend_shaders ? blend_shaders[i] : 0;

      pan_preload_emit_blend(i, rt_view, preload_shader, blend_shader, &out[i]);
   }
}
#endif

#if PAN_ARCH <= 7
static void
pan_preload_emit_rsd(const struct pan_preload_shader_data *preload_shader,
                     struct pan_preload_views *views, uint64_t *blend_shaders,
                     struct mali_renderer_state_packed *out)
{
   UNUSED bool zs = (views->z || views->s);
   bool ms = pan_preload_is_ms(views);

   pan_pack(out, RENDERER_STATE, cfg) {
      assert(preload_shader->address);
      pan_shader_prepare_rsd(&preload_shader->info, preload_shader->address, &cfg);

      cfg.multisample_misc.sample_mask = 0xFFFF;
      cfg.multisample_misc.multisample_enable = ms;
      cfg.multisample_misc.evaluate_per_sample = ms;
      cfg.multisample_misc.depth_write_mask = views->z != NULL;
      cfg.multisample_misc.depth_function = MALI_FUNC_ALWAYS;

      cfg.stencil_mask_misc.stencil_enable = views->s != NULL;
      cfg.stencil_mask_misc.stencil_mask_front = 0xFF;
      cfg.stencil_mask_misc.stencil_mask_back = 0xFF;
      cfg.stencil_front.compare_function = MALI_FUNC_ALWAYS;
      cfg.stencil_front.stencil_fail = MALI_STENCIL_OP_REPLACE;
      cfg.stencil_front.depth_fail = MALI_STENCIL_OP_REPLACE;
      cfg.stencil_front.depth_pass = MALI_STENCIL_OP_REPLACE;
      cfg.stencil_front.mask = 0xFF;
      cfg.stencil_back = cfg.stencil_front;

#if PAN_ARCH >= 6
      if (zs) {
         /* Writing Z/S requires late updates */
         cfg.properties.zs_update_operation = MALI_PIXEL_KILL_FORCE_LATE;
         cfg.properties.pixel_kill_operation = MALI_PIXEL_KILL_FORCE_LATE;
      } else {
         /* Skipping ATEST requires forcing Z/S */
         cfg.properties.zs_update_operation = MALI_PIXEL_KILL_FORCE_EARLY;
         cfg.properties.pixel_kill_operation = MALI_PIXEL_KILL_FORCE_EARLY;
      }

      /* However, while shaders writing Z/S can normally be killed, on v6
       * for frame shaders it can cause GPU timeouts, so only allow colour
       * preload shaders to be killed. */
      cfg.properties.allow_forward_pixel_to_kill = !zs;

      if (PAN_ARCH == 6)
         cfg.properties.allow_forward_pixel_to_be_killed = !zs;
#else

      uint64_t blend_shader =
         blend_shaders
            ? pan_last_nonnull(blend_shaders, MAX2(views->rt_count, 1))
            : 0;

      cfg.properties.work_register_count = 4;
      cfg.properties.force_early_z = !zs;
      cfg.stencil_mask_misc.alpha_test_compare_function = MALI_FUNC_ALWAYS;

      /* Set even on v5 for erratum workaround */
#if PAN_ARCH == 5
      cfg.legacy_blend_shader = blend_shader;
#else
      cfg.blend_shader = blend_shader;
      cfg.stencil_mask_misc.write_enable = true;
      cfg.stencil_mask_misc.dither_disable = true;
      cfg.multisample_misc.blend_shader = !!blend_shader;
      cfg.blend_shader = blend_shader;
      if (!cfg.multisample_misc.blend_shader) {
         cfg.blend_equation.rgb.a = MALI_BLEND_OPERAND_A_SRC;
         cfg.blend_equation.rgb.b = MALI_BLEND_OPERAND_B_SRC;
         cfg.blend_equation.rgb.c = MALI_BLEND_OPERAND_C_ZERO;
         cfg.blend_equation.alpha.a = MALI_BLEND_OPERAND_A_SRC;
         cfg.blend_equation.alpha.b = MALI_BLEND_OPERAND_B_SRC;
         cfg.blend_equation.alpha.c = MALI_BLEND_OPERAND_C_ZERO;
         cfg.blend_constant = 0;

         if (views->rts[0] != NULL) {
            cfg.stencil_mask_misc.srgb =
               util_format_is_srgb(views->rts[0]->format);
            cfg.blend_equation.color_mask = 0xf;
         }
      }
#endif
#endif
   }

#if PAN_ARCH >= 5
   pan_preload_emit_blends(preload_shader, views, blend_shaders,
                           (void*)((uint8_t*)out + pan_size(RENDERER_STATE)));
#endif
}
#endif

#if PAN_ARCH <= 5
static void
pan_preload_get_blend_shaders(struct pan_fb_preload_cache *cache,
                              unsigned rt_count,
                              const struct pan_image_view **rts,
                              const struct pan_preload_shader_data *preload_shader,
                              uint64_t *blend_shaders)
{
   if (!rt_count)
      return;

   struct pan_blend_state blend_state = {
      .rt_count = rt_count,
   };

   for (unsigned i = 0; i < rt_count; i++) {
      if (!rts[i] || pan_blendable_formats_v7[rts[i]->format].internal)
         continue;

      blend_state.rts[i] = (struct pan_blend_rt_state){
         .format = rts[i]->format,
         .nr_samples = pan_image_view_get_nr_samples(rts[i]),
         .equation =
            {
               .blend_enable = false,
               .color_mask = 0xf,
            },
      };

      pthread_mutex_lock(&cache->blend_shader_cache->lock);
      struct pan_blend_shader *b = GENX(pan_blend_get_shader_locked)(
         cache->blend_shader_cache, &blend_state,
         preload_shader->blend_types[i], nir_type_float32, /* unused */
         i);

      assert(b->work_reg_count <= 4);
      blend_shaders[i] = b->address;
      pthread_mutex_unlock(&cache->blend_shader_cache->lock);
   }
}
#endif

/*
 * Early Mali GPUs did not respect sampler LOD clamps or bias, so the Midgard
 * compiler inserts lowering code with a load_sampler_lod_parameters sysval
 * that we need to lower. Our samplers do not use LOD clamps or bias, so we
 * lower to the identity settings and let constant folding get rid of the
 * unnecessary lowering.
 */
static bool
lower_sampler_parameters(nir_builder *b, nir_intrinsic_instr *intr,
                         UNUSED void *data)
{
   if (intr->intrinsic != nir_intrinsic_load_sampler_lod_parameters)
      return false;

   const nir_const_value constants[4] = {
      nir_const_value_for_float(0.0f, 32),     /* min_lod */
      nir_const_value_for_float(INFINITY, 32), /* max_lod */
      nir_const_value_for_float(0.0f, 32),     /* lod_bias */
   };

   b->cursor = nir_after_instr(&intr->instr);
   nir_def_rewrite_uses(&intr->def, nir_build_imm(b, 3, 32, constants));
   return true;
}

static uint32_t
sampler_hw_index(uint32_t index)
{
   return PAN_ARCH >= 9 ? pan_res_handle(PAN_BLIT_TABLE_SAMPLER, index) : index;
}

static uint32_t
tex_hw_index(uint32_t index)
{
   return PAN_ARCH >= 9 ? pan_res_handle(PAN_BLIT_TABLE_TEXTURE, index) : index;
}

static uint32_t
attr_hw_index(uint32_t index)
{
   return PAN_ARCH >= 9 ? pan_res_handle(PAN_BLIT_TABLE_ATTRIBUTE, index)
                        : index;
}

static const struct pan_preload_shader_data *
pan_preload_get_shader(struct pan_fb_preload_cache *cache,
                       const struct pan_preload_shader_key *key)
{
   pthread_mutex_lock(&cache->shaders.lock);
   struct hash_entry *he =
      _mesa_hash_table_search(cache->shaders.preload, key);
   struct pan_preload_shader_data *shader = he ? he->data : NULL;

   if (shader)
      goto out;

   unsigned coord_comps = 0;
   unsigned sig_offset = 0;
   char sig[256];
   sig[0] = '\0';
   for (unsigned i = 0; i < ARRAY_SIZE(key->surfaces); i++) {
      const char *type_str, *dim_str;
      if (key->surfaces[i].type == nir_type_invalid)
         continue;

      switch (key->surfaces[i].type) {
      case nir_type_float32:
         type_str = "float";
         break;
      case nir_type_uint32:
         type_str = "uint";
         break;
      case nir_type_int32:
         type_str = "int";
         break;
      default:
         UNREACHABLE("Invalid type\n");
      }

      switch (key->surfaces[i].dim) {
      case MALI_TEXTURE_DIMENSION_CUBE:
         dim_str = "cube";
         break;
      case MALI_TEXTURE_DIMENSION_1D:
         dim_str = "1D";
         break;
      case MALI_TEXTURE_DIMENSION_2D:
         dim_str = "2D";
         break;
      case MALI_TEXTURE_DIMENSION_3D:
         dim_str = "3D";
         break;
      default:
         UNREACHABLE("Invalid dim\n");
      }

      coord_comps = MAX2(coord_comps, (key->surfaces[i].dim ?: 3) +
                                         (key->surfaces[i].array ? 1 : 0));

      if (sig_offset >= sizeof(sig))
         continue;

      sig_offset +=
         snprintf(sig + sig_offset, sizeof(sig) - sig_offset,
                  "%s[%s;%s;%s%s;samples=%d]",
                  sig_offset == 0 ? "" : ",",
                  gl_frag_result_name(key->surfaces[i].loc),
                  type_str, dim_str, key->surfaces[i].array ? "[]" : "",
                  key->surfaces[i].samples);
   }

   const nir_shader_compiler_options *compiler_options =
      pan_get_nir_shader_compiler_options(PAN_ARCH, false);
   nir_builder b = nir_builder_init_simple_shader(
      MESA_SHADER_FRAGMENT, compiler_options, "pan_preload(%s)", sig);

   nir_def *barycentric = nir_load_barycentric(
      &b, nir_intrinsic_load_barycentric_pixel, INTERP_MODE_SMOOTH);
   nir_def *coord = nir_load_interpolated_input(
      &b, coord_comps, 32, barycentric, nir_imm_int(&b, 0),
      .base = attr_hw_index(0), .dest_type = nir_type_float32,
      .io_semantics.location = VARYING_SLOT_VAR0, .io_semantics.num_slots = 1);

   unsigned active_count = 0;
   for (unsigned i = 0; i < ARRAY_SIZE(key->surfaces); i++) {
      if (key->surfaces[i].type == nir_type_invalid)
         continue;

      bool ms = key->surfaces[i].samples > 1;
      enum glsl_sampler_dim sampler_dim;

      switch (key->surfaces[i].dim) {
      case MALI_TEXTURE_DIMENSION_1D:
         sampler_dim = GLSL_SAMPLER_DIM_1D;
         break;
      case MALI_TEXTURE_DIMENSION_2D:
         sampler_dim = ms ? GLSL_SAMPLER_DIM_MS : GLSL_SAMPLER_DIM_2D;
         break;
      case MALI_TEXTURE_DIMENSION_3D:
         sampler_dim = GLSL_SAMPLER_DIM_3D;
         break;
      case MALI_TEXTURE_DIMENSION_CUBE:
         sampler_dim = GLSL_SAMPLER_DIM_CUBE;
         break;
      default:
         UNREACHABLE("Invalid dimension");
      }

      nir_tex_instr *tex = nir_tex_instr_create(b.shader, ms ? 3 : 1);

      tex->dest_type = key->surfaces[i].type;
      tex->texture_index = tex_hw_index(active_count);
      tex->sampler_index = sampler_hw_index(0);
      tex->is_array = key->surfaces[i].array;
      tex->sampler_dim = sampler_dim;

      if (ms) {
         tex->op = nir_texop_txf_ms;

         tex->src[0] =
            nir_tex_src_for_ssa(nir_tex_src_coord, nir_f2i32(&b, coord));
         tex->coord_components = coord_comps;

         tex->src[1] =
            nir_tex_src_for_ssa(nir_tex_src_ms_index, nir_load_sample_id(&b));

         tex->src[2] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(&b, 0));
      } else {
         tex->op = nir_texop_txl;

         tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, coord);
         tex->coord_components = coord_comps;
      }

      nir_def_init(&tex->instr, &tex->def, 4, 32);
      nir_builder_instr_insert(&b, &tex->instr);

      nir_def *res = &tex->def;

      if (key->surfaces[i].loc >= FRAG_RESULT_DATA0) {
         nir_store_output(
            &b, res, nir_imm_int(&b, 0), .base = active_count,
            .src_type = key->surfaces[i].type,
            .io_semantics.location = key->surfaces[i].loc,
            .io_semantics.num_slots = 1,
            .write_mask = nir_component_mask(res->num_components));
      } else {
         unsigned c = key->surfaces[i].loc == FRAG_RESULT_STENCIL ? 1 : 0;
         nir_store_output(
            &b, nir_channel(&b, res, c), nir_imm_int(&b, 0),
            .base = active_count, .src_type = key->surfaces[i].type,
            .io_semantics.location = key->surfaces[i].loc,
            .io_semantics.num_slots = 1, .write_mask = nir_component_mask(1));
      }
      active_count++;
   }

   struct pan_compile_inputs inputs = {
      .gpu_id = cache->gpu_id,
      .gpu_variant = cache->gpu_variant,
      .is_blit = true,
      .no_idvs = true,
   };
   struct util_dynarray binary = UTIL_DYNARRAY_INIT;

   shader = rzalloc(cache->shaders.preload, struct pan_preload_shader_data);

   nir_shader_gather_info(b.shader, nir_shader_get_entrypoint(b.shader));

   for (unsigned i = 0; i < active_count; ++i)
      BITSET_SET(b.shader->info.textures_used, i);

   pan_preprocess_nir(b.shader, inputs.gpu_id);
   pan_postprocess_nir(b.shader, inputs.gpu_id);

   if (PAN_ARCH == 4) {
      NIR_PASS(_, b.shader, nir_shader_intrinsics_pass,
               lower_sampler_parameters, nir_metadata_control_flow, NULL);
   }

   pan_shader_compile(b.shader, &inputs, &binary, &shader->info);

   shader->key = *key;
   shader->address =
      pan_pool_upload_aligned(cache->shaders.pool, binary.data,
                              binary.size, PAN_ARCH >= 6 ? 128 : 64);

   util_dynarray_fini(&binary);
   ralloc_free(b.shader);

#if PAN_ARCH >= 6
   for (unsigned i = 0; i < ARRAY_SIZE(shader->blend_ret_offsets); i++) {
      shader->blend_ret_offsets[i] =
         shader->info.bifrost.blend[i].return_offset;
      shader->blend_types[i] = shader->info.bifrost.blend[i].type;
   }
#endif

   _mesa_hash_table_insert(cache->shaders.preload, &shader->key, shader);

out:
   pthread_mutex_unlock(&cache->shaders.lock);
   return shader;
}

static struct pan_preload_shader_key
pan_preload_get_key(struct pan_preload_views *views)
{
   struct pan_preload_shader_key key = {0};

   if (views->z) {
      key.surfaces[0].loc = FRAG_RESULT_DEPTH;
      key.surfaces[0].type = nir_type_float32;
      key.surfaces[0].samples = pan_image_view_get_nr_samples(views->z);
      key.surfaces[0].dim = views->z->dim;
      key.surfaces[0].array = views->z->first_layer != views->z->last_layer;
   }

   if (views->s) {
      key.surfaces[1].loc = FRAG_RESULT_STENCIL;
      key.surfaces[1].type = nir_type_uint32;
      key.surfaces[1].samples = pan_image_view_get_nr_samples(views->s);
      key.surfaces[1].dim = views->s->dim;
      key.surfaces[1].array = views->s->first_layer != views->s->last_layer;
   }

   for (unsigned i = 0; i < views->rt_count; i++) {
      if (!views->rts[i])
         continue;

      key.surfaces[i].loc = FRAG_RESULT_DATA0 + i;
      key.surfaces[i].type =
         util_format_is_pure_uint(views->rts[i]->format) ? nir_type_uint32
         : util_format_is_pure_sint(views->rts[i]->format)
            ? nir_type_int32
            : nir_type_float32;
      key.surfaces[i].samples =
         pan_image_view_get_nr_samples(views->rts[i]);
      key.surfaces[i].dim = views->rts[i]->dim;
      key.surfaces[i].array =
         views->rts[i]->first_layer != views->rts[i]->last_layer;
   }

   return key;
}

#if PAN_ARCH <= 7
static uint64_t
pan_preload_get_rsd(struct pan_fb_preload_cache *cache,
                    struct pan_preload_views *views)
{
   struct pan_preload_rsd_key rsd_key = {0};

   assert(!views->rt_count || (!views->z && !views->s));

   struct pan_preload_shader_key preload_key = pan_preload_get_key(views);

   if (views->z) {
      rsd_key.z.format = views->z->format;
      rsd_key.z.type = preload_key.surfaces[0].type;
      rsd_key.z.samples = preload_key.surfaces[0].samples;
      rsd_key.z.dim = preload_key.surfaces[0].dim;
      rsd_key.z.array = preload_key.surfaces[0].array;
   }

   if (views->s) {
      rsd_key.s.format = views->s->format;
      rsd_key.s.type = preload_key.surfaces[1].type;
      rsd_key.s.samples = preload_key.surfaces[1].samples;
      rsd_key.s.dim = preload_key.surfaces[1].dim;
      rsd_key.s.array = preload_key.surfaces[1].array;
   }

   for (unsigned i = 0; i < views->rt_count; i++) {
      if (!views->rts[i])
         continue;

      rsd_key.rts[i].format = views->rts[i]->format;
      rsd_key.rts[i].type = preload_key.surfaces[i].type;
      rsd_key.rts[i].samples = preload_key.surfaces[i].samples;
      rsd_key.rts[i].dim = preload_key.surfaces[i].dim;
      rsd_key.rts[i].array = preload_key.surfaces[i].array;
   }

   pthread_mutex_lock(&cache->rsds.lock);
   struct hash_entry *he =
      _mesa_hash_table_search(cache->rsds.rsds, &rsd_key);
   struct pan_preload_rsd_data *rsd = he ? he->data : NULL;
   if (rsd)
      goto out;

   rsd = rzalloc(cache->rsds.rsds, struct pan_preload_rsd_data);
   rsd->key = rsd_key;

#if PAN_ARCH == 4
   struct pan_ptr rsd_ptr =
      pan_pool_alloc_desc(cache->rsds.pool, RENDERER_STATE);
#else
   unsigned bd_count = PAN_ARCH >= 5 ? MAX2(views->rt_count, 1) : 0;
   struct pan_ptr rsd_ptr =
      pan_pool_alloc_desc_aggregate(cache->rsds.pool, PAN_DESC(RENDERER_STATE),
                                    PAN_DESC_ARRAY(bd_count, BLEND));
#endif

   if (!rsd_ptr.cpu)
      return 0;

   uint64_t blend_shaders[8] = {0};

   const struct pan_preload_shader_data *preload_shader =
      pan_preload_get_shader(cache, &preload_key);

#if PAN_ARCH <= 5
   pan_preload_get_blend_shaders(cache,
                                 views->rt_count, views->rts, preload_shader,
                                 blend_shaders);
#endif

   pan_preload_emit_rsd(preload_shader, views, blend_shaders, rsd_ptr.cpu);
   rsd->address = rsd_ptr.gpu;
   _mesa_hash_table_insert(cache->rsds.rsds, &rsd->key, rsd);

out:
   pthread_mutex_unlock(&cache->rsds.lock);
   return rsd->address;
}
#endif

static struct pan_preload_views
pan_preload_get_views(const struct pan_fb_info *fb, bool zs,
                      struct pan_image_view *patched_s)
{
   struct pan_preload_views views = {0};

   if (zs) {
      if (fb->zs.preload.z)
         views.z = fb->zs.view.zs;

      if (fb->zs.preload.s) {
         const struct pan_image_view *view = fb->zs.view.s ?: fb->zs.view.zs;
         enum pipe_format fmt = util_format_get_depth_only(view->format);

         switch (view->format) {
         case PIPE_FORMAT_Z24_UNORM_S8_UINT:
            fmt = PIPE_FORMAT_X24S8_UINT;
            break;
         case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
            fmt = PIPE_FORMAT_X32_S8X24_UINT;
            break;
         default:
            fmt = view->format;
            break;
         }

         if (fmt != view->format) {
            *patched_s = *view;
            patched_s->format = fmt;
            views.s = patched_s;
         } else {
            views.s = view;
         }
      }
   } else {
      for (unsigned i = 0; i < fb->rt_count; i++) {
         if (fb->rts[i].preload)
            views.rts[i] = fb->rts[i].view;
      }

      views.rt_count = fb->rt_count;
   }

   return views;
}

static bool
pan_preload_needed(const struct pan_fb_info *fb, bool zs)
{
   if (zs) {
      if (fb->zs.preload.z || fb->zs.preload.s)
         return true;
   } else {
      for (unsigned i = 0; i < fb->rt_count; i++) {
         if (fb->rts[i].preload)
            return true;
      }
   }

   return false;
}

static uint64_t
pan_preload_emit_varying(struct pan_pool *pool)
{
   struct pan_ptr varying = pan_pool_alloc_desc(pool, ATTRIBUTE);

   if (!varying.cpu)
      return 0;

   pan_cast_and_pack(varying.cpu, ATTRIBUTE, cfg) {
      cfg.buffer_index = 0;
      cfg.offset_enable = PAN_ARCH <= 5;
      cfg.format =
         GENX(pan_format_from_pipe_format)(PIPE_FORMAT_R32G32B32_FLOAT)->hw;

#if PAN_ARCH >= 9
      cfg.attribute_type = MALI_ATTRIBUTE_TYPE_1D;
      cfg.table = PAN_BLIT_TABLE_ATTRIBUTE_BUFFER;
      cfg.frequency = MALI_ATTRIBUTE_FREQUENCY_VERTEX;
      cfg.stride = 4 * sizeof(float);
#endif
   }

   return varying.gpu;
}

static uint64_t
pan_preload_emit_varying_buffer(struct pan_pool *pool, uint64_t coordinates)
{
#if PAN_ARCH >= 9
   struct pan_ptr varying_buffer = pan_pool_alloc_desc(pool, BUFFER);

   if (!varying_buffer.cpu)
      return 0;

   pan_cast_and_pack(varying_buffer.cpu, BUFFER, cfg) {
      cfg.address = coordinates;
      cfg.size = 4 * sizeof(float) * 4;
   }
#else
   /* Bifrost needs an empty desc to mark end of prefetching */
   bool padding_buffer = PAN_ARCH >= 6;

   struct pan_ptr varying_buffer = pan_pool_alloc_desc_array(
      pool, (padding_buffer ? 2 : 1), ATTRIBUTE_BUFFER);

   if (!varying_buffer.cpu)
      return 0;

   pan_cast_and_pack(varying_buffer.cpu, ATTRIBUTE_BUFFER, cfg) {
      cfg.pointer = coordinates;
      cfg.stride = 4 * sizeof(float);
      cfg.size = cfg.stride * 4;
   }

   if (padding_buffer) {
      pan_cast_and_pack(varying_buffer.cpu + pan_size(ATTRIBUTE_BUFFER),
                        ATTRIBUTE_BUFFER, cfg)
         ;
   }
#endif

   return varying_buffer.gpu;
}

static uint64_t
pan_preload_emit_sampler(struct pan_pool *pool, bool nearest_filter)
{
   struct pan_ptr sampler = pan_pool_alloc_desc(pool, SAMPLER);

   if (!sampler.cpu)
      return 0;

   pan_cast_and_pack(sampler.cpu, SAMPLER, cfg) {
      cfg.seamless_cube_map = false;
      cfg.normalized_coordinates = false;
      cfg.minify_nearest = nearest_filter;
      cfg.magnify_nearest = nearest_filter;
   }

   return sampler.gpu;
}

static uint64_t
pan_preload_emit_textures(struct pan_pool *pool, const struct pan_fb_info *fb,
                          bool zs, unsigned *tex_count_out)
{
   const struct pan_image_view *views[8];
   struct pan_image_view patched_views[8];
   unsigned tex_count = 0;
   unsigned patched_count = 0;

   if (zs) {
      if (fb->zs.preload.z) {
         const struct pan_image_view *view = fb->zs.view.zs;
#if PAN_ARCH >= 7
         struct pan_image_view *pview = &patched_views[patched_count++];
         *pview = *view;
         /* v7+ doesn't have an _RRRR component order. */
         GENX(pan_texture_swizzle_replicate_x)(pview);
         view = pview;
#endif
         views[tex_count++] = view;
      }

      if (fb->zs.preload.s) {
         const struct pan_image_view *view = fb->zs.view.s ?: fb->zs.view.zs;
         enum pipe_format fmt = util_format_get_depth_only(view->format);

         switch (view->format) {
         case PIPE_FORMAT_Z24_UNORM_S8_UINT:
            fmt = PIPE_FORMAT_X24S8_UINT;
            break;
         case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
            fmt = PIPE_FORMAT_X32_S8X24_UINT;
            break;
         default:
            fmt = view->format;
            break;
         }

#if PAN_ARCH >= 7
         struct pan_image_view *pview = &patched_views[patched_count++];
         *pview = *view;
         pview->format = fmt;
         /* v7+ doesn't have an _RRRR component order. */
         GENX(pan_texture_swizzle_replicate_x)(pview);
         view = pview;
#else
         if (fmt != view->format) {
            struct pan_image_view *pview = &patched_views[patched_count++];
            *pview = *view;
            pview->format = fmt;
            view = pview;
         }
#endif
         views[tex_count++] = view;
      }
   } else {
      for (unsigned i = 0; i < fb->rt_count; i++) {
         if (fb->rts[i].preload) {
            const struct pan_image_view *view = fb->rts[i].view;
#if PAN_ARCH == 7
            /* v7 requires AFBC reswizzle. */
            if (!pan_format_is_yuv(view->format) &&
                pan_afbc_supports_format(PAN_ARCH, view->format)) {
               struct pan_image_view *pview = &patched_views[patched_count++];
               *pview = *view;
               GENX(pan_texture_afbc_reswizzle)(pview);
               view = pview;
            }
#endif
            views[tex_count++] = view;
         }
      }
   }

   *tex_count_out = tex_count;

#if PAN_ARCH >= 6
   struct pan_ptr textures =
      pan_pool_alloc_desc_array(pool, tex_count, TEXTURE);

   if (!textures.cpu)
      return 0;

   for (unsigned i = 0; i < tex_count; i++) {
      void *texture = textures.cpu + (pan_size(TEXTURE) * i);
      size_t payload_size =
         GENX(pan_texture_estimate_payload_size)(views[i]);
      struct pan_ptr surfaces = pan_pool_alloc_aligned(pool, payload_size, 64);

      GENX(pan_sampled_texture_emit)(views[i], texture, &surfaces);
   }

   return textures.gpu;
#else
   uint64_t textures[8] = {0};

   for (unsigned i = 0; i < tex_count; i++) {
      size_t sz = pan_size(TEXTURE) +
                  GENX(pan_texture_estimate_payload_size)(views[i]);
      struct pan_ptr texture =
         pan_pool_alloc_aligned(pool, sz, pan_alignment(TEXTURE));
      struct pan_ptr surfaces = {
         .cpu = texture.cpu + pan_size(TEXTURE),
         .gpu = texture.gpu + pan_size(TEXTURE),
      };

      GENX(pan_sampled_texture_emit)(views[i], texture.cpu, &surfaces);
      textures[i] = texture.gpu;
   }

   return pan_pool_upload_aligned(pool, textures, tex_count * sizeof(uint64_t),
                                  sizeof(uint64_t));
#endif
}

#if PAN_ARCH >= 8
/* TODO: cache */
static uint64_t
pan_preload_emit_zs(struct pan_pool *pool, bool z, bool s)
{
   struct pan_ptr zsd = pan_pool_alloc_desc(pool, DEPTH_STENCIL);

   if (!zsd.cpu)
      return 0;

   pan_cast_and_pack(zsd.cpu, DEPTH_STENCIL, cfg) {
      cfg.depth_function = MALI_FUNC_ALWAYS;
      cfg.depth_write_enable = z;

      if (z)
         cfg.depth_source = MALI_DEPTH_SOURCE_SHADER;

      cfg.stencil_test_enable = s;
      cfg.stencil_from_shader = s;

      cfg.front_compare_function = MALI_FUNC_ALWAYS;
      cfg.front_stencil_fail = MALI_STENCIL_OP_REPLACE;
      cfg.front_depth_fail = MALI_STENCIL_OP_REPLACE;
      cfg.front_depth_pass = MALI_STENCIL_OP_REPLACE;
      cfg.front_write_mask = 0xFF;
      cfg.front_value_mask = 0xFF;

      cfg.back_compare_function = MALI_FUNC_ALWAYS;
      cfg.back_stencil_fail = MALI_STENCIL_OP_REPLACE;
      cfg.back_depth_fail = MALI_STENCIL_OP_REPLACE;
      cfg.back_depth_pass = MALI_STENCIL_OP_REPLACE;
      cfg.back_write_mask = 0xFF;
      cfg.back_value_mask = 0xFF;

      cfg.depth_cull_enable = false;
   }

   return zsd.gpu;
}
#else
static uint64_t
pan_preload_emit_viewport(struct pan_pool *pool, uint16_t minx, uint16_t miny,
                          uint16_t maxx, uint16_t maxy)
{
   struct pan_ptr vp = pan_pool_alloc_desc(pool, VIEWPORT);

   if (!vp.cpu)
      return 0;

   pan_cast_and_pack(vp.cpu, VIEWPORT, cfg) {
      cfg.scissor_minimum_x = minx;
      cfg.scissor_minimum_y = miny;
      cfg.scissor_maximum_x = maxx;
      cfg.scissor_maximum_y = maxy;
   }

   return vp.gpu;
}
#endif

static void
pan_preload_emit_dcd(struct pan_fb_preload_cache *cache, struct pan_pool *pool,
                     struct pan_fb_info *fb, bool zs, uint64_t coordinates,
                     uint64_t tsd, struct mali_draw_packed *out,
                     bool always_write)
{
   unsigned tex_count = 0;
   uint64_t textures = pan_preload_emit_textures(pool, fb, zs, &tex_count);
   uint64_t samplers = pan_preload_emit_sampler(pool, true);
   uint64_t varyings = pan_preload_emit_varying(pool);
   uint64_t varying_buffers =
      pan_preload_emit_varying_buffer(pool, coordinates);

   /* Tiles updated by preload shaders are still considered clean (separate
    * for colour and Z/S), allowing us to suppress unnecessary writeback
    */
   UNUSED bool clean_fragment_write = !always_write;

   /* Image view used when patching stencil formats for combined
    * depth/stencil preloads.
    */
   struct pan_image_view patched_s;

   struct pan_preload_views views = pan_preload_get_views(fb, zs, &patched_s);

#if PAN_ARCH <= 7
   pan_pack(out, DRAW, cfg) {
      uint16_t minx = 0, miny = 0, maxx, maxy;

      if (PAN_ARCH == 4) {
         maxx = fb->width - 1;
         maxy = fb->height - 1;
      } else {
         /* Align on 32x32 tiles */
         minx = fb->draw_extent.minx & ~31;
         miny = fb->draw_extent.miny & ~31;
         maxx = MIN2(ALIGN_POT(fb->draw_extent.maxx + 1, 32), fb->width) - 1;
         maxy = MIN2(ALIGN_POT(fb->draw_extent.maxy + 1, 32), fb->height) - 1;
      }

      cfg.thread_storage = tsd;
      cfg.state = pan_preload_get_rsd(cache, &views);

      cfg.position = coordinates;
      cfg.viewport = pan_preload_emit_viewport(pool, minx, miny, maxx, maxy);

      cfg.varyings = varyings;
      cfg.varying_buffers = varying_buffers;
      cfg.textures = textures;
      cfg.samplers = samplers;

#if PAN_ARCH >= 6
      cfg.clean_fragment_write = clean_fragment_write;
#endif
   }
#else
   struct pan_ptr T;
   unsigned nr_tables = ALIGN_POT(PAN_BLIT_NUM_RESOURCE_TABLES,
                                  MALI_RESOURCE_TABLE_SIZE_ALIGNMENT);

   /* Although individual resources need only 16 byte alignment, the
    * resource table as a whole must be 64-byte aligned.
    */
   T = pan_pool_alloc_aligned(pool, nr_tables * pan_size(RESOURCE), 64);
   memset(T.cpu, 0, nr_tables * pan_size(RESOURCE));

   pan_make_resource_table(T, PAN_BLIT_TABLE_TEXTURE, textures, tex_count);
   pan_make_resource_table(T, PAN_BLIT_TABLE_SAMPLER, samplers, 1);
   pan_make_resource_table(T, PAN_BLIT_TABLE_ATTRIBUTE, varyings, 1);
   pan_make_resource_table(T, PAN_BLIT_TABLE_ATTRIBUTE_BUFFER, varying_buffers,
                           1);

   struct pan_preload_shader_key key = pan_preload_get_key(&views);
   const struct pan_preload_shader_data *preload_shader =
      pan_preload_get_shader(cache, &key);

   bool z = fb->zs.preload.z;
   bool s = fb->zs.preload.s;
   bool ms = pan_preload_is_ms(&views);

   struct pan_ptr spd = pan_pool_alloc_desc(pool, SHADER_PROGRAM);

   if (!spd.cpu) {
      mesa_loge("pan_pool_alloc_desc failed");
      return;
   }

   pan_cast_and_pack(spd.cpu, SHADER_PROGRAM, cfg) {
      cfg.stage = MALI_SHADER_STAGE_FRAGMENT;
      cfg.fragment_coverage_bitmask_type = MALI_COVERAGE_BITMASK_TYPE_GL;
      cfg.register_allocation = MALI_SHADER_REGISTER_ALLOCATION_32_PER_THREAD;
      cfg.binary = preload_shader->address;
      cfg.preload.r48_r63 = preload_shader->info.preload >> 48;
   }

   unsigned bd_count = views.rt_count;
   struct pan_ptr blend = pan_pool_alloc_desc_array(pool, bd_count, BLEND);

   if (!blend.cpu) {
      mesa_loge("pan_pool_alloc_desc_array failed");
      return;
   }

   if (!zs) {
      pan_preload_emit_blends(preload_shader, &views, NULL, blend.cpu);
   }

   pan_pack(out, DRAW, cfg) {
      if (zs) {
         /* ZS_EMIT requires late update/kill */
         cfg.flags_0.zs_update_operation = MALI_PIXEL_KILL_FORCE_LATE;
         cfg.flags_0.pixel_kill_operation = MALI_PIXEL_KILL_FORCE_LATE;
         cfg.blend_count = 0;
      } else {
         /* Skipping ATEST requires forcing Z/S */
         cfg.flags_0.zs_update_operation = MALI_PIXEL_KILL_FORCE_EARLY;
         cfg.flags_0.pixel_kill_operation = MALI_PIXEL_KILL_FORCE_EARLY;

         cfg.blend = blend.gpu;
         cfg.blend_count = bd_count;
         cfg.flags_1.render_target_mask = 0x1;
      }

      cfg.flags_0.allow_forward_pixel_to_kill = !zs;
      cfg.flags_0.allow_forward_pixel_to_be_killed = true;
      cfg.depth_stencil = pan_preload_emit_zs(pool, z, s);
      cfg.flags_1.sample_mask = 0xFFFF;
      cfg.flags_0.multisample_enable = ms;
      cfg.flags_0.evaluate_per_sample = ms;
      cfg.flags_0.clean_fragment_write = clean_fragment_write;

#if PAN_ARCH >= 12
      cfg.fragment_resources = T.gpu | nr_tables;
      cfg.fragment_shader = spd.gpu;
      cfg.thread_storage = tsd;
#else
      cfg.maximum_z = 1.0;
      cfg.shader.resources = T.gpu | nr_tables;
      cfg.shader.shader = spd.gpu;
      cfg.shader.thread_storage = tsd;
#endif
   }
#endif
}

#if PAN_ARCH >= 6
static void
pan_preload_fb_alloc_pre_post_dcds(struct pan_pool *desc_pool,
                                   struct pan_fb_info *fb)
{
   if (fb->bifrost.pre_post.dcds.gpu)
      return;

   fb->bifrost.pre_post.dcds = pan_pool_alloc_desc_array(desc_pool, 3, DRAW);
}

static void
pan_preload_emit_pre_frame_dcd(struct pan_fb_preload_cache *cache,
                               struct pan_pool *desc_pool,
                               struct pan_fb_info *fb, bool zs, uint64_t coords,
                               uint64_t tsd)
{
   unsigned dcd_idx = zs ? 1 : 0;
   pan_preload_fb_alloc_pre_post_dcds(desc_pool, fb);
   if (!fb->bifrost.pre_post.dcds.cpu) {
      mesa_loge("pan_preload_fb_alloc_pre_post_dcds failed");
      return;
   }

   void *dcd = fb->bifrost.pre_post.dcds.cpu + (dcd_idx * pan_size(DRAW));

   /* We only use crc_rt to determine whether to force writes for updating
    * the CRCs, so use a conservative tile size (16x16).
    */
   int crc_rt = GENX(pan_select_crc_rt)(fb, 16 * 16);

   bool always_write = false;

   /* If CRC data is currently invalid and this batch will make it valid,
    * write even clean tiles to make sure CRC data is updated. */
   if (crc_rt >= 0) {
      bool *valid = fb->rts[crc_rt].crc_valid;
      bool full = !fb->draw_extent.minx && !fb->draw_extent.miny &&
                  fb->draw_extent.maxx == (fb->width - 1) &&
                  fb->draw_extent.maxy == (fb->height - 1);

      if (full && !(*valid))
         always_write = true;
   }

   pan_preload_emit_dcd(cache, desc_pool, fb, zs, coords, tsd, dcd,
                        always_write);
   if (zs) {
      enum pipe_format fmt = fb->zs.view.zs
                                ? fb->zs.view.zs->planes[0].image->props.format
                                : fb->zs.view.s->planes[0].image->props.format;
      /* On some GPUs (e.g. G31), we must use SHADER_MODE_ALWAYS rather than
       * SHADER_MODE_INTERSECT for full screen operations. Since the full
       * screen rectangle will always intersect, this won't affect
       * performance. The UNUSED tag is because some PAN_ARCH variants do not
       * need this test.
       */
      UNUSED bool always = !fb->draw_extent.minx && !fb->draw_extent.miny &&
                           fb->draw_extent.maxx == (fb->width - 1) &&
                           fb->draw_extent.maxy == (fb->height - 1);

      /* If we're dealing with a combined ZS resource and only one
       * component is cleared, we need to reload the whole surface
       * because the zs_clean_tile_write_enable flag is set in that
       * case.
       */
      if (util_format_is_depth_and_stencil(fmt) &&
          fb->zs.clear.z != fb->zs.clear.s)
         always = true;

      /* We could use INTERSECT on Bifrost v7-v12 too, but
       * EARLY_ZS_ALWAYS has the advantage of reloading the ZS tile
       * buffer one or more tiles ahead, making ZS data immediately
       * available for any ZS tests taking place in other shaders.
       * Thing's haven't been benchmarked to determine what's
       * preferable (saving bandwidth vs having ZS preloaded
       * earlier), so let's leave it like that for now.
       *
       * On v13+, we don't have EARLY_ZS_ALWAYS instead we use
       * PREPASS_ALWAYS / PREPASS_INTERSECT.
       */
#if PAN_ARCH >= 13
      fb->bifrost.pre_post.modes[dcd_idx] =
         always ? MALI_PRE_POST_FRAME_SHADER_MODE_PREPASS_ALWAYS
                : MALI_PRE_POST_FRAME_SHADER_MODE_PREPASS_INTERSECT;
#elif PAN_ARCH > 7 && PAN_ARCH <= 12
      fb->bifrost.pre_post.modes[dcd_idx] =
         MALI_PRE_POST_FRAME_SHADER_MODE_EARLY_ZS_ALWAYS;
#else
      /* EARLY_ZS_ALWAYS was introduced in 7.2, so we have to check the
       * GPU id to find if it's supported, not just PAN_ARCH.
       * The PAN_ARCH check is redundant but allows the compiler to optimize
       * when PAN_ARCH < 7.
       */
      unsigned arch_major = PAN_ARCH_MAJOR(cache->gpu_id);
      unsigned arch_minor = PAN_ARCH_MINOR(cache->gpu_id);
      if (PAN_ARCH >= 7 && (arch_major == 7 && arch_minor >= 2))
         fb->bifrost.pre_post.modes[dcd_idx] =
            MALI_PRE_POST_FRAME_SHADER_MODE_EARLY_ZS_ALWAYS;
      else
         fb->bifrost.pre_post.modes[dcd_idx] =
            always ? MALI_PRE_POST_FRAME_SHADER_MODE_ALWAYS
                   : MALI_PRE_POST_FRAME_SHADER_MODE_INTERSECT;
#endif
   } else {
      fb->bifrost.pre_post.modes[dcd_idx] =
         always_write ? MALI_PRE_POST_FRAME_SHADER_MODE_ALWAYS
                      : MALI_PRE_POST_FRAME_SHADER_MODE_INTERSECT;
   }
}
#else
static struct pan_ptr
pan_preload_emit_tiler_job(struct pan_fb_preload_cache *cache,
                           struct pan_pool *desc_pool, struct pan_fb_info *fb,
                           bool zs, uint64_t coords, uint64_t tsd)
{
   struct pan_ptr job = pan_pool_alloc_desc(desc_pool, TILER_JOB);

   if (!job.cpu)
      return (struct pan_ptr){0};

   pan_preload_emit_dcd(cache, desc_pool, fb, zs, coords, tsd,
                        pan_section_ptr(job.cpu, TILER_JOB, DRAW), false);

   pan_section_pack(job.cpu, TILER_JOB, PRIMITIVE, cfg) {
      cfg.draw_mode = MALI_DRAW_MODE_TRIANGLE_STRIP;
      cfg.index_count = 4;
      cfg.job_task_split = 6;
   }

   pan_section_pack(job.cpu, TILER_JOB, PRIMITIVE_SIZE, cfg) {
      cfg.fixed_sized = 1.0f;
   }

   void *invoc = pan_section_ptr(job.cpu, TILER_JOB, INVOCATION);
   pan_pack_work_groups_compute(invoc, 1, 4, 1, 1, 1, 1, true, false);

   return job;
}
#endif

static struct pan_ptr
pan_preload_fb_part(struct pan_fb_preload_cache *cache, struct pan_pool *pool,
                    struct pan_fb_info *fb, bool zs, uint64_t coords,
                    uint64_t tsd)
{
   struct pan_ptr job = {0};

#if PAN_ARCH >= 6
   pan_preload_emit_pre_frame_dcd(cache, pool, fb, zs, coords, tsd);
#else
   job = pan_preload_emit_tiler_job(cache, pool, fb, zs, coords, tsd);
#endif
   return job;
}

unsigned
GENX(pan_preload_fb)(struct pan_fb_preload_cache *cache, struct pan_pool *pool,
                     struct pan_fb_info *fb, uint64_t tsd, struct pan_ptr *jobs)
{
   PAN_TRACE_FUNC(PAN_TRACE_GL_FB_PRELOAD);

   bool preload_zs = pan_preload_needed(fb, true);
   bool preload_rts = pan_preload_needed(fb, false);
   uint64_t coords;

   if (!preload_zs && !preload_rts)
      return 0;

   float rect[] = {
      0.0,       0.0,        0, 1.0,
      fb->width, 0.0,        0, 1.0,
      0.0,       fb->height, 0, 1.0,
      fb->width, fb->height, 0, 1.0,
   };

   coords = pan_pool_upload_aligned(pool, rect, sizeof(rect), 64);

   unsigned njobs = 0;
   if (preload_zs) {
      struct pan_ptr job =
         pan_preload_fb_part(cache, pool, fb, true, coords, tsd);
      if (jobs && job.cpu)
         jobs[njobs++] = job;
   }

   if (preload_rts) {
      struct pan_ptr job =
         pan_preload_fb_part(cache, pool, fb, false, coords, tsd);
      if (jobs && job.cpu)
         jobs[njobs++] = job;
   }

   return njobs;
}

DERIVE_HASH_TABLE(pan_preload_shader_key);
DERIVE_HASH_TABLE(pan_preload_rsd_key);

static void
pan_preload_prefill_preload_shader_cache(struct pan_fb_preload_cache *cache)
{
   static const struct pan_preload_shader_key prefill[] = {
      {
         .surfaces[0] =
            {
               .loc = FRAG_RESULT_DEPTH,
               .type = nir_type_float32,
               .dim = MALI_TEXTURE_DIMENSION_2D,
               .samples = 1,
            },
      },
      {
         .surfaces[1] =
            {
               .loc = FRAG_RESULT_STENCIL,
               .type = nir_type_uint32,
               .dim = MALI_TEXTURE_DIMENSION_2D,
               .samples = 1,
            },
      },
      {
         .surfaces[0] =
            {
               .loc = FRAG_RESULT_DATA0,
               .type = nir_type_float32,
               .dim = MALI_TEXTURE_DIMENSION_2D,
               .samples = 1,
            },
      },
   };

   for (unsigned i = 0; i < ARRAY_SIZE(prefill); i++)
      pan_preload_get_shader(cache, &prefill[i]);
}

void
GENX(pan_fb_preload_cache_init)(
   struct pan_fb_preload_cache *cache, uint64_t gpu_id, uint32_t gpu_variant,
   struct pan_blend_shader_cache *blend_shader_cache, struct pan_pool *bin_pool,
   struct pan_pool *desc_pool)
{
   cache->gpu_id = gpu_id;
   cache->gpu_variant = gpu_variant;
   cache->shaders.preload = pan_preload_shader_key_table_create(NULL);
   cache->shaders.pool = bin_pool;
   pthread_mutex_init(&cache->shaders.lock, NULL);
   pan_preload_prefill_preload_shader_cache(cache);

   cache->rsds.pool = desc_pool;
   cache->rsds.rsds = pan_preload_rsd_key_table_create(NULL);
   pthread_mutex_init(&cache->rsds.lock, NULL);
   cache->blend_shader_cache = blend_shader_cache;
}

void
GENX(pan_fb_preload_cache_cleanup)(struct pan_fb_preload_cache *cache)
{
   _mesa_hash_table_destroy(cache->shaders.preload, NULL);
   pthread_mutex_destroy(&cache->shaders.lock);
   _mesa_hash_table_destroy(cache->rsds.rsds, NULL);
   pthread_mutex_destroy(&cache->rsds.lock);
}
