/*
 * Copyright © 2022 Collabora Ltd
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "vk_meta_private.h"

#include "vk_command_buffer.h"
#include "vk_device.h"
#include "vk_format.h"
#include "vk_pipeline.h"
#include "vk_render_pass.h"

#include "nir_builder.h"

struct vk_meta_blit_key {
   enum vk_meta_object_key_type key_type;
   enum glsl_sampler_dim dim;
   VkSampleCountFlagBits src_samples;
   VkResolveModeFlagBits resolve_mode;
   VkResolveModeFlagBits stencil_resolve_mode;
   bool stencil_as_discard;
   VkFormat dst_format;
   VkImageAspectFlags aspects;
};

static enum glsl_sampler_dim
vk_image_sampler_dim(const struct vk_image *image)
{
   switch (image->image_type) {
   case VK_IMAGE_TYPE_1D: return GLSL_SAMPLER_DIM_1D;
   case VK_IMAGE_TYPE_2D:
      if (image->samples > 1)
         return GLSL_SAMPLER_DIM_MS;
      else
         return GLSL_SAMPLER_DIM_2D;
   case VK_IMAGE_TYPE_3D: return GLSL_SAMPLER_DIM_3D;
   default: UNREACHABLE("Invalid image type");
   }
}

enum blit_desc_binding {
   BLIT_DESC_BINDING_SAMPLER,
   BLIT_DESC_BINDING_COLOR,
   BLIT_DESC_BINDING_DEPTH,
   BLIT_DESC_BINDING_STENCIL,
};

static enum blit_desc_binding
aspect_to_tex_binding(VkImageAspectFlagBits aspect)
{
   switch (aspect) {
   case VK_IMAGE_ASPECT_COLOR_BIT: return BLIT_DESC_BINDING_COLOR;
   case VK_IMAGE_ASPECT_DEPTH_BIT: return BLIT_DESC_BINDING_DEPTH;
   case VK_IMAGE_ASPECT_STENCIL_BIT: return BLIT_DESC_BINDING_STENCIL;
   default: UNREACHABLE("Unsupported aspect");
   }
}

struct vk_meta_blit_push_data {
   float x_off, y_off, x_scale, y_scale;
   float z_off, z_scale;
   uint32_t pad;
   uint32_t stencil_bit;
};

static inline void
compute_off_scale(uint32_t src_level_size,
                  bool normalize_src_coords,
                  uint32_t src0, uint32_t src1,
                  uint32_t dst0, uint32_t dst1,
                  uint32_t *dst0_out, uint32_t *dst1_out,
                  float *off_out, float *scale_out)
{
   assert(src0 <= src_level_size && src1 <= src_level_size);

   if (dst0 < dst1) {
      *dst0_out = dst0;
      *dst1_out = dst1;
   } else {
      *dst0_out = dst1;
      *dst1_out = dst0;

      /* Flip the source region */
      SWAP(src0, src1);
   }

   double src_region_size = (double)src1 - (double)src0;
   assert(src_region_size != 0);

   double dst_region_size = (double)*dst1_out - (double)*dst0_out;
   assert(dst_region_size > 0);

   /* The offset and scal returned by this function will be passed to a fma()
    * to convert from destination coordinates to source coordinates so the
    * scale is applied first, followed by the offset.  This means that the
    * offset needs to be in (possibly normalized) source coordinates.
    *
    * The destination coordinates are in integer pixels since they're either
    * gl_FragCoord.xy or a gl_Layer for Z.  The source coordinates may or may
    * not be normalized to the range [0, 1].  For X/Y, they're always
    * normalized but for Z they're normalized for 3D source textures and
    * left in integer space for array slices.
    */
   double src_scale = normalize_src_coords ? (double)src_level_size : 1.0;
   double src_offset = src0 / src_scale;
   double dst_scale = src_region_size / (src_scale * dst_region_size);
   double dst_offset = (double)*dst0_out * dst_scale;

   *off_out = src_offset - dst_offset;
   *scale_out = dst_scale;
}

static inline nir_def *
load_struct_var(nir_builder *b, nir_variable *var, uint32_t field)
{
   nir_deref_instr *deref =
      nir_build_deref_struct(b, nir_build_deref_var(b, var), field);
   return nir_load_deref(b, deref);
}

static nir_def *
build_tex_resolve(nir_builder *b, nir_deref_instr *t,
                  nir_def *coord,
                  VkSampleCountFlagBits samples,
                  VkResolveModeFlagBits resolve_mode)
{
   nir_def *accum = nir_txf_ms(b, coord, nir_imm_int(b, 0), .texture_deref = t);
   if (resolve_mode == VK_RESOLVE_MODE_SAMPLE_ZERO_BIT)
      return accum;

   const enum glsl_base_type base_type =
      glsl_get_sampler_result_type(t->type);

   for (unsigned i = 1; i < samples; i++) {
      nir_def *val = nir_txf_ms(b, coord, nir_imm_int(b, i), .texture_deref = t);
      switch (resolve_mode) {
      case VK_RESOLVE_MODE_AVERAGE_BIT:
         assert(base_type == GLSL_TYPE_FLOAT);
         accum = nir_fadd(b, accum, val);
         break;

      case VK_RESOLVE_MODE_MIN_BIT:
         switch (base_type) {
         case GLSL_TYPE_UINT:
            accum = nir_umin(b, accum, val);
            break;
         case GLSL_TYPE_INT:
            accum = nir_imin(b, accum, val);
            break;
         case GLSL_TYPE_FLOAT:
            accum = nir_fmin(b, accum, val);
            break;
         default:
            UNREACHABLE("Invalid sample result type");
         }
         break;

      case VK_RESOLVE_MODE_MAX_BIT:
         switch (base_type) {
         case GLSL_TYPE_UINT:
            accum = nir_umax(b, accum, val);
            break;
         case GLSL_TYPE_INT:
            accum = nir_imax(b, accum, val);
            break;
         case GLSL_TYPE_FLOAT:
            accum = nir_fmax(b, accum, val);
            break;
         default:
            UNREACHABLE("Invalid sample result type");
         }
         break;

      default:
         UNREACHABLE("Unsupported resolve mode");
      }
   }

   if (resolve_mode == VK_RESOLVE_MODE_AVERAGE_BIT)
      accum = nir_fmul_imm(b, accum, 1.0 / samples);

   return accum;
}

static nir_shader *
build_blit_shader(const struct vk_meta_blit_key *key)
{
   nir_builder build;
   if (key->resolve_mode || key->stencil_resolve_mode) {
      build = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL,
                                             "vk-meta-resolve");
   } else {
      build = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
                                             NULL, "vk-meta-blit");
   }
   nir_builder *b = &build;

   struct glsl_struct_field push_fields[] = {
      { .type = glsl_vec4_type(), .name = "xy_xform", .offset = 0 },
      { .type = glsl_vec4_type(), .name = "z_xform", .offset = 16 },
   };
   const struct glsl_type *push_iface_type =
      glsl_interface_type(push_fields, ARRAY_SIZE(push_fields),
                          GLSL_INTERFACE_PACKING_STD140,
                          false /* row_major */, "push");
   nir_variable *push = nir_variable_create(b->shader, nir_var_mem_push_const,
                                            push_iface_type, "push");

   nir_def *xy_xform = load_struct_var(b, push, 0);
   nir_def *xy_off = nir_channels(b, xy_xform, 3 << 0);
   nir_def *xy_scale = nir_channels(b, xy_xform, 3 << 2);

   nir_def *out_coord_xy = nir_load_frag_coord(b);
   out_coord_xy = nir_trim_vector(b, out_coord_xy, 2);
   nir_def *src_coord_xy = nir_ffma_weak(b, out_coord_xy, xy_scale, xy_off);

   nir_def *z_xform = load_struct_var(b, push, 1);
   nir_def *z_off = nir_channel(b, z_xform, 0);
   nir_def *z_scale = nir_channel(b, z_xform, 1);

   nir_def *out_layer = nir_load_layer_id(b);
   /* Add 0.5 to get center-pixel sampling. */
   nir_def *out_coord_z = nir_fadd_imm(b, nir_u2f32(b, out_layer), 0.5);
   nir_def *src_coord_z = nir_ffma_weak(b, out_coord_z, z_scale, z_off);

   /* We use center-pixel coordinates for the transform calculation but
    * texelFetch() will round the array index to the nearest integer.
    * Subtract 0.5 to re-center on zero.  It's important that we add 0.5 and
    * then subtract it back off again because, even if our source is a 2D
    * array image, z_scale may still be -1.0.  If we don't add and subtract
    * 0.5, we'll end up off by a pixel.
    */
   if (key->dim != GLSL_SAMPLER_DIM_3D)
      src_coord_z = nir_fadd_imm(b, src_coord_z, -0.5);

   nir_def *src_coord;
   if (key->dim == GLSL_SAMPLER_DIM_1D) {
      src_coord = nir_vec2(b, nir_channel(b, src_coord_xy, 0),
                              src_coord_z);
   } else {
      src_coord = nir_vec3(b, nir_channel(b, src_coord_xy, 0),
                              nir_channel(b, src_coord_xy, 1),
                              src_coord_z);
   }

   nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
                                               glsl_bare_sampler_type(), NULL);
   sampler->data.descriptor_set = 0;
   sampler->data.binding = BLIT_DESC_BINDING_SAMPLER;
   nir_deref_instr *s = nir_build_deref_var(b, sampler);

   u_foreach_bit(a, key->aspects) {
      VkImageAspectFlagBits aspect = (1 << a);

      enum glsl_base_type base_type;
      unsigned out_location, out_comps;
      const char *tex_name, *out_name;
      VkResolveModeFlagBits resolve_mode;
      switch (aspect) {
      case VK_IMAGE_ASPECT_COLOR_BIT:
         tex_name = "color_tex";
         if (vk_format_is_sint(key->dst_format))
            base_type = GLSL_TYPE_INT;
         else if (vk_format_is_uint(key->dst_format))
            base_type = GLSL_TYPE_UINT;
         else
            base_type = GLSL_TYPE_FLOAT;
         resolve_mode = key->resolve_mode;
         out_name = "gl_FragData[0]";
         out_location = FRAG_RESULT_DATA0;
         out_comps = 4;
         break;
      case VK_IMAGE_ASPECT_DEPTH_BIT:
         tex_name = "depth_tex";
         base_type = GLSL_TYPE_FLOAT;
         resolve_mode = key->resolve_mode;
         out_name = "gl_FragDepth";
         out_location = FRAG_RESULT_DEPTH;
         out_comps = 1;
         break;
      case VK_IMAGE_ASPECT_STENCIL_BIT:
         tex_name = "stencil_tex";
         base_type = GLSL_TYPE_UINT;
         resolve_mode = key->stencil_resolve_mode;
         out_name = "gl_FragStencilRef";
         out_location = FRAG_RESULT_STENCIL;
         out_comps = 1;
         break;
      default:
         UNREACHABLE("Unsupported aspect");
      }

      const bool is_array = key->dim != GLSL_SAMPLER_DIM_3D;
      const struct glsl_type *texture_type =
         glsl_sampler_type(key->dim, false, is_array, base_type);
      nir_variable *texture = nir_variable_create(b->shader, nir_var_uniform,
                                                  texture_type, tex_name);
      texture->data.descriptor_set = 0;
      texture->data.binding = aspect_to_tex_binding(aspect);
      nir_deref_instr *t = nir_build_deref_var(b, texture);

      nir_def *val;
      if (resolve_mode == VK_RESOLVE_MODE_NONE) {
         val = nir_tex(b, src_coord, .lod = nir_imm_float(b, 0),
                       .texture_deref = t, .sampler_deref = s);
      } else {
         val = build_tex_resolve(b, t, nir_f2u32(b, src_coord),
                                 key->src_samples, resolve_mode);
      }
      val = nir_trim_vector(b, val, out_comps);

      if (key->stencil_as_discard) {
         assert(key->aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
         nir_def *stencil_bit = nir_channel(b, z_xform, 3);
         nir_demote_if(b, nir_ieq(b, nir_iand(b, val, stencil_bit),
                                     nir_imm_int(b, 0)));
      } else {
         const struct glsl_type *out_type =
            glsl_vector_type(base_type, out_comps);
         nir_variable *out = nir_variable_create(b->shader, nir_var_shader_out,
                                                 out_type, out_name);
         out->data.location = out_location;

         nir_store_var(b, out, val, BITFIELD_MASK(out_comps));
      }
   }

   return b->shader;
}

static VkResult
get_blit_pipeline_layout(struct vk_device *device,
                         struct vk_meta_device *meta,
                         VkPipelineLayout *layout_out)
{
   enum vk_meta_object_key_type key = VK_META_OBJECT_KEY_BLIT;

   const VkDescriptorSetLayoutBinding bindings[] = {{
      .binding = BLIT_DESC_BINDING_SAMPLER,
      .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER,
      .descriptorCount = 1,
      .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
   }, {
      .binding = BLIT_DESC_BINDING_COLOR,
      .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
      .descriptorCount = 1,
      .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
   }, {
      .binding = BLIT_DESC_BINDING_DEPTH,
      .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
      .descriptorCount = 1,
      .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
   }, {
      .binding = BLIT_DESC_BINDING_STENCIL,
      .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
      .descriptorCount = 1,
      .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
   }};

   const VkDescriptorSetLayoutCreateInfo desc_info = {
      .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
      .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
      .bindingCount = ARRAY_SIZE(bindings),
      .pBindings = bindings,
   };

   const VkPushConstantRange push_range = {
      .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
      .offset = 0,
      .size = sizeof(struct vk_meta_blit_push_data),
   };

   return vk_meta_get_pipeline_layout(device, meta, &desc_info, &push_range,
                                      &key, sizeof(key), layout_out);
}

static VkResult
get_blit_pipeline(struct vk_device *device,
                  struct vk_meta_device *meta,
                  const struct vk_meta_blit_key *key,
                  VkPipelineLayout layout,
                  VkPipeline *pipeline_out)
{
   VkPipeline from_cache = vk_meta_lookup_pipeline(meta, key, sizeof(*key));
   if (from_cache != VK_NULL_HANDLE) {
      *pipeline_out = from_cache;
      return VK_SUCCESS;
   }

   const VkPipelineShaderStageNirCreateInfoMESA fs_nir_info = {
      .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_NIR_CREATE_INFO_MESA,
      .nir = build_blit_shader(key),
   };
   const VkPipelineShaderStageCreateInfo fs_info = {
      .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
      .pNext = &fs_nir_info,
      .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
      .pName = "main",
   };

   VkPipelineDepthStencilStateCreateInfo ds_info = {
      .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
   };
   VkDynamicState dyn_tmp;
   VkPipelineDynamicStateCreateInfo dyn_info = {
      .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
   };
   struct vk_meta_rendering_info render = {
      .samples = 1,
   };
   if (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) {
      render.color_attachment_count = 1;
      render.color_attachment_formats[0] = key->dst_format;
      render.color_attachment_write_masks[0] =
         VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
         VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
   }
   if (key->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
      ds_info.depthTestEnable = VK_TRUE;
      ds_info.depthWriteEnable = VK_TRUE;
      ds_info.depthCompareOp = VK_COMPARE_OP_ALWAYS;
      render.depth_attachment_format = key->dst_format;
   }
   if (key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
      ds_info.stencilTestEnable = VK_TRUE;
      ds_info.front.compareOp = VK_COMPARE_OP_ALWAYS;
      ds_info.front.passOp = VK_STENCIL_OP_REPLACE;
      ds_info.front.compareMask = ~0u;
      ds_info.front.writeMask = ~0u;
      ds_info.front.reference = ~0;
      ds_info.back = ds_info.front;
      if (key->stencil_as_discard) {
         dyn_tmp = VK_DYNAMIC_STATE_STENCIL_WRITE_MASK;
         dyn_info.dynamicStateCount = 1;
         dyn_info.pDynamicStates = &dyn_tmp;
      }
      render.stencil_attachment_format = key->dst_format;
   }

   const VkGraphicsPipelineCreateInfo info = {
      .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
      .stageCount = 1,
      .pStages = &fs_info,
      .pDepthStencilState = &ds_info,
      .pDynamicState = &dyn_info,
      .layout = layout,
   };

   VkResult result = vk_meta_create_graphics_pipeline(device, meta, &info,
                                                      &render,
                                                      key, sizeof(*key),
                                                      pipeline_out);
   ralloc_free(fs_nir_info.nir);

   return result;
}

static VkResult
get_blit_sampler(struct vk_device *device,
                 struct vk_meta_device *meta,
                 VkFilter filter,
                 VkSampler *sampler_out)
{
   struct {
      enum vk_meta_object_key_type key_type;
      VkFilter filter;
   } key;

   memset(&key, 0, sizeof(key));
   key.key_type = VK_META_OBJECT_KEY_BLIT;
   key.filter = filter;

   VkSampler from_cache = vk_meta_lookup_sampler(meta, &key, sizeof(key));
   if (from_cache != VK_NULL_HANDLE) {
      *sampler_out = from_cache;
      return VK_SUCCESS;
   }

   const VkSamplerCreateInfo info = {
      .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
      .magFilter = filter,
      .minFilter = filter,
      .mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST,
      .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
      .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
      .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
      .unnormalizedCoordinates = VK_FALSE,
   };

   return vk_meta_create_sampler(device, meta, &info,
                                 &key, sizeof(key), sampler_out);
}

static void
do_blit(struct vk_command_buffer *cmd,
        struct vk_meta_device *meta,
        struct vk_image *src_image,
        VkFormat src_format,
        VkImageLayout src_image_layout,
        VkImageSubresourceLayers src_subres,
        struct vk_image *dst_image,
        VkFormat dst_format,
        VkImageLayout dst_image_layout,
        VkImageSubresourceLayers dst_subres,
        VkSampler sampler,
        struct vk_meta_blit_key *key,
        struct vk_meta_blit_push_data *push,
        const struct vk_meta_rect *dst_rect,
        uint32_t dst_layer_count)
{
   struct vk_device *device = cmd->base.device;
   const struct vk_device_dispatch_table *disp = &device->dispatch_table;
   VkResult result;

   VkPipelineLayout pipeline_layout;
   result = get_blit_pipeline_layout(device, meta, &pipeline_layout);
   if (unlikely(result != VK_SUCCESS)) {
      vk_command_buffer_set_error(cmd, result);
      return;
   }

   uint32_t desc_count = 0;
   VkDescriptorImageInfo image_infos[3];
   VkWriteDescriptorSet desc_writes[3];

   if (sampler != VK_NULL_HANDLE) {
      image_infos[desc_count] = (VkDescriptorImageInfo) {
         .sampler = sampler,
      };
      desc_writes[desc_count] = (VkWriteDescriptorSet) {
         .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
         .dstBinding = BLIT_DESC_BINDING_SAMPLER,
         .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER,
         .descriptorCount = 1,
         .pImageInfo = &image_infos[desc_count],
      };
      desc_count++;
   }

   u_foreach_bit(a, src_subres.aspectMask) {
      VkImageAspectFlagBits aspect = (1 << a);

      VkImageView src_view;
      const VkImageViewUsageCreateInfo src_view_usage = {
         .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
         .usage = VK_IMAGE_USAGE_SAMPLED_BIT,
      };
      const VkImageViewCreateInfo src_view_info = {
         .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
         .pNext = &src_view_usage,
         .flags = VK_IMAGE_VIEW_CREATE_DRIVER_INTERNAL_BIT_MESA,
         .image = vk_image_to_handle(src_image),
         .viewType = vk_image_sampled_view_type(src_image),
         .format = src_format,
         .subresourceRange = {
            .aspectMask = aspect,
            .baseMipLevel = src_subres.mipLevel,
            .levelCount = 1,
            .baseArrayLayer = src_subres.baseArrayLayer,
            .layerCount = src_subres.layerCount,
         },
      };
      result = vk_meta_create_image_view(cmd, meta, &src_view_info,
                                         &src_view);
      if (unlikely(result != VK_SUCCESS)) {
         vk_command_buffer_set_error(cmd, result);
         return;
      }

      assert(desc_count < ARRAY_SIZE(image_infos));
      assert(desc_count < ARRAY_SIZE(desc_writes));
      image_infos[desc_count] = (VkDescriptorImageInfo) {
         .imageView = src_view,
      };
      desc_writes[desc_count] = (VkWriteDescriptorSet) {
         .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
         .dstBinding = aspect_to_tex_binding(aspect),
         .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
         .descriptorCount = 1,
         .pImageInfo = &image_infos[desc_count],
      };
      desc_count++;
   }

   disp->CmdPushDescriptorSetKHR(vk_command_buffer_to_handle(cmd),
                                 VK_PIPELINE_BIND_POINT_GRAPHICS,
                                 pipeline_layout, 0,
                                 desc_count, desc_writes);

   assert(dst_subres.aspectMask == src_subres.aspectMask);
   VkImageAspectFlags aspects_left = dst_subres.aspectMask;

   while (aspects_left) {
      key->aspects = aspects_left;

      /* If we need to write stencil via iterative discard, it has to be
       * written by itself because otherwise the discards would also throw
       * away color or depth data.
       */
      if ((key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
          key->aspects != VK_IMAGE_ASPECT_STENCIL_BIT &&
          !meta->use_stencil_export)
         key->aspects &= ~VK_IMAGE_ASPECT_STENCIL_BIT;

      key->stencil_as_discard = key->aspects == VK_IMAGE_ASPECT_STENCIL_BIT &&
                                !meta->use_stencil_export;

      VkImageView dst_view;
      const VkImageViewUsageCreateInfo dst_view_usage = {
         .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
         .usage = (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) ?
                  VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT :
                  VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT,
      };
      const VkImageViewCreateInfo dst_view_info = {
         .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
         .pNext = &dst_view_usage,
         .flags = VK_IMAGE_VIEW_CREATE_DRIVER_INTERNAL_BIT_MESA,
         .image = vk_image_to_handle(dst_image),
         .viewType = vk_image_sampled_view_type(dst_image),
         .format = dst_format,
         .subresourceRange = {
            .aspectMask = dst_subres.aspectMask,
            .baseMipLevel = dst_subres.mipLevel,
            .levelCount = 1,
            .baseArrayLayer = dst_subres.baseArrayLayer,
            .layerCount = dst_subres.layerCount,
         },
      };
      result = vk_meta_create_image_view(cmd, meta, &dst_view_info,
                                         &dst_view);
      if (unlikely(result != VK_SUCCESS)) {
         vk_command_buffer_set_error(cmd, result);
         return;
      }

      const VkRenderingAttachmentInfo vk_att = {
         .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
         .imageView = dst_view,
         .imageLayout = dst_image_layout,
         .loadOp = key->stencil_as_discard ? VK_ATTACHMENT_LOAD_OP_CLEAR :
                                             VK_ATTACHMENT_LOAD_OP_DONT_CARE,
         .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
      };
      VkRenderingInfo vk_render = {
         .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
         .renderArea = {
            .offset = {
               dst_rect->x0,
               dst_rect->y0
            },
            .extent = {
               dst_rect->x1 - dst_rect->x0,
               dst_rect->y1 - dst_rect->y0
            },
         },
         .layerCount = dst_rect->layer + dst_layer_count,
      };

      if (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) {
         vk_render.colorAttachmentCount = 1;
         vk_render.pColorAttachments = &vk_att;
      }
      if (key->aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
         vk_render.pDepthAttachment = &vk_att;
      if (key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT)
         vk_render.pStencilAttachment = &vk_att;

      disp->CmdBeginRendering(vk_command_buffer_to_handle(cmd), &vk_render);

      VkPipeline pipeline;
      result = get_blit_pipeline(device, meta, key,
                                 pipeline_layout, &pipeline);
      if (unlikely(result != VK_SUCCESS)) {
         vk_command_buffer_set_error(cmd, result);
         return;
      }

      disp->CmdBindPipeline(vk_command_buffer_to_handle(cmd),
                            VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);

      if (key->stencil_as_discard) {
         for (uint32_t i = 0; i < 8; i++) {
            push->stencil_bit = BITFIELD_BIT(i);
            disp->CmdPushConstants(vk_command_buffer_to_handle(cmd),
                                   pipeline_layout,
                                   VK_SHADER_STAGE_FRAGMENT_BIT,
                                   0, sizeof(*push), push);

            disp->CmdSetStencilWriteMask(vk_command_buffer_to_handle(cmd),
                                         VK_STENCIL_FACE_FRONT_AND_BACK,
                                         push->stencil_bit);

            meta->cmd_draw_volume(cmd, meta, dst_rect, dst_layer_count);
         }
      } else {
         disp->CmdPushConstants(vk_command_buffer_to_handle(cmd),
                                pipeline_layout,
                                VK_SHADER_STAGE_FRAGMENT_BIT,
                                0, sizeof(*push), push);

         meta->cmd_draw_volume(cmd, meta, dst_rect, dst_layer_count);
      }

      disp->CmdEndRendering(vk_command_buffer_to_handle(cmd));

      aspects_left &= ~key->aspects;
   }
}

void
vk_meta_blit_image(struct vk_command_buffer *cmd,
                   struct vk_meta_device *meta,
                   struct vk_image *src_image,
                   VkFormat src_format,
                   VkImageLayout src_image_layout,
                   struct vk_image *dst_image,
                   VkFormat dst_format,
                   VkImageLayout dst_image_layout,
                   uint32_t region_count,
                   const VkImageBlit2 *regions,
                   VkFilter filter)
{
   struct vk_device *device = cmd->base.device;
   VkResult result;

   VkSampler sampler;
   result = get_blit_sampler(device, meta, filter, &sampler);
   if (unlikely(result != VK_SUCCESS)) {
      vk_command_buffer_set_error(cmd, result);
      return;
   }

   struct vk_meta_blit_key key;
   memset(&key, 0, sizeof(key));
   key.key_type = VK_META_OBJECT_KEY_BLIT;
   key.src_samples = src_image->samples;
   key.dim = vk_image_sampler_dim(src_image);
   key.dst_format = dst_format;

   for (uint32_t r = 0; r < region_count; r++) {
      struct vk_meta_blit_push_data push = {0};
      struct vk_meta_rect dst_rect = {0};

      uint32_t src_level = regions[r].srcSubresource.mipLevel;
      VkExtent3D src_extent = vk_image_mip_level_extent(src_image, src_level);

      compute_off_scale(src_extent.width, true,
                        regions[r].srcOffsets[0].x,
                        regions[r].srcOffsets[1].x,
                        regions[r].dstOffsets[0].x,
                        regions[r].dstOffsets[1].x,
                        &dst_rect.x0, &dst_rect.x1,
                        &push.x_off, &push.x_scale);
      compute_off_scale(src_extent.height, true,
                        regions[r].srcOffsets[0].y,
                        regions[r].srcOffsets[1].y,
                        regions[r].dstOffsets[0].y,
                        regions[r].dstOffsets[1].y,
                        &dst_rect.y0, &dst_rect.y1,
                        &push.y_off, &push.y_scale);

      VkImageSubresourceLayers src_subres = regions[r].srcSubresource;
      src_subres.layerCount =
         vk_image_subresource_layer_count(src_image, &src_subres);

      VkImageSubresourceLayers dst_subres = regions[r].dstSubresource;
      dst_subres.layerCount =
         vk_image_subresource_layer_count(dst_image, &dst_subres);

      uint32_t src_level_layers = src_image->image_type == VK_IMAGE_TYPE_3D
                                  ? src_extent.depth : src_subres.layerCount;

      uint32_t src_z_layer[2];
      if (src_image->image_type == VK_IMAGE_TYPE_3D) {
         src_z_layer[0] = regions[r].srcOffsets[0].z;
         src_z_layer[1] = regions[r].srcOffsets[1].z;
      } else {
         /* baseArrayLayer is applied by the image view */
         src_z_layer[0] = 0;
         src_z_layer[1] = src_subres.layerCount;
      }

      uint32_t dst_z_layer[2];
      if (dst_image->image_type == VK_IMAGE_TYPE_3D) {
         dst_z_layer[0] = regions[r].dstOffsets[0].z;
         dst_z_layer[1] = regions[r].dstOffsets[1].z;
      } else {
         /* baseArrayLayer is applied by the image view */
         dst_z_layer[0] = 0;
         dst_z_layer[1] = dst_subres.layerCount;
      }

      compute_off_scale(src_level_layers,
                        src_image->image_type == VK_IMAGE_TYPE_3D,
                        src_z_layer[0], src_z_layer[1],
                        dst_z_layer[0], dst_z_layer[1],
                        &dst_z_layer[0], &dst_z_layer[1],
                        &push.z_off, &push.z_scale);

      dst_rect.layer = dst_z_layer[0];
      uint32_t dst_layer_count = dst_z_layer[1] - dst_z_layer[0];

      do_blit(cmd, meta,
              src_image, src_format, src_image_layout, src_subres,
              dst_image, dst_format, dst_image_layout, dst_subres,
              sampler, &key, &push, &dst_rect, dst_layer_count);
   }
}

void
vk_meta_blit_image2(struct vk_command_buffer *cmd,
                    struct vk_meta_device *meta,
                    const VkBlitImageInfo2 *blit)
{
   VK_FROM_HANDLE(vk_image, src_image, blit->srcImage);
   VK_FROM_HANDLE(vk_image, dst_image, blit->dstImage);

   vk_meta_blit_image(cmd, meta,
                      src_image, src_image->format, blit->srcImageLayout,
                      dst_image, dst_image->format, blit->dstImageLayout,
                      blit->regionCount, blit->pRegions, blit->filter);
}

void
vk_meta_resolve_image(struct vk_command_buffer *cmd,
                      struct vk_meta_device *meta,
                      struct vk_image *src_image,
                      VkFormat src_format,
                      VkImageLayout src_image_layout,
                      struct vk_image *dst_image,
                      VkFormat dst_format,
                      VkImageLayout dst_image_layout,
                      uint32_t region_count,
                      const VkImageResolve2 *regions,
                      VkResolveModeFlagBits resolve_mode,
                      VkResolveModeFlagBits stencil_resolve_mode)
{
   struct vk_meta_blit_key key;
   memset(&key, 0, sizeof(key));
   key.key_type = VK_META_OBJECT_KEY_BLIT;
   key.dim = vk_image_sampler_dim(src_image);
   key.src_samples = src_image->samples;
   key.resolve_mode = resolve_mode;
   key.stencil_resolve_mode = stencil_resolve_mode;
   key.dst_format = dst_format;

   for (uint32_t r = 0; r < region_count; r++) {
      struct vk_meta_blit_push_data push = {
         .x_off = regions[r].srcOffset.x - regions[r].dstOffset.x,
         .y_off = regions[r].srcOffset.y - regions[r].dstOffset.y,
         .x_scale = 1,
         .y_scale = 1,
         .z_scale = 1,
      };
      struct vk_meta_rect dst_rect = {
         .x0 = regions[r].dstOffset.x,
         .y0 = regions[r].dstOffset.y,
         .x1 = regions[r].dstOffset.x + regions[r].extent.width,
         .y1 = regions[r].dstOffset.y + regions[r].extent.height,
      };

      VkImageSubresourceLayers src_subres = regions[r].srcSubresource;
      src_subres.layerCount =
         vk_image_subresource_layer_count(src_image, &src_subres);

      VkImageSubresourceLayers dst_subres = regions[r].dstSubresource;
      dst_subres.layerCount =
         vk_image_subresource_layer_count(dst_image, &dst_subres);

      assert(src_image->image_type != VK_IMAGE_TYPE_3D);

      uint32_t layer_count;
      if (dst_image->image_type == VK_IMAGE_TYPE_3D) {
         /* For 2D images, baseArrayLayer is applied by the image view.  For
          * 3D, we have to handle it by adjusting the layer and the Z xform.
          */
         dst_rect.layer = regions[r].dstOffset.z;
         push.z_off = -(float)regions[r].dstOffset.z;

         /* There is no separate destination extent.  The extent is specified
          * in terms of source image pixels.  dstSubresource.layerCount is
          * also useless since it's always 1.  Instead, we have to rely on
          * src_subresource.layerCount.  To avoid rendering outside of the
          * destination, be defensive and clamp to the destination image size.
          */
         VkExtent3D dst_extent =
            vk_image_mip_level_extent(dst_image, dst_subres.mipLevel);
         assert(regions[r].dstOffset.z < dst_extent.depth);
         uint32_t dst_remaining_layers =
            dst_extent.depth - regions[r].dstOffset.z;
         layer_count = MIN2(dst_subres.layerCount, dst_remaining_layers);
      } else {
         /* From the Vulkan 1.4.354 spec:
          *
          *    VUID-VkImageResolve2-layerCount-08803
          *
          *    "If neither of the layerCount members of srcSubresource or
          *    dstSubresource are VK_REMAINING_ARRAY_LAYERS, the layerCount
          *    member of srcSubresource and dstSubresource must match"
          *
          *    VUID-VkImageResolve2-layerCount-08804
          *
          *    "If one of the layerCount members of srcSubresource or
          *    dstSubresource is VK_REMAINING_ARRAY_LAYERS, the other member
          *    must be either VK_REMAINING_ARRAY_LAYERS or equal to the
          *    arrayLayers member of the VkImageCreateInfo used to create the
          *    image minus baseArrayLayer"
          *
          * Technically, the second one may allow for images with two
          * diferent layer counts to both use VK_REMAINING_ARRAY_LAYERS
          * so we may have a mismatch.
          *
          * We defensively take the destination layer count so we don't
          * render out-of-bounds in the destination.  If we end up out of
          * bounds on the source image, the sampler will handle that.
          */
         layer_count = dst_subres.layerCount;
      }

      do_blit(cmd, meta,
              src_image, src_format, src_image_layout, src_subres,
              dst_image, dst_format, dst_image_layout, dst_subres,
              VK_NULL_HANDLE, &key, &push, &dst_rect, layer_count);
   }
}

void
vk_meta_resolve_image2(struct vk_command_buffer *cmd,
                       struct vk_meta_device *meta,
                       const VkResolveImageInfo2 *resolve)
{
   VK_FROM_HANDLE(vk_image, src_image, resolve->srcImage);
   VK_FROM_HANDLE(vk_image, dst_image, resolve->dstImage);

   /* Color resolve default to be based on the format */
   VkResolveModeFlagBits resolve_mode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
   if (vk_format_is_color(src_image->format) &&
       !vk_format_is_int(src_image->format))
      resolve_mode = VK_RESOLVE_MODE_AVERAGE_BIT;

   VkResolveImageFlagBitsKHR resolve_flags = 0;
   VkResolveModeFlagBits stencil_resolve_mode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;

   const VkResolveImageModeInfoKHR *mode_info =
      vk_find_struct_const(resolve->pNext, RESOLVE_IMAGE_MODE_INFO_KHR);
   if (mode_info != NULL) {
      resolve_flags = mode_info->flags;
      resolve_mode = mode_info->resolveMode;
      stencil_resolve_mode = mode_info->stencilResolveMode;
   }

   VkFormat src_format = src_image->format;
   VkFormat dst_format = dst_image->format;
   if (resolve_flags & VK_RESOLVE_IMAGE_SKIP_TRANSFER_FUNCTION_BIT_KHR) {
      src_format = vk_format_srgb_to_linear(src_format);
      dst_format = vk_format_srgb_to_linear(dst_format);
   }

   vk_meta_resolve_image(cmd, meta,
                         src_image, src_format, resolve->srcImageLayout,
                         dst_image, dst_format, resolve->dstImageLayout,
                         resolve->regionCount, resolve->pRegions,
                         resolve_mode, stencil_resolve_mode);
}

static void
vk_meta_resolve_attachment(struct vk_command_buffer *cmd,
                           struct vk_meta_device *meta,
                           struct vk_image_view *src_view,
                           VkImageLayout src_image_layout,
                           struct vk_image_view *dst_view,
                           VkImageLayout dst_image_layout,
                           VkImageAspectFlags resolve_aspects,
                           VkRenderingAttachmentFlagsKHR att_flags,
                           VkResolveModeFlagBits resolve_mode,
                           VkResolveModeFlagBits stencil_resolve_mode,
                           VkRect2D area, uint32_t layer_count,
                           uint32_t view_mask)
{
   VkImageResolve2 region = {
      .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2,
      .srcSubresource = {
         .aspectMask = resolve_aspects,
         .mipLevel = src_view->base_mip_level,
      },
      .srcOffset = { area.offset.x, area.offset.y, 0},
      .dstSubresource = {
         .aspectMask = resolve_aspects,
         .mipLevel = dst_view->base_mip_level,
      },
      .dstOffset = { area.offset.x, area.offset.y, 0},
      .extent = { area.extent.width, area.extent.height, 1},
   };

   VkFormat src_format = src_view->format;
   VkFormat dst_format = dst_view->format;
   if (att_flags & VK_RENDERING_ATTACHMENT_RESOLVE_SKIP_TRANSFER_FUNCTION_BIT_KHR) {
      src_format = vk_format_srgb_to_linear(src_format);
      dst_format = vk_format_srgb_to_linear(dst_format);
   }

   if (view_mask) {
      u_foreach_bit(v, view_mask) {
         region.srcSubresource.baseArrayLayer = src_view->base_array_layer + v;
         region.srcSubresource.layerCount = 1;
         region.dstSubresource.baseArrayLayer = dst_view->base_array_layer + v;
         region.dstSubresource.layerCount = 1;

         vk_meta_resolve_image(cmd, meta,
                               src_view->image, src_format, src_image_layout,
                               dst_view->image, dst_format, dst_image_layout,
                               1, &region, resolve_mode, stencil_resolve_mode);
      }
   } else {
      region.srcSubresource.baseArrayLayer = src_view->base_array_layer;
      region.srcSubresource.layerCount = layer_count;
      region.dstSubresource.baseArrayLayer = dst_view->base_array_layer;
      region.dstSubresource.layerCount = layer_count;

      vk_meta_resolve_image(cmd, meta,
                            src_view->image, src_format, src_image_layout,
                            dst_view->image, dst_format, dst_image_layout,
                            1, &region, resolve_mode, stencil_resolve_mode);
   }
}

void
vk_meta_resolve_rendering(struct vk_command_buffer *cmd,
                          struct vk_meta_device *meta,
                          const VkRenderingInfo *pRenderingInfo)
{
   for (uint32_t c = 0; c < pRenderingInfo->colorAttachmentCount; c++) {
      const VkRenderingAttachmentInfo *att =
         &pRenderingInfo->pColorAttachments[c];
      if (att->resolveMode == VK_RESOLVE_MODE_NONE)
         continue;

      VK_FROM_HANDLE(vk_image_view, view, att->imageView);
      VK_FROM_HANDLE(vk_image_view, res_view, att->resolveImageView);

      vk_meta_resolve_attachment(cmd, meta, view, att->imageLayout,
                                 res_view, att->resolveImageLayout,
                                 VK_IMAGE_ASPECT_COLOR_BIT,
                                 vk_get_rendering_attachment_flags(att),
                                 att->resolveMode, VK_RESOLVE_MODE_NONE,
                                 pRenderingInfo->renderArea,
                                 pRenderingInfo->layerCount,
                                 pRenderingInfo->viewMask);
   }

   const VkRenderingAttachmentInfo *d_att = pRenderingInfo->pDepthAttachment;
   if (d_att && d_att->resolveMode == VK_RESOLVE_MODE_NONE)
      d_att = NULL;

   const VkRenderingAttachmentInfo *s_att = pRenderingInfo->pStencilAttachment;
   if (s_att && s_att->resolveMode == VK_RESOLVE_MODE_NONE)
      s_att = NULL;

   if (s_att != NULL || d_att != NULL) {
      if (s_att != NULL && d_att != NULL &&
          s_att->imageView == d_att->imageView &&
          s_att->resolveImageView == d_att->resolveImageView) {
         VK_FROM_HANDLE(vk_image_view, view, d_att->imageView);
         VK_FROM_HANDLE(vk_image_view, res_view, d_att->resolveImageView);

         vk_meta_resolve_attachment(cmd, meta, view, d_att->imageLayout,
                                    res_view, d_att->resolveImageLayout,
                                    VK_IMAGE_ASPECT_DEPTH_BIT |
                                    VK_IMAGE_ASPECT_STENCIL_BIT,
                                    0, /* Flags don't affect Z/S resolves */
                                    d_att->resolveMode, s_att->resolveMode,
                                    pRenderingInfo->renderArea,
                                    pRenderingInfo->layerCount,
                                    pRenderingInfo->viewMask);
      } else {
         if (d_att != NULL) {
            VK_FROM_HANDLE(vk_image_view, view, d_att->imageView);
            VK_FROM_HANDLE(vk_image_view, res_view, d_att->resolveImageView);

            vk_meta_resolve_attachment(cmd, meta, view, d_att->imageLayout,
                                       res_view, d_att->resolveImageLayout,
                                       VK_IMAGE_ASPECT_DEPTH_BIT,
                                       0, /* Flags don't affect Z/S resolves */
                                       d_att->resolveMode, VK_RESOLVE_MODE_NONE,
                                       pRenderingInfo->renderArea,
                                       pRenderingInfo->layerCount,
                                       pRenderingInfo->viewMask);
         }

         if (s_att != NULL) {
            VK_FROM_HANDLE(vk_image_view, view, s_att->imageView);
            VK_FROM_HANDLE(vk_image_view, res_view, s_att->resolveImageView);

            vk_meta_resolve_attachment(cmd, meta, view, s_att->imageLayout,
                                       res_view, s_att->resolveImageLayout,
                                       VK_IMAGE_ASPECT_STENCIL_BIT,
                                       0, /* Flags don't affect Z/S resolves */
                                       VK_RESOLVE_MODE_NONE, s_att->resolveMode,
                                       pRenderingInfo->renderArea,
                                       pRenderingInfo->layerCount,
                                       pRenderingInfo->viewMask);
         }
      }
   }
}
