// Copyright 2020 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include "iree/hal/device.h"

#include <inttypes.h>
#include <string.h>

#include "iree/hal/allocator.h"
#include "iree/hal/buffer.h"
#include "iree/hal/command_buffer.h"
#include "iree/hal/detail.h"
#include "iree/hal/resource.h"

//===----------------------------------------------------------------------===//
// iree_hal_device_t
//===----------------------------------------------------------------------===//

#define _VTABLE_DISPATCH(device, method_name) \
  IREE_HAL_VTABLE_DISPATCH(device, iree_hal_device, method_name)

IREE_HAL_API_RETAIN_RELEASE(device);

IREE_API_EXPORT iree_string_view_t
iree_hal_device_id(iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  return _VTABLE_DISPATCH(device, id)(device);
}

IREE_API_EXPORT iree_allocator_t
iree_hal_device_host_allocator(iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  return _VTABLE_DISPATCH(device, host_allocator)(device);
}

IREE_API_EXPORT iree_hal_allocator_t* iree_hal_device_allocator(
    iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  return _VTABLE_DISPATCH(device, device_allocator)(device);
}

IREE_API_EXPORT void iree_hal_device_replace_allocator(
    iree_hal_device_t* device, iree_hal_allocator_t* new_allocator) {
  IREE_ASSERT_ARGUMENT(device);
  _VTABLE_DISPATCH(device, replace_device_allocator)(device, new_allocator);
}

IREE_API_EXPORT void iree_hal_device_replace_channel_provider(
    iree_hal_device_t* device, iree_hal_channel_provider_t* new_provider) {
  IREE_ASSERT_ARGUMENT(device);
  _VTABLE_DISPATCH(device, replace_channel_provider)(device, new_provider);
}

IREE_API_EXPORT
iree_status_t iree_hal_device_trim(iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, trim)(device);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_query_i64(
    iree_hal_device_t* device, iree_string_view_t category,
    iree_string_view_t key, int64_t* out_value) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(out_value);
  return _VTABLE_DISPATCH(device, query_i64)(device, category, key, out_value);
}

IREE_API_EXPORT iree_status_t iree_hal_device_query_capabilities(
    iree_hal_device_t* device,
    iree_hal_device_capabilities_t* out_capabilities) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(out_capabilities);
  IREE_TRACE_ZONE_BEGIN(z0);
  IREE_RETURN_IF_ERROR(
      _VTABLE_DISPATCH(device, query_capabilities)(device, out_capabilities));
  IREE_TRACE_ZONE_END(z0);
  return iree_ok_status();
}

IREE_API_EXPORT const iree_hal_device_topology_info_t*
iree_hal_device_topology_info(iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  return _VTABLE_DISPATCH(device, topology_info)(device);
}

IREE_API_EXPORT iree_status_t iree_hal_device_refine_topology_edge(
    iree_hal_device_t* src_device, iree_hal_device_t* dst_device,
    iree_hal_topology_edge_t* edge) {
  IREE_ASSERT_ARGUMENT(src_device);
  IREE_ASSERT_ARGUMENT(dst_device);
  IREE_ASSERT_ARGUMENT(edge);
  return _VTABLE_DISPATCH(src_device, refine_topology_edge)(src_device,
                                                            dst_device, edge);
}

IREE_API_EXPORT iree_status_t iree_hal_device_assign_topology_info(
    iree_hal_device_t* device,
    const iree_hal_device_topology_info_t* topology_info) {
  IREE_ASSERT_ARGUMENT(device);
  return _VTABLE_DISPATCH(device, assign_topology_info)(device, topology_info);
}

IREE_API_EXPORT iree_hal_semaphore_compatibility_t
iree_hal_device_query_semaphore_compatibility(iree_hal_device_t* device,
                                              iree_hal_semaphore_t* semaphore) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(semaphore);
  return _VTABLE_DISPATCH(device, query_semaphore_compatibility)(device,
                                                                 semaphore);
}

IREE_API_EXPORT iree_status_t iree_hal_device_query_queue_pool_backend(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    iree_hal_queue_pool_backend_t* out_backend) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(out_backend);
  memset(out_backend, 0, sizeof(*out_backend));
  const iree_hal_device_topology_info_t* topology_info =
      iree_hal_device_topology_info(device);
  if (!topology_info->topology || !topology_info->frontier.tracker) {
    return iree_make_status(
        IREE_STATUS_FAILED_PRECONDITION,
        "device queue pool backends are unavailable before the device is "
        "assigned to a device group");
  }
  return _VTABLE_DISPATCH(device, query_queue_pool_backend)(
      device, queue_affinity, out_backend);
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_alloca(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_pool_t* pool, iree_hal_buffer_params_t params,
    iree_device_size_t allocation_size, iree_hal_alloca_flags_t flags,
    iree_hal_buffer_t** IREE_RESTRICT out_buffer) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(out_buffer);
  *out_buffer = NULL;
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_alloca)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list, pool,
      params, allocation_size, flags, out_buffer);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_dealloca(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_buffer_t* buffer, iree_hal_dealloca_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(buffer);
  IREE_TRACE_ZONE_BEGIN(z0);

  // If the buffer has an origin then use it for the deallocation. Some buffers
  // may not have an origin and we'll use the provided device.
  const iree_hal_buffer_placement_t placement =
      iree_hal_buffer_allocation_placement(buffer);
  if (iree_all_bits_set(flags, IREE_HAL_DEALLOCA_FLAG_PREFER_ORIGIN) &&
      placement.device != NULL) {
    device = placement.device;
    queue_affinity = placement.queue_affinity;
  }

  // If the buffer was not allocated asynchronously then this is just a barrier.
  iree_status_t status = iree_ok_status();
  if (iree_all_bits_set(placement.flags,
                        IREE_HAL_BUFFER_PLACEMENT_FLAG_ASYNCHRONOUS) &&
      !iree_all_bits_set(
          placement.flags,
          IREE_HAL_BUFFER_PLACEMENT_FLAG_INDETERMINATE_LIFETIME)) {
    status = _VTABLE_DISPATCH(device, queue_dealloca)(
        device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
        buffer, flags);
  } else {
    status = iree_hal_device_queue_barrier(
        device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
        IREE_HAL_EXECUTE_FLAG_NONE);
  }

  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_fill(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_buffer_t* target_buffer, iree_device_size_t target_offset,
    iree_device_size_t length, const void* pattern,
    iree_host_size_t pattern_length, iree_hal_fill_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(pattern);
  IREE_ASSERT_ARGUMENT(target_buffer);
  IREE_TRACE_ZONE_BEGIN(z0);
  IREE_TRACE_ZONE_APPEND_VALUE_I64(z0, (int64_t)length);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_fill)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
      target_buffer, target_offset, length, pattern, pattern_length, flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_update(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    const void* source_buffer, iree_host_size_t source_offset,
    iree_hal_buffer_t* target_buffer, iree_device_size_t target_offset,
    iree_device_size_t length, iree_hal_update_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(source_buffer);
  IREE_ASSERT_ARGUMENT(target_buffer);
  IREE_TRACE_ZONE_BEGIN(z0);
  IREE_TRACE_ZONE_APPEND_VALUE_I64(z0, (int64_t)length);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_update)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
      source_buffer, source_offset, target_buffer, target_offset, length,
      flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_copy(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_buffer_t* source_buffer, iree_device_size_t source_offset,
    iree_hal_buffer_t* target_buffer, iree_device_size_t target_offset,
    iree_device_size_t length, iree_hal_copy_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(source_buffer);
  IREE_ASSERT_ARGUMENT(target_buffer);
  IREE_TRACE_ZONE_BEGIN(z0);
  IREE_TRACE_ZONE_APPEND_VALUE_I64(z0, (int64_t)length);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_copy)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
      source_buffer, source_offset, target_buffer, target_offset, length,
      flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_read(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_file_t* source_file, uint64_t source_offset,
    iree_hal_buffer_t* target_buffer, iree_device_size_t target_offset,
    iree_device_size_t length, iree_hal_read_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(source_file);
  IREE_ASSERT_ARGUMENT(target_buffer);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_read)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
      source_file, source_offset, target_buffer, target_offset, length, flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_write(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_buffer_t* source_buffer, iree_device_size_t source_offset,
    iree_hal_file_t* target_file, uint64_t target_offset,
    iree_device_size_t length, iree_hal_write_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(source_buffer);
  IREE_ASSERT_ARGUMENT(target_file);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_write)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
      source_buffer, source_offset, target_file, target_offset, length, flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_host_call(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_host_call_t call, const uint64_t args[4],
    iree_hal_host_call_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(call.fn);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_host_call)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list, call,
      args, flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_dispatch(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_executable_t* executable, iree_hal_executable_function_t function,
    const iree_hal_dispatch_config_t config, iree_const_byte_span_t constants,
    const iree_hal_buffer_ref_list_t bindings,
    iree_hal_dispatch_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(executable);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, queue_dispatch)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
      executable, function, config, constants, bindings, flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_execute(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_command_buffer_t* command_buffer,
    iree_hal_buffer_binding_table_t binding_table,
    iree_hal_execute_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(
      !wait_semaphore_list.count ||
      (wait_semaphore_list.semaphores && wait_semaphore_list.payload_values));
  IREE_ASSERT_ARGUMENT(!signal_semaphore_list.count ||
                       (signal_semaphore_list.semaphores &&
                        signal_semaphore_list.payload_values));
  IREE_TRACE_ZONE_BEGIN(z0);

  // TODO(benvanik): move into devices instead? then a synchronous/inline device
  // could assert the waits are resolved instead of blanket failing on an
  // already-resolved semaphore. This would make using stream-ordered
  // allocations easier.
  if (wait_semaphore_list.count > 0 && command_buffer &&
      iree_all_bits_set(iree_hal_command_buffer_mode(command_buffer),
                        IREE_HAL_COMMAND_BUFFER_MODE_ALLOW_INLINE_EXECUTION)) {
    // Inline command buffers are not allowed to wait (as they could have
    // already been executed!). This is a requirement of the API so we
    // validate it across all backends even if they don't support inline
    // execution and ignore it.
    IREE_TRACE_ZONE_END(z0);
    return iree_make_status(
        IREE_STATUS_INVALID_ARGUMENT,
        "inline command buffer submitted with a wait; inline command "
        "buffers must be ready to execute immediately");
  }

  // Validate command buffer bindings against the provided binding tables.
  // This will error out if a binding table is required but not provided or if
  // any binding in the table does not match the requirements of the command
  // buffer as recorded.
  if (command_buffer) {
    IREE_RETURN_AND_END_ZONE_IF_ERROR(
        z0, iree_hal_command_buffer_validate_submission(command_buffer,
                                                        binding_table));
  }

  iree_status_t status = _VTABLE_DISPATCH(device, queue_execute)(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list,
      command_buffer, binding_table, flags);

  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_barrier(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity,
    const iree_hal_semaphore_list_t wait_semaphore_list,
    const iree_hal_semaphore_list_t signal_semaphore_list,
    iree_hal_execute_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = iree_hal_device_queue_execute(
      device, queue_affinity, wait_semaphore_list, signal_semaphore_list, NULL,
      iree_hal_buffer_binding_table_empty(), flags);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_queue_flush(
    iree_hal_device_t* device, iree_hal_queue_affinity_t queue_affinity) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status =
      _VTABLE_DISPATCH(device, queue_flush)(device, queue_affinity);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_wait_semaphores(
    iree_hal_device_t* device, iree_async_wait_mode_t wait_mode,
    const iree_hal_semaphore_list_t semaphore_list, iree_timeout_t timeout,
    iree_async_wait_flags_t flags) {
  IREE_ASSERT_ARGUMENT(device);
  if (semaphore_list.count == 0) return iree_ok_status();
  IREE_TRACE_ZONE_BEGIN(z0);
  // HAL semaphores embed async semaphores at offset 0 (toll-free bridge).
  iree_status_t status = iree_async_semaphore_multi_wait(
      wait_mode, (iree_async_semaphore_t**)semaphore_list.semaphores,
      semaphore_list.payload_values, semaphore_list.count, timeout, flags,
      iree_allocator_system());
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_profiling_begin(
    iree_hal_device_t* device,
    const iree_hal_device_profiling_options_t* options) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(options);

  const iree_hal_device_profiling_flags_t supported_flags =
      IREE_HAL_DEVICE_PROFILING_FLAG_LIGHTWEIGHT_STATISTICS;
  if (iree_any_bit_set(options->flags, ~supported_flags)) {
    return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
                            "unsupported profile option flags 0x%x",
                            options->flags & ~supported_flags);
  }
  const iree_hal_profile_capture_filter_flags_t supported_filter_flags =
      IREE_HAL_PROFILE_CAPTURE_FILTER_FLAG_EXECUTABLE_FUNCTION_PATTERN |
      IREE_HAL_PROFILE_CAPTURE_FILTER_FLAG_COMMAND_BUFFER_ID |
      IREE_HAL_PROFILE_CAPTURE_FILTER_FLAG_COMMAND_INDEX |
      IREE_HAL_PROFILE_CAPTURE_FILTER_FLAG_PHYSICAL_DEVICE_ORDINAL |
      IREE_HAL_PROFILE_CAPTURE_FILTER_FLAG_QUEUE_ORDINAL;
  if (iree_any_bit_set(options->capture_filter.flags,
                       ~supported_filter_flags)) {
    return iree_make_status(
        IREE_STATUS_INVALID_ARGUMENT,
        "unsupported profile capture filter bits 0x%x",
        options->capture_filter.flags & ~supported_filter_flags);
  }
  if (options->capture_filter.reserved0 != 0) {
    return iree_make_status(
        IREE_STATUS_INVALID_ARGUMENT,
        "profile capture filter reserved fields must be zero");
  }
  if (iree_any_bit_set(
          options->capture_filter.flags,
          IREE_HAL_PROFILE_CAPTURE_FILTER_FLAG_EXECUTABLE_FUNCTION_PATTERN) &&
      iree_string_view_is_empty(
          options->capture_filter.executable_function_pattern)) {
    return iree_make_status(
        IREE_STATUS_INVALID_ARGUMENT,
        "profile capture executable function filter must not be empty");
  }
  if (iree_any_bit_set(
          options->capture_filter.flags,
          IREE_HAL_PROFILE_CAPTURE_FILTER_FLAG_COMMAND_BUFFER_ID) &&
      options->capture_filter.command_buffer_id == 0) {
    return iree_make_status(
        IREE_STATUS_INVALID_ARGUMENT,
        "profile capture command buffer filter must be nonzero");
  }

  if (options->counter_set_count != 0) {
    if (!options->counter_sets) {
      return iree_make_status(
          IREE_STATUS_INVALID_ARGUMENT,
          "hardware counter set selections require a counter_sets array");
    }
    if (!iree_hal_device_profiling_options_requests_counters(options)) {
      return iree_make_status(
          IREE_STATUS_INVALID_ARGUMENT,
          "hardware counter set selections require a counter profiling data "
          "family");
    }
    for (iree_host_size_t i = 0; i < options->counter_set_count; ++i) {
      const iree_hal_profile_counter_set_selection_t* counter_set =
          &options->counter_sets[i];
      if (counter_set->counter_name_count == 0 || !counter_set->counter_names) {
        return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
                                "hardware counter set %" PRIhsz
                                " must request at least one counter name",
                                i);
      }
      for (iree_host_size_t j = 0; j < counter_set->counter_name_count; ++j) {
        iree_string_view_t counter_name = counter_set->counter_names[j];
        if (iree_string_view_is_empty(counter_name) || !counter_name.data) {
          return iree_make_status(
              IREE_STATUS_INVALID_ARGUMENT,
              "hardware counter set %" PRIhsz
              " has an empty counter name at index %" PRIhsz,
              i, j);
        }
      }
    }
  }
  if (iree_hal_device_profiling_options_requests_counters(options) &&
      options->counter_set_count == 0) {
    return iree_make_status(
        IREE_STATUS_INVALID_ARGUMENT,
        "counter profiling requires at least one counter set selection");
  }

  const bool data_requested =
      options->data_families != IREE_HAL_DEVICE_PROFILING_DATA_NONE ||
      iree_hal_device_profiling_options_requests_lightweight_statistics(
          options);
  if (data_requested && !options->sink) {
    return iree_make_status(
        IREE_STATUS_INVALID_ARGUMENT,
        "HAL-native profiling with requested data requires a profile sink");
  }
  if (!data_requested) {
    return iree_ok_status();
  }

  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status =
      _VTABLE_DISPATCH(device, profiling_begin)(device, options);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t
iree_hal_device_profiling_flush(iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, profiling_flush)(device);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t
iree_hal_device_profiling_end(iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, profiling_end)(device);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t iree_hal_device_external_capture_begin(
    iree_hal_device_t* device,
    const iree_hal_device_external_capture_options_t* options) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_ASSERT_ARGUMENT(options);
  if (iree_string_view_is_empty(options->provider)) {
    return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
                            "external capture provider must be specified");
  }

  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status =
      _VTABLE_DISPATCH(device, external_capture_begin)(device, options);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_status_t
iree_hal_device_external_capture_end(iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(device);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = _VTABLE_DISPATCH(device, external_capture_end)(device);
  IREE_TRACE_ZONE_END(z0);
  return status;
}

//===----------------------------------------------------------------------===//
// iree_hal_device_list_t
//===----------------------------------------------------------------------===//

IREE_API_EXPORT iree_status_t iree_hal_device_list_allocate(
    iree_host_size_t capacity, iree_allocator_t host_allocator,
    iree_hal_device_list_t** out_list) {
  IREE_ASSERT_ARGUMENT(out_list);
  *out_list = NULL;
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_hal_device_list_t* list = NULL;
  iree_host_size_t total_size =
      sizeof(*list) + capacity * sizeof(list->devices[0]);
  IREE_RETURN_AND_END_ZONE_IF_ERROR(
      z0, iree_allocator_malloc(host_allocator, total_size, (void**)&list));
  list->host_allocator = host_allocator;
  list->capacity = capacity;
  list->count = 0;
  *out_list = list;
  IREE_TRACE_ZONE_END(z0);
  return iree_ok_status();
}

IREE_API_EXPORT void iree_hal_device_list_free(iree_hal_device_list_t* list) {
  if (!list) return;
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_allocator_t host_allocator = list->host_allocator;
  for (iree_host_size_t i = 0; i < list->count; ++i) {
    iree_hal_device_release(list->devices[i]);
  }
  iree_allocator_free(host_allocator, list);
  IREE_TRACE_ZONE_END(z0);
}

IREE_API_EXPORT iree_status_t iree_hal_device_list_push_back(
    iree_hal_device_list_t* list, iree_hal_device_t* device) {
  IREE_ASSERT_ARGUMENT(list);
  IREE_ASSERT_ARGUMENT(device);
  IREE_TRACE_ZONE_BEGIN(z0);
  iree_status_t status = iree_ok_status();
  if (list->count + 1 <= list->capacity) {
    iree_hal_device_retain(device);
    list->devices[list->count++] = device;
  } else {
    status = iree_make_status(IREE_STATUS_RESOURCE_EXHAUSTED,
                              "list capacity %" PRIhsz
                              " reached; no more devices can be added",
                              list->capacity);
  }
  IREE_TRACE_ZONE_END(z0);
  return status;
}

IREE_API_EXPORT iree_hal_device_t* iree_hal_device_list_at(
    const iree_hal_device_list_t* list, iree_host_size_t i) {
  IREE_ASSERT_ARGUMENT(list);
  return i < list->count ? list->devices[i] : NULL;
}
