// Copyright 2019 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#ifndef IREE_VM_REF_H_
#define IREE_VM_REF_H_

#include <assert.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

#include "iree/base/api.h"
#include "iree/base/internal/atomics.h"

#ifdef __cplusplus
extern "C" {
#endif  // __cplusplus

typedef struct iree_vm_instance_t iree_vm_instance_t;

// Enable additional expensive checks to track down ref counting issues.
// #define IREE_VM_REF_PARANOID

// Enable printing every VM-aware retain/release to stderr.
// #define IREE_VM_REF_LOG
// #define IREE_VM_REF_LOG_INCLUDE_BACKTRACES

// Enable use-after-move detection for refs. When enabled, moved-from refs are
// poisoned with a sentinel value and assertions fire if they're accessed before
// being reassigned. This helps catch incorrect MOVE bit computation in the
// compiler (where a ref is marked as last-use when it's actually used again).
// #define IREE_VM_REF_DEBUG_MOVE

// Enable refs-clean verification for debugging. When enabled, the runtime
// asserts that all ref registers are zeroed on normal return, verifying the
// compiler's guarantee that all refs are explicitly released via MOVE bits
// and vm.discard.refs operations. Helps catch ref lifetime bugs in the
// compiler.
// #define IREE_VM_REF_DEBUG_CLEAN

// Number of least significant bits available in an iree_vm_ref_type_t value.
// These can be used for any purpose.
#define IREE_VM_REF_TYPE_TAG_BITS 3
#define IREE_VM_REF_TYPE_TAG_BIT_MASK 0b111

// Number of bits available in an iree_vm_ref_type_t value to hold the pointer.
#define IREE_VM_REF_TYPE_PTR_BITS \
  (sizeof(uintptr_t) * 8 - IREE_VM_REF_TYPE_TAG_BITS)
#define IREE_VM_REF_TYPE_PTR_BIT_MASK (~IREE_VM_REF_TYPE_TAG_BIT_MASK)

// (1 << IREE_VM_REF_TYPE_TAG_BITS) hardcoded for MSVC which cannot have
// expressions in its alignas in older versions.
#define IREE_VM_REF_TYPE_DESCRIPTOR_ALIGNMENT 8

// Alignment required for the reference counter.
// Used to pack the offset bits to fit in IREE_VM_REF_TYPE_TAG_BITS.
#define IREE_VM_REF_COUNTER_ALIGNMENT sizeof(iree_atomic_ref_count_t)

// Defines the type of the reference-counted pointer.
// This is used to verify that operations dealing with the variant ref struct
// are correct at runtime. We don't allow control over the ref types from the
// VM ops and as such we can use the type specified as a safe way to avoid
// reinterpreting memory incorrectly.
enum iree_vm_ref_type_bits_t {
  IREE_VM_REF_TYPE_NULL = 0,

  // NOTE: these type values are assigned dynamically right now. Treat them as
  // opaque and unstable across process invocations.

  // Wildcard type that indicates that a value may be a ref type but of an
  // unspecified internal type. Note that we mask off the bottom bits to allow
  // for tagging (all ref type values )
  IREE_VM_REF_TYPE_ANY = UINTPTR_MAX ^ IREE_VM_REF_TYPE_TAG_BIT_MASK,

#if defined(IREE_VM_REF_DEBUG_MOVE)
  // Poison value used to mark moved-from refs in debug mode.
  // Uses a distinctive pattern (0xDEAD...) that will cause obvious crashes if
  // dereferenced and is easily recognizable in debuggers.
  IREE_VM_REF_TYPE_POISONED = (uintptr_t)0xDEADCAFEDEADF00Dull,
#endif  // IREE_VM_REF_DEBUG_MOVE
};

typedef void(IREE_API_PTR* iree_vm_ref_destroy_t)(void* ptr);

// Describes a type for the VM.
typedef iree_alignas(IREE_VM_REF_TYPE_DESCRIPTOR_ALIGNMENT) struct
    iree_vm_ref_type_descriptor_t {
  // Function called when references of this type reach 0 and should be
  // destroyed.
  iree_vm_ref_destroy_t destroy;
  // Unretained type name that can be used for debugging.
  iree_string_view_t type_name;
  // Offset from ptr in units of IREE_VM_REF_COUNTER_ALIGNMENT to the start of
  // an iree_atomic_ref_count_t representing the current reference count.
  uintptr_t offsetof_counter : IREE_VM_REF_TYPE_TAG_BITS;
  uintptr_t reserved : IREE_VM_REF_TYPE_PTR_BITS;
} iree_vm_ref_type_descriptor_t;

// Type-erased reference counted type descriptor.
typedef uintptr_t iree_vm_ref_type_t;

// Makes an opaque reference to a type descriptor.
static inline iree_vm_ref_type_t iree_vm_make_ref_type(
    const iree_vm_ref_type_descriptor_t* descriptor) {
  return (iree_vm_ref_type_t)descriptor |
         (iree_vm_ref_type_t)descriptor->offsetof_counter;
}

// Returns the type name for the given type, if found.
IREE_API_EXPORT iree_string_view_t
iree_vm_ref_type_name(iree_vm_ref_type_t type);

// Returns the type descriptor backing the given |type|.
// The descriptor is an implementation detail.
static inline const iree_vm_ref_type_descriptor_t* iree_vm_ref_type_descriptor(
    iree_vm_ref_type_t type) {
  return (const iree_vm_ref_type_descriptor_t*)(type &
                                                IREE_VM_REF_TYPE_PTR_BIT_MASK);
}

// Base for iree_vm_ref_t object targets.
//
// Usage (C):
//  typedef struct my_type_t {
//    iree_vm_ref_object_t ref_object;
//    int my_fields;
//  } my_type_t;
//  void my_type_destroy(void* ptr) {
//    free(ptr);
//  }
//  static iree_vm_ref_type_descriptor_t my_type_descriptor;
//  my_type_descriptor.type_name = iree_string_view_t{"my_type", 7};
//  my_type_descriptor.destroy = my_type_destroy;
//  my_type_descriptor.offsetof_counter =
//      offsetof(my_type_t, ref_object.counter) / IREE_VM_REF_COUNTER_ALIGNMENT;
//  iree_vm_ref_register_defined_type(&my_type_descriptor);
//
// Usage (C++):
//  Prefer using iree::vm::RefObject as a base type.
typedef struct iree_vm_ref_object_t {
  iree_atomic_ref_count_t counter;
} iree_vm_ref_object_t;

// A pointer reference to a reference-counted object.
// The counter is stored within the target object itself ala intrusive_ptr.
//
// NOTE: we try to keep this small so that we aren't wasting stack space or
// copying around too much when we pass it to functions by value. This also
// helps make the CPU caches happier as we need no indirections to check the
// type and adjusting the counter happens without needing to query a descriptor.
// Ideally the iree_vm_ref_t is in-cache on the stack and the target ptr is
// either in cache from a previous use or will be used again after manipulating
// its ref count.
typedef struct iree_vm_ref_t {
  // Pointer to the object. Type is resolved based on the |type| field.
  // Will be NULL if the reference points to nothing.
  void* ptr;
  // Registered type of the object pointed to by ptr.
  iree_vm_ref_type_t type;
} iree_vm_ref_t;
static_assert(
    sizeof(iree_vm_ref_t) <= sizeof(void*) * 2,
    "iree_vm_ref_t dominates stack space usage and should be kept tiny");

// Directly retains the object with base |ptr| with the given |type|.
//
// Note that this avoids any kind of type checking; for untrusted inputs use
// the iree_vm_ref_t-based methods.
IREE_API_EXPORT void iree_vm_ref_object_retain(void* ptr,
                                               iree_vm_ref_type_t type);

// Directly release the object with base |ptr| with the given |type|,
// possibly destroying it if it is the last reference. Assume that |ptr| is
// invalid after this function returns.
//
// Note that this avoids any kind of type checking; for untrusted inputs use
// the iree_vm_ref_t-based methods.
IREE_API_EXPORT void iree_vm_ref_object_release(void* ptr,
                                                iree_vm_ref_type_t type);

// Returns a NULL ref wrapper.
static inline iree_vm_ref_t iree_vm_ref_null(void) {
  iree_vm_ref_t ref = {0};
  return ref;
}

// Wraps a raw pointer in a iree_vm_ref_t reference and assigns it to |out_ref|.
// |out_ref| will be released if it already contains a reference. The target
// object will not be retained and must come in with a count >= 1.
//
// Usage (C):
//  my_type_t* my_type = (my_type_t*)malloc(sizeof(my_type_t));
//  my_type.ref_object.counter = IREE_ATOMIC_VAR_INIT(1);
//  iree_vm_ref_t my_ref;
//  iree_vm_ref_wrap_assign(my_type, IREE_VM_REF_TYPE_MY_TYPE, &my_ref);
//  iree_vm_ref_release(&my_ref);
//
// Usage (C++):
//  iree_vm_ref_t my_ref;
//  iree_vm_ref_wrap_assign(new MyType(), IREE_VM_REF_TYPE_MY_TYPE, &my_ref);
//  iree_vm_ref_release(&my_ref);
IREE_API_EXPORT iree_status_t iree_vm_ref_wrap_assign(void* ptr,
                                                      iree_vm_ref_type_t type,
                                                      iree_vm_ref_t* out_ref);

// Wraps a raw pointer in a iree_vm_ref_t reference and retains it in |out_ref|.
// |out_ref| will be released if it already contains a reference.
IREE_API_EXPORT iree_status_t iree_vm_ref_wrap_retain(void* ptr,
                                                      iree_vm_ref_type_t type,
                                                      iree_vm_ref_t* out_ref);

// Checks that the given reference-counted pointer |ref| is of |type|.
static inline iree_status_t iree_vm_ref_check(const iree_vm_ref_t ref,
                                              iree_vm_ref_type_t type) {
  return IREE_LIKELY(ref.type == type)
             ? iree_ok_status()
             : iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
                                ref.type == IREE_VM_REF_TYPE_NULL
                                    ? "ref is null"
                                    : "ref type mismatch");
}

// Retains the reference-counted pointer |ref|.
// Unsafe to call on unaligned refs.
IREE_API_EXPORT void iree_vm_ref_retain_inplace(iree_vm_ref_t* ref);

// Retains the reference-counted pointer |ref|.
// |out_ref| will be released if it already contains a reference.
// Safe to call on unaligned refs.
IREE_API_EXPORT void iree_vm_ref_retain(iree_vm_ref_t* ref,
                                        iree_vm_ref_t* out_ref);

// Retains the reference-counted pointer |ref| and checks that it is of |type|.
// |out_ref| will be released if it already contains a reference.
// Safe to call on unaligned refs.
IREE_API_EXPORT iree_status_t iree_vm_ref_retain_checked(
    iree_vm_ref_t* ref, iree_vm_ref_type_t type, iree_vm_ref_t* out_ref);

// Retains or moves |ref| to |out_ref|.
// |out_ref| will be released if it already contains a reference.
// Safe to call on unaligned refs.
IREE_API_EXPORT void iree_vm_ref_retain_or_move(int is_move, iree_vm_ref_t* ref,
                                                iree_vm_ref_t* out_ref);

// Retains or moves |ref| to |out_ref| and checks that |ref| is of |type|.
// |out_ref| will be released if it already contains a reference.
// Safe to call on unaligned refs.
IREE_API_EXPORT iree_status_t iree_vm_ref_retain_or_move_checked(
    int is_move, iree_vm_ref_t* ref, iree_vm_ref_type_t type,
    iree_vm_ref_t* out_ref);

// Releases the reference-counted pointer |ref|, possibly freeing it.
// Safe to call on unaligned refs.
IREE_API_EXPORT void iree_vm_ref_release(iree_vm_ref_t* ref);

// Assigns the reference-counted pointer |ref| without incrementing the count.
// |out_ref| will be released if it already contains a reference.
// Safe to call on unaligned refs.
IREE_API_EXPORT void iree_vm_ref_assign(iree_vm_ref_t* ref,
                                        iree_vm_ref_t* out_ref);

// Moves one reference to another without changing the reference count.
// |out_ref| will be released if it already contains a reference.
// Safe to call on unaligned refs.
IREE_API_EXPORT void iree_vm_ref_move(iree_vm_ref_t* ref,
                                      iree_vm_ref_t* out_ref);

// Returns true if the given |ref| is NULL.
// Safe to call on unaligned refs.
IREE_API_EXPORT bool iree_vm_ref_is_null(const iree_vm_ref_t* ref);

// Returns true if the two references point at the same value (or are both
// null).
// Safe to call on unaligned refs.
IREE_API_EXPORT bool iree_vm_ref_equal(const iree_vm_ref_t* lhs,
                                       const iree_vm_ref_t* rhs);

#ifdef __cplusplus
}  // extern "C"
#endif  // __cplusplus

//===----------------------------------------------------------------------===//
// Type adapter utilities for interfacing with the VM
//===----------------------------------------------------------------------===//

#if defined(IREE_VM_REF_PARANOID)
#define IREE_VM_REF_ASSERT(expr) IREE_ASSERT(expr)
#else
#define IREE_VM_REF_ASSERT(expr)
#endif  // IREE_VM_REF_PARANOID

// TODO(benvanik): make these macros standard/document them.
#define IREE_VM_DECLARE_TYPE_ADAPTERS(name, T)                              \
  IREE_API_EXPORT_VARIABLE iree_vm_ref_type_t name##_registration;          \
  static inline iree_vm_ref_type_t name##_type(void) {                      \
    IREE_VM_REF_ASSERT(name##_registration);                                \
    return name##_registration;                                             \
  }                                                                         \
  static inline bool name##_isa(const iree_vm_ref_t ref) {                  \
    IREE_VM_REF_ASSERT(name##_registration);                                \
    return name##_registration == ref.type;                                 \
  }                                                                         \
  IREE_API_EXPORT iree_vm_ref_t name##_retain_ref(T* value);                \
  IREE_API_EXPORT iree_vm_ref_t name##_move_ref(T* value);                  \
  static inline T* name##_deref(const iree_vm_ref_t ref) {                  \
    IREE_VM_REF_ASSERT(name##_registration);                                \
    return IREE_LIKELY(name##_isa(ref)) ? (T*)ref.ptr : NULL;               \
  }                                                                         \
  IREE_API_EXPORT iree_status_t name##_check_deref(const iree_vm_ref_t ref, \
                                                   T** out_ptr);            \
  IREE_API_EXPORT iree_status_t name##_check_deref_or_null(                 \
      const iree_vm_ref_t ref, T** out_ptr);                                \
  IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T)

// TODO(benvanik): make these macros standard/document them.
#define IREE_VM_DEFINE_TYPE_ADAPTERS(name, T)                               \
  iree_vm_ref_type_t name##_registration = 0;                               \
  IREE_API_EXPORT iree_vm_ref_t name##_retain_ref(T* value) {               \
    IREE_VM_REF_ASSERT(name##_registration);                                \
    iree_vm_ref_t ref = {0};                                                \
    iree_vm_ref_wrap_retain(value, name##_type(), &ref);                    \
    return ref;                                                             \
  }                                                                         \
  IREE_API_EXPORT iree_vm_ref_t name##_move_ref(T* value) {                 \
    IREE_VM_REF_ASSERT(name##_registration);                                \
    iree_vm_ref_t ref = {0};                                                \
    iree_vm_ref_wrap_assign(value, name##_type(), &ref);                    \
    return ref;                                                             \
  }                                                                         \
  IREE_API_EXPORT iree_status_t name##_check_deref(const iree_vm_ref_t ref, \
                                                   T** out_ptr) {           \
    IREE_VM_REF_ASSERT(name##_registration);                                \
    IREE_RETURN_IF_ERROR(iree_vm_ref_check(ref, name##_type()));            \
    *out_ptr = (T*)ref.ptr;                                                 \
    return iree_ok_status();                                                \
  }                                                                         \
  IREE_API_EXPORT iree_status_t name##_check_deref_or_null(                 \
      const iree_vm_ref_t ref, T** out_ptr) {                               \
    IREE_VM_REF_ASSERT(name##_registration);                                \
    if (ref.type != IREE_VM_REF_TYPE_NULL) {                                \
      IREE_RETURN_IF_ERROR(iree_vm_ref_check(ref, name##_type()));          \
      *out_ptr = (T*)ref.ptr;                                               \
    } else {                                                                \
      *out_ptr = NULL;                                                      \
    }                                                                       \
    return iree_ok_status();                                                \
  }

// Optional C++ iree::vm::ref<T> wrapper.
#ifdef __cplusplus
#include "iree/vm/ref_cc.h"
#else
#define IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T)
#define IREE_VM_DECLARE_CC_TYPE_ADAPTERS(name, T)
#endif  // __cplusplus

#endif  // IREE_VM_REF_H_
