// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// SPDX-License-Identifier: Apache-2.0

#pragma once

#include <Python.h>
#include <cuda.h>
#include <nvrtc.h>
#include <cstdint>
#include <memory>

// Forward declaration for NVVM - avoids nvvm.h dependency
// Use void* to match cuda.bindings.cynvvm's typedef
using nvvmProgram = void*;

// Forward declaration for nvJitLink - avoids nvJitLink.h dependency
// Use void* to match cuda.bindings.cynvjitlink's typedef
using nvJitLink_t = void*;

namespace cuda_core {

// ============================================================================
// TaggedHandle - make void*-based handle types distinct for overloading
//
// Both nvvmProgram and nvJitLink_t are void*, so shared_ptr<const void*>
// would be the same C++ type for both. TaggedHandle<T, Tag> wraps the raw
// value with a unique tag type, making each shared_ptr type distinct.
// ============================================================================

template<typename T, int Tag>
struct TaggedHandle {
    T raw;
};

using NvvmProgramValue = TaggedHandle<nvvmProgram, 0>;
using NvJitLinkValue = TaggedHandle<nvJitLink_t, 1>;

// CUtexObject, CUsurfObject and CUdeviceptr are all `unsigned long long`, so
// shared_ptr<const CUtexObject> et al. would be the *same* C++ type as
// DevicePtrHandle (and each other), collapsing the as_cu/as_intptr/as_py
// overload sets. Tag them to keep each handle type distinct, exactly as the
// NVVM / nvJitLink handles above do.
using TexObjectValue = TaggedHandle<CUtexObject, 2>;
using SurfObjectValue = TaggedHandle<CUsurfObject, 3>;

// ============================================================================
// Thread-local error handling
// ============================================================================

// Get and clear the last CUDA error (like cudaGetLastError)
CUresult get_last_error() noexcept;

// Get the last CUDA error without clearing it (like cudaPeekAtLastError)
CUresult peek_last_error() noexcept;

// Explicitly clear the last error
void clear_last_error() noexcept;

// ============================================================================
// CUDA driver function pointers
//
// These are populated by _resource_handles.pyx at module import time using
// function pointers extracted from cuda.bindings.cydriver.__pyx_capi__.
// ============================================================================

extern decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain;
extern decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease;
extern decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent;
extern decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate;
extern decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy;
extern decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx;
extern decltype(&cuDevResourceGenerateDesc) p_cuDevResourceGenerateDesc;

extern decltype(&cuGreenCtxStreamCreate) p_cuGreenCtxStreamCreate;

extern decltype(&cuStreamCreateWithPriority) p_cuStreamCreateWithPriority;
extern decltype(&cuStreamDestroy) p_cuStreamDestroy;

extern decltype(&cuEventCreate) p_cuEventCreate;
extern decltype(&cuEventDestroy) p_cuEventDestroy;
extern decltype(&cuIpcOpenEventHandle) p_cuIpcOpenEventHandle;

extern decltype(&cuDeviceGetCount) p_cuDeviceGetCount;

extern decltype(&cuMemPoolSetAccess) p_cuMemPoolSetAccess;
extern decltype(&cuMemPoolDestroy) p_cuMemPoolDestroy;
extern decltype(&cuMemPoolCreate) p_cuMemPoolCreate;
extern decltype(&cuDeviceGetMemPool) p_cuDeviceGetMemPool;
extern decltype(&cuMemPoolImportFromShareableHandle) p_cuMemPoolImportFromShareableHandle;

extern decltype(&cuMemAllocFromPoolAsync) p_cuMemAllocFromPoolAsync;
extern decltype(&cuMemAllocAsync) p_cuMemAllocAsync;
extern decltype(&cuMemAlloc) p_cuMemAlloc;
extern decltype(&cuMemAllocHost) p_cuMemAllocHost;

extern decltype(&cuMemFreeAsync) p_cuMemFreeAsync;
extern decltype(&cuMemFree) p_cuMemFree;
extern decltype(&cuMemFreeHost) p_cuMemFreeHost;

extern decltype(&cuMemPoolImportPointer) p_cuMemPoolImportPointer;

// Library
extern decltype(&cuLibraryLoadFromFile) p_cuLibraryLoadFromFile;
extern decltype(&cuLibraryLoadData) p_cuLibraryLoadData;
extern decltype(&cuLibraryUnload) p_cuLibraryUnload;
extern decltype(&cuLibraryGetKernel) p_cuLibraryGetKernel;

// Graph
extern decltype(&cuGraphDestroy) p_cuGraphDestroy;
extern decltype(&cuGraphInstantiateWithParams) p_cuGraphInstantiateWithParams;
extern decltype(&cuGraphExecUpdate) p_cuGraphExecUpdate;
extern decltype(&cuGraphExecDestroy) p_cuGraphExecDestroy;
extern decltype(&cuUserObjectCreate) p_cuUserObjectCreate;
extern decltype(&cuUserObjectRelease) p_cuUserObjectRelease;
extern decltype(&cuGraphRetainUserObject) p_cuGraphRetainUserObject;
extern decltype(&cuGraphReleaseUserObject) p_cuGraphReleaseUserObject;
extern decltype(&cuGraphNodeFindInClone) p_cuGraphNodeFindInClone;
extern decltype(&cuGraphChildGraphNodeGetGraph) p_cuGraphChildGraphNodeGetGraph;

// Linker
extern decltype(&cuLinkDestroy) p_cuLinkDestroy;

// Graphics interop
extern decltype(&cuGraphicsUnmapResources) p_cuGraphicsUnmapResources;
extern decltype(&cuGraphicsUnregisterResource) p_cuGraphicsUnregisterResource;

// Texture / surface / array (PR #467)
extern decltype(&cuArray3DCreate) p_cuArray3DCreate;
extern decltype(&cuArrayDestroy) p_cuArrayDestroy;
extern decltype(&cuMipmappedArrayCreate) p_cuMipmappedArrayCreate;
extern decltype(&cuMipmappedArrayDestroy) p_cuMipmappedArrayDestroy;
extern decltype(&cuMipmappedArrayGetLevel) p_cuMipmappedArrayGetLevel;
extern decltype(&cuTexObjectCreate) p_cuTexObjectCreate;
extern decltype(&cuTexObjectDestroy) p_cuTexObjectDestroy;
extern decltype(&cuSurfObjectCreate) p_cuSurfObjectCreate;
extern decltype(&cuSurfObjectDestroy) p_cuSurfObjectDestroy;

// SM resource split (13.1+ — may be null on older drivers/bindings)
#if CUDA_VERSION >= 13010
extern decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit;
#else
// cuDevSmResourceSplit doesn't exist in CUDA < 13.1 headers, so use a
// void* placeholder. The pointer is always null when built against 12.x.
extern void* p_cuDevSmResourceSplit;
#endif

// ============================================================================
// NVRTC function pointers
//
// These are populated by _resource_handles.pyx at module import time using
// function pointers extracted from cuda.bindings.cynvrtc.__pyx_capi__.
// ============================================================================

extern decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram;

// ============================================================================
// NVVM function pointers
//
// These are populated by _resource_handles.pyx at module import time using
// function pointers extracted from cuda.bindings.cynvvm.__pyx_capi__.
// Note: May be null if NVVM is not available at runtime.
// ============================================================================

// Function pointer type for nvvmDestroyProgram (avoids nvvm.h dependency)
// Signature: nvvmResult nvvmDestroyProgram(nvvmProgram *prog)
using NvvmDestroyProgramFn = int (*)(nvvmProgram*);
extern NvvmDestroyProgramFn p_nvvmDestroyProgram;

// ============================================================================
// nvJitLink function pointers
//
// These are populated by _resource_handles.pyx at module import time using
// function pointers extracted from cuda.bindings.cynvjitlink.__pyx_capi__.
// Note: May be null if nvJitLink is not available at runtime.
// ============================================================================

// Function pointer type for nvJitLinkDestroy (avoids nvJitLink.h dependency)
// Signature: nvJitLinkResult nvJitLinkDestroy(nvJitLinkHandle *handle)
using NvJitLinkDestroyFn = int (*)(nvJitLink_t*);
extern NvJitLinkDestroyFn p_nvJitLinkDestroy;

// ============================================================================
// Handle type aliases - expose only the raw CUDA resource
// ============================================================================

using ContextHandle = std::shared_ptr<const CUcontext>;
using GreenCtxHandle = std::shared_ptr<const CUgreenCtx>;
using StreamHandle = std::shared_ptr<const CUstream>;
using EventHandle = std::shared_ptr<const CUevent>;
using MemoryPoolHandle = std::shared_ptr<const CUmemoryPool>;
using LibraryHandle = std::shared_ptr<const CUlibrary>;
using KernelHandle = std::shared_ptr<const CUkernel>;
using GraphHandle = std::shared_ptr<const CUgraph>;
using GraphExecHandle = std::shared_ptr<const CUgraphExec>;
using GraphNodeHandle = std::shared_ptr<const CUgraphNode>;
using GraphicsResourceHandle = std::shared_ptr<const CUgraphicsResource>;
using NvrtcProgramHandle = std::shared_ptr<const nvrtcProgram>;
using NvvmProgramHandle = std::shared_ptr<const NvvmProgramValue>;
using NvJitLinkHandle = std::shared_ptr<const NvJitLinkValue>;
using CuLinkHandle = std::shared_ptr<const CUlinkState>;
using FileDescriptorHandle = std::shared_ptr<const int>;
using OpaqueArrayHandle = std::shared_ptr<const CUarray>;
using MipmappedArrayHandle = std::shared_ptr<const CUmipmappedArray>;
using TexObjectHandle = std::shared_ptr<const TexObjectValue>;
using SurfObjectHandle = std::shared_ptr<const SurfObjectValue>;


// ============================================================================
// Context handle functions
// ============================================================================

// Function to create a non-owning context handle (references existing context).
ContextHandle create_context_handle_ref(CUcontext ctx);

// Create a context handle for the CUcontext view of the provided green context.
// The returned ContextHandle keeps the green context alive, but the CUcontext
// view is non-owning and is not destroyed independently.
ContextHandle create_context_handle_from_green_ctx(const GreenCtxHandle& h_green_ctx);

// Return the green context dependency associated with a ContextHandle, if any.
GreenCtxHandle get_context_green_ctx(const ContextHandle& h) noexcept;

// Create an owning green context handle from a list of device resources.
GreenCtxHandle create_green_ctx_handle(CUdevResource* resources, unsigned int nbResources,
                                       CUdevice dev, unsigned int flags);

// Create a non-owning green context handle.
GreenCtxHandle create_green_ctx_handle_ref(CUgreenCtx ctx);

// Get handle to the primary context for a device (with thread-local caching)
// Returns empty handle on error (caller must check)
ContextHandle get_primary_context(int device_id);

// Get handle to the current CUDA context
// Returns empty handle if no context is current (caller must check)
ContextHandle get_current_context();

// ============================================================================
// Stream handle functions
// ============================================================================

// Create an owning stream handle by calling cuStreamCreateWithPriority.
// The stream structurally depends on the provided context handle.
// When the last reference is released, cuStreamDestroy is called automatically.
// Returns empty handle on error (caller must check).
StreamHandle create_stream_handle(const ContextHandle& h_ctx, unsigned int flags, int priority);

// Create a non-owning stream handle (references existing stream).
// Use for borrowed streams (from foreign code) or built-in streams.
// The stream will NOT be destroyed when the handle is released.
// Caller is responsible for keeping the stream's context alive.
StreamHandle create_stream_handle_ref(CUstream stream);

// Create a non-owning stream handle that prevents a Python owner from being GC'd.
// The owner's refcount is incremented; decremented when handle is released.
// The owner is responsible for keeping the stream's context alive.
StreamHandle create_stream_handle_with_owner(CUstream stream, PyObject* owner);

// Destroy a Python-backed CUDA user object by decref'ing it when safe.
// If Python is finalized or finalizing, the object is intentionally leaked.
void py_object_user_object_destroy(void* py_object) noexcept;

// Initialize the process-lifetime CUDA user-object cleanup queue. Called once
// from module initialization while Python is fully initialized.
void initialize_deferred_cleanup();
void retry_deferred_cleanup() noexcept;

// Return the context dependency associated with a stream handle, if any.
ContextHandle get_stream_context(const StreamHandle& h) noexcept;

// Get non-owning handle to the legacy default stream (CU_STREAM_LEGACY)
// Note: Legacy stream has no specific context dependency.
StreamHandle get_legacy_stream();

// Get non-owning handle to the per-thread default stream (CU_STREAM_PER_THREAD)
// Note: Per-thread stream has no specific context dependency.
StreamHandle get_per_thread_stream();

// ============================================================================
// Event handle functions
// ============================================================================

// Create an owning event handle by calling cuEventCreate.
// The event structurally depends on the provided context handle.
// Metadata fields are stored in the EventBox for later retrieval.
// When the last reference is released, cuEventDestroy is called automatically.
// Returns empty handle on error (caller must check).
EventHandle create_event_handle(const ContextHandle& h_ctx, unsigned int flags,
                                bool timing_enabled, bool is_blocking_sync,
                                bool ipc_enabled, int device_id);

// Create an owning event handle without context dependency.
// Use for temporary events that are created and destroyed in the same scope.
// When the last reference is released, cuEventDestroy is called automatically.
// Returns empty handle on error (caller must check).
EventHandle create_event_handle_noctx(unsigned int flags);

// Create an owning event handle from an IPC handle.
// The originating process owns the event and its context.
// When the last reference is released, cuEventDestroy is called automatically.
// Returns empty handle on error (caller must check).
EventHandle create_event_handle_ipc(const CUipcEventHandle& ipc_handle,
                                    bool is_blocking_sync);

// Create a non-owning event handle (references existing event).
// Use for events that are managed by the CUDA graph or another owner.
// The event will NOT be destroyed when the handle is released.
// Metadata defaults to unknown (timing_enabled=false, device_id=-1).
EventHandle create_event_handle_ref(CUevent event);

// Event metadata accessors (read from EventBox via pointer arithmetic)
bool get_event_timing_enabled(const EventHandle& h) noexcept;
bool get_event_is_blocking_sync(const EventHandle& h) noexcept;
bool get_event_ipc_enabled(const EventHandle& h) noexcept;
int get_event_device_id(const EventHandle& h) noexcept;
ContextHandle get_event_context(const EventHandle& h) noexcept;

// ============================================================================
// Memory pool handle functions
// ============================================================================

// Create an owning memory pool handle by calling cuMemPoolCreate.
// Memory pools are device-scoped (not context-scoped).
// When the last reference is released, cuMemPoolDestroy is called automatically.
// Returns empty handle on error (caller must check).
MemoryPoolHandle create_mempool_handle(const CUmemPoolProps& props);

// Create a non-owning memory pool handle (references existing pool).
// Use for device default/current pools that are managed by the driver.
// The pool will NOT be destroyed when the handle is released.
MemoryPoolHandle create_mempool_handle_ref(CUmemoryPool pool);

// Get non-owning handle to the current memory pool for a device.
// Returns empty handle on error (caller must check).
MemoryPoolHandle get_device_mempool(int device_id);

// Create an owning memory pool handle from an IPC import.
// The file descriptor is NOT owned by this handle (caller manages FD separately).
// When the last reference is released, cuMemPoolDestroy is called automatically.
// Returns empty handle on error (caller must check).
MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type);

// ============================================================================
// Device pointer handle functions
// ============================================================================

using DevicePtrHandle = std::shared_ptr<const CUdeviceptr>;

// Allocate device memory from a pool asynchronously via cuMemAllocFromPoolAsync.
// The pointer structurally depends on the provided pool handle (captured in deleter).
// When the last reference is released, cuMemFreeAsync is called on the stored stream.
// Returns empty handle on error (caller must check).
DevicePtrHandle deviceptr_alloc_from_pool(
    size_t size,
    const MemoryPoolHandle& h_pool,
    const StreamHandle& h_stream);

// Allocate device memory asynchronously via cuMemAllocAsync.
// When the last reference is released, cuMemFreeAsync is called on the stored stream.
// Returns empty handle on error (caller must check).
DevicePtrHandle deviceptr_alloc_async(size_t size, const StreamHandle& h_stream);

// Allocate device memory synchronously via cuMemAlloc.
// When the last reference is released, cuMemFree is called.
// Returns empty handle on error (caller must check).
DevicePtrHandle deviceptr_alloc(size_t size);

// Allocate pinned host memory via cuMemAllocHost.
// When the last reference is released, cuMemFreeHost is called.
// Returns empty handle on error (caller must check).
DevicePtrHandle deviceptr_alloc_host(size_t size);

// Create a non-owning device pointer handle (references existing pointer).
// Use for foreign pointers (e.g., from external libraries).
// The pointer will NOT be freed when the handle is released.
DevicePtrHandle deviceptr_create_ref(CUdeviceptr ptr);

// Create a non-owning device pointer handle that prevents a Python owner from being GC'd.
// The owner's refcount is incremented; decremented when handle is released.
// The pointer will NOT be freed when the handle is released.
// If owner is nullptr, equivalent to deviceptr_create_ref.
DevicePtrHandle deviceptr_create_with_owner(CUdeviceptr ptr, PyObject* owner);

// Create a device pointer handle for a mapped graphics resource.
// The pointer structurally depends on the provided graphics resource handle.
// When the last reference is released, cuGraphicsUnmapResources is called on
// the stored stream, then the graphics resource may be unregistered when its
// own handle is released.
DevicePtrHandle deviceptr_create_mapped_graphics(
    CUdeviceptr ptr,
    const GraphicsResourceHandle& h_resource,
    const StreamHandle& h_stream);

// Callback type for MemoryResource deallocation.
// Called from the shared_ptr deleter when a handle created via
// deviceptr_create_with_mr is destroyed.  The implementation is responsible
// for converting raw C types to Python objects and calling
// mr.deallocate(ptr, size, stream).
using MRDeallocCallback = void (*)(PyObject* mr, CUdeviceptr ptr,
                                   size_t size, const StreamHandle& stream);

// Register the MR deallocation callback.
void register_mr_dealloc_callback(MRDeallocCallback cb);

// Create a device pointer handle whose destructor calls mr.deallocate()
// via the registered callback.  The mr's refcount is incremented and
// decremented when the handle is released.
// If mr is nullptr, equivalent to deviceptr_create_ref.
DevicePtrHandle deviceptr_create_with_mr(CUdeviceptr ptr, size_t size, PyObject* mr);

// Import a device pointer from IPC via cuMemPoolImportPointer.
// When the last reference is released, cuMemFreeAsync is called on the stored stream.
// Note: Does not yet implement reference counting for nvbug 5570902.
// On error, returns empty handle and sets thread-local error (use get_last_error()).
DevicePtrHandle deviceptr_import_ipc(
    const MemoryPoolHandle& h_pool,
    const void* export_data,
    const StreamHandle& h_stream);

// Access the deallocation stream for a device pointer handle (read-only).
// For non-owning handles, the stream is not used but can still be accessed.
StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept;

// Set the deallocation stream for a device pointer handle.
void set_deallocation_stream(const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept;

// ============================================================================
// Library handle functions
// ============================================================================

// Create an owning library handle by loading from a file path.
// When the last reference is released, cuLibraryUnload is called automatically.
// Returns empty handle on error (caller must check).
LibraryHandle create_library_handle_from_file(const char* path);

// Create an owning library handle by loading from memory data.
// The driver makes an internal copy of the data; caller can free it after return.
// When the last reference is released, cuLibraryUnload is called automatically.
// Returns empty handle on error (caller must check).
LibraryHandle create_library_handle_from_data(const void* data);

// Create a non-owning library handle (references existing library).
// Use for borrowed libraries (e.g., from foreign code).
// The library will NOT be unloaded when the handle is released.
LibraryHandle create_library_handle_ref(CUlibrary library);

// ============================================================================
// Kernel handle functions
// ============================================================================

// Get a kernel from a library by name.
// The kernel structurally depends on the provided library handle.
// Kernels have no explicit destroy - their lifetime is tied to the library.
// Returns empty handle on error (caller must check).
KernelHandle create_kernel_handle(const LibraryHandle& h_library, const char* name);

// Create a kernel handle from a raw CUkernel.
// If the kernel is already managed (in the registry), returns the owning
// handle with library dependency. Otherwise returns a non-owning ref.
KernelHandle create_kernel_handle_ref(CUkernel kernel);

// Get the library handle associated with a kernel (from KernelBox).
// Returns empty handle if the kernel has no library dependency.
LibraryHandle get_kernel_library(const KernelHandle& h) noexcept;

// ============================================================================
// Graph handle functions
// ============================================================================

// Create the owning handle for a root graph and its hierarchy.
GraphHandle create_graph_handle(CUgraph graph);

// Create the canonical handle for a graph whose CUDA lifetime is owned by a
// node in h_parent.
GraphHandle create_child_graph_handle(
    CUgraph child_graph, const GraphHandle& h_parent, CUgraphNode owner_node);

// ============================================================================
// Graph node attachments
//
// Each resource-bearing node has one attachment with an immutable owner bundle,
// retained on its CUgraph as a CUDA user object.
//
// Attachment mutations use prepare -> CUDA mutation -> commit. Preparation
// graph-retains a replacement and preallocates its map entry when needed; an
// empty replacement stages removal. Dropping an uncommitted PreparedAttachment
// rolls back any staged retain. Commit updates metadata before releasing the
// previous graph reference.
// graph_get_attachment lets callers carry unchanged owners into partial
// updates. The clone and invalidation helpers synchronize non-owning metadata
// after CUDA copies or destroys graph state.
// ============================================================================

// Type-erased shared owner of an attached resource. Typed handles such as
// EventHandle and KernelHandle convert to OpaqueHandle by assignment, reusing
// their existing control block; the helpers below build OpaqueHandles for the
// two cases that need a custom deleter.
using OpaqueHandle = std::shared_ptr<const void>;

// Build an OpaqueHandle from a Python object: increments its refcount now and
// decrements it (under the GIL) on release. The caller must hold the GIL.
OpaqueHandle make_opaque_py(PyObject* obj);

// Build an OpaqueHandle from a malloc'd buffer: std::free on release.
OpaqueHandle make_opaque_malloc(void* buf);

struct PreparedAttachmentState;
using PreparedAttachmentRollback =
    void (*)(PreparedAttachmentState*) noexcept;
struct PreparedAttachmentDeleter {
    PreparedAttachmentRollback rollback = nullptr;

    void operator()(PreparedAttachmentState* state) const noexcept {
        rollback(state);
    }
};
using PreparedAttachment =
    std::unique_ptr<PreparedAttachmentState, PreparedAttachmentDeleter>;

struct PreparedChildGraphUpdateState;
// Opaque unpublished hierarchy transaction; releasing it discards staged
// metadata unless graph_commit_child_graph_update publishes the replacement.
using PreparedChildGraphUpdate =
    std::shared_ptr<PreparedChildGraphUpdateState>;

struct PreparedExecAttachmentState;
using PreparedExecAttachmentRollback =
    void (*)(PreparedExecAttachmentState*) noexcept;
struct PreparedExecAttachmentDeleter {
    PreparedExecAttachmentRollback rollback = nullptr;

    void operator()(PreparedExecAttachmentState* state) const noexcept {
        rollback(state);
    }
};
// Opaque append transaction. Releasing it rolls back newly appended owners
// unless graph_commit_exec_attachment has kept them.
using PreparedExecAttachment =
    std::unique_ptr<PreparedExecAttachmentState, PreparedExecAttachmentDeleter>;

// Copy requested owners from node's current attachment. Pass nullptr to ignore
// either owner; a missing attachment produces empty handles.
CUresult graph_get_attachment(
    const GraphHandle& h_graph,
    CUgraphNode node,
    OpaqueHandle* owner0,
    OpaqueHandle* owner1);

// Create and graph-retain a replacement attachment before a CUDA mutation.
// Destruction rolls the prepared attachment back unless it is committed.
CUresult graph_prepare_attachment(
    const GraphHandle& h_graph,
    OpaqueHandle owner0,
    OpaqueHandle owner1,
    PreparedAttachment* out_prepared);

// Publish a prepared attachment after the CUDA mutation succeeds. A null node
// retains the attachment anonymously without publishing node metadata.
CUresult graph_commit_attachment(
    PreparedAttachment& prepared,
    CUgraphNode node);

// Copy attachment metadata from a source graph hierarchy into its CUDA clone.
CUresult graph_clone_attachments(
    const GraphHandle& h_clone,
    const GraphHandle& h_source);

// Stage a complete metadata replacement before CUDA replaces an embedded
// graph. Dropping the prepared state leaves the current hierarchy unchanged.
CUresult graph_prepare_child_graph_update(
    const GraphHandle& h_parent,
    const GraphHandle& h_old_child,
    CUgraphNode owner_node,
    const GraphHandle& h_source,
    PreparedChildGraphUpdate* out_prepared);

// Rekey staged metadata to CUDA's replacement clone, retire the old embedded
// hierarchy, and publish the replacement handle.
CUresult graph_commit_child_graph_update(
    PreparedChildGraphUpdate& prepared,
    GraphHandle* out_child);

// Invalidate cuda.core state for child graphs CUDA destroyed with owner_node.
void invalidate_child_graph_state(
    const GraphHandle& h_parent,
    CUgraphNode owner_node) noexcept;

// ============================================================================
// Graph exec handle functions
// ============================================================================

// Create an owning exec handle by calling cuGraphInstantiateWithParams.
// A fresh attachment accumulator is retained on h_source first, because CUDA
// propagates user object references only at instantiation; an exec cannot
// receive them afterwards. The exec is the sole owner once this returns.
// When the last reference is released, cuGraphExecDestroy is called
// automatically.
// Returns empty handle on error (caller must check). The caller reads
// params->result_out for the specific instantiation failure and
// get_last_error() for a driver status.
GraphExecHandle create_graph_exec_handle(
    const GraphHandle& h_source,
    CUDA_GRAPH_INSTANTIATE_PARAMS* params);

// Update h_exec in place by calling cuGraphExecUpdate, and publish a fresh
// accumulator when CUDA accepts the update. Writes result_info for the caller.
CUresult graph_exec_update(
    const GraphExecHandle& h_exec,
    const GraphHandle& h_source,
    CUgraphExecUpdateResultInfo* result_info);

// Append owners before an executable-node mutation. The accumulator grows
// because CUDA cannot attach user objects to an exec after instantiation, so
// old owners stay reachable. Dropping the transaction restores the accumulator
// to its original size.
CUresult graph_prepare_exec_attachment(
    const GraphExecHandle& h_exec,
    OpaqueHandle owner0,
    OpaqueHandle owner1,
    PreparedExecAttachment* out_prepared);

// Keep the owners added by graph_prepare_exec_attachment.
void graph_commit_exec_attachment(
    PreparedExecAttachment& prepared) noexcept;

// ============================================================================
// Graph node handle functions
// ============================================================================

// Create a node handle. Nodes are owned by their parent graph (not
// independently destroyable). The GraphHandle dependency ensures the
// graph outlives any node reference.
GraphNodeHandle create_graph_node_handle(CUgraphNode node, const GraphHandle& h_graph);

// Extract the owning graph handle from a node handle.
GraphHandle graph_node_get_graph(const GraphNodeHandle& h) noexcept;

// Zero the CUgraphNode resource inside the handle, marking it invalid.
void invalidate_graph_node(const GraphNodeHandle& h) noexcept;

// ============================================================================
// Graphics resource handle functions
// ============================================================================

// Create an owning graphics resource handle.
// When the last reference is released, cuGraphicsUnregisterResource is called automatically.
// Use for CUgraphicsResource handles obtained from cuGraphicsGLRegisterBuffer etc.
GraphicsResourceHandle create_graphics_resource_handle(CUgraphicsResource resource);

// ============================================================================
// NVRTC Program handle functions
// ============================================================================

// Create an owning NVRTC program handle.
// When the last reference is released, nvrtcDestroyProgram is called.
// Use this to wrap a program created via nvrtcCreateProgram.
NvrtcProgramHandle create_nvrtc_program_handle(nvrtcProgram prog);

// Create a non-owning NVRTC program handle (references existing program).
// The program will NOT be destroyed when the handle is released.
NvrtcProgramHandle create_nvrtc_program_handle_ref(nvrtcProgram prog);

// ============================================================================
// NVVM Program handle functions
// ============================================================================

// Create an owning NVVM program handle.
// When the last reference is released, nvvmDestroyProgram is called.
// Use this to wrap a program created via nvvmCreateProgram.
// Note: If NVVM is not available (p_nvvmDestroyProgram is null), the deleter is a no-op.
NvvmProgramHandle create_nvvm_program_handle(nvvmProgram prog);

// Create a non-owning NVVM program handle (references existing program).
// The program will NOT be destroyed when the handle is released.
NvvmProgramHandle create_nvvm_program_handle_ref(nvvmProgram prog);

// ============================================================================
// nvJitLink handle functions
// ============================================================================

// Create an owning nvJitLink handle.
// When the last reference is released, nvJitLinkDestroy is called.
// Use this to wrap a handle created via nvJitLinkCreate.
// Note: If nvJitLink is not available (p_nvJitLinkDestroy is null), the deleter is a no-op.
NvJitLinkHandle create_nvjitlink_handle(nvJitLink_t handle);

// Create a non-owning nvJitLink handle (references existing handle).
// The handle will NOT be destroyed when the last reference is released.
NvJitLinkHandle create_nvjitlink_handle_ref(nvJitLink_t handle);

// ============================================================================
// cuLink handle functions
// ============================================================================

// Create an owning cuLink handle.
// When the last reference is released, cuLinkDestroy is called.
// Use this to wrap a CUlinkState created via cuLinkCreate.
CuLinkHandle create_culink_handle(CUlinkState state);

// Create a non-owning cuLink handle (references existing CUlinkState).
// The handle will NOT be destroyed when the last reference is released.
CuLinkHandle create_culink_handle_ref(CUlinkState state);

// ============================================================================
// File descriptor handle functions
// ============================================================================

// Create an owning file descriptor handle.
// When the last reference is released, POSIX close() is called.
FileDescriptorHandle create_fd_handle(int fd);

// Create a non-owning file descriptor handle (caller manages the fd).
FileDescriptorHandle create_fd_handle_ref(int fd);

// ============================================================================
// Array / mipmapped-array / texture / surface handle functions (PR #467)
//
// These resources are managed exactly like every other cuda.core resource:
// the owning handle's deleter calls the matching cu*Destroy with the GIL
// released, structural dependencies are embedded in the box (so a backing
// resource always outlives a texture/surface/level built on it), and
// creation returns an empty handle + thread-local error on failure.
// ============================================================================

// Create an owning CUDA array via cuArray3DCreate.
// When the last reference is released, cuArrayDestroy is called automatically.
// Returns empty handle on error (caller must check).
OpaqueArrayHandle create_array_handle(const CUDA_ARRAY3D_DESCRIPTOR& desc);

// Create a non-owning array handle (references an existing CUarray).
// Use for arrays owned elsewhere (e.g. graphics interop). Never destroyed here.
OpaqueArrayHandle create_array_handle_ref(CUarray arr);

// Create an owning array handle adopting an existing CUarray.
// When the last reference is released, cuArrayDestroy is called automatically.
OpaqueArrayHandle create_array_handle_owning(CUarray arr);

// Create a non-owning handle to a mipmap level via cuMipmappedArrayGetLevel.
// The level CUarray is owned by the mipmap; the parent MipmappedArrayHandle is
// embedded in the box so it outlives the level view. No destroy in the deleter.
// Returns empty handle on error (caller must check).
OpaqueArrayHandle create_array_level_handle(const MipmappedArrayHandle& h_mip, unsigned int level);

// Create an owning mipmapped array via cuMipmappedArrayCreate.
// When the last reference is released, cuMipmappedArrayDestroy is called.
// Returns empty handle on error (caller must check).
MipmappedArrayHandle create_mipmapped_array_handle(const CUDA_ARRAY3D_DESCRIPTOR& desc,
                                                   unsigned int num_levels);

// Create an owning texture object via cuTexObjectCreate, embedding the backing
// resource handle (array / mipmapped array / linear-or-pitch2d device pointer)
// so the backing always outlives the texture. cuTexObjectDestroy runs in the
// deleter. Returns empty handle on error (caller must check).
TexObjectHandle create_tex_object_handle_array(const CUDA_RESOURCE_DESC& res,
                                               const CUDA_TEXTURE_DESC& tex,
                                               const OpaqueArrayHandle& h_backing);
TexObjectHandle create_tex_object_handle_mipmap(const CUDA_RESOURCE_DESC& res,
                                                const CUDA_TEXTURE_DESC& tex,
                                                const MipmappedArrayHandle& h_backing);
TexObjectHandle create_tex_object_handle_linear(const CUDA_RESOURCE_DESC& res,
                                                const CUDA_TEXTURE_DESC& tex,
                                                const DevicePtrHandle& h_backing);

// Create an owning surface object via cuSurfObjectCreate, embedding the backing
// array handle so it outlives the surface. cuSurfObjectDestroy runs in the
// deleter. Returns empty handle on error (caller must check).
SurfObjectHandle create_surf_object_handle(const CUDA_RESOURCE_DESC& res,
                                           const OpaqueArrayHandle& h_backing);

// ============================================================================
// Overloaded helper functions to extract raw resources from handles
// ============================================================================

// as_cu() - extract the raw CUDA handle
inline CUcontext as_cu(const ContextHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUgreenCtx as_cu(const GreenCtxHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUstream as_cu(const StreamHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUevent as_cu(const EventHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUmemoryPool as_cu(const MemoryPoolHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUdeviceptr as_cu(const DevicePtrHandle& h) noexcept {
    return h ? *h : 0;
}

inline CUlibrary as_cu(const LibraryHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUmodule as_cu(const CUmodule& h) noexcept {
    return h;
}

inline CUkernel as_cu(const KernelHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUgraph as_cu(const GraphHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUgraphExec as_cu(const GraphExecHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUgraphNode as_cu(const GraphNodeHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUgraphicsResource as_cu(const GraphicsResourceHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline nvrtcProgram as_cu(const NvrtcProgramHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline nvvmProgram as_cu(const NvvmProgramHandle& h) noexcept {
    return h ? h->raw : nullptr;
}

inline nvJitLink_t as_cu(const NvJitLinkHandle& h) noexcept {
    return h ? h->raw : nullptr;
}

inline CUlinkState as_cu(const CuLinkHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUarray as_cu(const OpaqueArrayHandle& h) noexcept {
    return h ? *h : nullptr;
}

inline CUmipmappedArray as_cu(const MipmappedArrayHandle& h) noexcept {
    return h ? *h : nullptr;
}

// CUtexObject / CUsurfObject are integer-valued (like CUdeviceptr); null is 0.
// The raw value lives in the tagged wrapper's `raw` field.
inline CUtexObject as_cu(const TexObjectHandle& h) noexcept {
    return h ? h->raw : 0;
}

inline CUsurfObject as_cu(const SurfObjectHandle& h) noexcept {
    return h ? h->raw : 0;
}

// as_intptr() - extract handle as intptr_t for Python interop
// Using signed intptr_t per C standard convention and issue #1342
inline std::intptr_t as_intptr(const ContextHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const GreenCtxHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const StreamHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const EventHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const MemoryPoolHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const DevicePtrHandle& h) noexcept {
    return static_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const LibraryHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const CUmodule& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const KernelHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const GraphHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const GraphExecHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const GraphNodeHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const GraphicsResourceHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const NvrtcProgramHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const NvvmProgramHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const NvJitLinkHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const CuLinkHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const FileDescriptorHandle& h) noexcept {
    return h ? static_cast<std::intptr_t>(*h) : -1;
}

inline std::intptr_t as_intptr(const OpaqueArrayHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const MipmappedArrayHandle& h) noexcept {
    return reinterpret_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const TexObjectHandle& h) noexcept {
    return static_cast<std::intptr_t>(as_cu(h));
}

inline std::intptr_t as_intptr(const SurfObjectHandle& h) noexcept {
    return static_cast<std::intptr_t>(as_cu(h));
}

// as_py() - convert handle to Python wrapper object (returns new reference)
#if PY_VERSION_HEX < 0x030D0000
extern "C" int _Py_IsFinalizing(void);
#endif

// Best-effort probe for interpreter shutdown.
//
// In CPython this is not a hard guarantee: finalization can begin after this
// returns false but before a later PyGILState_Ensure() or other Python C-API
// call.
//
// If that race is lost on a non-finalizer thread, CPython's behavior is
// version-dependent: on older supported versions (3.10-3.13) it may abruptly
// terminate the current thread (historically via PyThread_exit_thread(),
// without normal C++ unwinding), while on newer versions (3.14+) it may hang
// the thread until process exit.
//
// We still use this check because the policy in this layer is to avoid Python
// work once shutdown is underway and accept an intentional leak or skipped
// Python conversion in that edge case rather than add more complex deferral
// machinery.
inline bool py_is_finalizing() noexcept {
#if PY_VERSION_HEX >= 0x030D0000
    return Py_IsFinalizing();
#else
    return _Py_IsFinalizing() != 0;
#endif
}

namespace detail {
// n.b. class lookup is not cached to avoid deadlock hazard, see DESIGN.md
inline PyObject* make_py(const char* module_name, const char* class_name, std::intptr_t value) noexcept {
    if (py_is_finalizing()) {
        Py_RETURN_NONE;
    }
    PyObject* mod = PyImport_ImportModule(module_name);
    if (!mod) return nullptr;
    PyObject* cls = PyObject_GetAttrString(mod, class_name);
    Py_DECREF(mod);
    if (!cls) return nullptr;
    PyObject* result = PyObject_CallFunction(cls, "L", value);
    Py_DECREF(cls);
    return result;
}
}  // namespace detail

inline PyObject* as_py(const ContextHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUcontext", as_intptr(h));
}

inline PyObject* as_py(const GreenCtxHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUgreenCtx", as_intptr(h));
}

inline PyObject* as_py(const StreamHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUstream", as_intptr(h));
}

inline PyObject* as_py(const EventHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUevent", as_intptr(h));
}

inline PyObject* as_py(const MemoryPoolHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUmemoryPool", as_intptr(h));
}

inline PyObject* as_py(const DevicePtrHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUdeviceptr", as_intptr(h));
}

inline PyObject* as_py(const LibraryHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUlibrary", as_intptr(h));
}

inline PyObject* as_py(const CUmodule& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUmodule", as_intptr(h));
}

inline PyObject* as_py(const KernelHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUkernel", as_intptr(h));
}

inline PyObject* as_py(const GraphHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUgraph", as_intptr(h));
}

inline PyObject* as_py(const GraphExecHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUgraphExec", as_intptr(h));
}

inline PyObject* as_py(const GraphNodeHandle& h) noexcept {
    if (!as_intptr(h)) {
        Py_RETURN_NONE;
    }
    return detail::make_py("cuda.bindings.driver", "CUgraphNode", as_intptr(h));
}

inline PyObject* as_py(const NvrtcProgramHandle& h) noexcept {
    return detail::make_py("cuda.bindings.nvrtc", "nvrtcProgram", as_intptr(h));
}

inline PyObject* as_py(const NvvmProgramHandle& h) noexcept {
    // NVVM bindings use raw integers, not wrapper classes
    return PyLong_FromSsize_t(as_intptr(h));
}

inline PyObject* as_py(const NvJitLinkHandle& h) noexcept {
    // nvJitLink bindings use raw integers, not wrapper classes
    return PyLong_FromSsize_t(as_intptr(h));
}

inline PyObject* as_py(const CuLinkHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUlinkState", as_intptr(h));
}

inline PyObject* as_py(const GraphicsResourceHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUgraphicsResource", as_intptr(h));
}

inline PyObject* as_py(const FileDescriptorHandle& h) noexcept {
    return PyLong_FromSsize_t(as_intptr(h));
}

inline PyObject* as_py(const OpaqueArrayHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUarray", as_intptr(h));
}

inline PyObject* as_py(const MipmappedArrayHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUmipmappedArray", as_intptr(h));
}

inline PyObject* as_py(const TexObjectHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUtexObject", as_intptr(h));
}

inline PyObject* as_py(const SurfObjectHandle& h) noexcept {
    return detail::make_py("cuda.bindings.driver", "CUsurfObject", as_intptr(h));
}

// ============================================================================
// SM resource split wrapper (13.1+)
//
// Calls through p_cuDevSmResourceSplit if available, otherwise returns
// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the
// cydriver cdef function, which would fail at module init on cuda-bindings
// < 13.1 (see https://github.com/NVIDIA/cuda-python/issues/2063).
// ============================================================================

// groupParams is void* so the Cython declaration doesn't reference
// CU_DEV_SM_RESOURCE_GROUP_PARAMS (absent from cuda-bindings 13.0 .pxd).
CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups,
                           const CUdevResource* input, CUdevResource* remainder,
                           unsigned int flags, void* groupParams);

// Returns true if the cuDevSmResourceSplit function pointer is available.
bool has_sm_resource_split() noexcept;

}  // namespace cuda_core
