/* * Copyright 2014-2023 NVIDIA Corporation. All rights reserved. * * NOTICE TO LICENSEE: * * This source code and/or documentation ("Licensed Deliverables") are * subject to NVIDIA intellectual property rights under U.S. and * international Copyright laws. * * These Licensed Deliverables contained herein is PROPRIETARY and * CONFIDENTIAL to NVIDIA and is being provided under the terms and * conditions of a form of NVIDIA software license agreement by and * between NVIDIA and Licensee ("License Agreement") or electronically * accepted by Licensee. Notwithstanding any terms or conditions to * the contrary in the License Agreement, reproduction or disclosure * of the Licensed Deliverables to any third party without the express * written consent of NVIDIA is prohibited. * * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE * OF THESE LICENSED DELIVERABLES. * * U.S. Government End Users. These Licensed Deliverables are a * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT * 1995), consisting of "commercial computer software" and "commercial * computer software documentation" as such terms are used in 48 * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government * only as a commercial end item. Consistent with 48 C.F.R.12.212 and * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all * U.S. Government End Users acquire the Licensed Deliverables with * only those rights set forth herein. * * Any use of the Licensed Deliverables in individual and commercial * software must include, in the user documentation and internal * comments to the code, the above Disclaimer and U.S. Government End * Users Notice. */ /** * @file cudnn_graph.h * @brief cuDNN Graph library: core definitions, enums, backend descriptor operations, and handle management. * @since cuDNN 9.0.0 */ /* * cudnn_graph : cuDNN's basic definitions operations. */ #if !defined(CUDNN_GRAPH_H_) #define CUDNN_GRAPH_H_ /* See: C5105_WINDOWS_SDK_WORKAROUND in src/common/include/cudnn_dls.h */ #ifdef _WIN32 #pragma warning(push) #pragma warning(disable: 5105) #endif #include #include #ifdef _WIN32 #pragma warning(pop) #endif #include #include "cudnn_version.h" /* These version numbers are autogenerated, do not edit manually. */ #define CUDNN_GRAPH_MAJOR 9 #define CUDNN_GRAPH_MINOR 24 #define CUDNN_GRAPH_PATCH 0 #if (CUDNN_GRAPH_MAJOR != CUDNN_MAJOR) || (CUDNN_GRAPH_MINOR != CUDNN_MINOR) || (CUDNN_GRAPH_PATCH != CUDNN_PATCHLEVEL) #error Version mismatch in cuDNN GRAPH!!! #endif #ifndef CUDNNWINAPI #ifdef _WIN32 #define CUDNNWINAPI __stdcall #else #define CUDNNWINAPI #endif #endif /* Warnings for deprecated API-s are enabled using the CUDNN_WARN_DEPRECATED macro */ #if defined(CUDNN_WARN_DEPRECATED) && (defined(__GNUC__) || defined(__clang__)) /* GCC, Intel C/C++, Cray C/C++, CLANG, IBM XL C/C++ little endian */ #define CUDNN_DEPRECATED __attribute__((deprecated)) #define CUDNN_DEPRECATED_ENUM __attribute__((deprecated)) #elif defined(CUDNN_WARN_DEPRECATED) && defined(_MSC_VER) /* Microsoft Visual C++ */ #define CUDNN_DEPRECATED __declspec(deprecated) #define CUDNN_DEPRECATED_ENUM __declspec(deprecated) #elif defined(CUDNN_WARN_DEPRECATED) && (__cplusplus >= 201402L) /* C++14 compilers */ #define CUDNN_DEPRECATED [[deprecated]] #define CUDNN_DEPRECATED_ENUM [[deprecated]] #else /* No support for the deprecated attribute */ #define CUDNN_DEPRECATED #define CUDNN_DEPRECATED_ENUM #endif #if defined(__cplusplus) extern "C" { #endif struct cudnnContext; /** @brief Opaque pointer to cuDNN library context. * @since cuDNN 9.0.0 */ typedef struct cudnnContext *cudnnHandle_t; /** @brief Returns cuDNN library version (MAJOR*10000 + MINOR*100 + PATCH). * @return The cuDNN version as an encoded integer. * @since cuDNN 9.0.0 */ size_t CUDNNWINAPI cudnnGetVersion(void); /** @brief Returns max supported GPU compute capability. * @return The maximum supported device version. * @since cuDNN 9.0.0 */ size_t CUDNNWINAPI cudnnGetMaxDeviceVersion(void); /** @brief Returns CUDA Runtime version linked against cuDNN. * @return The CUDA Runtime version. * @since cuDNN 9.0.0 */ /* Returns CUDA Runtime version statically linked against cudnn */ size_t CUDNNWINAPI cudnnGetCudartVersion(void); /** @brief Return status codes for cuDNN API calls. * * Status codes are grouped by category: 0=success, 1xxx=initialization/version, * 2xxx=bad parameter, 3xxx=not supported, 4xxx=internal error, 5xxx=execution failed. * @since cuDNN 9.0.0 */ /* * CUDNN return codes */ typedef enum { CUDNN_STATUS_SUCCESS = 0, /**< Operation completed successfully. @since cuDNN 9.0.0 */ /* Uncategorized errors */ CUDNN_STATUS_NOT_INITIALIZED = 1001, /**< cuDNN library not initialized. @since cuDNN 9.0.0 */ CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH = 1002, /**< Sub-library version mismatch. @since cuDNN 9.0.0 */ CUDNN_STATUS_SERIALIZATION_VERSION_MISMATCH = 1003, /**< Serialization version mismatch. @since cuDNN 9.0.0 */ CUDNN_STATUS_DEPRECATED = 1004, /**< Deprecated feature was used. @since cuDNN 9.0.0 */ CUDNN_STATUS_LICENSE_ERROR = 1005, /**< License error. @since cuDNN 9.0.0 */ CUDNN_STATUS_RUNTIME_IN_PROGRESS = 1006, /**< Runtime operation in progress. @since cuDNN 9.0.0 */ CUDNN_STATUS_RUNTIME_FP_OVERFLOW = 1007, /**< Floating-point overflow at runtime. @since cuDNN 9.0.0 */ CUDNN_STATUS_SUBLIBRARY_LOADING_FAILED = 1008, /**< Sub-library loading failed. @since cuDNN 9.2.0 */ CUDNN_STATUS_BAD_PARAM = 2000, /**< Invalid parameter value. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_NULL_POINTER = 2002, /**< Null pointer passed as parameter. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_MISALIGNED_POINTER = 2003, /**< Misaligned pointer passed. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_NOT_FINALIZED = 2004, /**< Descriptor not finalized. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_OUT_OF_BOUND = 2005, /**< Parameter out of bounds. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_SIZE_INSUFFICIENT = 2006, /**< Insufficient buffer size. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_STREAM_MISMATCH = 2007, /**< CUDA stream mismatch. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_SHAPE_MISMATCH = 2008, /**< Tensor shape mismatch. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_DUPLICATED_ENTRIES = 2009, /**< Duplicated entries detected. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_ATTRIBUTE_TYPE = 2010, /**< Wrong attribute type. @since cuDNN 9.0.0 */ CUDNN_STATUS_BAD_PARAM_CUDA_GRAPH_MISMATCH = 2011, /**< CUDA graph mismatch. @since cuDNN 9.5.0 */ CUDNN_STATUS_BAD_PARAM_DESCRIPTOR_TYPE = 2012, /**< Wrong descriptor type. @since cuDNN 9.6.0 */ CUDNN_STATUS_NOT_SUPPORTED = 3000, /**< Operation not supported. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_GRAPH_PATTERN = 3001, /**< Graph pattern not supported. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_SHAPE = 3002, /**< Shape not supported. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_DATA_TYPE = 3003, /**< Data type not supported. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_LAYOUT = 3004, /**< Layout not supported. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_INCOMPATIBLE_CUDA_DRIVER = 3005, /**< Incompatible CUDA driver. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_INCOMPATIBLE_CUDART = 3006, /**< Incompatible CUDA runtime. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_ARCH_MISMATCH = 3007, /**< GPU architecture mismatch. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_RUNTIME_PREREQUISITE_MISSING = 3008, /**< Runtime prerequisite missing. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_SUBLIBRARY_UNAVAILABLE = 3009, /**< Sub-library unavailable. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_SHARED_MEMORY_INSUFFICIENT = 3010, /**< Insufficient shared memory. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_PADDING = 3011, /**< Padding mode not supported. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_BAD_LAUNCH_PARAM = 3012, /**< Bad launch parameters. @since cuDNN 9.0.0 */ CUDNN_STATUS_NOT_SUPPORTED_CUDA_GRAPH_NATIVE_API = 3013, /**< CUDA graph native API not supported. @since cuDNN 9.5.0 */ CUDNN_STATUS_NOT_SUPPORTED_INVALID_DYNAMIC_SHAPE = 3014, /**< Invalid dynamic shape. @since cuDNN 9.18.0 */ CUDNN_STATUS_INTERNAL_ERROR = 4000, /**< Internal error. @since cuDNN 9.0.0 */ CUDNN_STATUS_INTERNAL_ERROR_COMPILATION_FAILED = 4001, /**< Kernel compilation failed. @since cuDNN 9.0.0 */ CUDNN_STATUS_INTERNAL_ERROR_UNEXPECTED_VALUE = 4002, /**< Unexpected internal value. @since cuDNN 9.0.0 */ CUDNN_STATUS_INTERNAL_ERROR_HOST_ALLOCATION_FAILED = 4003, /**< Host memory allocation failed. @since cuDNN 9.0.0 */ CUDNN_STATUS_INTERNAL_ERROR_DEVICE_ALLOCATION_FAILED = 4004, /**< Device memory allocation failed. @since cuDNN 9.0.0 */ CUDNN_STATUS_INTERNAL_ERROR_BAD_LAUNCH_PARAM = 4005, /**< Bad internal launch parameters. @since cuDNN 9.0.0 */ CUDNN_STATUS_INTERNAL_ERROR_TEXTURE_CREATION_FAILED = 4006, /**< Texture creation failed. @since cuDNN 9.0.0 */ CUDNN_STATUS_EXECUTION_FAILED = 5000, /**< Execution failed. @since cuDNN 9.0.0 */ CUDNN_STATUS_EXECUTION_FAILED_CUDA_DRIVER = 5001, /**< CUDA driver execution failure. @since cuDNN 9.0.0 */ CUDNN_STATUS_EXECUTION_FAILED_CUBLAS = 5002, /**< cuBLAS execution failure. @since cuDNN 9.0.0 */ CUDNN_STATUS_EXECUTION_FAILED_CUDART = 5003, /**< CUDA runtime execution failure. @since cuDNN 9.0.0 */ CUDNN_STATUS_EXECUTION_FAILED_CURAND = 5004, /**< cuRAND execution failure. @since cuDNN 9.0.0 */ CUDNN_STATUS_ALLOC_FAILED CUDNN_DEPRECATED_ENUM = CUDNN_STATUS_INTERNAL_ERROR_HOST_ALLOCATION_FAILED, /**< @deprecated Use CUDNN_STATUS_INTERNAL_ERROR_HOST_ALLOCATION_FAILED. @since cuDNN 9.0.0 */ CUDNN_STATUS_INVALID_VALUE CUDNN_DEPRECATED_ENUM = 2001 /* please transition to CUDNN_STATUS_BAD_PARAM instead */, /**< @deprecated Use CUDNN_STATUS_BAD_PARAM. @since cuDNN 9.0.0 */ CUDNN_STATUS_ARCH_MISMATCH CUDNN_DEPRECATED_ENUM = CUDNN_STATUS_NOT_SUPPORTED_ARCH_MISMATCH, /**< @deprecated Use CUDNN_STATUS_NOT_SUPPORTED_ARCH_MISMATCH. @since cuDNN 9.0.0 */ CUDNN_STATUS_MAPPING_ERROR CUDNN_DEPRECATED_ENUM = CUDNN_STATUS_INTERNAL_ERROR_TEXTURE_CREATION_FAILED, /**< @deprecated Use CUDNN_STATUS_INTERNAL_ERROR_TEXTURE_CREATION_FAILED. @since cuDNN 9.0.0 */ CUDNN_STATUS_RUNTIME_PREREQUISITE_MISSING CUDNN_DEPRECATED_ENUM = CUDNN_STATUS_NOT_SUPPORTED_RUNTIME_PREREQUISITE_MISSING, /**< @deprecated Use CUDNN_STATUS_NOT_SUPPORTED_RUNTIME_PREREQUISITE_MISSING. @since cuDNN 9.0.0 */ CUDNN_STATUS_VERSION_MISMATCH CUDNN_DEPRECATED_ENUM = CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH, /**< @deprecated Use CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH. @since cuDNN 9.0.0 */ } cudnnStatus_t; #define CUDNN_STATUS_FULL_ERROR_CODE(category, specific_err) ((cudnnStatus_t)(0 + (category) + (specific_err))) #define CUDNN_STATUS_CATEGORY(full_error_code) ((full_error_code) / 1000 * 1000) #define CUDNN_STATUS_SPECIFIC_ERROR(full_error_code) ((full_error_code) % 1000) /** @brief Converts status code to human-readable string. * @param[in] status The cuDNN status code to convert. * @return Pointer to a static string describing the status code. * @since cuDNN 9.0.0 */ /* human-readable error messages */ const char *CUDNNWINAPI cudnnGetErrorString(cudnnStatus_t status); /** @brief Retrieves most recent error message. Thread-safe. * @param[out] message Buffer to receive the error message string. * @param[in] max_size Maximum number of bytes to write into @p message. * @since cuDNN 9.0.0 */ void CUDNNWINAPI cudnnGetLastErrorString(char *message, size_t max_size); /* Forward definition in this version only */ typedef struct cudnnRuntimeTag_t cudnnRuntimeTag_t CUDNN_DEPRECATED; /** @brief Error query modes for cudnnQueryRuntimeError. * @deprecated * @since cuDNN 9.0.0 */ typedef enum { CUDNN_ERRQUERY_RAWCODE = 0, /**< Return raw error code. @since cuDNN 9.0.0 */ CUDNN_ERRQUERY_NONBLOCKING = 1, /**< Non-blocking error query. @since cuDNN 9.0.0 */ CUDNN_ERRQUERY_BLOCKING = 2, /**< Blocking error query. @since cuDNN 9.0.0 */ } cudnnErrQueryMode_t; /** @brief Queries remote kernel error state. * @deprecated Use cudnnGetLastErrorString instead. * @param[in] handle cuDNN handle. * @param[out] rstatus Pointer to receive the runtime status. * @param[in] mode Error query mode. * @param[out] tag Runtime tag (unused, may be NULL). * @return cuDNN status code. * @since cuDNN 9.0.0 */ CUDNN_DEPRECATED cudnnStatus_t CUDNNWINAPI cudnnQueryRuntimeError(cudnnHandle_t handle, cudnnStatus_t *rstatus, cudnnErrQueryMode_t mode, cudnnRuntimeTag_t *tag); /** @brief Queries library property (major, minor, or patch version). * @param[in] type The property type to query (MAJOR_VERSION, MINOR_VERSION, or PATCH_LEVEL). * @param[out] value Pointer to receive the property value. * @return cuDNN status code. * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnGetProperty(libraryPropertyType type, int *value); /** @brief Creates cuDNN context. Must precede all other cuDNN library calls. * @param[out] handle Pointer to receive the newly created cuDNN handle. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @retval CUDNN_STATUS_NOT_INITIALIZED * @retval CUDNN_STATUS_NOT_SUPPORTED_ARCH_MISMATCH * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnCreate(cudnnHandle_t *handle); /** @brief Destroys cuDNN context. Calls cudaDeviceSynchronize. * @param[in] handle The cuDNN handle to destroy. * @return cuDNN status code. * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnDestroy(cudnnHandle_t handle); /** @brief Associates CUDA stream with cuDNN handle. * @param[in] handle cuDNN handle. * @param[in] streamId CUDA stream to associate. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnSetStream(cudnnHandle_t handle, cudaStream_t streamId); /** @brief Retrieves CUDA stream from cuDNN handle. * @param[in] handle cuDNN handle. * @param[out] streamId Pointer to receive the associated CUDA stream. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnGetStream(cudnnHandle_t handle, cudaStream_t *streamId); /** @brief Supported data types for cuDNN tensors and operations. * @since cuDNN 9.0.0 */ /* * CUDNN data type */ typedef enum { CUDNN_DATA_FLOAT = 0, /**< 32-bit IEEE floating point. @since cuDNN 9.0.0 */ CUDNN_DATA_DOUBLE = 1, /**< 64-bit IEEE floating point. @since cuDNN 9.0.0 */ CUDNN_DATA_HALF = 2, /**< 16-bit IEEE floating point (FP16). @since cuDNN 9.0.0 */ CUDNN_DATA_INT8 = 3, /**< 8-bit signed integer. @since cuDNN 9.0.0 */ CUDNN_DATA_INT32 = 4, /**< 32-bit signed integer. @since cuDNN 9.0.0 */ CUDNN_DATA_INT8x4 CUDNN_DEPRECATED_ENUM = 5, /**< @deprecated Vectorized 4x INT8. @since cuDNN 9.0.0 */ CUDNN_DATA_UINT8 = 6, /**< 8-bit unsigned integer. @since cuDNN 9.0.0 */ CUDNN_DATA_UINT8x4 CUDNN_DEPRECATED_ENUM = 7, /**< @deprecated Vectorized 4x UINT8. @since cuDNN 9.0.0 */ CUDNN_DATA_INT8x32 CUDNN_DEPRECATED_ENUM = 8, /**< @deprecated Vectorized 32x INT8. @since cuDNN 9.0.0 */ CUDNN_DATA_BFLOAT16 = 9, /**< Brain floating point (BF16). @since cuDNN 9.0.0 */ CUDNN_DATA_INT64 = 10, /**< 64-bit signed integer. @since cuDNN 9.0.0 */ CUDNN_DATA_BOOLEAN = 11, /**< Boolean type. @since cuDNN 9.0.0 */ CUDNN_DATA_FP8_E4M3 = 12, /**< FP8 with 4-bit exponent, 3-bit mantissa. @since cuDNN 9.0.0 */ CUDNN_DATA_FP8_E5M2 = 13, /**< FP8 with 5-bit exponent, 2-bit mantissa. @since cuDNN 9.0.0 */ CUDNN_DATA_FAST_FLOAT_FOR_FP8 = 14, /**< Fast float accumulator type for FP8 compute paths. @since cuDNN 9.0.0 */ CUDNN_DATA_FP8_E8M0 = 15, /**< Pure-exponent scale format (8-bit exponent, 0-bit mantissa) for block scaling. @since cuDNN 9.7.0 */ CUDNN_DATA_FP4_E2M1 = 16, /**< FP4 with 2-bit exponent, 1-bit mantissa. @since cuDNN 9.7.0 */ CUDNN_DATA_INT4 = 17, /**< 4-bit signed integer. @since cuDNN 9.11.0 */ CUDNN_DATA_UINT4 = 18, /**< 4-bit unsigned integer. @since cuDNN 9.11.0 */ CUDNN_DATA_UINT32 = 19, /**< 32-bit unsigned integer. @since cuDNN 9.11.0 */ CUDNN_DATA_COMPLEX_FP32 = 20, /**< Complex 32-bit floating point. @since cuDNN 9.14.0 */ CUDNN_DATA_COMPLEX_FP64 = 21, /**< Complex 64-bit floating point. @since cuDNN 9.14.0 */ } cudnnDataType_t; /** @brief Math precision modes for cuDNN operations. * @since cuDNN 9.0.0 */ /* * CUDNN math type */ typedef enum { CUDNN_DEFAULT_MATH = 0, /**< Default math mode. @since cuDNN 9.0.0 */ CUDNN_TENSOR_OP_MATH = 1, /**< Tensor Core math. @since cuDNN 9.0.0 */ CUDNN_TENSOR_OP_MATH_ALLOW_CONVERSION = 2, /**< Tensor Core math with type conversion. @since cuDNN 9.0.0 */ CUDNN_FMA_MATH = 3, /**< FMA (fused multiply-add) math only. @since cuDNN 9.0.0 */ } cudnnMathType_t; /** @brief NaN propagation modes. * @deprecated * @since cuDNN 9.0.0 */ /* * CUDNN propagate Nan */ typedef enum { CUDNN_NOT_PROPAGATE_NAN CUDNN_DEPRECATED_ENUM = 0, /**< @deprecated Do not propagate NaN. @since cuDNN 9.0.0 */ CUDNN_PROPAGATE_NAN CUDNN_DEPRECATED_ENUM = 1, /**< @deprecated Propagate NaN values. @since cuDNN 9.0.0 */ } cudnnNanPropagation_t; /** @brief CTC gradient modes controlling behavior for out-of-bounds (OOB) samples. * @since cuDNN 9.0.0 */ /* * Behavior for OOB samples. OOB samples are samples where L+R > T is encountered during the gradient calculation. If * gradMode is set to CUDNN_CTC_SKIP_OOB_GRADIENTS, then the CTC loss function does not write to the gradient buffer for * that sample. Instead, the current values, even not finite, are retained. If gradMode is set to * CUDNN_CTC_ZERO_OOB_GRADIENTS, then the gradient for that sample is set to zero. This guarantees a finite gradient. */ typedef enum { CUDNN_CTC_ZERO_OOB_GRADIENTS = 0, /**< Zero the gradient for OOB samples (guarantees finite). @since cuDNN 9.0.0 */ CUDNN_CTC_SKIP_OOB_GRADIENTS = 1, /**< Skip writing gradient for OOB samples. @since cuDNN 9.0.0 */ } cudnnCTCGradMode_t; /** @brief Tensor memory layout formats. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_TENSOR_NCHW = 0, /**< Row major layout (wStride = 1, hStride = w). @since cuDNN 9.0.0 */ CUDNN_TENSOR_NHWC = 1, /**< Feature maps interleaved (cStride = 1). @since cuDNN 9.0.0 */ CUDNN_TENSOR_NCHW_VECT_C = 2, /**< Vectorized channel layout, vector length in data type. @since cuDNN 9.0.0 */ } cudnnTensorFormat_t; /** @brief Reduction operations for tensor reduction. * @since cuDNN 9.0.0 */ /* * CUDNN ReduceTensor op type */ typedef enum { CUDNN_REDUCE_TENSOR_ADD = 0, /**< Sum reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_MUL = 1, /**< Product reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_MIN = 2, /**< Minimum value reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_MAX = 3, /**< Maximum value reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_AMAX = 4, /**< Maximum absolute value reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_AVG = 5, /**< Average reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_NORM1 = 6, /**< L1 norm reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_NORM2 = 7, /**< L2 norm reduction. @since cuDNN 9.0.0 */ CUDNN_REDUCE_TENSOR_MUL_NO_ZEROS = 8, /**< Product reduction ignoring zeros. @since cuDNN 9.0.0 */ } cudnnReduceTensorOp_t; /** @brief Activation function modes. * @deprecated Use pointwise operations instead. * @since cuDNN 9.0.0 */ /* * activation mode */ typedef enum { CUDNN_ACTIVATION_SIGMOID = 0, /**< @deprecated Sigmoid activation. @since cuDNN 9.0.0 */ CUDNN_ACTIVATION_RELU = 1, /**< @deprecated ReLU activation. @since cuDNN 9.0.0 */ CUDNN_ACTIVATION_TANH = 2, /**< @deprecated Tanh activation. @since cuDNN 9.0.0 */ CUDNN_ACTIVATION_CLIPPED_RELU = 3, /**< @deprecated Clipped ReLU activation. @since cuDNN 9.0.0 */ CUDNN_ACTIVATION_ELU = 4, /**< @deprecated ELU activation. @since cuDNN 9.0.0 */ CUDNN_ACTIVATION_IDENTITY = 5, /**< @deprecated Identity (pass-through). @since cuDNN 9.0.0 */ CUDNN_ACTIVATION_SWISH = 6 /**< @deprecated Swish activation. @since cuDNN 9.0.0 */ } cudnnActivationMode_t CUDNN_DEPRECATED; /** @brief Debug severity levels for cuDNN callback messages. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_SEV_FATAL = 0, /**< Fatal error severity. @since cuDNN 9.0.0 */ CUDNN_SEV_ERROR = 1, /**< Error severity. @since cuDNN 9.0.0 */ CUDNN_SEV_WARNING = 2, /**< Warning severity. @since cuDNN 9.0.0 */ CUDNN_SEV_INFO = 3, /**< Informational severity. @since cuDNN 9.0.0 */ } cudnnSeverity_t; /* Message masks to be used with cudnnSetCallback() */ #define CUDNN_SEV_ERROR_EN (1U << CUDNN_SEV_ERROR) #define CUDNN_SEV_WARNING_EN (1U << CUDNN_SEV_WARNING) #define CUDNN_SEV_INFO_EN (1U << CUDNN_SEV_INFO) /** @brief Debug callback metadata containing version, status, timestamps, handle, stream, PID, TID, and device ID. * @since cuDNN 9.0.0 */ /* struct containing useful informaiton for each API call */ typedef struct cudnnDebugStruct { unsigned cudnn_version; /**< cuDNN library version. */ cudnnStatus_t cudnnStatus; /**< Status code for this API call. */ unsigned time_sec; /**< Epoch time in seconds. */ unsigned time_usec; /**< Microseconds part of epoch time. */ unsigned time_delta; /**< Time since start in seconds. */ cudnnHandle_t handle; /**< cuDNN handle. */ cudaStream_t stream; /**< CUDA stream ID. */ unsigned long long pid; /**< Process ID. */ unsigned long long tid; /**< Thread ID. */ int cudaDeviceId; /**< CUDA device ID. */ int reserved[15]; /**< Reserved for future use. */ } cudnnDebug_t; /** @brief Callback function type for cuDNN debug messages. * @since cuDNN 9.0.0 */ typedef void (*cudnnCallback_t)(cudnnSeverity_t sev, void *udata, const cudnnDebug_t *dbg, const char *msg); /** @brief Registers debug callback with message mask. * @param[in] mask Bitmask of severity levels to enable (see CUDNN_SEV_*_EN). * @param[in] udata User data pointer passed to callback. * @param[in] fptr Callback function pointer. * @return cuDNN status code. * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnSetCallback(unsigned mask, void *udata, cudnnCallback_t fptr); /** @brief Retrieves registered debug callback and its configuration. * @param[out] mask Pointer to receive the current severity mask. * @param[out] udata Pointer to receive the user data pointer. * @param[out] fptr Pointer to receive the callback function pointer. * @return cuDNN status code. * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnGetCallback(unsigned *mask, void **udata, cudnnCallback_t *fptr); /** @brief Cross-library version checker. * * This function is implemented differently in each sub-library. Each sublib * checks whether its own version matches that of its dependencies. * @retval CUDNN_STATUS_SUCCESS if the version check passes. * @retval CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH if the versions are inconsistent. * @since cuDNN 9.0.0 */ /* * \brief Cross-library version checker. * This function is implemented differently in each sub-library. Each sublib * checks whether its own version matches that of its dependencies. * \returns CUDNN_STATUS_SUCCESS if the version check passes, * CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH if the versions are inconsistent. */ cudnnStatus_t CUDNNWINAPI cudnnGraphVersionCheck(void); /* Maximum supported number of tensor dimensions */ #define CUDNN_DIM_MAX 8 /** @brief Convolution operation modes. * @since cuDNN 9.0.0 */ /* * convolution mode */ typedef enum { CUDNN_CONVOLUTION = 0, /**< Standard convolution. @since cuDNN 9.0.0 */ CUDNN_CROSS_CORRELATION = 1 /**< Cross-correlation. @since cuDNN 9.0.0 */ } cudnnConvolutionMode_t; /** @brief Tensor reorder type. * @deprecated * @since cuDNN 9.0.0 */ /* * CUDNN Reorder */ typedef enum { CUDNN_DEFAULT_REORDER = 0, /**< @deprecated Default reordering behavior. @since cuDNN 9.0.0 */ CUDNN_NO_REORDER = 1, /**< @deprecated No reordering. @since cuDNN 9.0.0 */ } cudnnReorderType_t CUDNN_DEPRECATED; /** @brief Opaque pointer to a cuDNN backend descriptor. * @since cuDNN 9.0.0 */ typedef void *cudnnBackendDescriptor_t; /** @brief Integer fraction with numerator and denominator. * @since cuDNN 9.0.0 */ typedef struct cudnnFractionStruct { int64_t numerator; /**< Fraction numerator. */ int64_t denominator; /**< Fraction denominator. */ } cudnnFraction_t; /** @brief Pointwise operation modes including binary, unary, activation forward/backward, * comparison, logical, and special operations. * @since cuDNN 9.0.0 */ typedef enum { /* Binary operations (0-9) */ CUDNN_POINTWISE_ADD = 0, /**< Element-wise addition. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_ADD_SQUARE = 5, /**< Element-wise add-and-square. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_DIV = 6, /**< Element-wise division. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_MAX = 3, /**< Element-wise maximum. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_MIN = 2, /**< Element-wise minimum. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_MOD = 7, /**< Element-wise modulo. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_MUL = 1, /**< Element-wise multiplication. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_POW = 8, /**< Element-wise power. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SUB = 9, /**< Element-wise subtraction. @since cuDNN 9.0.0 */ /* Unary operations (10-23) */ CUDNN_POINTWISE_ABS = 10, /**< Absolute value. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_CEIL = 11, /**< Ceiling. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_COS = 12, /**< Cosine. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_EXP = 13, /**< Exponential. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_FLOOR = 14, /**< Floor. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_LOG = 15, /**< Natural logarithm. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_NEG = 16, /**< Negation. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_RSQRT = 17, /**< Reciprocal square root. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SIN = 18, /**< Sine. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SQRT = 4, /**< Square root. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_TAN = 19, /**< Tangent. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_ERF = 20, /**< Error function. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_IDENTITY = 21, /**< Identity (no-op); enables implicit data type conversion between tensors. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_RECIPROCAL = 22, /**< Reciprocal. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_ATAN2 = 23, /**< Two-argument arctangent. @since cuDNN 9.1.0 */ /* Activation forward (100-107) */ CUDNN_POINTWISE_RELU_FWD = 100, /**< ReLU forward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_TANH_FWD = 101, /**< Tanh forward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SIGMOID_FWD = 102, /**< Sigmoid forward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_ELU_FWD = 103, /**< ELU forward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_GELU_FWD = 104, /**< GELU forward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SOFTPLUS_FWD = 105, /**< Softplus forward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SWISH_FWD = 106, /**< Swish forward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_GELU_APPROX_TANH_FWD = 107, /**< GELU forward using tanh approximation: 0.5*x*(1+tanh[sqrt(2/pi)*(x+0.044715*x^3)]). @since cuDNN 9.0.0 */ /* Activation backward (200-207) */ CUDNN_POINTWISE_RELU_BWD = 200, /**< ReLU backward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_TANH_BWD = 201, /**< Tanh backward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SIGMOID_BWD = 202, /**< Sigmoid backward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_ELU_BWD = 203, /**< ELU backward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_GELU_BWD = 204, /**< GELU backward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SOFTPLUS_BWD = 205, /**< Softplus backward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_SWISH_BWD = 206, /**< Swish backward. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_GELU_APPROX_TANH_BWD = 207, /**< GELU backward using tanh approximation. @since cuDNN 9.0.0 */ /* Comparison operations (300-305) */ CUDNN_POINTWISE_CMP_EQ = 300, /**< Equal comparison. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_CMP_NEQ = 301, /**< Not-equal comparison. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_CMP_GT = 302, /**< Greater-than comparison. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_CMP_GE = 303, /**< Greater-or-equal comparison. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_CMP_LT = 304, /**< Less-than comparison. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_CMP_LE = 305, /**< Less-or-equal comparison. @since cuDNN 9.0.0 */ /* Logical operations (400-402) */ CUDNN_POINTWISE_LOGICAL_AND = 400, /**< Logical AND. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_LOGICAL_OR = 401, /**< Logical OR. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_LOGICAL_NOT = 402, /**< Logical NOT. @since cuDNN 9.0.0 */ /* Special operations (500+) */ CUDNN_POINTWISE_GEN_INDEX = 501, /**< Generates a tensor of index values along a given axis. @since cuDNN 9.0.0 */ CUDNN_POINTWISE_BINARY_SELECT = 601, /**< Ternary select: y = predicate ? x : b, using three input tensors. @since cuDNN 9.0.0 */ } cudnnPointwiseMode_t; /** @brief Resampling modes for pooling and interpolation. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_RESAMPLE_NEAREST = 0, /**< Nearest-neighbor interpolation. @since cuDNN 9.0.0 */ CUDNN_RESAMPLE_BILINEAR = 1, /**< Bilinear interpolation. @since cuDNN 9.0.0 */ CUDNN_RESAMPLE_AVGPOOL = 2, /**< Average pooling (include padding). @since cuDNN 9.0.0 */ CUDNN_RESAMPLE_AVGPOOL_INCLUDE_PADDING = 2, /**< Average pooling including padding in divisor. @since cuDNN 9.0.0 */ CUDNN_RESAMPLE_AVGPOOL_EXCLUDE_PADDING = 4, /**< Average pooling excluding padding from divisor. @since cuDNN 9.0.0 */ CUDNN_RESAMPLE_MAXPOOL = 3, /**< Max pooling. @since cuDNN 9.0.0 */ } cudnnResampleMode_t; /** @brief Signal synchronization modes. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_SIGNAL_SET = 0, /**< Set signal. @since cuDNN 9.0.0 */ CUDNN_SIGNAL_WAIT = 1, /**< Wait on signal. @since cuDNN 9.0.0 */ } cudnnSignalMode_t; /** @brief Statistics generation mode. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_GENSTATS_SUM_SQSUM = 0, /**< Generate sum and sum-of-squares statistics. @since cuDNN 9.0.0 */ } cudnnGenStatsMode_t; /** @brief Batch normalization finalize statistics mode. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_BN_FINALIZE_STATISTICS_TRAINING = 0, /**< Training mode finalization. @since cuDNN 9.0.0 */ CUDNN_BN_FINALIZE_STATISTICS_INFERENCE = 1, /**< Inference mode finalization. @since cuDNN 9.0.0 */ } cudnnBnFinalizeStatsMode_t; /** @brief Random number generator distribution types. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_RNG_DISTRIBUTION_BERNOULLI = 0, /**< Bernoulli distribution. @since cuDNN 9.0.0 */ CUDNN_RNG_DISTRIBUTION_UNIFORM = 1, /**< Uniform distribution. @since cuDNN 9.0.0 */ CUDNN_RNG_DISTRIBUTION_NORMAL = 2, /**< Normal (Gaussian) distribution. @since cuDNN 9.0.0 */ } cudnnRngDistribution_t; /** @brief Mixture-of-Experts grouped matmul modes. * @since cuDNN 9.15.0 */ typedef enum { CUDNN_MOE_GROUPED_MATMUL_MODE_NONE = 0, /**< No gather/scatter. @since cuDNN 9.15.0 */ CUDNN_MOE_GROUPED_MATMUL_MODE_GATHER = 1, /**< Gather mode. @since cuDNN 9.15.0 */ CUDNN_MOE_GROUPED_MATMUL_MODE_SCATTER = 2, /**< Scatter mode. @since cuDNN 9.15.0 */ } cudnnMoeGroupedMatmulMode_t; /** @brief Backend attribute names for configuring and querying backend descriptors. * * Attribute names are grouped by descriptor type and numeric range: * - 0-9: Pointwise attributes * - 100-106: Convolution attributes * - 200-204: Engine heuristic attributes * - 300-304: Engine config attributes * - 400-407: Execution plan attributes * - 500-503: Intermediate info attributes * - 600-601: Knob choice attributes * - 700-717: Operation convolution attributes * - 750-758: Operation pointwise attributes * - 770-796: Operation gen-stats and BN finalize attributes * - 800-804: Operation graph attributes * - 900-913: Tensor attributes * - 1000-1012: Variant pack attributes * - 1100+: Layout info, knob info, engine, matmul, reduction, resample, etc. * @since cuDNN 9.0.0 */ typedef enum { /* Pointwise attributes */ CUDNN_ATTR_POINTWISE_MODE = 0, /**< Pointwise operation type. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_MATH_PREC = 1, /**< Computation precision for pointwise ops. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_NAN_PROPAGATION CUDNN_DEPRECATED_ENUM = 2, /**< NaN handling behavior. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP = 3, /**< Lower clipping threshold for ReLU. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_RELU_UPPER_CLIP = 4, /**< Upper clipping threshold for ReLU. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP_SLOPE = 5, /**< Slope below lower clip for leaky ReLU. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_ELU_ALPHA = 6, /**< Alpha parameter for ELU activation. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_SOFTPLUS_BETA = 7, /**< Beta parameter for softplus function. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_SWISH_BETA = 8, /**< Beta parameter for swish activation. @since cuDNN 9.0.0 */ CUDNN_ATTR_POINTWISE_AXIS = 9, /**< Axis for axis-dependent pointwise operations. @since cuDNN 9.0.0 */ /* Convolution attributes */ CUDNN_ATTR_CONVOLUTION_COMP_TYPE = 100, /**< Computation data type for convolution. @since cuDNN 9.0.0 */ CUDNN_ATTR_CONVOLUTION_CONV_MODE = 101, /**< Convolution vs cross-correlation mode. @since cuDNN 9.0.0 */ CUDNN_ATTR_CONVOLUTION_DILATIONS = 102, /**< Dilation factors per spatial dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES = 103, /**< Filter stride values per spatial dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_CONVOLUTION_POST_PADDINGS = 104, /**< Post-paddings per spatial dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS = 105, /**< Pre-paddings per spatial dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS = 106, /**< Number of spatial dimensions. @since cuDNN 9.0.0 */ /* Engine heuristic attributes */ CUDNN_ATTR_ENGINEHEUR_MODE = 200, /**< Heuristic algorithm selection mode. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH = 201, /**< Associated operation graph descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINEHEUR_RESULTS = 202, /**< Array of resulting engine configurations. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINEHEUR_SM_COUNT_TARGET = 203, /**< Target streaming multiprocessor count. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINEHEUR_DEVICEPROP = 204, /**< Device properties for heuristic query. @since cuDNN 9.8.0 */ /* Engine config attributes */ CUDNN_ATTR_ENGINECFG_ENGINE = 300, /**< Selected engine from heuristic results. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINECFG_INTERMEDIATE_INFO = 301, /**< Intermediate tensor information. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINECFG_KNOB_CHOICES = 302, /**< Performance tuning knob selections. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINECFG_WORKSPACE_SIZE = 303, /**< Required workspace memory size. @since cuDNN 9.2.0 */ CUDNN_ATTR_ENGINECFG_SHARED_MEMORY_USED = 304, /**< Shared memory used by engine. @since cuDNN 9.2.0 */ /* Execution plan attributes */ CUDNN_ATTR_EXECUTION_PLAN_HANDLE CUDNN_DEPRECATED_ENUM = 400, /**< Associated cuDNN handle. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG = 401, /**< Engine configuration to execute. @since cuDNN 9.0.0 */ CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE = 402, /**< Total workspace size requirement. @since cuDNN 9.0.0 */ CUDNN_ATTR_EXECUTION_PLAN_COMPUTED_INTERMEDIATE_UIDS = 403, /**< UIDs of computed intermediates. @since cuDNN 9.0.0 */ CUDNN_ATTR_EXECUTION_PLAN_RUN_ONLY_INTERMEDIATE_UIDS = 404, /**< Run-only intermediate UIDs. @since cuDNN 9.0.0 */ CUDNN_ATTR_EXECUTION_PLAN_JSON_REPRESENTATION = 405, /**< Human-readable execution plan JSON. @since cuDNN 9.0.0 */ CUDNN_ATTR_EXECUTION_PLAN_KERNEL_CACHE = 406, /**< Compiled kernel cache descriptor. @since cuDNN 9.4.0 */ CUDNN_ATTR_EXECUTION_PLAN_DEVICEPROP = 407, /**< Device properties for execution plan. @since cuDNN 9.8.0 */ /* Intermediate info attributes */ CUDNN_ATTR_INTERMEDIATE_INFO_UNIQUE_ID = 500, /**< Unique identifier for intermediate tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_INTERMEDIATE_INFO_SIZE = 501, /**< Memory size requirement in bytes. @since cuDNN 9.0.0 */ CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_DATA_UIDS = 502, /**< Data dependency UIDs. @since cuDNN 9.0.0 */ CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_ATTRIBUTES = 503, /**< Attribute dependencies. @since cuDNN 9.0.0 */ /* Knob choice attributes */ CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE = 600, /**< Type of performance tuning knob. @since cuDNN 9.0.0 */ CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE = 601, /**< Selected value for the knob. @since cuDNN 9.0.0 */ /* Operation convolution attributes */ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA = 700, /**< Forward convolution scaling factor alpha. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA = 701, /**< Forward convolution scaling factor beta. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC = 702, /**< Forward convolution descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_W = 703, /**< Forward convolution filter weight tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_X = 704, /**< Forward convolution input tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y = 705, /**< Forward convolution output tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_ALPHA = 706, /**< Backward data scaling factor alpha. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_BETA = 707, /**< Backward data scaling factor beta. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_CONV_DESC = 708, /**< Backward data convolution descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_W = 709, /**< Backward data filter weights. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DX = 710, /**< Backward data input gradient output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DY = 711, /**< Backward data output gradient input. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_ALPHA = 712, /**< Backward filter scaling factor alpha. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_BETA = 713, /**< Backward filter scaling factor beta. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_CONV_DESC = 714, /**< Backward filter convolution descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DW = 715, /**< Backward filter gradient output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_X = 716, /**< Backward filter input feature maps. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DY = 717, /**< Backward filter output gradients. @since cuDNN 9.0.0 */ /* Operation pointwise attributes */ CUDNN_ATTR_OPERATION_POINTWISE_PW_DESCRIPTOR = 750, /**< Pointwise descriptor reference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_XDESC = 751, /**< First input tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_BDESC = 752, /**< Bias or second operand descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_YDESC = 753, /**< Output tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_ALPHA1 = 754, /**< First scaling constant. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_ALPHA2 = 755, /**< Second scaling constant. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_DXDESC = 756, /**< Input gradient descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_DYDESC = 757, /**< Output gradient descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_POINTWISE_TDESC = 758, /**< Intermediate tensor descriptor. @since cuDNN 9.0.0 */ /* Operation gen-stats attributes */ CUDNN_ATTR_OPERATION_GENSTATS_MODE = 770, /**< Statistics computation mode. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_GENSTATS_MATH_PREC = 771, /**< Computation precision for statistics. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_GENSTATS_XDESC = 772, /**< Input data tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_GENSTATS_SUMDESC = 773, /**< Sum output descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_GENSTATS_SQSUMDESC = 774, /**< Sum of squares output descriptor. @since cuDNN 9.0.0 */ /* Operation batch normalization finalize attributes */ CUDNN_ATTR_OPERATION_BN_FINALIZE_STATS_MODE = 780, /**< Training vs inference mode. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_MATH_PREC = 781, /**< Computation precision. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SUM_DESC = 782, /**< Sum of batch norm outputs. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SQ_SUM_DESC = 783, /**< Sum of squared outputs. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_SCALE_DESC = 784, /**< Batch norm scale parameter. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_BIAS_DESC = 785, /**< Batch norm bias parameter. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_MEAN_DESC = 786, /**< Previous running mean. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_VAR_DESC = 787, /**< Previous running variance. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_MEAN_DESC = 788, /**< Updated running mean output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_VAR_DESC = 789, /**< Updated running variance output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_MEAN_DESC = 790, /**< Cached mean for backward pass. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_INV_STD_DESC = 791, /**< Cached inverse std dev for backward. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_SCALE_DESC = 792, /**< Equivalent scale for fused inference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_BIAS_DESC = 793, /**< Equivalent bias for fused inference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_ACCUM_COUNT_DESC = 794, /**< Accumulation sample counter. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_EPSILON_DESC = 795, /**< Numerical stability epsilon. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_FINALIZE_EXP_AVERATE_FACTOR_DESC = 796, /**< Exponential averaging factor. @since cuDNN 9.0.0 */ /* Operation graph attributes */ CUDNN_ATTR_OPERATIONGRAPH_HANDLE CUDNN_DEPRECATED_ENUM = 800, /**< Associated cuDNN handle. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATIONGRAPH_OPS = 801, /**< Array of operation descriptors in the graph. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATIONGRAPH_ENGINE_GLOBAL_COUNT = 802, /**< Total number of engines available. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATIONGRAPH_IS_DYNAMIC_SHAPE_ENABLED = 803, /**< Dynamic shape support flag. @since cuDNN 9.4.0 */ CUDNN_ATTR_OPERATIONGRAPH_IS_SAME_TOPOLOGY = 804, /**< Same topology reuse flag. @since cuDNN 9.6.0 */ CUDNN_ATTR_OPERATIONGRAPH_IS_OVERRIDE_SHAPE_ENABLED = 805, /**< Dynamic shape support with execute time override @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATIONGRAPH_MODE = 806, /**< Optional cudnnBackendOperationGraphMode_t — see enum doc. @since cuDNN 9.NEXT */ /* Tensor attributes */ CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT = 900, /**< Memory alignment requirement in bytes. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_DATA_TYPE = 901, /**< Element data type. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_DIMENSIONS = 902, /**< Dimension sizes array. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_STRIDES = 903, /**< Memory strides per dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_VECTOR_COUNT = 904, /**< Vectorization element count. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_VECTORIZED_DIMENSION = 905, /**< Which dimension is vectorized. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_UNIQUE_ID = 906, /**< Unique identifier for graph connectivity. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_IS_VIRTUAL = 907, /**< Virtual (intermediate) vs I/O tensor flag. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_IS_BY_VALUE = 908, /**< Constant scalar vs device pointer flag. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_REORDERING_MODE = 909, /**< Memory layout transformation type. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_CONSTANT_VALUE = 910, /**< Compile-time constant value for by-value tensors. @since cuDNN 9.22.0 */ CUDNN_ATTR_TENSOR_RAGGED_OFFSET_DESC = 913, /**< Ragged tensor offset descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_TENSOR_RAGGED_OFFSET_MULTIPLIER = 914, /**< Ragged tensor offset unit size. @since cuDNN 9.24.0 */ /* Variant pack attributes */ CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS = 1000, /**< Tensor UIDs in this variant pack. @since cuDNN 9.0.0 */ CUDNN_ATTR_VARIANT_PACK_DATA_POINTERS = 1001, /**< GPU memory data pointers array. @since cuDNN 9.0.0 */ CUDNN_ATTR_VARIANT_PACK_INTERMEDIATES = 1002, /**< Intermediate tensor pointers. @since cuDNN 9.0.0 */ CUDNN_ATTR_VARIANT_PACK_WORKSPACE = 1003, /**< Workspace memory pointer. @since cuDNN 9.0.0 */ CUDNN_ATTR_VARIANT_PACK_OVERRIDE_UNIQUE_IDS = 1010, /**< Override tensor UIDs for dynamic shapes. @since cuDNN 9.18.0 */ CUDNN_ATTR_VARIANT_PACK_OVERRIDE_SHAPES = 1011, /**< Override shapes for dynamic shapes. @since cuDNN 9.18.0 */ CUDNN_ATTR_VARIANT_PACK_OVERRIDE_STRIDES = 1012, /**< Override strides for dynamic shapes. @since cuDNN 9.18.0 */ /* Layout info attributes */ CUDNN_ATTR_LAYOUT_INFO_TENSOR_UID = 1100, /**< Associated tensor identifier. @since cuDNN 9.0.0 */ CUDNN_ATTR_LAYOUT_INFO_TYPES = 1101, /**< Available memory layout types. @since cuDNN 9.0.0 */ /* Knob info attributes */ CUDNN_ATTR_KNOB_INFO_TYPE = 1200, /**< Knob type being described. @since cuDNN 9.0.0 */ CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE = 1201, /**< Upper bound for knob value. @since cuDNN 9.0.0 */ CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE = 1202, /**< Lower bound for knob value. @since cuDNN 9.0.0 */ CUDNN_ATTR_KNOB_INFO_STRIDE = 1203, /**< Valid increment between knob values. @since cuDNN 9.0.0 */ /* Engine attributes */ CUDNN_ATTR_ENGINE_OPERATION_GRAPH = 1300, /**< Operation graph this engine processes. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINE_GLOBAL_INDEX = 1301, /**< Engine index in the global engine list. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINE_KNOB_INFO = 1302, /**< Available knob configuration options. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINE_NUMERICAL_NOTE = 1303, /**< Numerical properties (tensor cores, precision). @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINE_LAYOUT_INFO = 1304, /**< Preferred tensor memory layouts. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE = 1305, /**< Runtime behavior characteristics. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINE_SM_COUNT_TARGET = 1306, /**< Streaming multiprocessor target count. @since cuDNN 9.0.0 */ CUDNN_ATTR_ENGINE_DEVICEPROP = 1307, /**< Device properties descriptor. @since cuDNN 9.8.0 */ CUDNN_ATTR_ENGINE_DISABLE_CLUSTER_COOPERATIVE = 1308, /**< Disable cluster cooperative kernels. @since cuDNN 9.17.0 */ /* Matmul attributes */ CUDNN_ATTR_MATMUL_COMP_TYPE = 1500, /**< Computation precision type for matmul. @since cuDNN 9.0.0 */ CUDNN_ATTR_MATMUL_PADDING_VALUE = 1503, /**< Padding value for incomplete blocks. @since cuDNN 9.0.0 */ /* Operation matmul attributes */ CUDNN_ATTR_OPERATION_MATMUL_ADESC = 1520, /**< First input matrix (A) tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_MATMUL_BDESC = 1521, /**< Second input matrix (B) tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_MATMUL_CDESC = 1522, /**< Output matrix (C) tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_MATMUL_DESC = 1523, /**< MatMul operation configuration descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_MATMUL_IRREGULARLY_STRIDED_BATCH_COUNT CUDNN_DEPRECATED_ENUM = 1524, /**< Irregular batch count. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_MATMUL_GEMM_M_OVERRIDE_DESC = 1525, /**< Override for output rows (M). @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_MATMUL_GEMM_N_OVERRIDE_DESC = 1526, /**< Override for output columns (N). @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_MATMUL_GEMM_K_OVERRIDE_DESC = 1527, /**< Override for contraction dim (K). @since cuDNN 9.0.0 */ /* Reduction attributes */ CUDNN_ATTR_REDUCTION_OPERATOR = 1600, /**< Reduction operation type (ADD, MUL, MIN, etc.). @since cuDNN 9.0.0 */ CUDNN_ATTR_REDUCTION_COMP_TYPE = 1601, /**< Computation data type for reduction. @since cuDNN 9.0.0 */ CUDNN_ATTR_REDUCTION_IS_DETERMINISTIC = 1602, /**< Whether reduction must be deterministic. @since cuDNN 9.11.0 */ /* Operation reduction attributes */ CUDNN_ATTR_OPERATION_REDUCTION_XDESC = 1610, /**< Reduction input tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_REDUCTION_YDESC = 1611, /**< Reduction output tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_REDUCTION_DESC = 1612, /**< Reduction descriptor reference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_REDUCTION_GROUP_OFFSET_DESC = 1613, /**< Optional group offset tensor descriptor for grouped reduction. @since cuDNN 9.24.0 */ /* Operation BN backward weights attributes */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MATH_PREC = 1620, /**< Computation precision. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MEAN_DESC = 1621, /**< Cached batch mean from forward. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_INVSTD_DESC = 1622, /**< Cached inverse std dev from forward. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_BN_SCALE_DESC = 1623, /**< Batch norm scale parameter. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_X_DESC = 1624, /**< Forward input tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DY_DESC = 1625, /**< Output gradient tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_SCALE_DESC = 1626, /**< Scale gradient output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_BIAS_DESC = 1627, /**< Bias gradient output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_DY_SCALE_DESC = 1628, /**< Equivalent output gradient scale. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_X_SCALE_DESC = 1629, /**< Equivalent input scale. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_BIAS = 1630, /**< Equivalent bias value. @since cuDNN 9.0.0 */ /* Resample attributes */ CUDNN_ATTR_RESAMPLE_MODE = 1700, /**< Resampling interpolation method. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_COMP_TYPE = 1701, /**< Computation precision for resampling. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_SPATIAL_DIMS = 1702, /**< Number of spatial dimensions. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_POST_PADDINGS = 1703, /**< Post-paddings per spatial dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_PRE_PADDINGS = 1704, /**< Pre-paddings per spatial dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_STRIDES = 1705, /**< Stride factors per spatial dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_WINDOW_DIMS = 1706, /**< Filter window sizes per dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_NAN_PROPAGATION = 1707, /**< NaN handling behavior. @since cuDNN 9.0.0 */ CUDNN_ATTR_RESAMPLE_PADDING_MODE = 1708, /**< Padding strategy (zero, neg_inf, edge). @since cuDNN 9.0.0 */ /* Operation resample forward attributes */ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_XDESC = 1710, /**< Forward resample input tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_YDESC = 1711, /**< Forward resample output tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_IDXDESC = 1712, /**< Max pooling index tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_ALPHA CUDNN_DEPRECATED_ENUM = 1713, /**< Output scaling factor. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_BETA CUDNN_DEPRECATED_ENUM = 1714, /**< Accumulation scaling factor. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_DESC = 1716, /**< Resample descriptor reference. @since cuDNN 9.0.0 */ /* Operation resample backward attributes */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_DXDESC = 1720, /**< Backward resample input gradient. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_DYDESC = 1721, /**< Backward resample output gradient. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_IDXDESC = 1722, /**< Index tensor from forward pass. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_ALPHA CUDNN_DEPRECATED_ENUM = 1723, /**< Gradient scaling factor. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_BETA CUDNN_DEPRECATED_ENUM = 1724, /**< Accumulation scaling. @deprecated @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_DESC = 1725, /**< Resample descriptor reference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_XDESC = 1726, /**< Forward input reference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESAMPLE_BWD_YDESC = 1727, /**< Forward output reference. @since cuDNN 9.0.0 */ /* Operation concat attributes */ CUDNN_ATTR_OPERATION_CONCAT_AXIS = 1800, /**< Concatenation axis dimension. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONCAT_INPUT_DESCS = 1801, /**< Array of input tensor descriptors. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONCAT_INPLACE_INDEX = 1802, /**< In-place output tensor selection index. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_CONCAT_OUTPUT_DESC = 1803, /**< Concatenated output descriptor. @since cuDNN 9.0.0 */ /* Operation signal attributes */ CUDNN_ATTR_OPERATION_SIGNAL_MODE = 1900, /**< Signal set vs wait mode. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_SIGNAL_FLAGDESC = 1901, /**< Flag variable tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_SIGNAL_VALUE = 1902, /**< Signal value for comparison. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_SIGNAL_XDESC = 1903, /**< Input tensor for signal set. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_SIGNAL_YDESC = 1904, /**< Output tensor descriptor. @since cuDNN 9.0.0 */ /* Operation paged cache load attributes */ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_CONTAINER_DESC = 1950, /**< Cache container descriptor. @since cuDNN 9.4.0 */ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_YDESC = 1951, /**< Output tensor descriptor. @since cuDNN 9.4.0 */ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_SEQUENCE_DESC = 1952, /**< Load sequence specification. @since cuDNN 9.4.0 */ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_PAGE_TABLE_DESC = 1953, /**< Page table mapping descriptor. @since cuDNN 9.4.0 */ /* Operation norm forward attributes */ CUDNN_ATTR_OPERATION_NORM_FWD_MODE = 2000, /**< Normalization algorithm type. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE = 2001, /**< Training vs inference phase. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC = 2002, /**< Input tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC = 2003, /**< Computed or cached mean tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC = 2004, /**< Computed or cached inverse variance. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC = 2005, /**< Learnable scale parameter (gamma). @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC = 2006, /**< Learnable bias parameter (beta). @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC = 2007, /**< Numerical stability constant. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_EXP_AVG_FACTOR_DESC = 2008, /**< Momentum for running statistics. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_MEAN_DESC = 2009, /**< Input running mean. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_VAR_DESC = 2010, /**< Input running variance. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_MEAN_DESC = 2011, /**< Updated running mean output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_VAR_DESC = 2012, /**< Updated running variance output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC = 2013, /**< Output tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_FWD_PEER_STAT_DESCS = 2014, /**< Peer statistics for multi-GPU sync. @since cuDNN 9.0.0 */ /* Operation norm backward attributes */ CUDNN_ATTR_OPERATION_NORM_BWD_MODE = 2100, /**< Normalization algorithm type. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_XDESC = 2101, /**< Forward input tensor reference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC = 2102, /**< Cached mean from forward pass. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC = 2103, /**< Cached inverse std dev. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC = 2104, /**< Output gradient tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC = 2105, /**< Forward scale parameter. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_EPSILON_DESC = 2106, /**< Numerical stability epsilon. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC = 2107, /**< Scale gradient output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC = 2108, /**< Bias gradient output. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC = 2109, /**< Input gradient output tensor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_NORM_BWD_PEER_STAT_DESCS = 2110, /**< Peer gradient statistics for multi-GPU. @since cuDNN 9.0.0 */ /* Operation reshape attributes */ CUDNN_ATTR_OPERATION_RESHAPE_XDESC = 2200, /**< Reshape input tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESHAPE_YDESC = 2201, /**< Reshape output tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RESHAPE_MODE = 2202, /**< Reshape mode (view-only or logical). @since cuDNN 9.22.0 */ /* Operation transpose attributes */ CUDNN_ATTR_OPERATION_TRANSPOSE_XDESC = 3200, /**< Transpose input tensor descriptor. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_TRANSPOSE_YDESC = 3201, /**< Transpose output tensor descriptor. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_TRANSPOSE_PERMUTATION = 3202, /**< Transpose permutation array. @since cuDNN 9.22.0 */ /* Operation slice attributes */ CUDNN_ATTR_OPERATION_SLICE_XDESC = 3300, /**< Slice input tensor descriptor. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_SLICE_YDESC = 3301, /**< Slice output tensor descriptor. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_SLICE_START_INDICES = 3302, /**< Slice start indices. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_SLICE_LIMIT_INDICES = 3303, /**< Slice limit indices. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_SLICE_STRIDES = 3304, /**< Slice strides. @since cuDNN 9.22.0 */ /* Operation RoPE forward attributes */ CUDNN_ATTR_OPERATION_ROPE_FWD_XDESC = 3400, /**< RoPE input tensor descriptor [B,H,S,D]. @since cuDNN 9.24.0 */ CUDNN_ATTR_OPERATION_ROPE_FWD_YDESC = 3401, /**< RoPE output tensor descriptor [B,H,S,D]. @since cuDNN 9.24.0 */ CUDNN_ATTR_OPERATION_ROPE_FWD_FREQSDESC = 3402, /**< RoPE frequency angles tensor [S,1,1,D]. Kernel computes sincosf internally. @since cuDNN 9.24.0 */ CUDNN_ATTR_OPERATION_ROPE_FWD_OUTPUT_SCALE = 3403, /**< RoPE host-side fp32 scalar multiplied into cos/sin (default 1.0). Used to fold YARN mscale and/or SDPA attn_scale. @since cuDNN 9.24.0 */ CUDNN_ATTR_OPERATION_ROPE_FWD_ROPE_DIM = 3404, /**< RoPE rotation width in last dim (int64, default = head_dim). When < head_dim, the last rope_dim slots get rotated; the first (head_dim - rope_dim) are scaled-pass-through (used for DSv3-style MLA). @since cuDNN 9.24.0 */ /* Operation RoPE backward attributes */ CUDNN_ATTR_OPERATION_ROPE_BWD_DYDESC = 3410, /**< RoPE bwd input gradient (dY) descriptor [B,H,S,D]. @since cuDNN 9.25.0 */ CUDNN_ATTR_OPERATION_ROPE_BWD_DXDESC = 3411, /**< RoPE bwd output gradient (dX) descriptor [B,H,S,D]. @since cuDNN 9.25.0 */ CUDNN_ATTR_OPERATION_ROPE_BWD_FREQSDESC = 3412, /**< RoPE bwd frequency angles tensor [S,1,1,D]; same semantics as fwd. @since cuDNN 9.25.0 */ CUDNN_ATTR_OPERATION_ROPE_BWD_OUTPUT_SCALE = 3413, /**< RoPE bwd host fp32 scalar (default 1.0). Symmetric to fwd: cos/sin pre-scaled, passthrough scaled. @since cuDNN 9.25.0 */ CUDNN_ATTR_OPERATION_ROPE_BWD_ROPE_DIM = 3414, /**< RoPE bwd rotation width in last dim (int64, default = head_dim). @since cuDNN 9.25.0 */ /* Operation expand band matrix attributes */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_XDESC = 2250, /**< Band matrix input tensor. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_YDESC = 2251, /**< Expanded output tensor. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_LOWER_BANDWIDTH = 2252, /**< Lower bandwidth of the band. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_UPPER_BANDWIDTH = 2253, /**< Upper bandwidth of the band. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_AXIS = 2254, /**< Axis along which to expand. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_PAD_VALUE = 2255, /**< Padding value outside the band. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_KV_TOKEN_OFFSET_DESC = 2256, /**< KV token offset descriptor. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_SPECULATIVE_MASK_DESC = 2257, /**< Speculative decoding mask. @since cuDNN 9.13.0 */ /* Operation contract band matrix attributes */ CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_XDESC = 2270, /**< Full matrix input tensor. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_YDESC = 2271, /**< Contracted band output tensor. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_LOWER_BANDWIDTH = 2272, /**< Lower bandwidth. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_UPPER_BANDWIDTH = 2273, /**< Upper bandwidth. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_AXIS = 2274, /**< Axis along which to contract. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_PAD_VALUE = 2275, /**< Padding value. @since cuDNN 9.10.0 */ CUDNN_ATTR_OPERATION_CONTRACT_BAND_MAX_TOKEN_VALUE = 2276, /**< Maximum token value for contraction. @since cuDNN 9.10.0 */ /* RNG attributes */ CUDNN_ATTR_RNG_DISTRIBUTION = 2300, /**< Random distribution type selection. @since cuDNN 9.0.0 */ CUDNN_ATTR_RNG_NORMAL_DIST_MEAN = 2301, /**< Normal distribution mean parameter. @since cuDNN 9.0.0 */ CUDNN_ATTR_RNG_NORMAL_DIST_STANDARD_DEVIATION = 2302, /**< Normal distribution std deviation. @since cuDNN 9.0.0 */ CUDNN_ATTR_RNG_UNIFORM_DIST_MAXIMUM = 2303, /**< Uniform distribution upper bound. @since cuDNN 9.0.0 */ CUDNN_ATTR_RNG_UNIFORM_DIST_MINIMUM = 2304, /**< Uniform distribution lower bound. @since cuDNN 9.0.0 */ CUDNN_ATTR_RNG_BERNOULLI_DIST_PROBABILITY = 2305, /**< Bernoulli probability of 1. @since cuDNN 9.0.0 */ /* Operation RNG attributes */ CUDNN_ATTR_OPERATION_RNG_YDESC = 2310, /**< RNG output tensor descriptor. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RNG_SEED = 2311, /**< RNG seed value. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RNG_DESC = 2312, /**< RNG descriptor reference. @since cuDNN 9.0.0 */ CUDNN_ATTR_OPERATION_RNG_OFFSET_DESC = 2313, /**< RNG offset/state descriptor. @since cuDNN 9.0.0 */ /* Kernel cache attributes */ CUDNN_ATTR_KERNEL_CACHE_OPERATION_GRAPH = 2400, /**< Operation graph for kernel cache. @since cuDNN 9.5.0 */ CUDNN_ATTR_KERNEL_CACHE_IS_ENGINECFG_KERNEL_CACHED = 2401, /**< Whether kernel is cached. @since cuDNN 9.4.0 */ CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION = 2402, /**< Kernel cache JSON serialization. @since cuDNN 9.10.0 */ /* Operation block-scale quantize attributes */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_XDESC = 2500, /**< Input float tensor to quantize. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_YDESC = 2501, /**< Quantized output tensor. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_SCALE_DESC = 2502, /**< Per-block scaling factors output. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_MATH_PREC = 2503, /**< Computation precision. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_BLOCK_SIZE = 2504, /**< Quantization block size. @since cuDNN 9.7.0 */ /* Operation block-scale dequantize attributes */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_XDESC = 2600, /**< Quantized input tensor. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_SCALE_DESC = 2601, /**< Per-block scale factors. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_YDESC = 2602, /**< Dequantized output tensor. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_MATH_PREC = 2603, /**< Computation precision. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_BLOCK_SIZE = 2604, /**< Dequantization block size. @since cuDNN 9.7.0 */ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_NEG_SCALE = 2605, /**< Negative scale handling. @since cuDNN 9.13.0 */ /* Device property attributes */ CUDNN_ATTR_DEVICEPROP_DEVICE_ID = 2700, /**< CUDA device identifier. @since cuDNN 9.8.0 */ CUDNN_ATTR_DEVICEPROP_HANDLE = 2701, /**< Associated cuDNN handle. @since cuDNN 9.8.0 */ CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION = 2702, /**< Device properties JSON. @since cuDNN 9.8.0 */ /* Operation SDPA forward attributes */ CUDNN_ATTR_OPERATION_SDPA_FWD_QDESC = 2800, /**< Query tensor descriptor. @since cuDNN 9.13.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_KDESC = 2801, /**< Key tensor descriptor. @since cuDNN 9.13.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_VDESC = 2802, /**< Value tensor descriptor. @since cuDNN 9.13.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_ODESC = 2803, /**< Output tensor descriptor. @since cuDNN 9.13.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC = 2804, /**< Statistics output descriptor. @since cuDNN 9.13.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_SCALEDESC = 2805, /**< Attention scaling factor descriptor. @since cuDNN 9.13.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_BLOCK_MASK_DESC = 2806, /**< Block-sparse attention mask. @since cuDNN 9.14.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_KDESC = 2807, /**< Paged attention key page table. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_VDESC = 2808, /**< Paged attention value page table. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC = 2809, /**< Query sequence length tensor. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC = 2810, /**< Key-value sequence length tensor. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH = 2811, /**< Forward SDPA subgraph. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_INPUT_UID = 2812, /**< Subgraph input tensor UID. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_OUTPUT_UID = 2813, /**< Subgraph output tensor UID. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_SOFTMAX_DESC = 2814, /**< Softmax descriptor. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_SEED_DESC = 2815, /**< Dropout seed tensor descriptor. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_OFFSET_DESC = 2816, /**< Dropout offset tensor descriptor. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_RNG_DUMP_DESC = 2817, /**< Dropout RNG dump tensor descriptor. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_PROBABILITY = 2818, /**< Dropout probability. @since cuDNN 9.21.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_UNFUSE_FMA = 2819, /**< Unfuse FMA in softmax for SM100. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_QDESC = 2820, /**< Query cumulative sequence length tensor. @since cuDNN 9.24.0 */ CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_KVDESC = 2821, /**< Key-value cumulative sequence length tensor. @since cuDNN 9.24.0 */ /* Operation SDPA backward attributes */ CUDNN_ATTR_OPERATION_SDPA_BWD_QDESC = 2851, /**< Query tensor descriptor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_KDESC = 2852, /**< Key tensor descriptor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_VDESC = 2853, /**< Value tensor descriptor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_ODESC = 2854, /**< Forward output tensor descriptor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_STATSDESC = 2855, /**< Forward statistics descriptor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_SCALEDESC = 2856, /**< Attention scaling factor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_SEQ_LEN_QDESC = 2857, /**< Query sequence length tensor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_SEQ_LEN_KVDESC = 2858, /**< Key-value sequence length tensor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_DQDESC = 2859, /**< Query gradient output tensor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_DKDESC = 2860, /**< Key gradient output tensor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_DVDESC = 2861, /**< Value gradient output tensor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_DODDESC = 2862, /**< Output gradient input tensor. @since cuDNN 9.17.0 */ CUDNN_ATTR_OPERATION_SDPA_BWD_SINK_DESC = 2863, /**< Backward sink descriptor. @since UNPUBLISHED */ CUDNN_ATTR_OPERATION_SDPA_BWD_DSINK_DESC = 2864, /**< Backward sink gradient descriptor. @since UNPUBLISHED */ CUDNN_ATTR_OPERATION_SDPA_BWD_MAX_TOTAL_SEQ_LEN_Q = 2865, /**< Max total query sequence length. @since UNPUBLISHED */ CUDNN_ATTR_OPERATION_SDPA_BWD_MAX_TOTAL_SEQ_LEN_KV = 2866, /**< Max total KV sequence length. @since UNPUBLISHED */ CUDNN_ATTR_OPERATION_SDPA_BWD_SUBGRAPH = 2867, /**< Backward SDPA subgraph. @since UNPUBLISHED */ CUDNN_ATTR_OPERATION_SDPA_BWD_SUBGRAPH_INPUT_UID = 2868, /**< Subgraph input tensor UID. @since UNPUBLISHED */ CUDNN_ATTR_OPERATION_SDPA_BWD_SUBGRAPH_OUTPUT_UID = 2869, /**< Subgraph output tensor UID. @since UNPUBLISHED */ /* Operation MoE grouped matmul attributes */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MODE = 2900, /**< Gather/scatter mode for MoE. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MATH_PREC = 2901, /**< Computation precision. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_DESC = 2902, /**< Token tensor descriptor. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_WEIGHT_DESC = 2903, /**< Expert weight tensor descriptor. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_FIRST_TOKEN_OFFSET_DESC = 2904, /**< First token offset per expert. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_OUTPUT_DESC = 2905, /**< Output tensor descriptor. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_INDEX_DESC = 2906, /**< Token routing index descriptor. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_KS_DESC = 2907, /**< Token routing weights descriptor. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOP_K = 2908, /**< Top-K experts per token. @since cuDNN 9.15.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_MATH_PREC = 2951, /**< Backward math precision for MoE grouped matmul. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_TOKEN_DESC = 2952, /**< Backward token descriptor for MoE grouped matmul. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DWEIGHT_DESC = 2953, /**< Backward weight descriptor for MoE grouped matmul. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_FIRST_TOKEN_OFFSET_DESC = 2954, /**< Backward first token offset descriptor for MoE grouped matmul. @since cuDNN 9.22.0 */ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DOUTPUT_DESC = 2955, /**< Backward output descriptor for MoE grouped matmul. @since cuDNN 9.22.0 */ /* Operation diagonal band mask attributes */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_XDESC = 3000, /**< Input tensor descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_KVDESC = 3001, /**< KV sequence length tensor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_QDESC = 3002, /**< Query sequence length tensor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_LEFT_BOUND_DESC = 3003, /**< Left bound offset descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SHIFT_RIGHT_BOUND_DESC = 3004, /**< Right bound shift descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_BDESC = 3005, /**< Band descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_YDESC = 3006, /**< Output mask tensor descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_COMPARISON_MODE = 3007, /**< Mask comparison mode. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_KVDESC = 3008, /**< Key-value cumulative sequence length tensor. @since cuDNN 9.24.0 */ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_QDESC = 3009, /**< Query cumulative sequence length tensor. @since cuDNN 9.24.0 */ /* Operation softmax attributes */ CUDNN_ATTR_OPERATION_SOFTMAX_XDESC = 3100, /**< Softmax input tensor descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_SOFTMAX_YDESC = 3101, /**< Softmax output tensor descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_SOFTMAX_STATS_DESC = 3102, /**< Softmax statistics output. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_SOFTMAX_MAX_DESC = 3103, /**< Row-wise max values descriptor. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_SOFTMAX_SUM_EXP_DESC = 3104, /**< Row-wise sum of exponentials. @since cuDNN 9.20.0 */ CUDNN_ATTR_OPERATION_SOFTMAX_SINK_DESC = 3105, /**< Softmax sink descriptor. @since cuDNN 9.20.0 */ } cudnnBackendAttributeName_t; /** @brief Attribute data types used by the cuDNN backend API for get/set operations. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_TYPE_HANDLE = 0, /**< cudnnHandle_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_DATA_TYPE = 1, /**< cudnnDataType_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_BOOLEAN = 2, /**< Boolean value. @since cuDNN 9.0.0 */ CUDNN_TYPE_INT64 = 3, /**< 64-bit signed integer value. @since cuDNN 9.0.0 */ CUDNN_TYPE_FLOAT = 4, /**< 32-bit float value. @since cuDNN 9.0.0 */ CUDNN_TYPE_DOUBLE = 5, /**< 64-bit double value. @since cuDNN 9.0.0 */ CUDNN_TYPE_VOID_PTR = 6, /**< Void pointer value. @since cuDNN 9.0.0 */ CUDNN_TYPE_CONVOLUTION_MODE = 7, /**< cudnnConvolutionMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_HEUR_MODE = 8, /**< cudnnBackendHeurMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_KNOB_TYPE = 9, /**< cudnnBackendKnobType_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_NAN_PROPOGATION CUDNN_DEPRECATED_ENUM = 10, /**< cudnnNanPropagation_t value. @deprecated @since cuDNN 9.0.0 */ CUDNN_TYPE_NUMERICAL_NOTE = 11, /**< cudnnBackendNumericalNote_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_LAYOUT_TYPE = 12, /**< cudnnBackendLayoutType_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_ATTRIB_NAME = 13, /**< cudnnBackendAttributeName_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_POINTWISE_MODE = 14, /**< cudnnPointwiseMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_BACKEND_DESCRIPTOR = 15, /**< cudnnBackendDescriptor_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_GENSTATS_MODE = 16, /**< cudnnGenStatsMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_BN_FINALIZE_STATS_MODE = 17, /**< cudnnBnFinalizeStatsMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_REDUCTION_OPERATOR_TYPE = 18, /**< cudnnReduceTensorOp_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_BEHAVIOR_NOTE = 19, /**< cudnnBackendBehaviorNote_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_TENSOR_REORDERING_MODE = 20, /**< cudnnBackendTensorReordering_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_RESAMPLE_MODE = 21, /**< cudnnResampleMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_PADDING_MODE = 22, /**< cudnnPaddingMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_INT32 = 23, /**< 32-bit signed integer value. @since cuDNN 9.0.0 */ CUDNN_TYPE_CHAR = 24, /**< Character value. @since cuDNN 9.0.0 */ CUDNN_TYPE_SIGNAL_MODE = 25, /**< cudnnSignalMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_FRACTION = 26, /**< cudnnFraction_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_NORM_MODE = 27, /**< cudnnBackendNormMode_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_NORM_FWD_PHASE = 28, /**< cudnnBackendNormFwdPhase_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_RNG_DISTRIBUTION = 29, /**< cudnnRngDistribution_t value. @since cuDNN 9.0.0 */ CUDNN_TYPE_MOE_GROUPED_MATMUL_MODE = 30, /**< cudnnMoeGroupedMatmulMode_t value. @since cuDNN 9.15.0 */ CUDNN_TYPE_RESHAPE_MODE = 31, /**< cudnnBackendReshapeMode_t value. @since cuDNN 9.22.0 */ CUDNN_TYPE_OPERATIONGRAPH_MODE = 32, /**< cudnnBackendOperationGraphMode_t value. @since cuDNN 9.NEXT */ } cudnnBackendAttributeType_t; /** @brief Backend descriptor types identifying the kind of descriptor to create. * @since cuDNN 9.0.0 */ typedef enum { /** * @brief Specifies parameters for a pointwise operator. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_POINTWISE_DESCRIPTOR, &desc); * the cuDNN backend pointwise descriptor specifies the parameters for a * pointwise operator like mode, math precision, nan propagation, and so on. * * Supported attributes (prefix CUDNN_ATTR_POINTWISE_): * * - CUDNN_ATTR_POINTWISE_MODE (CUDNN_TYPE_POINTWISE_MODE, 1 element) * Mode of the pointwise operation. Required attribute. * * - CUDNN_ATTR_POINTWISE_MATH_PREC (CUDNN_TYPE_DATA_TYPE, 1 element) * The math precision of the computation. Required attribute. * * - CUDNN_ATTR_POINTWISE_NAN_PROPAGATION (CUDNN_TYPE_NAN_PROPOGATION, 1 element) * Specifies a method by which to propagate NaNs. Required only for * comparison based pointwise modes, like ReLU. Current support only * includes enum value CUDNN_PROPAGATE_NAN. Default value is * CUDNN_NOT_PROPAGATE_NAN. * * - CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP (CUDNN_TYPE_DOUBLE/CUDNN_TYPE_FLOAT, 1 element) * Sets the lower clip value for ReLU. If (value < lower_clip) * value = lower_clip + lower_clip_slope * (value - lower_clip). * Default value is 0.0f. * * - CUDNN_ATTR_POINTWISE_RELU_UPPER_CLIP (CUDNN_TYPE_DOUBLE/CUDNN_TYPE_FLOAT, 1 element) * Sets the upper clip value for ReLU. If (value > upper_clip) * value = upper_clip. Default value is Numeric limit max. * * - CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP_SLOPE (CUDNN_TYPE_DOUBLE/CUDNN_TYPE_FLOAT, 1 element) * Sets the lower clip slope value for ReLU. If (value < lower_clip) * value = lower_clip + lower_clip_slope * (value - lower_clip). * Default value is 0.0f. * * - CUDNN_ATTR_POINTWISE_ELU_ALPHA (CUDNN_TYPE_DOUBLE/CUDNN_TYPE_FLOAT, 1 element) * Sets the alpha value for ELU. If (value < 0.0) * value = alpha * (e^value - 1.0). Default value is 1.0f. * * - CUDNN_ATTR_POINTWISE_SOFTPLUS_BETA (CUDNN_TYPE_DOUBLE/CUDNN_TYPE_FLOAT, 1 element) * Sets the beta value for softplus. If value = log(1 + e^(beta * value)) / beta. * Default value is 1.0f. * * - CUDNN_ATTR_POINTWISE_SWISH_BETA (CUDNN_TYPE_DOUBLE/CUDNN_TYPE_FLOAT, 1 element) * Sets the beta value for swish. If value = value / (1 + e^(-beta * value)). * Default value is 1.0f. * * - CUDNN_ATTR_POINTWISE_AXIS (CUDNN_TYPE_INT64, 1 element) * Sets the axis value for GEN_INDEX. The index will be generated for * this axis. Default value is -1. Needs to lie between * [0, input_dim_size-1]. For example, if your input has dimensions * [N,C,H,W], the axis can be set to anything in [0,3]. * * Finalization: * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_POINTWISE_DESCRIPTOR = 0, /**< Pointwise op config: mode, math precision, activation params. @since cuDNN 9.0.0 */ /** * @brief Specifies parameters for a convolution operator. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR, &desc); * the cuDNN backend convolution descriptor specifies the parameters for a * convolution operator for both forward and backward propagation: compute * data type, convolution mode, filter dilation and stride, and padding on * both sides. * * Supported attributes (prefix CUDNN_ATTR_CONVOLUTION_): * * - CUDNN_ATTR_CONVOLUTION_COMP_TYPE (CUDNN_TYPE_DATA_TYPE, 1 element) * The compute type of the convolution operator. Required attribute. * * - CUDNN_ATTR_CONVOLUTION_MODE (CUDNN_TYPE_CONVOLUTION_MODE, 1 element) * Convolution or cross-correlation mode. Required attribute. * * - CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS (CUDNN_TYPE_INT64, 1 element) * The number of spatial dimensions, expected array length for each of * dilations, filter strides, and padding arrays. Required attribute. * * - CUDNN_ATTR_CONVOLUTION_DILATIONS (CUDNN_TYPE_INT64, 1..CUDNN_MAX_DIMS elements) * Filter dilation. Required attribute. * * - CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES (CUDNN_TYPE_INT64, 1..CUDNN_MAX_DIMS elements) * Filter stride. Required attribute. * * - CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS (CUDNN_TYPE_INT64, 1..CUDNN_MAX_DIMS elements) * Padding at the beginning of each spatial dimension. Required attribute. * * - CUDNN_ATTR_CONVOLUTION_POST_PADDINGS (CUDNN_TYPE_INT64, 1..CUDNN_MAX_DIMS elements) * Padding at the end of each spatial dimension. Required attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM - An elemCount argument for setting * CUDNN_ATTR_CONVOLUTION_DILATIONS, CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, * CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, and CUDNN_ATTR_CONVOLUTION_POST_PADDINGS * is not equal to the value set for CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR = 1, /**< Convolution config: compute type, mode, dilation, stride, padding. @since cuDNN 9.0.0 */ /** * @brief Describes an engine to compute an operation graph. * * * Created with descriptor type value CUDNN_BACKEND_ENGINE_DESCRIPTOR, cuDNN * backend engine descriptor describes an engine to compute an operation * graph. An engine is a grouping of kernels with similar compute and * numerical attributes. * * Supported attributes (prefix CUDNN_ATTR_ENGINE_): * * - CUDNN_ATTR_ENGINE_OPERATION_GRAPH (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The operation graph to compute. Descriptor type * CUDNN_BACKEND_OPERATIONGRAPH_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_ENGINE_GLOBAL_INDEX (CUDNN_TYPE_INT64, 1 element) * The index for the engine. Valid values are between 0 and * CUDNN_ATTR_OPERATIONGRAPH_ENGINE_GLOBAL_COUNT-1. Required attribute. * * - CUDNN_ATTR_ENGINE_KNOB_INFO (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The descriptors of performance knobs of the engine. Descriptor type * CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR. Read-only attribute. * * - CUDNN_ATTR_ENGINE_NUMERICAL_NOTE (CUDNN_TYPE_NUMERICAL_NOTE, 0+ elements) * The numerical attributes of the engine. Read-only attribute. * * - CUDNN_ATTR_ENGINE_LAYOUT_INFO (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The preferred tensor layouts of the engine. Descriptor type * CUDNN_BACKEND_LAYOUT_INFO_DESCRIPTOR. Read-only attribute. * * - CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE (CUDNN_TYPE_BEHAVIOR_NOTE, 0+ elements) * The behavior attributes of the engine. Read-only attribute. * * - CUDNN_ATTR_ENGINE_SM_COUNT_TARGET (CUDNN_TYPE_INT32, 1 element) * The number of SMs to target. Valid values are between 0 and the * number of SMs on the device, where 0 is default meaning all the SMs * will be used. Optional attribute. * * - CUDNN_ATTR_ENGINE_DEVICEPROP (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The descriptor of the device that this engine descriptor targets. * Descriptor type CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR. Optional attribute. * * Finalization: * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. * CUDNN_STATUS_NOT_SUPPORTED - The descriptor attribute set is not supported * by the current version of cuDNN. For example, the value of * CUDNN_ATTR_ENGINE_GLOBAL_INDEX is not in a valid range. * CUDNN_STATUS_BAD_PARAM - The descriptor attribute set is inconsistent or * in an unexpected state. For example, the operation graph descriptor set * is not already finalized. */ CUDNN_BACKEND_ENGINE_DESCRIPTOR = 2, /**< Engine (kernel grouping) to compute an operation graph. @since cuDNN 9.0.0 */ /** * @brief Consists of an engine descriptor and an array of knob choice descriptors. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_ENGINECFG_DESCRIPTOR, &desc); * the cuDNN backend engine configuration descriptor consists of an engine * descriptor and an array of knob choice descriptors. Users can query from * engine config information about intermediates: computational intermediate * results that can be reused between executions. * * Supported attributes: * * - CUDNN_ATTR_ENGINECFG_ENGINE (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The backend engine. Descriptor type CUDNN_BACKEND_ENGINE_DESCRIPTOR. * Required attribute. * * - CUDNN_ATTR_ENGINECFG_KNOB_CHOICES (CUDNN_TYPE_BACKEND_DESCRIPTOR, 0+ elements) * The engine tuning knobs and choices. Descriptor type * CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR. * * - CUDNN_ATTR_ENGINECFG_INTERMEDIATE_INFO (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * Information of the computational intermediate of this engine config. * Descriptor type CUDNN_BACKEND_INTERMEDIATE_INFO_DESCRIPTOR. * Read-only attribute. Currently unsupported. Placeholder for future * implementation. * * - CUDNN_ATTR_ENGINECFG_WORKSPACE_SIZE (CUDNN_TYPE_INT64, 1 element) * The size of the workspace buffer required to execute this engine config. * Read-only attribute. * * Finalization: * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. * CUDNN_STATUS_NOT_SUPPORTED - The descriptor attribute set is not supported * by the current version of cuDNN. For example, the value knob. */ CUDNN_BACKEND_ENGINECFG_DESCRIPTOR = 3, /**< Engine configuration: engine descriptor plus knob choices. @since cuDNN 9.0.0 */ /** * @brief Allows users to obtain engine configurations ranked by performance heuristics. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR, &desc); * the cuDNN backend engine heuristics descriptor allows users to obtain for * an operation graph engine configuration descriptors ranked by performance * according to cuDNN's heuristics. * * Supported attributes: * * - CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The operation graph for which heuristics result in a query. * Required attribute. * * - CUDNN_ATTR_ENGINEHEUR_MODE (CUDNN_TYPE_HEUR_MODE, 1 element) * The heuristic mode to query the result. Required attribute. * * - CUDNN_ATTR_ENGINEHEUR_RESULTS (CUDNN_TYPE_BACKEND_DESCRIPTOR, 0+ elements) * The result of the heuristics query. Descriptor type * CUDNN_BACKEND_ENGINECFG_DESCRIPTOR. Get-only attribute. * * - CUDNN_ATTR_ENGINEHEUR_SM_COUNT_TARGET (CUDNN_TYPE_INT32, 1 element) * The number of SMs to target. Valid values are between 0 and the * number of SMs on the device, where 0 is default meaning all the SMs * will be used. Optional attribute. * * - CUDNN_ATTR_ENGINEHEUR_DEVICEPROP (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The descriptor of the device that this engine heuristics descriptor * targets. Descriptor type CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR. * Optional attribute. * * Finalization: * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR = 4, /**< Engine configurations ranked by performance via cuDNN heuristics. @since cuDNN 9.0.0 */ /** * @brief Specifies an execution plan consisting of a handle, engine config, and optional intermediates. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR, &desc); * the cuDNN backend execution plan descriptor allows the user to specify an * execution plan, consists of a cuDNN handle, an engine configuration, and * optionally an array of intermediates to compute. * * Supported attributes: * * - CUDNN_ATTR_EXECUTION_PLAN_HANDLE (CUDNN_TYPE_HANDLE, 1 element) * A cuDNN handle. Required attribute. * * - CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * An engine configuration to execute. Descriptor type * CUDNN_BACKEND_ENGINECFG_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_EXECUTION_PLAN_RUN_ONLY_INTERMEDIATE_UIDS (CUDNN_TYPE_INT64, 0+ elements) * Unique identifiers of intermediates to compute. Optional attribute. * If set, the execution plan will only compute the specified intermediate * and not any of the output tensors on the operation graph in the engine * configuration. * * - CUDNN_ATTR_EXECUTION_PLAN_COMPUTED_INTERMEDIATE_UIDS (CUDNN_TYPE_INT64, 0+ elements) * Unique identifiers of precomputed intermediates. Optional attribute. * If set, the plan will expect and use pointers for each intermediate in * the variant pack descriptor during execution. Currently unsupported. * Placeholder for future implementation. * * - CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE (CUDNN_TYPE_INT64, 1 element) * The size of the workspace buffer required to execute this plan. * Read-only attribute. * * - CUDNN_ATTR_EXECUTION_PLAN_JSON_REPRESENTATION (CUDNN_TYPE_CHAR, many elements) * The JSON representation of the serialized execution plan. Serialization * and deserialization can be done by getting and setting this attribute, * respectively. Element count is the same as the size of a null-terminated * string of the json representation of the execution plan. * * - CUDNN_ATTR_EXECUTION_PLAN_KERNEL_CACHE (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The kernel cache that the execution plan can refer to in order to * accelerate the finalization for runtime fusion engines by reusing a * previously compiled identical kernel implementation. Descriptor type * CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR. * * - CUDNN_ATTR_EXECUTION_PLAN_DEVICEPROP (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The descriptor of the device that this execution plan targets. * Descriptor type CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR. Optional attribute. * * Finalization: * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR = 5, /**< Finalized execution plan: engine config, workspace size, optional kernel cache. @since cuDNN 9.0.0 */ /** * @brief Read-only descriptor containing information about an execution intermediate. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_INTERMEDIATE_INFO_DESCRIPTOR, &desc); * the cuDNN backend intermediate descriptor is a read-only descriptor that * contains information about an execution intermediate. An execution * intermediate is some intermediate computation for an engine config in * device memory that can be reused between plan execution to amortize the * kernel. Each intermediate is identified by a unique ID. Users can query * for the device memory size of the intermediate. An intermediate can depend * on the data of one or more tensors identified by the tensor UIDs or one * more attribute of the operation graph. * * This is a read-only descriptor. Users cannot set the descriptor attributes * or finalize the descriptor. User query for a finalized descriptor from an * engine config descriptor. * * Supported attributes: * * - CUDNN_ATTR_INTERMEDIATE_INFO_UNIQUE_ID (CUDNN_TYPE_INT64, 1 element) * A unique identifier of the intermediate. Read-only attribute. * * - CUDNN_ATTR_INTERMEDIATE_INFO_SIZE (CUDNN_TYPE_INT64, 1 element) * The required device memory size for the intermediate. Read-only attribute. * * - CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_DATA_UIDS (CUDNN_TYPE_INT64, 0+ elements) * UID of tensors on which the intermediate depends. Read-only attribute. * * - CUDNN_ATTR_INTERMEDIATE_INFO_DEPENDENT_ATTRIBUTES * Currently unsupported. Placeholder for future implementation. * * Finalization: * User does not finalize this descriptor. cudnnBackendFinalize(desc) with * a backend intermediate descriptor returns CUDNN_STATUS_NOT_SUPPORTED. */ CUDNN_BACKEND_INTERMEDIATE_INFO_DESCRIPTOR = 6, /**< Read-only info about a reusable execution intermediate. @since cuDNN 9.0.0 */ /** * @brief Consists of the type and value of a performance knob setting. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR, &desc); * the cuDNN backend knob choice descriptor consists of the type of knobs to * be set and the value to which the knob is set. * * Supported attributes: * * - CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE (CUDNN_TYPE_KNOB_TYPE, 1 element) * The type of knobs to be set. Required attribute. * * - CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE (CUDNN_TYPE_INT64, 1 element) * The value of the knobs to be set. Required attribute. * * Finalization: * CUDNN_STATUS_SUCCESS - The knob choice descriptor was finalized successfully. */ CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR = 7, /**< Type and value of an engine performance tuning knob. @since cuDNN 9.0.0 */ /** * @brief * * 9. CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR * */ CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR = 8, /**< Read-only info about an engine knob: type, min/max, and stride. @since cuDNN 9.0.0 */ /** * @brief Provides information on the preferred layout for a tensor. * * * Created with descriptor type value CUDNN_BACKEND_LAYOUT_INFO_DESCRIPTOR, * cuDNN backend layout info descriptor provides information on the preferred * layout for a tensor. * * Supported attributes: * * - CUDNN_ATTR_LAYOUT_INFO_TENSOR_UID (CUDNN_TYPE_INT64, 1 element) * The UID of the tensor. Read-only attribute. * * - CUDNN_ATTR_LAYOUT_INFO_TYPES (CUDNN_TYPE_LAYOUT_TYPE, 0+ elements) * The preferred layout of the tensor (cudnnBackendLayoutType_t). * Read-only attribute. * * Finalization: * This descriptor is read-only; it is retrieved and finalized from a cuDNN * backend engine configuration descriptor. Users cannot set its attribute * or finalize it. */ CUDNN_BACKEND_LAYOUT_INFO_DESCRIPTOR = 9, /**< Read-only info on the preferred memory layout for a tensor. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for forward convolution. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR, &desc); * the cuDNN backend convolution forward operation descriptor specifies an * operation node for forward convolution to compute the response tensor y of * image tensor x convoluted with filter tensor w with output scaling alpha * and residual add with beta scaling. That is, the equation: * y = alpha * (w * x) + beta * y * where * is the convolution operator in the forward direction. * * Supported attributes (prefix CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_): * * - CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA (CUDNN_TYPE_FLOAT or CUDNN_TYPE_DOUBLE, 1+ elements) * The alpha value. Required to be set before finalization. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA (CUDNN_TYPE_FLOAT or CUDNN_TYPE_DOUBLE, 1+ elements) * The beta value. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The convolution operator descriptor. Descriptor type * CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_W (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The convolution filter tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_X (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The image tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The response tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. * * Notes on tensor dimension binding during finalization: * The CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS attribute of * CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC is the number of * spatial dimension of the convolution. The number of dimensions for * tensor X, W, and Y must be larger than the number of spatial dimensions * by 2 or 3 depending on how users choose to specify the convolution tensors. * * If the number of tensor dimension is the number of spatial dimensions plus 2: * - X tensor dimension and stride arrays are [N, GC, ...] * - W tensor dimension and stride arrays are [GK, C, ...] * - Y tensor dimension and stride arrays are [N, GK, ...] * Where the ellipsis ... are shorthand for spatial dimensions of each tensor, * G is the number of convolution groups, and C and K are the number of input * and output feature maps per group. In this interpretation, it is assumed * that the memory layout for each group is packed. * cudnnBackendFinalize() asserts the tensors dimensions and strides are * consistent with this interpretation or it returns CUDNN_STATUS_BAD_PARAM. * * Finalization: * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are * encountered. For example, the X, W, and Y tensors do not constitute * a valid convolution operation under the convolution operator. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR = 10, /**< Forward convolution: y = alpha * conv(w, x) + beta * y. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for convolution backward filter. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR, &desc); * the cuDNN backend convolution backward filter operation descriptor * specifies an operation node for convolution backward filter to compute the * gradient of filter dw with image tensor x and gradient of response dy with * output alpha scaling and residue add with beta scaling. That is, the equation: * dx = alpha * (x ~* dy) + beta * dx * where ~* denotes the convolution backward filter operator. * * Supported attributes (prefix CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_): * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_ALPHA (CUDNN_TYPE_FLOAT or CUDNN_TYPE_DOUBLE, 1+ elements) * The alpha value. Required attribute. Required to be set before finalization. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_BETA (CUDNN_TYPE_FLOAT or CUDNN_TYPE_DOUBLE, 1+ elements) * The beta value. Required attribute. Required to be set before finalization. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_CONV_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The convolution operator descriptor. Descriptor type * CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR. Required attribute. Required to * be set before finalization. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DW (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The convolution filter tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. Required to be * set before finalization. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_X (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The image gradient tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. Required to be * set before finalization. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DY (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The response gradient tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. Required to be * set before finalization. * * Notes on tensor dimension binding during finalization: * In finalizing the convolution operation, the tensor dimensions of the * tensor X, DW, and DY are bound based on the same interpretations as the * X, W, and Y tensor dimensions described in the * CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR section. * * Finalization: * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are * encountered. For example, the X, DW, and DY tensors do not constitute * a valid convolution operation under the convolution operator. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR = 11, /**< Backward filter convolution: computes dw from x and dy. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for convolution backward data. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR, &desc); * the cuDNN backend convolution backward data operation descriptor specifies * an operation node for convolution backward data to compute the gradient of * input data dx with filter tensor w and gradient of response dy with output * alpha scaling and residue add with beta scaling. That is, the equation: * dx = alpha * (w _* dy) + beta * dx * where _* denotes the convolution backward data operator. * * Supported attributes (prefix CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_): * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_ALPHA (CUDNN_TYPE_FLOAT or CUDNN_TYPE_DOUBLE, 1+ elements) * The alpha value. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_BETA (CUDNN_TYPE_FLOAT or CUDNN_TYPE_DOUBLE, 1+ elements) * The beta value. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_CONV_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The convolution operator descriptor. Descriptor type * CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_W (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The convolution filter tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DX (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The image gradient tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. * * - CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DY (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * The response gradient tensor descriptor. Descriptor type * CUDNN_BACKEND_TENSOR_DESCRIPTOR. Required attribute. * * Notes on tensor dimension binding during finalization: * In finalizing the convolution operation, the tensor dimensions of the * tensor DX, W, and DY are bound based on the same interpretations as the * X, W, and Y tensor dimensions described in the * CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR section. * * Finalization: * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are * encountered. For example, the DX, W, and DY tensors do not constitute * a valid convolution operation under the convolution operator. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR = 12, /**< Backward data convolution: computes dx from w and dy. @since cuDNN 9.0.0 */ /** * @brief Represents a pointwise operation: Y = op(alpha1 * X) or Y = op(alpha1 * X, alpha2 * B). * * * Represents a pointwise operation that implements the equation * Y = op(alpha1 * X) or Y = op(alpha1 * X, alpha2 * B) depending on the * operation type. The actual type of operation represented by op() above * depends on the CUDNN_ATTR_OPERATION_POINTWISE_PW_DESCRIPTOR attribute in * the descriptor. This operation descriptor supports operations with * single-input single-output. * * For a list of supported operations, refer to the cudnnPointwiseMode_t section. * * For dual-input pointwise operations, broadcasting is assumed when a tensor * dimension in one of the tensors is 1 while the other tensors corresponding * dimension is not 1. * * For three-input single-output pointwise operations, we do not support * broadcasting in any tensor. * * This opaque struct can be created with * cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR). * * Supported attributes: * * - CUDNN_ATTR_OPERATION_POINTWISE_PW_DESCRIPTOR * Sets the descriptor containing the mathematical settings of the * pointwise operation. This attribute is required. * * - CUDNN_ATTR_OPERATION_POINTWISE_XDESC * Sets the descriptor for the input tensor X. This attribute is required * for pointwise mathematical functions or activation forward propagation * computations. * * - CUDNN_ATTR_OPERATION_POINTWISE_BDESC * If the operation requires two inputs, such as add or multiply, this * attribute sets the second input tensor B. If the operation requires * only 1 input, this field is not used and should not be set. * * - CUDNN_ATTR_OPERATION_POINTWISE_YDESC * Sets the descriptor for the output tensor Y. This attribute is * required for pointwise mathematical functions or activation forward * propagation computations. * * - CUDNN_ATTR_OPERATION_POINTWISE_TDESC * Sets the descriptor for the tensor T. This attribute is required for * CUDNN_ATTR_POINTWISE_MODE set to CUDNN_POINTWISE_BINARY_SELECT and * acts as the mask based on which the selection is done. * * - CUDNN_ATTR_OPERATION_POINTWISE_ALPHA1 * Sets the scalar alpha1 value in the equation. Can be in float or half. * This attribute is optional, if not set, the default value is 1.0. * * - CUDNN_ATTR_OPERATION_POINTWISE_ALPHA2 * If the operation requires 2 inputs, such as add or multiply. This * attribute sets the scalar alpha2 value in the equation. Can be in * float or half. This attribute is optional, if not set, the default * value is 1.0. If the operation requires only 1 input, this field is * not used and should not be set. * * - CUDNN_ATTR_OPERATION_POINTWISE_DXDESC * Sets the descriptor for the output tensor dX. This attribute is * required for pointwise activation back propagation computations. * * - CUDNN_ATTR_OPERATION_POINTWISE_DYDESC * Sets the descriptor for the input tensor dY. This attribute is * required for pointwise activation back propagation computations. * * Finalization: * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are * encountered. Some examples include: * - The number of dimensions do not match between the input and output tensors. * - The input/output tensor dimensions do not agree with the above * described automatic broadcasting rules. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR = 13, /**< Pointwise operation: Y = op(alpha1*X) or Y = op(alpha1*X, alpha2*B). @since cuDNN 9.0.0 */ /** * @brief Represents an operation that generates per-channel statistics. * * * Represents an operation that will generate per-channel statistics. The * specific statistics that will be generated depends on the * CUDNN_ATTR_OPERATION_GENSTATS_MODE attribute in the descriptor. Currently, * only CUDNN_GENSTATS_SUM_SQSUM is supported for the * CUDNN_ATTR_OPERATION_GENSTATS_MODE. It will generate the sum and quadratic * sum of per-channel elements of the input tensor x. The output dimension * should be all 1 except the C dimension. Also, the C dimension of outputs * should equal the C dimension of the input. This opaque struct can be * created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_GEN_STATS_DESCRIPTOR). * * Supported attributes: * * - CUDNN_ATTR_OPERATION_GENSTATS_MODE * Sets the CUDNN_TYPE_GENSTATS_MODE of the operation. This attribute * is required. * * - CUDNN_ATTR_OPERATION_GENSTATS_MATH_PREC * The math precision of the computation. This attribute is required. * * - CUDNN_ATTR_OPERATION_GENSTATS_XDESC * Sets the descriptor for the input tensor X. This attribute is required. * * - CUDNN_ATTR_OPERATION_GENSTATS_SUMDESC * Sets the descriptor for the output tensor sum. This attribute is required. * * - CUDNN_ATTR_OPERATION_GENSTATS_SQSUMDESC * Sets the descriptor for the output tensor quadratic sum. This * attribute is required. * * Finalization: * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are * encountered. Some examples include: * - The number of dimensions do not match between the input and output tensors. * - The input/output tensor dimensions do not agree with the above description. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_GEN_STATS_DESCRIPTOR = 14, /**< Generates per-channel statistics (sum and sum-of-squares). @since cuDNN 9.0.0 */ /** * @brief Describes an operation graph of one or more operations connected by virtual tensors. * * * Created with descriptor type value CUDNN_BACKEND_OPERATIONGRAPH_DESCRIPTOR, * cuDNN backend operation graph descriptor describes an operation graph, a small * network of one or more operations connected by virtual tensors. Operation graph * defines users' computation case or mathematical expression that they wish to compute. * * Supported attributes: * - CUDNN_ATTR_OPERATIONGRAPH_HANDLE (CUDNN_TYPE_HANDLE, 1 element) * A cuDNN handle. * Required attribute. * * - CUDNN_ATTR_OPERATIONGRAPH_OPS (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1+ elements of type CUDNN_BACKEND_OPERATION_*_DESCRIPTOR) * Operation nodes to form the operation graph. * Required attribute. * * - CUDNN_ATTR_OPERATIONGRAPH_ENGINE_GLOBAL_COUNT (CUDNN_TYPE_INT64, 1 element) * The number of engines to support the operation graph. * Read-only attribute. * * - CUDNN_ATTR_OPERATIONGRAPH_ENGINE_SUPPORTED_COUNT (CUDNN_TYPE_INT64, 1 element) * The number of engines that support the operation graph. * Read-only attribute. * Currently unsupported. Placeholder for future implementation. * * - CUDNN_ATTR_OPERATIONGRAPH_IS_DYNAMIC_SHAPE_ENABLED (CUDNN_TYPE_BOOLEAN, 1 element) * Whether dynamic shape is enabled for the operation graph. The rest of the * backend API will treat the graph as a dynamic shape graph and enable this feature. * * Finalization: * CUDNN_STATUS_BAD_PARAM - An invalid attribute value was encountered. Some examples include: * - One of the backend descriptors in CUDNN_ATTR_OPERATIONGRAPH_OPS is not finalized. * - The value CUDNN_ATTR_OPERATIONGRAPH_HANDLE is not a valid cuDNN handle. * CUDNN_STATUS_NOT_SUPPORTED - An unsupported attribute value was encountered. * For example, the combination of operations of attribute CUDNN_ATTR_OPERATIONGRAPH_OPS * is not supported. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATIONGRAPH_DESCRIPTOR = 15, /**< Operation graph: a DAG of operations connected by virtual tensors. @since cuDNN 9.0.0 */ /** * @brief Sets up pointers to device buffers for non-virtual tensors, workspace, and computation intermediates. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR, &desc); * the cuDNN backend variant pack plan allows users to set up pointers to device buffers * to various non-virtual tensors, identified by unique identifiers, of the operation graph, * workspace, and computation intermediates. * * Supported attributes: * - CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS (CUDNN_TYPE_INT64, 0+ elements) * A unique identifier of tensor for each data pointer. * Required attribute. * * - CUDNN_ATTR_VARIANT_PACK_DATA_POINTERS (CUDNN_TYPE_VOID_PTR, 0+ elements) * Tensor data device pointers. * Required attribute. * * - CUDNN_ATTR_VARIANT_PACK_INTERMEDIATES (CUDNN_TYPE_VOID_PTR, 0+ elements) * Intermediate device pointers. * Currently unsupported. Placeholder for future implementation. * * - CUDNN_ATTR_VARIANT_PACK_WORKSPACE (CUDNN_TYPE_VOID_PTR, 1 element) * Workspace to device pointer. * Required attribute. * * Finalization: * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR = 16, /**< Binds device pointers to non-virtual tensors, workspace, and intermediates. @since cuDNN 9.0.0 */ /** * @brief Specifies the memory storage of a generic tensor. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &desc); * the cuDNN backend tensor allows users to specify the memory storage of a generic tensor. * A tensor is identified by a unique identifier and described by its data type, its data * byte-alignment requirements, and the extents and strides of its dimensions. Optionally, * a tensor element can be vector in one of its dimensions. A tensor can also be set to be * virtual when it is an intermediate variable in a computation graph and not mapped to * physical global memory storage. * * Supported attributes: * - CUDNN_ATTR_TENSOR_UNIQUE_ID (CUDNN_TYPE_INT64, 1 element) * An integer that uniquely identifies the tensor. * Required attribute. * * - CUDNN_ATTR_TENSOR_DATA_TYPE (CUDNN_TYPE_DATA_TYPE, 1 element) * Data type of tensor. * Required attribute. * * - CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT (CUDNN_TYPE_INT64, 1 element) * Byte alignment of pointers for this tensor. * Required attribute. * * - CUDNN_ATTR_TENSOR_DIMENSIONS (CUDNN_TYPE_INT64, at most CUDNN_MAX_DIMS elements) * Tensor dimensions. * Required attribute. * * - CUDNN_ATTR_TENSOR_STRIDES (CUDNN_TYPE_INT64, at most CUDNN_MAX_DIMS elements) * Tensor strides. * Required attribute. * * - CUDNN_ATTR_TENSOR_VECTOR_COUNT (CUDNN_TYPE_INT64, 1 element) * Size of vectorization. * Default value is 1. * * - CUDNN_ATTR_TENSOR_VECTORIZED_DIMENSION (CUDNN_TYPE_INT64, 1 element) * Index of the vectorized dimension. * Required to be set before finalization if CUDNN_ATTR_TENSOR_VECTOR_COUNT is set * to a value different than its default; otherwise it's ignored. * * - CUDNN_ATTR_TENSOR_IS_VIRTUAL (CUDNN_TYPE_BOOLEAN, 1 element) * Indicates whether the tensor is virtual. A virtual tensor is an intermediate tensor * in the operation graph that exists in transient and not read from or written to in * global device memory. * Default value is false. * * - CUDNN_ATTR_TENSOR_RAGGED_OFFSET_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element) * A ragged tensor, that is, a tensor with nested variable length lists as inner * dimensions, will have another tensor called the ragged offset descriptor that * contains offsets in memory to the next variable length list. * Default value is None. * * - CUDNN_ATTR_TENSOR_RAGGED_OFFSET_MULTIPLIER (CUDNN_TYPE_INT64, 1 element) * Unit size of the offsets stored in CUDNN_ATTR_TENSOR_RAGGED_OFFSET_DESC, * as a multiple of the tensor's element size (*not* in bytes). * Consumers of the ragged offset tensor multiply its values by this * multiplier before use. Only a limited number of engines support this attribute. * If set, CUDNN_ATTR_TENSOR_RAGGED_OFFSET_DESC must also be set. * Default value is 1 (meaning the ragged offset tensor is in units of elements * of the tensor's data type). * * Finalization: * CUDNN_STATUS_BAD_PARAM - An invalid attribute value was encountered. Some examples include: * - Any of the tensor dimensions or strides is not positive. * - The value of the tensor alignment attribute is not divisible by the size of the data type. * CUDNN_STATUS_NOT_SUPPORTED - An unsupported attribute value was encountered. Some examples include: * - The data type attribute is CUDNN_DATA_INT8x4, CUDNN_DATA_UINT8x4, or CUDNN_DATA_INT8x32. * - The data type attribute is CUDNN_DATA_INT8 and CUDNN_ATTR_TENSOR_VECTOR_COUNT value is not 1, 4, or 32. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_TENSOR_DESCRIPTOR = 17, /**< Tensor: data type, dimensions, strides, alignment, unique ID, virtual flag. @since cuDNN 9.0.0 */ /** * @brief Specifies metadata needed for the matmul operation. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_MATMUL_DESCRIPTOR, &desc); * the cuDNN backend matmul descriptor specifies any metadata needed for the matmul operation. * * Supported attributes: * - CUDNN_ATTR_MATMUL_COMP_TYPE (CUDNN_TYPE_DATA_TYPE, 1 element) * The compute precision used for the matmul operation. * Required attribute. * * Finalization: * Return values of cudnnBackendFinalize(desc) where desc is a cuDNN backend matmul descriptor: * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_MATMUL_DESCRIPTOR = 18, /**< Matrix multiply config: compute type and padding value. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for matmul to compute C = A * B. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_MATMUL_DESCRIPTOR, &desc); * the cuDNN backend matmul operation descriptor specifies an operation node for matmul to * compute the matrix product C by multiplying Matrix A and Matrix B, as shown in the * following equation: C=AB * * When using the matmul operation, the matrices are expected to be at least rank-2 tensors. * The last two dimensions are expected to correspond to either M, K or N. All the preceding * dimensions are interpreted as batch dimensions. If there are zero batch dimensions then * the requirements are as follows: * * Zero Batch Dimensions: * Single Matmul: A is M x K, B is K x N, C is M x N. * * Single Batch Dimension: * Single Matmul: A is 1 x M x K, B is 1 x K x N, C is 1 x M x N. * Batch Matmul: A is B x M x K, B is B x K x N, C is B x M x N. * Broadcast A: A is (B/c) x M x K, B is B x K x N, C is B x M x N. * Broadcast B: A is B x M x K, B is (B/c) x K x N, C is B x M x N. * * Where: * B indicates the batch size. * M is the number of rows of the Matrix A. * K is the number of columns of the input Matrix A (which is the same as the number * of rows as the input Matrix B). * N is the number of columns of the input Matrix B. * c is a constant integer and a factor of B. * * If either the batch size of Matrix A or B is set to B/c, this indicates that the matrix * will be broadcasted in the batch matmul. The resulting output Matrix C will be a tensor * of B x M x N. * * The above broadcasting convention is extended to all the batch dimensions. Concretely, * for tensors with three batch dimensions: * Multiple Batched Matmul: A is B1 x 1 x B3 x M x K, B is 1 x B2 x (B3/c) x K x N, * C is B1 x B2 x B3 x M x N. * * The functionality of having multiple batch dimensions allows you to have layouts where * the batch is not packed at a single stride. This case is especially seen in multihead * attention. c is only allowed to be B (leading to a batch dimension for 1) for matmul * and matmul fusions. The other possible values of c are supported for Grouped Query * Attention in the cuDNN Fused Flash Attention. * * The addressing of the matrix elements from a given tensor can be specified using strides * in the tensor descriptor. The strides represent the spacing between elements for each * tensor dimension. Considering a matrix tensor A (B x M x N) with strides [BS, MS, NS], * it indicates that the actual matrix element A[x, y, z] is found at * (A_base_address + x * BS + y * MS + z * NS) from the linear memory space allocated for * tensor A. With our current support, the innermost dimension must be packed, which * requires either MS=1 or NS=1. Otherwise, there are no other technical constraints with * regard to how the strides can be specified in a tensor descriptor as it should follow * the aforementioned addressing formula and the strides as specified by the user. * * This representation provides support for some common usages, such as leading dimension * and matrix transpose as we will explain through the following examples. * * 1. The most basic case is a fully packed row-major batch matrix, without any * consideration of leading dimension or transpose. In this case, * BS = M*N, MS = N, and NS = 1. * 2. Matrix transpose can be achieved by exchanging the inner and outer dimensions * using strides. Namely: * a. To specify a non-transposed matrix: BS = M*N, MS = N, and NS = 1. * b. To specify matrix transpose: BS = M*N, MS = 1, and NS = M. * 3. Leading dimension, a widely used concept in BLAS-like APIs, describes the inner * dimension of the 2D array memory allocation (as opposed to the conceptual matrix * dimension). It resembles the stride in a way that it defines the spacing between * elements in the outer dimension. The most typical use cases where it shows difference * from the matrix inner dimension is when the matrix is only part of the data in the * allocated memory, addressing submatrices, or addressing matrices from an aligned * memory allocation. Therefore, the leading dimension LDA in a column-major matrix A * must satisfy LDA >= M, whereas in a row-major matrix A, it must satisfy LDA >= N. * To transition from the leading dimension concept to using strides, this entails * MS >= N and NS = 1 or MS = 1 and NS >= M. Keep in mind that, while these are some * practical use cases, these inequalities do not impose technical constraints with * respect to an acceptable specification of the strides. * * Other commonly used GEMM features, such as alpha/beta output blending, can also be * achieved using this matmul operation along with other pointwise operations. * * Supported attributes: * - CUDNN_ATTR_OPERATION_MATMUL_ADESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The Matrix A descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_MATMUL_BDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The Matrix B descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_MATMUL_CDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The Matrix C descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_MATMUL_IRREGULARLY_STRIDED_BATCH_COUNT (CUDNN_TYPE_INT64, 1 element) * Number of matmul operations to perform in the batch on matrix. * Default value is 1. * * - CUDNN_ATTR_OPERATION_MATMUL_GEMM_M_OVERRIDE_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The tensor gemm_m_override descriptor. Allows you to override the M dimension of a * batch matmul through this tensor. It is only supported as documented in the Fused * Attention fprop, Fused Attention bprop, Fused Flash Attention fprop, and Fused Flash * Attention bprop sections. * Optional attribute. * * - CUDNN_ATTR_OPERATION_MATMUL_GEMM_N_OVERRIDE_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The tensor gemm_n_override descriptor. Allows you to override the N dimension of a * batch matmul through this tensor. It is only supported as documented in the Fused * Attention fprop, Fused Attention bprop, Fused Flash Attention fprop, and Fused Flash * Attention bprop sections. * Optional attribute. * * - CUDNN_ATTR_OPERATION_MATMUL_GEMM_K_OVERRIDE_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The tensor gemm_k_override descriptor. Allows you to override the K dimension of a * batch matmul through this tensor. It is only supported as documented in the Fused * Attention fprop, Fused Attention bprop, Fused Flash Attention fprop, and Fused Flash * Attention bprop sections. * Optional attribute. * * - CUDNN_ATTR_OPERATION_MATMUL_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_MATMUL_DESCRIPTOR) * The matmul operation descriptor. * Required attribute. * * Finalization: * In the finalization of the matmul operation, the tensor dimensions of the Matrices * A, B, and C will be checked to ensure that they satisfy the requirements of matmul. * * CUDNN_STATUS_NOT_SUPPORTED - An unsupported attribute value was encountered. For * example, if not all of the Matrices A, B, and C are at least rank-2 tensors. * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are encountered. * Some examples include: * - The CUDNN_ATTR_OPERATION_MATMUL_IRREGULARLY_STRIDED_BATCH_COUNT specified is a * negative value. * - The CUDNN_ATTR_OPERATION_MATMUL_IRREGULARLY_STRIDED_BATCH_COUNT and one or more * of the batch sizes of the Matrices A, B, and C are not equal to one. That is to * say there is a conflict where both irregularly and regularly strided batched matmul * are specified, which is not a valid use case. * - The dimensions of the Matrices A, B, and C do not satisfy the matmul requirements. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_MATMUL_DESCRIPTOR = 19, /**< Matrix multiplication operation: C = A * B with optional overrides. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for the batch norm finalize operation. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_BN_FINALIZE_STATISTICS_DESCRIPTOR, &desc); * the cuDNN backend bn_finalize statistics operation descriptor specifies an operation node * for the batch norm finalize operation. * * In ResNet like models, a common technique to fuse batch norm with convolutions would * involve splitting the batch norm operation into three parts - genStats, finalize, and * apply (pointwise, scale, and bias). The genStats operation is usually fused with the * convolution operation that precedes the batch norm while the apply is fused with the * ReLU and convolution that follows the batch norm op. The batch norm finalize operation * is a buffer op between the two fusions that takes the batch norm scale, bias, sum, and * sqsum produced by the genStats operation as inputs and produces an equivalent scale and * bias as output. The equivalent scale and bias are then consumed in the apply phase. * Additionally, the bn_finalize operation also produces the running stats, mean, and * inverse standard deviation as outputs. * * Supported attributes: * - CUDNN_ATTR_OPERATION_BN_FINALIZE_STATS_MODE (CUDNN_TYPE_BN_FINALIZE_STATS_MODE, 1 element) * Sets inference or training mode for the bn_finalize operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_MATH_PREC * Math precision of the computation. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SUM_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input sum tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SQ_SUM_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input square sum tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_SCALE_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm input scale tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_BIAS_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm input bias tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_MEAN_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm input running mean descriptor. * Optional attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_VAR_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm input running variance descriptor. * Optional attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_MEAN_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm output running mean descriptor. * Optional attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_VAR_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm output running variance descriptor. * Optional attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_MEAN_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm output saved mean tensor descriptor. This is computed from the sum input * that's fed in from the preceding genStats operation. Storing out the saved mean helps * avoid recomputation in the backpropagation phase. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_INV_STD_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Batch norm output inverse standard deviation tensor descriptor. This is computed from * the sum and sqm sums input that's fed in from the preceding genStats operation. Storing * out the saved inv standard deviations helps avoid recomputation in the backpropagation phase. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_SCALE_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Output tensor descriptor for the equivalent scale tensor. The equivalent scale tensor * is typically fed as input to the batch norm apply computation (pointwise, scale, and * bias) that follows the batch norm finalize operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_BIAS_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Output tensor descriptor for the equivalent bias tensor. The equivalent bias tensor * is typically fed as input to the batch norm apply computation (pointwise, scale, and * bias) that follows the batch norm finalize operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_ACCUM_COUNT_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Scalar input tensor descriptor representing the number of elements accumulated over * while calculating the sum and sqsum inputs. The count usually equals N*H*W in case * of batch norm. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_EPSILON_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Scalar input tensor descriptor for the epsilon value used in batch norm variance * calculation. * Required attribute. * * - CUDNN_ATTR_OPERATION_BN_FINALIZE_EXP_AVERAGE_FACTOR_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Scalar input tensor descriptor for the exponential average value used in batch norm * running stats calculation. * Required attribute. */ CUDNN_BACKEND_OPERATION_BN_FINALIZE_STATISTICS_DESCRIPTOR = 20, /**< BN finalize: computes running stats, saved mean/invstd, eq scale/bias. @since cuDNN 9.0.0 */ /** * @brief Specifies metadata for the reduction operation including math operation and compute data type. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_REDUCTION_DESCRIPTOR, &desc); * the cuDNN backend reduction descriptor specifies any metadata, including the math * operation and compute data type, needed for the reduction operation. * * Supported attributes: * - CUDNN_ATTR_REDUCTION_OPERATOR (CUDNN_TYPE_REDUCTION_OPERATOR_TYPE, 1 element) * The math operation used for the reduction operation. * Required attribute. * * - CUDNN_ATTR_REDUCTION_COMP_TYPE (CUDNN_TYPE_DATA_TYPE, 1 element) * The compute precision used for the reduction operation. * Required attribute. * * Finalization: * CUDNN_STATUS_NOT_SUPPORTED - An unsupported attribute value was encountered. For * example, CUDNN_ATTR_REDUCTION_OPERATOR is not set to either of * CUDNN_REDUCE_TENSOR_ADD, CUDNN_REDUCE_TENSOR_MUL, CUDNN_REDUCE_TENSOR_MIN, * or CUDNN_REDUCE_TENSOR_MAX. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_REDUCTION_DESCRIPTOR = 21, /**< Reduction config: operator type (ADD/MUL/MIN/MAX/etc.) and compute type. @since cuDNN 9.0.0 */ /** * @brief Represents a reduction operation node that reduces input tensor X to output tensor Y. * * * The cuDNN backend reduction operation descriptor represents an operation node that * implements reducing values of an input tensor X in one or more dimensions to get an * output tensor Y. The math operation and compute data type used for reducing tensor * values is specified via CUDNN_ATTR_OPERATION_REDUCTION_DESC. * * This operation descriptor can be created with * cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR, &desc). * * The output tensor Y should be the size as that of input tensor X, except dimensions * where its size is 1. * * There is a special use case for Grouped Query Attention and Multi Query Attention in * cuDNN Fused Flash Attention where some dimensions in the output tensor Y can also be * factors of the corresponding dimensions in the input tensor X. * * Supported attributes: * - CUDNN_ATTR_OPERATION_REDUCTION_XDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The matrix X descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_REDUCTION_YDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The matrix Y descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_REDUCTION_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The reduction operation descriptor. * Required attribute. * * Finalization: * In the finalization of the reduction operation, the dimensions of tensors X and Y are * checked to ensure that they satisfy the requirements of the reduction operation. * * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are encountered. * For example, the dimensions of the tensors X and Y do not satisfy the requirements * of the reduction operation. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR = 22, /**< Reduces input tensor values along one or more dimensions. @since cuDNN 9.0.0 */ /** * @brief Batch normalization backward weights operation descriptor. * * * NOTE: This descriptor type (CUDNN_BACKEND_OPERATION_BN_BWD_WEIGHTS_DESCRIPTOR) is * listed in the cudnnBackendDescriptorType_t enumeration but does not have a dedicated * documentation section in the cuDNN backend API RST reference. The associated attributes * are defined in the cudnnBackendAttributeName_t enumeration with prefix * CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_ and include: * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MATH_PREC (1620) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MEAN_DESC (1621) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_INVSTD_DESC (1622) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_BN_SCALE_DESC (1623) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_X_DESC (1624) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DY_DESC (1625) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_SCALE_DESC (1626) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_BIAS_DESC (1627) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_DY_SCALE_DESC (1628) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_X_SCALE_DESC (1629) * - CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_BIAS (1630) */ CUDNN_BACKEND_OPERATION_BN_BWD_WEIGHTS_DESCRIPTOR = 23, /**< BN backward weights: computes dScale, dBias, and equivalent gradients. @since cuDNN 9.0.0 */ /** * @brief Specifies parameters for a resample operation (upsampling or downsampling). * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_RESAMPLE_DESCRIPTOR, &desc); * the cuDNN backend resample descriptor specifies the parameters for a resample operation * (upsampling or downsampling) in both forward and backward propagation. * * Supported attributes: * - CUDNN_ATTR_RESAMPLE_MODE (CUDNN_TYPE_RESAMPLE_MODE, 1 element) * Specifies mode of resampling, for example, average pool, nearest-neighbor, and so on. * Default value is CUDNN_RESAMPLE_NEAREST. * * - CUDNN_ATTR_RESAMPLE_COMP_TYPE (CUDNN_TYPE_DATA_TYPE, 1 element) * Compute data type for the resampling operator. * Default value is CUDNN_DATA_FLOAT. * * - CUDNN_ATTR_RESAMPLE_NAN_PROPAGATION (CUDNN_TYPE_NAN_PROPAGATION, 1 element) * Specifies a method by which to propagate NaNs. * Default value is CUDNN_NOT_PROPAGATE_NAN. * * - CUDNN_ATTR_RESAMPLE_SPATIAL_DIMS (CUDNN_TYPE_INT64, 1 element) * Specifies the number of spatial dimensions to perform the resampling over. * Required attribute. * * - CUDNN_ATTR_RESAMPLE_PADDING_MODE (CUDNN_TYPE_PADDING_MODE, 1 element) * Specifies which values to use for padding. * Default value is CUDNN_ZERO_PAD. * * - CUDNN_ATTR_RESAMPLE_STRIDES (CUDNN_TYPE_INT64 or CUDNN_TYPE_FRACTION, at most CUDNN_MAX_DIMS - 2) * Stride in each dimension for the kernel or filter. * Required attribute. * * - CUDNN_ATTR_RESAMPLE_PRE_PADDINGS (CUDNN_TYPE_INT64 or CUDNN_TYPE_FRACTION, at most CUDNN_MAX_DIMS - 2) * Padding added to the beginning of the input tensor in each dimension. * Required attribute. * * - CUDNN_ATTR_RESAMPLE_POST_PADDINGS (CUDNN_TYPE_INT64 or CUDNN_TYPE_FRACTION, at most CUDNN_MAX_DIMS - 2) * Padding added to the end of the input tensor in each dimension. * Required attribute. * * - CUDNN_ATTR_RESAMPLE_WINDOW_DIMS (CUDNN_TYPE_INT64 or CUDNN_TYPE_FRACTION, at most CUDNN_MAX_DIMS - 2) * Spatial dimensions of filter. * Required attribute. * * Finalization: * CUDNN_STATUS_NOT_SUPPORTED - An unsupported attribute value was encountered. Some * examples include: * - An elemCount argument for setting CUDNN_ATTR_RESAMPLE_WINDOW_DIMS, * CUDNN_ATTR_RESAMPLE_STRIDES, CUDNN_ATTR_RESAMPLE_PRE_PADDINGS, and * CUDNN_ATTR_RESAMPLE_POST_PADDINGS is not equal to the value set for * CUDNN_ATTR_RESAMPLE_SPATIAL_DIMS. * - CUDNN_ATTR_RESAMPLE_MODE is set to CUDNN_RESAMPLE_BILINEAR and any of the * CUDNN_ATTR_RESAMPLE_WINDOW_DIMS are not set to 2. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_RESAMPLE_DESCRIPTOR = 24, /**< Resample config: mode, spatial dims, padding, strides, window dims. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for forward resampling with alpha/beta blending. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_RESAMPLE_FWD_DESCRIPTOR, &desc); * the cuDNN backend resample forward operation descriptor specifies an operation node for * forward resampling. It computes the output tensor y of image tensor x resampled according * to CUDNN_ATTR_RESAMPLE_MODE, with output scaling alpha and residual add with beta scaling. * * The resampling mode acts independently on each spatial dimension. For spatial dimension i, * the output spatial dimension size y_i can be calculated by combining input image's spatial * dimension size x_i, post padding post_i, pre padding pre_i, stride s_i, and window size * w_i as: y_i = 1 + (x_i + post_i + pre_i - w_i) / s_i * * Supported attributes: * - CUDNN_ATTR_OPERATION_RESAMPLE_FWD_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_RESAMPLE_DESCRIPTOR) * Resample operation descriptor (CUDNN_BACKEND_RESAMPLE_DESCRIPTOR) instance contains * metadata about the operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_RESAMPLE_FWD_XDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_RESAMPLE_FWD_YDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Output tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_RESAMPLE_FWD_IDXDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Tensor containing maxpool or nearest neighbor resampling indices to be used in backprop. * Optional attribute (primarily used for use cases involving training). * * - CUDNN_ATTR_OPERATION_RESAMPLE_FWD_ALPHA (CUDNN_TYPE_DOUBLE or CUDNN_TYPE_FLOAT, 1 element) * Sets the alpha parameter used in blending. * Optional attribute. * Default value is 1.0. * * - CUDNN_ATTR_OPERATION_RESAMPLE_FWD_BETA (CUDNN_TYPE_DOUBLE or CUDNN_TYPE_FLOAT, 1 element) * Sets the beta parameter used in blending. * Optional attribute. * Default value is 0.0. * * Finalization: * In the finalization stage, the attributes are cross checked to make sure there are * no conflicts. The status below may be returned: * * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are encountered. * Some examples include: * - The output shape calculated based on the padding and strides does not match the * given output tensor dimensions. * - The shape of the YDESC and IDXDESC (if given) do not match. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_RESAMPLE_FWD_DESCRIPTOR = 25, /**< Forward resampling (pooling/interpolation) with alpha/beta scaling. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for backward resampling computing dx from dy. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_RESAMPLE_BWD_DESCRIPTOR, &desc); * the cuDNN backend resample backward operation descriptor specifies an operation node for * backward resampling. It computes the input tensor gradient dx from output tensor gradient * dy with backward resampling done according to CUDNN_ATTR_RESAMPLE_MODE with output scaling * alpha and residual add with beta scaling. * * Supported attributes: * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_RESAMPLE_DESCRIPTOR) * Resample operation descriptor (CUDNN_BACKEND_RESAMPLE_DESCRIPTOR) instance contains * metadata about the operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_DXDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input tensor gradient descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_DYDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Output tensor gradient descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_IDXDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Tensor containing maxpool or nearest neighbor resampling indices to be used in backprop. * Optional attribute. * * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_ALPHA (CUDNN_TYPE_DOUBLE or CUDNN_TYPE_FLOAT, 1 element) * Sets the alpha parameter used in blending. * Optional attribute. * Default value is 1.0. * * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_BETA (CUDNN_TYPE_DOUBLE or CUDNN_TYPE_FLOAT, 1 element) * Sets the beta parameter used in blending. * Optional attribute. * Default value is 0.0. * * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_XDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input tensor X descriptor. * Optional attribute. * Required for NCHW layout. * * - CUDNN_ATTR_OPERATION_RESAMPLE_BWD_YDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input tensor Y descriptor. * Optional attribute. * Required for NCHW layout. * * Finalization: * In the finalization stage, the attributes are cross checked to make sure there are * no conflicts. The status below may be returned: * * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are encountered. * Some examples include: * - The output shape calculated based on the padding and strides does not match the * given output tensor dimensions. * - The shape of YDESC and IDXDESC (if given) do not match. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_RESAMPLE_BWD_DESCRIPTOR = 26, /**< Backward resampling: computes dx from dy with alpha/beta scaling. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for concatenating tensors along a given axis. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_CONCAT_DESCRIPTOR, &desc); * the cuDNN backend concatenation operation descriptor specifies an operation node for * concatenating a given vector of tensors along a given concatenation axis. * * This operation also supports an in-place mode, where one of the input tensors is already * assumed to be at the correct location in the output tensor, that is, they share the same * device buffer. * * Supported attributes: * - CUDNN_ATTR_OPERATION_CONCAT_AXIS (CUDNN_TYPE_INT64) * The dimension which tensors are being concatenated over. * Required attribute. * * - CUDNN_ATTR_OPERATION_CONCAT_INPUT_DESCS (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1+ elements of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * A vector of input tensor descriptors, which are concatenated in the same order as * provided in this vector. * Required attribute. * * - CUDNN_ATTR_OPERATION_CONCAT_INPLACE_INDEX (CUDNN_TYPE_INT64) * The index of input tensor in the vector of input tensor descriptors that is already * present in-place in the output tensor. * Optional attribute. * * - CUDNN_ATTR_OPERATION_CONCAT_OUTPUT_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The output tensor descriptor for the result from concatenation of input tensors. * Required attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are encountered. * Some examples include: * - The tensors involved in the operation should have the same shape in all dimensions * except the dimension that they are being concatenated over. * - The output tensor shape in the concatenating dimension should equal the sum of * tensor shape of all input tensors in that same dimension. * - Concatenation axis should be a valid tensor dimension. * - If provided, the in-place input tensor index should be a valid index in the vector * of input tensor descriptors. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_CONCAT_DESCRIPTOR = 27, /**< Concatenates multiple tensors along a given axis. @since cuDNN 9.0.0 */ /** * @brief Specifies an operation node for updating or waiting on a flag variable. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_SIGNAL_DESCRIPTOR, &desc); * the cuDNN backend signal operation descriptor specifies an operation node for updating * or waiting on a flag variable. Signaling operations can be used to communicate between * cuDNN operation graphs, even with operation graphs in another GPU. * * This operation, to connect to other nodes in the graph, also has a pass-through input * tensor, which is not operated on and is just passed along to the output tensor. This * mandatory pass-through input tensor helps in determining the predecessor node after which * the signal operation should be executed. The optional output tensor helps in determining * the successor node before which the signal execution should have completed. It is also * guaranteed that for a non-virtual tensor as the output tensor, all writes for the tensor * will have taken place before the signal value is updated by the operation. * * Supported attributes: * - CUDNN_ATTR_OPERATION_SIGNAL_MODE (CUDNN_TYPE_SIGNAL_MODE) * The signaling mode to use. * Required attribute. * * - CUDNN_ATTR_OPERATION_SIGNAL_FLAGDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Flag tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_SIGNAL_VALUE (CUDNN_TYPE_INT64) * The scalar value to compare or update the flag variable with. * Required attribute. * * - CUDNN_ATTR_OPERATION_SIGNAL_XDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * A pass-through input tensor to enable connecting this signal operation to other nodes * in the graph. * Required attribute. * * - CUDNN_ATTR_OPERATION_SIGNAL_YDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * The output tensor for the pass-through input tensor. * Optional attribute. * * Finalization: * In the finalization stage, the attributes are cross checked to make sure there are * no conflicts. The status below may be returned: * * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are encountered. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_SIGNAL_DESCRIPTOR = 28, /**< Updates or waits on a flag variable for inter-graph signaling. @since cuDNN 9.0.0 */ /** * @brief Specifies a node for forward normalization producing normalized output Y from input X. * * * Created with cudnnBackendCreateDescriptor(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR, &desc); * the cuDNN backend normalization forward operation specifies a node for a forward * normalization that takes as input a tensor X and produces a normalized output Y with * the normalization mode set by the CUDNN_ATTR_OPERATION_NORM_FWD_MODE attribute. The * operation supports optional running stats computation and allows for storing the computed * means and variances for reuse in the backwards calculation depending on the setting of * the CUDNN_ATTR_OPERATION_NORM_FWD_PHASE attribute. * * Limitations: * - Does not support CUDNN_GROUP_NORM mode. * - Batch norm only supports forward training and not forward inference. * * Supported configurations: * CUDNN_NORM_FWD_TRAINING: * CUDNN_LAYER_NORM: Yes, CUDNN_INSTANCE_NORM: Yes, CUDNN_BATCH_NORM: Yes, * CUDNN_GROUP_NORM: No, CUDNN_RMS_NORM: Yes. * CUDNN_NORM_FWD_INFERENCE: * CUDNN_LAYER_NORM: Yes, CUDNN_INSTANCE_NORM: Yes, CUDNN_BATCH_NORM: No, * CUDNN_GROUP_NORM: No, CUDNN_RMS_NORM: Yes. * * Note: In addition to single-GPU, batch normalization supports running on single node * multi-GPUs, while other normalization modes only support running on a single GPU. For * more information, refer to the NormAddRelu pattern. * * Supported attributes: * - CUDNN_ATTR_OPERATION_NORM_FWD_MODE (CUDNN_TYPE_NORM_MODE, 1 element) * Chooses the normalization mode for the norm forward operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_PHASE (CUDNN_TYPE_NORM_FWD_PHASE, 1 element) * Selects the training or inference phase for the norm forward operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_XDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Estimated mean input tensor descriptor for the inference phase and the computed mean * output tensor descriptor for the training phase. * Optional attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Estimated inverse variance input tensor descriptor for the inference phase and the * computed inverse variance output tensor descriptor for the training phase. * Optional attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Normalization scale input tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Normalization bias input tensor descriptor. * Required attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Scalar input tensor descriptor for the epsilon value used in normalization calculation. * Note that the attribute CUDNN_ATTR_TENSOR_IS_BY_VALUE of this descriptor should be * set to true. * Required attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_EXP_AVG_FACTOR_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Scalar input tensor descriptor for the exponential average factor value used in * running stats computation. * Optional attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_MEAN_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input running mean tensor descriptor for the running stats computation in the * training phase. * Optional attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_VAR_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Input running variance tensor descriptor for the running stats computation in the * training phase. * Optional attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_MEAN_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Output running mean tensor descriptor for the running stats computation in the * training phase. * Optional attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_VAR_DESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Output running variance tensor descriptor for the running stats computation in the * training phase. * Optional attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_YDESC (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1 element of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Tensor descriptor for the output of the normalization operation. * Required attribute. * * - CUDNN_ATTR_OPERATION_NORM_FWD_PEER_STAT_DESCS (CUDNN_TYPE_BACKEND_DESCRIPTOR, 1+ elements of CUDNN_BACKEND_TENSOR_DESCRIPTOR) * Vector of tensor descriptors for the communication buffers used in multi-GPU * normalization. Typically, one buffer is provided for every GPU in the node. This * is an optional attribute only used for multi-GPU tensor stats reduction. * Optional attribute. * * Finalization: * In the finalization stage, the attributes are checked to ensure there are no conflicts. * * CUDNN_STATUS_BAD_PARAM - Invalid or inconsistent attribute values are encountered. * Some examples include: * - The output tensor dimensions do not match the input tensor dimensions. * - The channel count C for the mean, scale, bias, and inv_variance tensors do not match. * CUDNN_STATUS_SUCCESS - The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR = 29, /**< Forward normalization (layer/instance/batch/RMS) with optional running stats. @since cuDNN 9.0.0 */ /** * @brief the cuDNN backend normalization backward operation specifies a node for a backward normalization * * that takes as input the gradient tensor dY and outputs the gradient tensor dX and weight gradients * dScale and dBias. The normalization mode is set using the CUDNN_ATTR_OPERATION_NORM_BWD_MODE attribute. * * Limitations: * - Does not support CUDNN_GROUP_NORM mode. * * Supported Configurations: * CUDNN_LAYER_NORM : Yes * CUDNN_INSTANCE_NORM : Yes * CUDNN_BATCH_NORM : Yes * CUDNN_GROUP_NORM : No * CUDNN_RMS_NORM : Yes * * Note: In addition to single GPU, CUDNN_BATCH_NORM also supports single node multi-GPU batch norm, * while other normalization modes only support running on a single GPU. For more information, refer * to the DReluForkDNorm pattern. * * Attributes: * * CUDNN_ATTR_OPERATION_NORM_BWD_MODE * Chooses the normalization mode for the norm backward operation. * - CUDNN_TYPE_NORM_MODE; one element. * - Required attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_XDESC * Input tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC * Saved mean input tensor descriptor for reusing the mean computed during the forward computation * of the training phase. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC * Saved inverse variance input tensor descriptor for reusing the mean computed during the forward * computation of the training phase. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC * Gradient tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC * Normalization scale descriptor. Note that the bias descriptor is not necessary for the backward pass. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_EPSILON_DESC * Scalar input tensor descriptor for the epsilon value. The epsilon values are needed only if the * saved mean and variances are not passed as inputs to the operation. Note that the attribute * CUDNN_ATTR_TENSOR_IS_BY_VALUE of this descriptor should be set to true. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC * Scale gradient tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC * Bias gradient tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC * Input gradient tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_NORM_BWD_PEER_STAT_DESCS * Vector of tensor descriptors for the communication buffers used in multi-GPU normalization. * Typically, one buffer is provided for every GPU in the node. This is an optional attribute only * used for multi-GPU tensor stats reduction. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one or more elements of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM * Invalid or inconsistent attribute values are encountered. Some examples include: * - The tensor dimensions of the gradient tensors dY, dX, and input tensor X, do not match. * - The channel count C for the mean, scale, and inv_variance tensors do not match. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR = 30, /**< Backward normalization: computes dX, dScale, dBias from dY. @since cuDNN 9.0.0 */ /** * @brief Reshapes a tensor from one layout to another. * * * Note: No dedicated RST documentation section exists for this descriptor. The information * below is derived from the header (cudnn_graph.h) attribute definitions. * * Attributes: * * CUDNN_ATTR_OPERATION_RESHAPE_XDESC * Reshape input tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_RESHAPE_YDESC * Reshape output tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * Note: Documentation pending. * @since cuDNN 9.0.0 */ CUDNN_BACKEND_OPERATION_RESHAPE_DESCRIPTOR = 31, /**< Reshapes a tensor from one layout to another. @since cuDNN 9.0.0 */ /** * @brief the cuDNN backend Rng descriptor specifies any metadata, including the probability distribution * * that will be used to generate the tensor and the distribution's corresponding parameters. * * Attributes: * * CUDNN_ATTR_RNG_DISTRIBUTION * The probability distribution used for the rng operation. * - CUDNN_TYPE_RNG_DISTRIBUTION; one element. * - Default value is CUDNN_RNG_DISTRIBUTION_BERNOULLI. * * CUDNN_ATTR_RNG_NORMAL_DIST_MEAN * The mean value for the normal distribution, used if * CUDNN_ATTR_RNG_DISTRIBUTION = CUDNN_RNG_DISTRIBUTION_NORMAL. * - CUDNN_TYPE_DOUBLE; one element. * - Default value is -1. * * CUDNN_ATTR_RNG_NORMAL_DIST_STANDARD_DEVIATION * The standard deviation value for the normal distribution, used if * CUDNN_ATTR_RNG_DISTRIBUTION = CUDNN_RNG_DISTRIBUTION_NORMAL. * - CUDNN_TYPE_DOUBLE; one element. * - Default value is -1. * * Finalization: * Return values of cudnnBackendFinalize(desc) where desc is CUDNN_BACKEND_RNG_DESCRIPTOR are: * * CUDNN_STATUS_BAD_PARAM * An invalid attribute value was encountered. Some examples include: * - If CUDNN_ATTR_RNG_DISTRIBUTION = CUDNN_RNG_DISTRIBUTION_NORMAL and the standard * deviation supplied is negative. * - If CUDNN_ATTR_RNG_DISTRIBUTION = CUDNN_RNG_DISTRIBUTION_UNIFORM and the maximum * value of the range is lower than minimum value. * - If CUDNN_ATTR_RNG_DISTRIBUTION = CUDNN_RNG_DISTRIBUTION_BERNOULLI and the * probability supplied is negative. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. */ CUDNN_BACKEND_RNG_DESCRIPTOR = 32, /**< RNG config: distribution type (Bernoulli/uniform/normal) and parameters. @since cuDNN 9.0.0 */ /** * @brief the cuDNN backend Rng operation descriptor specifies an operation node for generating a tensor * * with random numbers based on the probability distribution specified in the Rng descriptor. * * The random numbers are generated using a Philox random number generator (RNG) as described in * Pytorch (https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/core/PhiloxRNGEngine.h). * The Philox object takes a seed value, a subsequence for starting the generation, and an offset * for the subsequence. Seed and offset can be set by using the attributes. The subsequence is * internally set, to ensure independent random numbers. * * Attributes: * * CUDNN_ATTR_OPERATION_RNG_DESC * Rng descriptor (CUDNN_BACKEND_RNG_DESCRIPTOR) instance containing metadata about the operation. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_RNG_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_RNG_YDESC * Output tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_RNG_SEED * Sets the seed for the random number generator which creates the Y tensor. It can be a host * INT64 value or a backend descriptor binded to a value on the device. Only supports a tensor * with all dimensions set to 1 and all strides set to 1. * - CUDNN_TYPE_INT64; one element or CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor * type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * - Default value is 0. * * CUDNN_ATTR_OPERATION_RNG_OFFSET_DESC * Tensor descriptor for the offset used in the RNG Philox object. Only supports a tensor with * all dimensions set to 1 and all strides set to 1. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM * CUDNN_ATTR_OPERATION_RNG_OFFSET_DESC or CUDNN_ATTR_OPERATION_RNG_SEED do not have all * dimensions and strides set to 1. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_RNG_DESCRIPTOR = 33, /**< Generates a tensor of random numbers from a specified distribution. @since cuDNN 9.0.0 */ /** * @brief the cuDNN backend kernel cache helps significantly reduce execution plan finalizing time for * * use cases that have same-topology dynamic shape operation graph by binding the previously * compiled applicable kernel to the execution plan instead of re-compiling a new one from scratch. * This is used with execution plans containing a graph with * CUDNN_ATTR_OPERATIONGRAPH_IS_DYNAMIC_SHAPE_ENABLED enabled. * * Attributes: * * CUDNN_ATTR_KERNEL_CACHE_IS_ENGINECFG_KERNEL_CACHED * An attribute used to query whether a given engine config is cached. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element. * - Read-only attribute using the cudnnBackendGetAttribute() API. The engine config in question * is to be passed into this attribute as a constant input through arrayOfElements and the * elementCount will serve as the resulting output, a value of zero meaning not cached and a * positive number meaning that it is cached. * - Required attribute. * * Finalization: * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. */ CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR = 34, /**< Caches compiled kernels to speed up plan finalization for dynamic-shape graphs. @since cuDNN 9.4.0 */ /** * @brief the cuDNN backend paged cache load operation descriptor is used with a fused flash attention fprop * * graph, and specifies an operation node for reconstructing the k- or v-cache. * * The k/v-cache is reconstructed by using a page table tensor to look up the location of a specific * sequence ID in a non-contiguous container tensor. Storing a k/v-cache non-contiguously enables * efficient memory management by avoiding fragmentation. For more information, refer to the * Paged Attention paper (https://arxiv.org/abs/2309.06180). * * Attributes: * * CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_YDESC * Virtual output tensor descriptor, containing the reconstructed k/v-cache. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Datatype: FP16, BF16, or FP8 * - Required attribute. * * CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_CONTAINER_DESC * A non-virtual tensor descriptor with dimensions [num_blocks,H,block_size,D] containing the * k/v-cache. The k/v-cache is divided into num_blocks of [H,block_size,D] tensors, where * block_size is a parameter chosen by the user. A smaller block_size leads to less fragmentation, * but also less parallelism. num_blocks is arbitrary and depends on the size of the allocated * k/v-cache. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Datatype: FP16, BF16, or FP8 * - Required attribute. * * CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_PAGE_TABLE_DESC * A non-virtual tensor descriptor of dimensions [B,1,ceil(max_seq_size/block_size),1] pointing * to the lookup table. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Datatype: INT32 * - Required attribute. * * CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_SEQUENCE_DESC * A non-virtual [B,1,1,1] tensor descriptor indicates which sequence numbers from the k/v-cache * are requested. For each batch, all items from the container will be copied from sequence 0 to * sequence number 1. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Datatype: INT32 or INT64 * - Sequence numbers are in the interval [1, max_seq_size] * - Required attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM * Types or dimensions of one or more of the input/output tensors are invalid. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. */ CUDNN_BACKEND_OPERATION_PAGED_CACHE_LOAD_DESCRIPTOR = 35, /**< Reconstructs K/V-cache pages in fused flash attention forward graphs. @since cuDNN 9.4.0 */ /** * @brief the cuDNN block scale quantize descriptor specifies the parameters for the block scale quantize * * operation to output block scaled tensors. * * Attributes: * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_XDESC * Input float tensor to quantize. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_YDESC * Quantized output tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_SCALE_DESC * Per-block scaling factors output. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_MATH_PREC * The math precision of the computation. * - CUDNN_TYPE_DATA_TYPE; one element. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_BLOCK_SIZE * The number of elements per block to perform block scaling. * - CUDNN_TYPE_INT32; one element. * - Required attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM * An invalid attribute value was encountered. Some examples include: * - Tensor shape mismatch between x, y, and scale tensors. * - Data type mismatch between y and scale tensors. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. * * @since cuDNN 9.7.0 */ CUDNN_BACKEND_OPERATION_BLOCK_SCALE_QUANTIZE_DESCRIPTOR = 36, /**< Block-scale quantization: converts float tensors to block-scaled format. @since cuDNN 9.7.0 */ /** * @brief the cuDNN block scale dequantize descriptor specifies the parameters for the block scale dequantize * * operation to take in block scaled tensors. * * Attributes: * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_XDESC * Quantized input tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_SCALE_DESC * Per-block scale factors. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_YDESC * Dequantized output tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_MATH_PREC * The math precision of the computation. * - CUDNN_TYPE_DATA_TYPE; one element. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_BLOCK_SIZE * The number of elements per block to perform block scaling. * - CUDNN_TYPE_INT32; one element. * - Required attribute. * * CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_NEG_SCALE * Negative scale handling. * - Optional attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM * An invalid attribute value was encountered. Some examples include: * - Tensor shape mismatch between x, scale, and y tensors. * - Data type mismatch between x and scale tensors. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. * * @since cuDNN 9.7.0 */ CUDNN_BACKEND_OPERATION_BLOCK_SCALE_DEQUANTIZE_DESCRIPTOR = 37, /**< Block-scale dequantization: converts block-scaled tensors back to float. @since cuDNN 9.7.0 */ /** * @brief the cuDNN device property descriptor specifies the properties of a device. * * * Attributes: * * CUDNN_ATTR_DEVICEPROP_DEVICE_ID * The CUDA device ID of the device that the descriptor targets. * - CUDNN_TYPE_INT32; one element. * - Optional attribute. * * CUDNN_ATTR_DEVICEPROP_HANDLE * The cuDNN handle of the device that the descriptor targets. * - CUDNN_TYPE_HANDLE; one element. * - Optional attribute. * * CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION * The JSON representation of the device that the descriptor targets. * - CUDNN_TYPE_CHAR; one element. * - Optional attribute. * * Finalization: * CUDNN_STATUS_BAD_PARAM * An invalid attribute value was encountered, for example, the provided JSON representation * is invalid. * CUDNN_STATUS_NOT_INITIALIZED * For some reason, querying the device properties failed. * CUDNN_STATUS_NOT_SUPPORTED_ARCH_MISMATCH * The target device is not supported. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. */ CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR = 38, /**< CUDA device properties: device ID, handle, JSON representation. @since cuDNN 9.8.0 */ /** * @brief Expands a band matrix into a full matrix representation. This operation takes a compact * * band-storage tensor and expands it along a specified axis into a full dense matrix, filling * positions outside the band with a configurable pad value. * * Note: No dedicated RST documentation section exists for this descriptor. The information * below is derived from the header (cudnn_graph.h) attribute definitions. * * Attributes: * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_XDESC * Band matrix input tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_YDESC * Expanded output tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_LOWER_BANDWIDTH * Lower bandwidth of the band. * - Required attribute. * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_UPPER_BANDWIDTH * Upper bandwidth of the band. * - Required attribute. * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_AXIS * Axis along which to expand. * - Required attribute. * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_PAD_VALUE * Padding value outside the band. * - Optional attribute. * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_KV_TOKEN_OFFSET_DESC * KV token offset descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_EXPAND_BAND_MATRIX_SPECULATIVE_MASK_DESC * Speculative decoding mask. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * Note: Documentation pending. * @since cuDNN 9.10.0 */ CUDNN_BACKEND_OPERATION_EXPAND_BAND_MATRIX_DESCRIPTOR = 39, /**< Expands a band matrix into a full matrix representation. @since cuDNN 9.10.0 */ /** * @brief Contracts a full matrix into a band matrix representation. This operation extracts the band * * portion of a dense matrix along a specified axis into compact band storage, using a configurable * pad value for out-of-bounds elements. * * Note: No dedicated RST documentation section exists for this descriptor. The information * below is derived from the header (cudnn_graph.h) attribute definitions. * * Attributes: * * CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_XDESC * Full matrix input tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_YDESC * Contracted band output tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_LOWER_BANDWIDTH * Lower bandwidth. * - Required attribute. * * CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_UPPER_BANDWIDTH * Upper bandwidth. * - Required attribute. * * CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_AXIS * Axis along which to contract. * - Required attribute. * * CUDNN_ATTR_OPERATION_CONTRACT_BAND_MATRIX_PAD_VALUE * Padding value. * - Optional attribute. * * CUDNN_ATTR_OPERATION_CONTRACT_BAND_MAX_TOKEN_VALUE * Maximum token value for contraction. * - Optional attribute. * * Note: Documentation pending. * @since cuDNN 9.10.0 */ CUDNN_BACKEND_OPERATION_CONTRACT_BAND_MATRIX_DESCRIPTOR = 40, /**< Contracts a full matrix into a band matrix representation. @since cuDNN 9.10.0 */ /** * @brief the cuDNN scaled dot-product attention forward operation descriptor represents a flash attention * * forward computation as a single operation, obviating in some cases the need to represent flash * attention as many separate operations. Only one basic flash attention operation is supported: * * O = softmax((Q * K^t) * scale) * V * * An optional padding mask is also supported. The padding mask does not have its own attribute, but * is implicitly enabled if CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC and * CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC are both specified. In this case, the formula becomes: * * O = softmax(padding_mask((Q * K^t) * scale)) * V * * Furthermore, a block mask is also supported. A block mask can be applied to the scores to mask out * entire blocks. In this case, the formula becomes: * * O = softmax(padding_mask(block_mask(Q * K^t) * scale)) * V * * For ease of use, access this operation through the frontend's attention API, which can automatically * select this operation depending on the features requested and the cuDNN backend version. The details * of this backend descriptor type are provided here for completeness. * * Attributes: * * CUDNN_ATTR_OPERATION_SDPA_FWD_QDESC * Input tensor descriptor representing the Q input to the above formula. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Tensor must be of dimensions (B, H_q, S_q, D_qk). * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_FWD_KDESC * Input tensor descriptor representing the K input to the above formula, or the K page container * if paged attention for K is enabled (if CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_KDESC is given). * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Tensor must be of dimensions: * - If paged attention is disabled: (B, H_k, S_kv, D_qk). Note that this attribute represents * a literal K tensor and not its transpose K^t. * - If paged attention is enabled: (num_blocks_k, H_k, bs_k, D_qk) (where bs_k is the K * block size). * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_FWD_VDESC * Input tensor descriptor representing the V input to the above formula, or the V page container * if paged attention is enabled (if CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_VDESC is given). * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Tensor must be of dimensions: * - If paged attention is disabled: (B, H_v, S_kv, D_v). * - If paged attention is enabled: (num_blocks_v, H_v, bs_v, D_v) (where bs_v is the V * block size). * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_FWD_ODESC * Output tensor descriptor representing the O output of the above formula. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Tensor must be of dimensions (B, H_q, S_q, D_v). * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC * Output tensor descriptor representing the stats output of the softmax in the above formula, * used during training. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Tensor must be of dimensions (B, H_q, S_q, 1). * - Optional attribute. * * CUDNN_ATTR_OPERATION_SDPA_FWD_SCALEDESC * Input tensor descriptor representing the scale factor in the above formula. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Single element must be of type CUDNN_DATA_FLOAT. * - Tensor must be of dimensions (1, 1, 1, 1). * - Tensor may be a device-side tensor, or a "by value" (host side) tensor. * - Optional attribute. If not given, a scale of 1.0 is used. * * CUDNN_ATTR_OPERATION_SDPA_FWD_BLOCK_MASK_DESC * Input tensor descriptor representing the block mask of the scores assuming block size 128x128. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Elements must be of type CUDNN_DATA_INT8. * - Tensor must be of dimensions (B, H_q, ceil(S_q / 128), ceil(S_kv / 128)). * - Optional attribute. * * CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_KDESC * Input tensor descriptor representing the page table for K. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Elements must be of type CUDNN_DATA_INT32. * - Tensor must be of dimensions (B, 1, ceil(S_kv / bs_k), 1). * - Optional attribute. * - If not given, paged attention for K is disabled. * - If given, paged attention for K is enabled, and the attribute * CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC must also be given. * * CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_VDESC * Input tensor descriptor representing the page table for V. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Elements must be of type CUDNN_DATA_INT32. * - Tensor must be of dimensions (B, 1, ceil(S_kv / bs_v), 1). * - Optional attribute. * - If not given, paged attention for V is disabled. * - If given, paged attention for V is enabled, and the attribute * CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC must also be given. * * CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC * Input tensor descriptor representing sequence lengths for Q. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Elements must be of type CUDNN_DATA_INT32 or CUDNN_DATA_INT64. * - Tensor must be of dimensions (B, 1, 1, 1). * - Optional attribute. * - If given and CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC is also given, a padding * mask is applied. * * CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC * Input tensor descriptor representing sequence lengths for K and V. If specified and * CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC is also specified, a padding mask is applied. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Elements must be of type CUDNN_DATA_INT32 or CUDNN_DATA_INT64. * - Tensor must be of dimensions (B, 1, 1, 1). * - Optional attribute, but required if paged attention is enabled. * - If given and CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC is also given, a padding * mask is applied. * * Finalization: * CUDNN_STATUS_BAD_PARAM * An invalid attribute value was encountered, for example, the provided input tensor sizes * don't match along the necessary dimensions to make the scaled dot-product attention * operation mathematically possible. * CUDNN_STATUS_SUCCESS * The descriptor was finalized successfully. * * @since cuDNN 9.13.0 */ CUDNN_BACKEND_OPERATION_SDPA_FWD_DESCRIPTOR = 41, /**< Scaled dot-product attention forward (fused flash attention). @since cuDNN 9.13.0 */ /** * @brief Mixture-of-Experts grouped matmul with token routing. Performs a grouped matrix multiplication * * where tokens are routed to different expert weight matrices based on routing indices. Supports * gather and scatter modes for efficient expert-parallel computation. * * Note: No dedicated RST documentation section exists for this descriptor. The information * below is derived from the header (cudnn_graph.h) attribute definitions. * * Attributes: * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MODE * Gather/scatter mode for MoE. * - CUDNN_TYPE_MOE_GROUPED_MATMUL_MODE; one element. * - Values: CUDNN_MOE_GROUPED_MATMUL_MODE_NONE (0), * CUDNN_MOE_GROUPED_MATMUL_MODE_GATHER (1), * CUDNN_MOE_GROUPED_MATMUL_MODE_SCATTER (2). * - Required attribute. * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MATH_PREC * Computation precision. * - CUDNN_TYPE_DATA_TYPE; one element. * - Required attribute. * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_DESC * Token tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_WEIGHT_DESC * Expert weight tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_FIRST_TOKEN_OFFSET_DESC * First token offset per expert. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_OUTPUT_DESC * Output tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_INDEX_DESC * Token routing index descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute (used with gather/scatter modes). * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_KS_DESC * Token routing weights descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOP_K * Top-K experts per token. * - Optional attribute. * * Note: Documentation pending. * @since cuDNN 9.15.0 */ CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_DESCRIPTOR = 42, /**< Mixture-of-Experts grouped matmul with token routing. @since cuDNN 9.15.0 */ /** * @brief Scaled dot-product attention backward operation. Computes gradients dQ, dK, and dV for the * * backward pass of flash attention given the forward pass outputs and the incoming gradient dO. * * Note: No dedicated RST documentation section exists for this descriptor. The information * below is derived from the header (cudnn_graph.h) attribute definitions. * * Attributes: * * CUDNN_ATTR_OPERATION_SDPA_BWD_QDESC * Query tensor descriptor (forward input). * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_KDESC * Key tensor descriptor (forward input). * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_VDESC * Value tensor descriptor (forward input). * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_ODESC * Forward output tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_STATSDESC * Forward statistics descriptor (softmax stats from the forward pass). * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_SCALEDESC * Attention scaling factor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_SEQ_LEN_QDESC * Query sequence length tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_SEQ_LEN_KVDESC * Key-value sequence length tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_DQDESC * Query gradient output tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_DKDESC * Key gradient output tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_DVDESC * Value gradient output tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SDPA_BWD_DODDESC * Output gradient input tensor (dO). * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * Note: Documentation pending. * @since cuDNN 9.17.0 */ CUDNN_BACKEND_OPERATION_SDPA_BWD_DESCRIPTOR = 43, /**< Scaled dot-product attention backward: computes dQ, dK, dV. @since cuDNN 9.17.0 */ /** * @brief Generates a diagonal band attention mask. Produces a mask tensor based on diagonal band * * boundaries (left and right bounds) relative to the query and key-value sequence positions, * commonly used to implement sliding window or local attention patterns. * * Note: No dedicated RST documentation section exists for this descriptor. The information * below is derived from the header (cudnn_graph.h) attribute definitions. * * Attributes: * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_XDESC * Input tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_KVDESC * KV sequence length tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_QDESC * Query sequence length tensor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_LEFT_BOUND_DESC * Left bound offset descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SHIFT_RIGHT_BOUND_DESC * Right bound shift descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_BDESC * Band descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_YDESC * Output mask tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_COMPARISON_MODE * Mask comparison mode. * - Required attribute. * * Note: Documentation pending. * @since cuDNN 9.20.0 */ CUDNN_BACKEND_OPERATION_DIAGONAL_BAND_MASK_DESCRIPTOR = 44, /**< Generates a diagonal band attention mask. @since cuDNN 9.20.0 */ /** * @brief Softmax operation with optional statistics output. Computes the softmax function over * * the input tensor and optionally outputs intermediate statistics (row-wise max, sum of * exponentials) for use in fused attention patterns. * * Note: No dedicated RST documentation section exists for this descriptor. The information * below is derived from the header (cudnn_graph.h) attribute definitions. * * Attributes: * * CUDNN_ATTR_OPERATION_SOFTMAX_XDESC * Softmax input tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SOFTMAX_YDESC * Softmax output tensor descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Required attribute. * * CUDNN_ATTR_OPERATION_SOFTMAX_STATS_DESC * Softmax statistics output. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_SOFTMAX_MAX_DESC * Row-wise max values descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_SOFTMAX_SUM_EXP_DESC * Row-wise sum of exponentials. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * CUDNN_ATTR_OPERATION_SOFTMAX_SINK_DESC * Softmax sink descriptor. * - CUDNN_TYPE_BACKEND_DESCRIPTOR; one element of descriptor type CUDNN_BACKEND_TENSOR_DESCRIPTOR. * - Optional attribute. * * Note: Documentation pending. * @since cuDNN 9.20.0 */ CUDNN_BACKEND_OPERATION_SOFTMAX_DESCRIPTOR = 45, /**< Softmax operation with optional statistics output. @since cuDNN 9.20.0 */ CUDNN_BACKEND_OPERATION_TRANSPOSE_DESCRIPTOR = 46, /**< Transpose operation. @since cuDNN 9.22.0 */ CUDNN_BACKEND_OPERATION_SLICE_DESCRIPTOR = 47, /**< Slice operation. @since cuDNN 9.22.0 */ CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_BWD_DESCRIPTOR = 48, /**< Backward MoE grouped matmul. @since cuDNN 9.22.0 */ CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR = 49, /**< Rotary Position Embedding (RoPE) forward. @since cuDNN 9.24.0 */ CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR = 50, /**< Rotary Position Embedding (RoPE) backward. @since cuDNN 9.25.0 */ } cudnnBackendDescriptorType_t; /** @brief Numerical behavior notes describing properties of engine implementations. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_NUMERICAL_NOTE_TENSOR_CORE = 0, /**< Engine uses Tensor Core hardware acceleration. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_DOWN_CONVERT_INPUTS = 1, /**< Engine down-converts inputs to lower precision for compute. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_REDUCED_PRECISION_REDUCTION = 2, /**< Engine uses reduced-precision accumulation in reductions. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_FFT = 3, /**< Engine uses FFT-based computation. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_NONDETERMINISTIC = 4, /**< Engine may produce non-deterministic results across runs. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_WINOGRAD = 5, /**< Engine uses Winograd transform. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_WINOGRAD_TILE_4x4 = 6, /**< Engine uses Winograd with 4x4 output tiles. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_WINOGRAD_TILE_6x6 = 7, /**< Engine uses Winograd with 6x6 output tiles. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_WINOGRAD_TILE_13x13 = 8, /**< Engine uses Winograd with 13x13 output tiles. @since cuDNN 9.0.0 */ CUDNN_NUMERICAL_NOTE_STRICT_NAN_PROP = 9, /**< Engine strictly propagates NaN values. @since cuDNN 9.1.0 */ CUDNN_NUMERICAL_NOTE_TYPE_COUNT = 10, /**< Number of numerical note types. @since cuDNN 9.0.0 */ } cudnnBackendNumericalNote_t; /** @brief Engine behavior notes describing runtime requirements and capabilities. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_BEHAVIOR_NOTE_RUNTIME_COMPILATION = 0, /**< Engine requires runtime compilation (NVRTC). @since cuDNN 9.0.0 */ CUDNN_BEHAVIOR_NOTE_REQUIRES_FILTER_INT8x32_REORDER = 1, /**< Engine requires INT8x32-reordered filter tensors. @since cuDNN 9.0.0 */ CUDNN_BEHAVIOR_NOTE_REQUIRES_BIAS_INT8x32_REORDER = 2, /**< Engine requires INT8x32-reordered bias tensors. @since cuDNN 9.0.0 */ CUDNN_BEHAVIOR_NOTE_SUPPORTS_CUDA_GRAPH_NATIVE_API = 3, /**< Engine supports native CUDA graph capture. @since cuDNN 9.5.0 */ CUDNN_BEHAVIOR_NOTE_CUBLASLT_DEPENDENCY = 4, /**< Engine depends on cuBLASLt library. @since cuDNN 9.15.0 */ CUDNN_BEHAVIOR_NOTE_TYPE_COUNT = 5, /**< Number of behavior note types. @since cuDNN 9.0.0 */ } cudnnBackendBehaviorNote_t; /** @brief Engine tuning knob types for fine-grained engine configuration. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_KNOB_TYPE_SPLIT_K CUDNN_DEPRECATED_ENUM = 0, /**< @deprecated Split-K factor. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SWIZZLE = 1, /**< Memory access swizzle pattern for coalescing. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_TILE_SIZE = 2, /**< Thread block tile size for computation. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_USE_TEX CUDNN_DEPRECATED_ENUM = 3, /**< @deprecated Use texture memory. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_EDGE = 4, /**< Edge/boundary handling mode. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_KBLOCK CUDNN_DEPRECATED_ENUM = 5, /**< @deprecated K-block size. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_LDGA CUDNN_DEPRECATED_ENUM = 6, /**< @deprecated Load granularity A. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_LDGB CUDNN_DEPRECATED_ENUM = 7, /**< @deprecated Load granularity B. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_CHUNK_K CUDNN_DEPRECATED_ENUM = 8, /**< @deprecated Chunk-K size. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SPLIT_H CUDNN_DEPRECATED_ENUM = 9, /**< @deprecated Split-H factor. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_WINO_TILE CUDNN_DEPRECATED_ENUM = 10, /**< @deprecated Winograd tile size. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_MULTIPLY = 11, /**< Multiplication factor for tiling. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SPLIT_K_BUF = 12, /**< Split-K with separate output buffers. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_TILEK = 13, /**< Tile size along K (contraction) dimension. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_STAGES = 14, /**< Number of pipeline stages. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_REDUCTION_MODE = 15, /**< Cross-thread/cross-block reduction strategy. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_CTA_SPLIT_K_MODE CUDNN_DEPRECATED_ENUM = 16, /**< @deprecated CTA split-K mode. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SPLIT_K_SLC = 17, /**< Split-K slice count. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_IDX_MODE = 18, /**< Index computation mode. @since cuDNN 9.7.0 */ CUDNN_KNOB_TYPE_SLICED CUDNN_DEPRECATED_ENUM = 19, /**< @deprecated Sliced mode. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SPLIT_RS CUDNN_DEPRECATED_ENUM = 20, /**< @deprecated Split-RS factor. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SINGLEBUFFER CUDNN_DEPRECATED_ENUM = 21, /**< @deprecated Single buffer mode. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_LDGC CUDNN_DEPRECATED_ENUM = 22, /**< @deprecated Load granularity C. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SPECFILT = 23, /**< Specialized filter processing mode. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_KERNEL_CFG = 24, /**< Kernel configuration selector. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_WORKSPACE = 25, /**< Workspace size preference. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_TILE_CGA CUDNN_DEPRECATED_ENUM = 26, /**< @deprecated CGA tile size. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_TILE_CGA_M = 27, /**< CGA cluster tile size along M dimension. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_TILE_CGA_N = 28, /**< CGA cluster tile size along N dimension. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_BLOCK_SIZE = 29, /**< Thread block size (threads per block). @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_OCCUPANCY = 30, /**< Target occupancy level. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_ARRAY_SIZE_PER_THREAD = 31, /**< Register array size per thread. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_NUM_C_PER_BLOCK CUDNN_DEPRECATED_ENUM = 32, /**< @deprecated Channels per block. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_SPLIT_COLS = 33, /**< Column split factor for parallelism. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_TILE_ROWS = 34, /**< Tile size along row dimension. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_TILE_COLS = 35, /**< Tile size along column dimension. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_LOAD_SIZE = 36, /**< Memory load granularity. @since cuDNN 9.0.0 */ CUDNN_KNOB_TYPE_CTA_COUNT = 37, /**< Number of CTAs (thread blocks) to launch. @since cuDNN 9.7.0 */ CUDNN_KNOB_TYPE_STREAM_K = 38, /**< Stream-K work distribution mode. @since cuDNN 9.7.0 */ CUDNN_KNOB_TYPE_SPLIT_P_SLC = 39, /**< Split-P slice count. @since cuDNN 9.7.0 */ CUDNN_KNOB_TYPE_TILE_M = 40, /**< Tile size along M (output rows) dimension. @since cuDNN 9.7.0 */ CUDNN_KNOB_TYPE_TILE_N = 41, /**< Tile size along N (output cols) dimension. @since cuDNN 9.7.0 */ CUDNN_KNOB_TYPE_WARP_SPEC_CFG = 42, /**< Warp specialization configuration. @since cuDNN 9.7.0 */ CUDNN_KNOB_TYPE_SWAP_AB = 43, /**< Swap A and B operands in matmul. @since cuDNN 9.18.0 */ CUDNN_KNOB_TYPE_INPUT_TMA_ENABLE = 44, /**< Enable TMA for input tensors. @since cuDNN 9.22.0 */ CUDNN_KNOB_TYPE_OUTPUT_TMA_ENABLE = 45, /**< Enable TMA for output tensors. @since cuDNN 9.22.0 */ CUDNN_KNOB_TYPE_COUNTS = 46, /**< Number of knob types. @since cuDNN 9.0.0 */ } cudnnBackendKnobType_t; /** @brief Preferred tensor layout types reported by engines. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_LAYOUT_TYPE_PREFERRED_NCHW = 0, /**< Prefers NCHW layout. @since cuDNN 9.0.0 */ CUDNN_LAYOUT_TYPE_PREFERRED_NHWC = 1, /**< Prefers NHWC layout. @since cuDNN 9.0.0 */ CUDNN_LAYOUT_TYPE_PREFERRED_PAD4CK = 2, /**< Prefers padded 4CK layout. @since cuDNN 9.0.0 */ CUDNN_LAYOUT_TYPE_PREFERRED_PAD8CK = 3, /**< Prefers padded 8CK layout. @since cuDNN 9.0.0 */ CUDNN_LAYOUT_TYPE_COUNT = 4, /**< Number of layout types. @since cuDNN 9.0.0 */ } cudnnBackendLayoutType_t; /** @brief Heuristic modes for engine selection. * * INSTANT provides fast heuristic lookup, B uses neural-net-based heuristics, * FALLBACK provides functional (non-optimized) results, A is an alias for INSTANT. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_HEUR_MODE_INSTANT = 0, /**< Fast decision-tree heuristic with minimal CPU overhead. @since cuDNN 9.0.0 */ CUDNN_HEUR_MODE_B = 1, /**< Neural-net heuristic; 10-100x CPU cost vs INSTANT, better GPU perf. No 3D/grouped/dilated conv. @since cuDNN 9.0.0 */ CUDNN_HEUR_MODE_FALLBACK = 2, /**< Functional fallback engines with no GPU performance guarantee. @since cuDNN 9.0.0 */ CUDNN_HEUR_MODE_A = 3, /**< Decision-tree heuristic (preferred over INSTANT). @since cuDNN 9.0.0 */ CUDNN_HEUR_MODES_COUNT = 4, /**< Number of heuristic modes. @since cuDNN 9.0.0 */ } cudnnBackendHeurMode_t; /** @brief Tensor reordering modes for specialized memory layouts. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_TENSOR_REORDERING_NONE = 0, /**< No tensor reordering applied. @since cuDNN 9.0.0 */ CUDNN_TENSOR_REORDERING_INT8x32 = 1, /**< INT8 data reordered into 32-element vectors. @since cuDNN 9.0.0 */ CUDNN_TENSOR_REORDERING_F16x16 = 2, /**< FP16 data reordered into 16-element vectors. @since cuDNN 9.0.0 */ CUDNN_TENSOR_REORDERING_F8_128x4 = 3, /**< FP8 data reordered into 128x4 blocks. @since cuDNN 9.7.0 */ } cudnnBackendTensorReordering_t; /** @brief Padding modes for convolution and pooling operations. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_ZERO_PAD = 0, /**< Pads with zeros. @since cuDNN 9.0.0 */ CUDNN_NEG_INF_PAD = 1, /**< Pads with negative infinity (for max pooling). @since cuDNN 9.0.0 */ CUDNN_EDGE_VAL_PAD = 2, /**< Pads by replicating edge values. @since cuDNN 9.0.0 */ } cudnnPaddingMode_t; /** @brief Normalization modes supported by norm forward/backward operations. * @since cuDNN 9.0.0 */ typedef enum { CUDNN_LAYER_NORM = 0, /**< Layer normalization: normalizes over feature dims per sample. @since cuDNN 9.0.0 */ CUDNN_INSTANCE_NORM = 1, /**< Instance normalization: normalizes per instance per channel. @since cuDNN 9.0.0 */ CUDNN_BATCH_NORM = 2, /**< Batch normalization: normalizes across the batch dimension. @since cuDNN 9.0.0 */ CUDNN_GROUP_NORM = 3, /**< Group normalization (unsupported; returns CUDNN_STATUS_INTERNAL_ERROR). @since cuDNN 9.0.0 */ CUDNN_RMS_NORM = 4, /**< Root mean square normalization: normalizes by RMS of activations. @since cuDNN 9.0.0 */ CUDNN_ADA_LAYER_NORM = 5, /**< Adaptive layer normalization with learned affine parameters. @since cuDNN 9.7.0 */ } cudnnBackendNormMode_t; /** @brief Normalization forward phase (inference or training). * @since cuDNN 9.0.0 */ typedef enum { CUDNN_NORM_FWD_INFERENCE = 0, /**< Inference phase: uses pre-computed running statistics. @since cuDNN 9.0.0 */ CUDNN_NORM_FWD_TRAINING = 1, /**< Training phase: computes batch statistics and updates running stats. @since cuDNN 9.0.0 */ } cudnnBackendNormFwdPhase_t; /** @brief Reshape operation mode. * @since cuDNN 9.22.0 */ typedef enum { CUDNN_RESHAPE_VIEW_ONLY = 0, /**< View-only reshape (no data movement). @since cuDNN 9.22.0 */ CUDNN_RESHAPE_LOGICAL = 1, /**< Logical reshape (may involve data movement). @since cuDNN 9.22.0 */ } cudnnBackendReshapeMode_t; /** @brief Mode of an operation graph. * * Optional. The default value (::CUDNN_OPERATIONGRAPH_MODE_AUTO) makes finalize * detect the mode automatically; this is the recommended setting and the only * safe choice for users constructing graphs by hand. * * Setting an explicit mode via ::CUDNN_ATTR_OPERATIONGRAPH_MODE is an advanced * hint primarily intended for cudnn_frontend, which knows the canonical shape of * the graph it just built. Setting a wrong mode silently bypasses pattern * matching and causes failures during plan creation or execution. If you are * not going through cudnn_frontend, leave this on AUTO. * * @since cuDNN 9.NEXT */ typedef enum { CUDNN_OPERATIONGRAPH_MODE_AUTO = -1, /**< Default; backend detects the mode. */ CUDNN_OPERATIONGRAPH_MODE_CONV_FORWARD = 0, CUDNN_OPERATIONGRAPH_MODE_CONV_BWD_DATA = 1, CUDNN_OPERATIONGRAPH_MODE_CONV_BWD_FILTER = 2, CUDNN_OPERATIONGRAPH_MODE_NORM_FWD_INFER = 17, CUDNN_OPERATIONGRAPH_MODE_NORM_FWD_TRAIN = 18, CUDNN_OPERATIONGRAPH_MODE_NORM_BWD = 19, CUDNN_OPERATIONGRAPH_MODE_GENERIC_CONV_FUSION = 20, CUDNN_OPERATIONGRAPH_MODE_RESAMPLE_FWD = 22, CUDNN_OPERATIONGRAPH_MODE_RESAMPLE_BWD = 23, CUDNN_OPERATIONGRAPH_MODE_BLAS_MATMUL = 25, CUDNN_OPERATIONGRAPH_MODE_GENERIC_MATMUL_FUSION = 26, CUDNN_OPERATIONGRAPH_MODE_GENERIC_POINTWISE_FUSION = 27, CUDNN_OPERATIONGRAPH_MODE_SDPA_FWD = 28, /**< Composite SDPA fwd (matmul + softmax + matmul). */ CUDNN_OPERATIONGRAPH_MODE_SDPA_BWD = 29, CUDNN_OPERATIONGRAPH_MODE_UNIFIED_SDPA_FWD = 30, /**< Unified SDPA fwd (single SdpaForwardOperation). */ CUDNN_OPERATIONGRAPH_MODE_UNIFIED_SDPA_BWD = 31, } cudnnBackendOperationGraphMode_t; /** @brief Allocates memory for a backend descriptor of the specified type. * @param[in] descriptorType The type of descriptor to create. * @param[out] descriptor Pointer to receive the newly created descriptor. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_NOT_SUPPORTED * @retval CUDNN_STATUS_ALLOC_FAILED * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendCreateDescriptor(cudnnBackendDescriptorType_t descriptorType, cudnnBackendDescriptor_t *descriptor); /** @brief Deallocates a backend descriptor and frees associated memory. * @param[in] descriptor The descriptor to destroy. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_ALLOC_FAILED * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendDestroyDescriptor(cudnnBackendDescriptor_t descriptor); /** @brief Repurposes pre-allocated memory for a backend descriptor. * @deprecated Since cuDNN 9.2. Use cudnnBackendCreateDescriptor instead. * @param[in] descriptor The descriptor to initialize. * @return cuDNN status code. * @since cuDNN 9.0.0 */ CUDNN_DEPRECATED cudnnStatus_t CUDNNWINAPI cudnnBackendInitialize(cudnnBackendDescriptor_t descriptor); /** @brief Validates and finalizes a descriptor. After finalization, attributes become read-only. * @param[in] descriptor The descriptor to finalize. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @retval CUDNN_STATUS_NOT_SUPPORTED * @retval CUDNN_STATUS_INTERNAL_ERROR * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendFinalize(cudnnBackendDescriptor_t descriptor); /** @brief Sets an attribute on an unfinalized backend descriptor. * @param[in] descriptor The target descriptor (must not be finalized). * @param[in] attributeName The attribute to set. * @param[in] attributeType The data type of the attribute values. * @param[in] elementCount Number of elements in @p arrayOfElements. * @param[in] arrayOfElements Pointer to the attribute value(s). * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_NOT_INITIALIZED * @retval CUDNN_STATUS_BAD_PARAM * @retval CUDNN_STATUS_NOT_SUPPORTED * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendSetAttribute(cudnnBackendDescriptor_t descriptor, cudnnBackendAttributeName_t attributeName, cudnnBackendAttributeType_t attributeType, int64_t elementCount, const void *arrayOfElements); /** @brief Retrieves an attribute from a finalized backend descriptor. * @param[in] descriptor The source descriptor (must be finalized). * @param[in] attributeName The attribute to query. * @param[in] attributeType The expected data type of the attribute. * @param[in] requestedElementCount Maximum number of elements to retrieve. * @param[out] elementCount Pointer to receive the actual element count. * @param[out] arrayOfElements Buffer to receive the attribute value(s). * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @retval CUDNN_STATUS_NOT_INITIALIZED * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendGetAttribute(cudnnBackendDescriptor_t const descriptor, cudnnBackendAttributeName_t attributeName, cudnnBackendAttributeType_t attributeType, int64_t requestedElementCount, int64_t *elementCount, void *arrayOfElements); /** @brief Queries runtime workspace size for an execution plan using a variant pack. * @param[in] handle cuDNN handle. * @param[in] executionPlan Finalized execution plan descriptor. * @param[in] variantPack Finalized variant pack descriptor. * @param[out] workspaceSizeInBytes Pointer to receive the required workspace size. * @since cuDNN 9.23.0 */ cudnnStatus_t CUDNNWINAPI cudnnGetExecutionPlanWorkspaceSize(cudnnHandle_t handle, cudnnBackendDescriptor_t executionPlan, cudnnBackendDescriptor_t variantPack, size_t *workspaceSizeInBytes); /** @brief Runs an execution plan with the given variant pack containing data pointers. * @param[in] handle cuDNN handle. * @param[in] executionPlan Finalized execution plan descriptor. * @param[in] variantPack Finalized variant pack descriptor with data pointers. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @retval CUDNN_STATUS_INTERNAL_ERROR * @retval CUDNN_STATUS_EXECUTION_FAILED * @since cuDNN 9.0.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendExecute(cudnnHandle_t handle, cudnnBackendDescriptor_t executionPlan, cudnnBackendDescriptor_t variantPack); /** @brief Populates a CUDA graph with nodes from an execution plan. * @param[in] handle cuDNN handle. * @param[in] executionPlan Finalized execution plan descriptor. * @param[in] variantPack Finalized variant pack descriptor. * @param[inout] graph CUDA graph to populate. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @retval CUDNN_STATUS_INTERNAL_ERROR * @retval CUDNN_STATUS_EXECUTION_FAILED * @retval CUDNN_STATUS_NOT_SUPPORTED * @since cuDNN 9.5.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendPopulateCudaGraph(cudnnHandle_t handle, cudnnBackendDescriptor_t executionPlan, cudnnBackendDescriptor_t variantPack, cudaGraph_t graph); /** @brief Updates an existing CUDA graph with new data pointers from a variant pack. * @param[in] handle cuDNN handle. * @param[in] executionPlan Finalized execution plan descriptor. * @param[in] variantPack Finalized variant pack with updated data pointers. * @param[inout] graph CUDA graph to update. * @retval CUDNN_STATUS_SUCCESS * @retval CUDNN_STATUS_BAD_PARAM * @retval CUDNN_STATUS_INTERNAL_ERROR * @retval CUDNN_STATUS_EXECUTION_FAILED * @retval CUDNN_STATUS_NOT_SUPPORTED * @since cuDNN 9.5.0 */ cudnnStatus_t CUDNNWINAPI cudnnBackendUpdateCudaGraph(cudnnHandle_t handle, cudnnBackendDescriptor_t executionPlan, cudnnBackendDescriptor_t variantPack, cudaGraph_t graph); #if defined(__cplusplus) } #endif #endif /* CUDNN_GRAPH_H_ */