Create rocSPARSE and corresponding spmv

.github/workflows/ci.yml

@@ -93,3 +93,21 @@ jobs:
     - name: Test
       run: |
         ./build/test/gtest/spblas-tests
+
+  rocsparse:
+    runs-on: 'gpu_amd'
+    steps:
+    - uses: actions/checkout@v4
+    - name: CMake
+      shell: bash -l {0}
+      run: |
+        module load cmake
+        cmake -B build -DENABLE_ROCSPARSE=ON -DCMAKE_PREFIX_PATH=/opt/rocm
+    - name: Build
+      shell: bash -l {0}
+      run: |
+        make -C build -j `nproc`
+    - name: Test
+      shell: bash -l {0}
+      run: |
+        ./build/test/gtest/spblas-tests

CMakeLists.txt

@@ -7,6 +7,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_FLAGS "-O3 -march=native")
 
 option(ENABLE_SANITIZERS "Enable Clang sanitizers" OFF)
+option(ENABLE_ROCSPARSE "Enable rocSPARSE" OFF)
 
 # Get includes, which declares the `spblas` library
 add_subdirectory(include)
@@ -33,6 +34,16 @@ if (ENABLE_ARMPL)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSPBLAS_ENABLE_ARMPL")
 endif()
 
+if (ENABLE_ROCSPARSE)
+  set(SPBLAS_GPU_BACKEND ON)
+  project(spblas LANGUAGES HIP)
+  find_package(hip REQUIRED)
+  find_package(rocsparse REQUIRED)
+  target_link_libraries(spblas INTERFACE roc::rocsparse hip::host)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSPBLAS_ENABLE_ROCSPARSE")
+  set(CMAKE_HIP_FLAGS "${CMAKE_CXX_FLAGS}")
+endif()
+
 # turn on/off debug logging
 if (LOG_LEVEL)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLOG_LEVEL=${LOG_LEVEL}") # SPBLAS_DEBUG | SPBLAS_WARNING | SPBLAS_TRACE | SPBLAS_INFO

README.md

@@ -104,7 +104,8 @@ brock@slothius:~/src/spblas-reference$ CXX=g++-13 cmake -B build
 
 ### Compiling with a Vendor Backend
 A vendor backend can be enabled by passing in an `-DENABLE_{BACKEND}=ON` switch
-to `cmake`.  Currently, oneMKL is the only supported backend.
+to `cmake`.  Currently, oneMKL, ArmPL, and rocSPARSE are the supported vendor
+backends.
 
 ### Compiling with oneMKL
 The most straightforward way to build with oneMKL is by installing the [oneAPI
@@ -118,6 +119,28 @@ brock@slothius:~/src/spblas-reference$ source /opt/intel/oneapi/setvars.sh
 brock@slothius:~/src/spblas-reference$ CXX=icpx cmake -B build -DENABLE_ONEMKL_SYCL=ON
 ```
 
+### Compiling with ArmPL
+In order to compile with ArmPL, the library must be installed and the
+environment variable `ARMPL_DIR` set.
+
+```bash
+# Set the environemnt variable `ARMPL_DIR` to the install location of ArmPL.
+brock@slothius:~/src/spblas-reference$ export ARMPL_DIR=/opt/arm/armpl_24.10_flang-new_clang_19
+# Compile with the ArmPL backend.
+brock@slothius:~/src/spblas-reference$ cmake -B build -DENABLE_ARMPL=ON
+```
+
+### Compiling with rocSPARSE
+In order to compile with rocSPARSE, ROCm must be installed and the install
+location of ROCm added to `CMAKE_PREFIX_PATH`.  Your package manager will likely
+take care of this for you, but you can also manually specify the location if you
+have ROCm installed in a non-standard location.
+
+```bash
+# Explicitly set the location of ROCm using `CMAKE_PREFIX_PATH`.
+brock@slothius:~/src/spblas-reference$ cmake -B build -DENABLE_ROCSPARSE=ON -DCMAKE_PREFIX_PATH=/opt/rocm-6.1.2
+```
+
 #### Compiling with GCC on Mac OS
 There is a known linking issue when compiling with GCC on recent versions of
 Mac OS.  This will cause a link error inside of `ld::AtomPlacement::findAtom()`.

examples/CMakeLists.txt

@@ -3,9 +3,18 @@ function(add_example example_name)
   target_link_libraries(${example_name} spblas fmt)
 endfunction()
 
-add_example(simple_spmv)
-add_example(simple_spmm)
-add_example(simple_spgemm)
-add_example(simple_sptrsv)
-add_example(matrix_opt_example)
-add_example(spmm_csc)
+if (NOT SPBLAS_GPU_BACKEND)
+  add_example(simple_spmv)
+  add_example(simple_spmm)
+  add_example(simple_spgemm)
+  add_example(simple_sptrsv)
+  add_example(matrix_opt_example)
+  add_example(spmm_csc)
+else()
+  find_package(rocthrust REQUIRED)
+  add_subdirectory(device)
+endif()
+
+if (ENABLE_ROCSPARSE)
+  add_subdirectory(rocsparse)
+endif()

examples/device/CMakeLists.txt

@@ -0,0 +1,13 @@
+function(add_device_example example_name)
+  if (ENABLE_ROCSPARSE)
+    set_source_files_properties(${example_name}.cpp PROPERTIES LANGUAGE HIP)
+  # elseif (ENABLE_CUSPARSE)
+    # cuSPARSE linking details will go here.
+  else()
+    message(FATAL_ERROR "Device backend not found.")
+  endif()
+  add_executable(${example_name} ${example_name}.cpp)
+  target_link_libraries(${example_name} spblas fmt)
+endfunction()
+
+add_device_example(simple_spmv)

examples/device/simple_spmv.cpp

@@ -0,0 +1,67 @@
+#include <iostream>
+#include <spblas/spblas.hpp>
+
+#include <thrust/device_vector.h>
+
+#include <fmt/core.h>
+#include <fmt/ranges.h>
+
+int main(int argc, char** argv) {
+  using value_t = float;
+  using index_t = spblas::index_t;
+  using offset_t = spblas::offset_t;
+
+  index_t m = 100;
+  index_t n = 100;
+  index_t nnz_in = 10;
+
+  fmt::print("\n\t###########################################################"
+             "######################");
+  fmt::print("\n\t### Running SpMV Example:");
+  fmt::print("\n\t###");
+  fmt::print("\n\t###   y = alpha * A * x");
+  fmt::print("\n\t###");
+  fmt::print("\n\t### with ");
+  fmt::print("\n\t### A, in CSR format, of size ({}, {}) with nnz = {}", m, n,
+             nnz_in);
+  fmt::print("\n\t### x, a dense vector, of size ({}, {})", n, 1);
+  fmt::print("\n\t### y, a dense vector, of size ({}, {})", m, 1);
+  fmt::print("\n\t### using float and spblas::index_t (size = {} bytes)",
+             sizeof(index_t));
+  fmt::print("\n\t###########################################################"
+             "######################");
+  fmt::print("\n");
+
+  auto&& [values, rowptr, colind, shape, nnz] =
+      spblas::generate_csr<value_t, index_t, offset_t>(m, n, nnz_in);
+
+  thrust::device_vector<value_t> d_values(values);
+  thrust::device_vector<offset_t> d_rowptr(rowptr);
+  thrust::device_vector<index_t> d_colind(colind);
+
+  spblas::csr_view<value_t, index_t, offset_t> a(
+      d_values.data().get(), d_rowptr.data().get(), d_colind.data().get(),
+      shape, nnz);
+
+  // Scale every value of `a` by 5 in place.
+  // scale(5.f, a);
+
+  std::vector<value_t> x(n, 1);
+  std::vector<value_t> y(m, 0);
+
+  thrust::device_vector<value_t> d_x(x);
+  thrust::device_vector<value_t> d_y(y);
+
+  std::span<value_t> x_span(d_x.data().get(), n);
+  std::span<value_t> y_span(d_y.data().get(), m);
+
+  // y = A * x
+  spblas::spmv_state_t state;
+  spblas::multiply(state, a, x_span, y_span);
+
+  thrust::copy(d_y.begin(), d_y.end(), y.begin());
+
+  fmt::print("\tExample is completed!\n");
+
+  return 0;
+}

examples/rocsparse/CMakeLists.txt

@@ -0,0 +1,6 @@
+function(add_rocm_example example_name)
+  add_executable(${example_name} ${example_name}.cpp)
+  target_link_libraries(${example_name} spblas fmt)
+endfunction()
+
+add_rocm_example(rocsparse_simple_spmv)

examples/rocsparse/rocsparse_simple_spmv.cpp

@@ -0,0 +1,88 @@
+#include <iostream>
+#include <spblas/spblas.hpp>
+
+#include <hip/hip_runtime.h>
+
+#include "util.hpp"
+
+#include <fmt/core.h>
+#include <fmt/ranges.h>
+
+int main(int argc, char** argv) {
+  using value_t = float;
+  using index_t = spblas::index_t;
+  using offset_t = spblas::offset_t;
+
+  index_t m = 100;
+  index_t n = 100;
+  index_t nnz_in = 10;
+
+  fmt::print("\n\t###########################################################"
+             "######################");
+  fmt::print("\n\t### Running SpMV Example:");
+  fmt::print("\n\t###");
+  fmt::print("\n\t###   y = alpha * A * x");
+  fmt::print("\n\t###");
+  fmt::print("\n\t### with ");
+  fmt::print("\n\t### A, in CSR format, of size ({}, {}) with nnz = {}", m, n,
+             nnz_in);
+  fmt::print("\n\t### x, a dense vector, of size ({}, {})", n, 1);
+  fmt::print("\n\t### y, a dense vector, of size ({}, {})", m, 1);
+  fmt::print("\n\t### using float and spblas::index_t (size = {} bytes)",
+             sizeof(spblas::index_t));
+  fmt::print("\n\t###########################################################"
+             "######################");
+  fmt::print("\n");
+
+  auto&& [values, rowptr, colind, shape, nnz] =
+      spblas::generate_csr<value_t, index_t, offset_t>(m, n, nnz_in);
+
+  value_t* d_values;
+  offset_t* d_rowptr;
+  index_t* d_colind;
+
+  HIP_CHECK(hipMalloc(&d_values, values.size() * sizeof(value_t)));
+  HIP_CHECK(hipMalloc(&d_rowptr, rowptr.size() * sizeof(offset_t)));
+  HIP_CHECK(hipMalloc(&d_colind, colind.size() * sizeof(index_t)));
+
+  HIP_CHECK(hipMemcpy(d_values, values.data(), values.size() * sizeof(value_t),
+                      hipMemcpyDefault));
+  HIP_CHECK(hipMemcpy(d_rowptr, rowptr.data(), rowptr.size() * sizeof(offset_t),
+                      hipMemcpyDefault));
+  HIP_CHECK(hipMemcpy(d_colind, colind.data(), colind.size() * sizeof(index_t),
+                      hipMemcpyDefault));
+
+  spblas::csr_view<value_t, index_t, offset_t> a(d_values, d_rowptr, d_colind,
+                                                 shape, nnz);
+
+  // Scale every value of `a` by 5 in place.
+  // scale(5.f, a);
+
+  std::vector<value_t> x(n, 1);
+  std::vector<value_t> y(m, 0);
+
+  value_t* d_x;
+  value_t* d_y;
+
+  HIP_CHECK(hipMalloc(&d_x, x.size() * sizeof(value_t)));
+  HIP_CHECK(hipMalloc(&d_y, y.size() * sizeof(value_t)));
+
+  HIP_CHECK(
+      hipMemcpy(d_x, x.data(), x.size() * sizeof(value_t), hipMemcpyDefault));
+  HIP_CHECK(
+      hipMemcpy(d_y, y.data(), y.size() * sizeof(value_t), hipMemcpyDefault));
+
+  std::span<value_t> x_span(d_x, n);
+  std::span<value_t> y_span(d_y, m);
+
+  // y = A * x
+  spblas::spmv_state_t state;
+  spblas::multiply(state, a, x_span, y_span);
+
+  HIP_CHECK(
+      hipMemcpy(y.data(), d_y, y.size() * sizeof(value_t), hipMemcpyDefault));
+
+  fmt::print("\tExample is completed!\n");
+
+  return 0;
+}

examples/rocsparse/util.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <hip/hip_runtime.h>
+
+#define HIP_CHECK(expression)                                                  \
+  do {                                                                         \
+    const hipError_t status = expression;                                      \
+    if (status != hipSuccess) {                                                \
+      std::cerr << "HIP error " << status << ": " << hipGetErrorString(status) \
+                << " at " << __FILE__ << ":" << __LINE__ << std::endl;         \
+    }                                                                          \
+  } while (false)

include/spblas/backend/backend.hpp

@@ -13,3 +13,7 @@
 #ifdef SPBLAS_ENABLE_ARMPL
 #include <spblas/vendor/armpl/armpl.hpp>
 #endif
+
+#ifdef SPBLAS_ENABLE_ROCSPARSE
+#include <spblas/vendor/rocsparse/rocsparse.hpp>
+#endif

include/spblas/detail/types.hpp

@@ -11,6 +11,10 @@
 #include <spblas/vendor/armpl/types.hpp>
 #endif
 
+#ifdef SPBLAS_ENABLE_ROCSPARSE
+#include <spblas/vendor/rocsparse/types.hpp>
+#endif
+
 namespace spblas {
 
 #ifndef SPBLAS_VENDOR_BACKEND

include/spblas/spblas.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
-#if defined(SPBLAS_ENABLE_ONEMKL_SYCL) || defined(SPBLAS_ENABLE_ARMPL)
+#if defined(SPBLAS_ENABLE_ONEMKL_SYCL) || defined(SPBLAS_ENABLE_ARMPL) ||      \
+    defined(SPBLAS_ENABLE_ROCSPARSE)
 #define SPBLAS_VENDOR_BACKEND true
 #endif