Changes to compile for single precision (#616)

* Use amrex max and min.

* Changes for single precision.

* Update PelePhysics.

* Actually test the single precision build.

* Use amrex clamp.

* Fix CI file.

* Formatting.

* Try to fix test_masscons.py.

* Fix clamp.

* Fix narrowing conversions.

Author: jrood-nrel

.github/workflows/ci.yml

@@ -83,10 +83,12 @@ jobs:
     needs: Formatting
     runs-on: ${{matrix.os}}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-24.04, macos-latest]
         build_type: [Release, Debug]
         enable_eb: [EB-OFF, EB-ON]
+        dim: [2, 3]
         include:
           - os: macos-latest
             install_deps: brew install open-mpi automake ccache
@@ -150,35 +152,28 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{matrix.ccache_cache}}
-          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}}-git-${{github.sha}}
+          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.dim}}d-${{matrix.build_type}}-${{matrix.enable_eb}}-git-${{github.sha}}
           restore-keys: |
-               ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}}-git-
+               ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.dim}}d-${{matrix.build_type}}-${{matrix.enable_eb}}-git-
       - name: Configure
         run: |
-          (for DIM in 2 3; do \
-            printf "\n-------- Configuring ${DIM}D --------\n"; \
-            cmake -B${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
-            -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
-            -DCMAKE_BUILD_TYPE:STRING=${{matrix.build_type}} \
-            -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-            -DPELE_DIM:STRING=${DIM} \
-            -DPELE_ENABLE_MPI:BOOL=ON \
-            -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
-            -DPELE_ENABLE_EB:BOOL=${{matrix.use_eb}} \
-            -DPELE_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
-            -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
-            ${{github.workspace}}; \
-            if [ $? -ne 0 ]; then exit 1; fi \
-          done)
+          cmake -B${{runner.workspace}}/build-${{matrix.dim}}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
+          -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install-${{matrix.dim}}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
+          -DCMAKE_BUILD_TYPE:STRING=${{matrix.build_type}} \
+          -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
+          -DPELE_DIM:STRING=${{matrix.dim}} \
+          -DPELE_ENABLE_MPI:BOOL=ON \
+          -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
+          -DPELE_ENABLE_EB:BOOL=${{matrix.use_eb}} \
+          -DPELE_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
+          -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
+          ${{github.workspace}}
       - name: Build
         run: |
           ccache -z
-          (for DIM in 2 3; do \
-            printf "\n-------- Building ${DIM}D --------\n"; \
-            cmake --build ${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
-              --parallel ${{env.NPROCS}} 2>&1 | tee -a ${{runner.workspace}}/build-output.txt; \
-            if [ ${PIPESTATUS[0]} -ne 0 ]; then exit 1; fi \
-          done)
+          cmake --build ${{runner.workspace}}/build-${{matrix.dim}}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
+            --parallel ${{env.NPROCS}} 2>&1 | tee -a ${{runner.workspace}}/build-output.txt; \
+          if [ ${PIPESTATUS[0]} -ne 0 ]; then exit 1; fi \
       - name: Ccache Report
         run: |
           ccache -s
@@ -195,12 +190,95 @@ jobs:
           exit ${return}
       - name: Test
         run: |
-          (for DIM in 2 3; do \
-            printf "\n-------- Testing ${DIM}D --------\n"; \
-            cd ${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}}; \
-            ctest ${{matrix.ctest_args}} -VV --output-on-failure; \
-            if [ $? -ne 0 ]; then exit 1; fi \
-          done)
+          cd ${{runner.workspace}}/build-${{matrix.dim}}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}}; \
+          ctest ${{matrix.ctest_args}} -VV --output-on-failure; \
+          if [ $? -ne 0 ]; then exit 1; fi \
+  CPU-CMake-SINGLE:
+    needs: Formatting
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        enable_eb: [EB-OFF, EB-ON]
+        dim: [2, 3]
+        include:
+          - enable_eb: EB-OFF
+            use_eb: "OFF"
+          - enable_eb: EB-ON
+            use_eb: "ON"
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Setup
+        run: |
+          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
+          echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
+          echo "CCACHE_COMPRESSLEVEL=10" >> $GITHUB_ENV
+          echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=50M" >> $GITHUB_ENV
+          cmake -E make_directory ${{runner.workspace}}/deps
+      - name: Dependencies
+        run: |
+          sudo apt-get install -y libopenmpi-dev openmpi-bin
+      - name: Install Ccache
+        run: |
+          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
+          tar xvf ccache-4.8-linux-x86_64.tar.xz
+          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
+      - name: Set Up Ccache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ccache
+          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.dim}}d-${{matrix.enable_eb}}-single-git-${{github.sha}}
+          restore-keys: |
+               ccache-${{github.workflow}}-${{github.job}}-${{matrix.dim}}d-${{matrix.enable_eb}}-single-git-
+      - name: Configure
+        run: |
+          cmake -B${{runner.workspace}}/build-${{matrix.dim}}d-${{matrix.enable_eb}}-single \
+          -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install-${{matrix.dim}}d-${{matrix.enable_eb}}-single \
+          -DCMAKE_BUILD_TYPE:STRING=Release \
+          -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
+          -DPELE_DIM:STRING=${{matrix.dim}} \
+          -DPELE_PRECISION:STRING=SINGLE \
+          -DCMAKE_CXX_FLAGS:STRING=-Wno-float-conversion \
+          -DPELE_ENABLE_MPI:BOOL=ON \
+          -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
+          -DPELE_ENABLE_EB:BOOL=${{matrix.use_eb}} \
+          -DPELE_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
+          -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
+          ${{github.workspace}}
+      - name: Build
+        run: |
+          ccache -z
+          cmake --build ${{runner.workspace}}/build-${{matrix.dim}}d-${{matrix.enable_eb}}-single \
+            --parallel ${{env.NPROCS}} 2>&1 | tee -a ${{runner.workspace}}/build-output.txt; \
+          if [ ${PIPESTATUS[0]} -ne 0 ]; then exit 1; fi
+      - name: Ccache Report
+        run: |
+          ccache -s
+          du -hs ~/.cache/ccache
+      - name: Report
+        run: |
+          echo "::add-matcher::.github/problem-matchers/gcc.json"
+          egrep "Warning:|Error:|warning:|error:" ${{runner.workspace}}/build-output.txt \
+            | egrep -v "Submodules/amrex|Submodules/sundials|Submodules/AMReX-Hydro" \
+            | egrep -v "ld: warning:" | egrep -v "lto-wrapper: warning:" | sort | uniq \
+            | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > ${{runner.workspace}}/build-output-warnings.txt
+          cat ${{runner.workspace}}/build-output-warnings.txt
+          export return=$(tail -n 1 ${{runner.workspace}}/build-output-warnings.txt | awk '{print $2}')
+          exit ${return}
+      #- name: Test
+      #  run: |
+      #    cd ${{runner.workspace}}/build-${{matrix.dim}}d-${{matrix.enable_eb}}-single; \
+      #    ctest -LE no-ci -VV --output-on-failure; \
+      #    if [ $? -ne 0 ]; then exit 1; fi
+
   GPU-Nvidia:
     name: GPU-CUDA
     needs: [Formatting, CPU-GNUmake]
@@ -247,9 +325,9 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cache/ccache
-          key: ccache-${{github.workflow}}-${{github.job}}-git-${{github.sha}}
+          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.enable_eb}}-git-${{github.sha}}
           restore-keys: |
-               ccache-${{github.workflow}}-${{github.job}}-git-
+               ccache-${{github.workflow}}-${{github.job}}-${{matrix.enable_eb}}-git-
       - name: Configure
         run: |
           export PATH=/usr/local/nvidia/bin:/usr/local/cuda-12.6/bin:${PATH}

CMake/SetPeleCompileFlags.cmake

@@ -25,6 +25,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
                               "-Wno-unused-variable")
 endif()
 
+list(APPEND PELE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 separate_arguments(PELE_CXX_FLAGS)
 target_compile_options(${pele_physics_lib_name} PUBLIC $<$<COMPILE_LANGUAGE:CXX>:${PELE_CXX_FLAGS}>)
 

Exec/Production/ChallengeProblem/pelelmex_prob.H

@@ -69,7 +69,7 @@ time_eta(
   eta_time = 0.5 * (std::tanh((time - start) / (duration * tau)) *
                       std::tanh((-time + start + duration) / (duration * tau)) +
                     1.0);
-  eta_time = std::min(std::max(eta_time, 0.0), 1.0);
+  eta_time = amrex::Clamp(eta_time, 0.0, 1.0);
   return eta_time;
 }
 

Exec/Production/NormalJet_OpenDomain/pelelmex_prob.H

@@ -20,7 +20,7 @@ time_func(
   eta_time = 0.5 * (std::tanh((time - start) / (duration * tau)) *
                       std::tanh((-time + start + duration) / (duration * tau)) +
                     1.0);
-  eta_time = std::min(std::max(eta_time, 0.0), 1.0);
+  eta_time = amrex::Clamp(eta_time, 0.0, 1.0);
   return eta_time;
 }
 

Exec/RegTests/HIT/pelelmex_prob.H

@@ -4,13 +4,16 @@
 #include <AMReX_Geometry.H>
 #include <AMReX_FArrayBox.H>
 #include <AMReX_SPACE.H>
+#include <AMReX_REAL.H>
 
 #include <PeleLMeX_ProblemSpecificFunctions.H>
 #include <PeleLMeX_Index.H>
 #include <PMFData.H>
 #include <PelePhysics.H>
 #include <Utilities.H>
 
+using namespace amrex::literals;
+
 namespace m2c = pele::physics::utilities::mks2cgs;
 namespace c2m = pele::physics::utilities::cgs2mks;
 
@@ -101,9 +104,9 @@ struct MyProblemSpecificFunctions : public DefaultProblemSpecificFunctions
     const amrex::Real* dx = geomdata.CellSize();
 
     amrex::Real x[3] = {
-      prob_lo[0] + static_cast<amrex::Real>(i + 0.5) * dx[0] * 100.0,
-      prob_lo[1] + static_cast<amrex::Real>(j + 0.5) * dx[1] * 100.0,
-      prob_lo[2] + static_cast<amrex::Real>(k + 0.5) * dx[2] * 100.0};
+      prob_lo[0] + static_cast<amrex::Real>(i + 0.5_rt) * dx[0] * 100.0_rt,
+      prob_lo[1] + static_cast<amrex::Real>(j + 0.5_rt) * dx[1] * 100.0_rt,
+      prob_lo[2] + static_cast<amrex::Real>(k + 0.5_rt) * dx[2] * 100.0_rt};
 
     if (prob_parm.doing_turbulent_forcing) {
       AMREX_ALWAYS_ASSERT(is_incompressible != 0);

Exec/RegTests/HIT/pelelmex_prob.cpp

@@ -54,7 +54,7 @@ read_csv(
   const size_t nx,
   const size_t ny,
   const size_t nz,
-  amrex::Vector<amrex::Real>& data)
+  amrex::Vector<double>& data) // needs to be double
 {
   std::ifstream infile(iname, std::ios::in);
   const std::string memfile = read_file(infile);
@@ -132,8 +132,7 @@ PeleLM::readProbParm() // NOLINT(readability-make-member-function-const)
     const size_t nx = PeleLM::prob_parm->input_resolution;
     const size_t ny = PeleLM::prob_parm->input_resolution;
     const size_t nz = PeleLM::prob_parm->input_resolution;
-    amrex::Vector<amrex::Real> data(
-      nx * ny * nz * 6); /* this needs to be double */
+    amrex::Vector<double> data(nx * ny * nz * 6); // needs to be double
     if (binfmt != 0) {
       read_binary(datafile, nx, ny, nz, 6, data);
     } else {

Exec/RegTests/PassiveScalar/pelelmex_prob.H

@@ -4,12 +4,15 @@
 #include <AMReX_Geometry.H>
 #include <AMReX_FArrayBox.H>
 #include <AMReX_SPACE.H>
+#include <AMReX_REAL.H>
 
 #include <PeleLMeX_ProblemSpecificFunctions.H>
 #include <PeleLMeX_Index.H>
 #include <PMFData.H>
 #include <PelePhysics.H>
 
+using namespace amrex::literals;
+
 struct MyProbParm : public ProbParmDefault
 {
   pele::physics::eos::EosParm<pele::physics::PhysicsType::eos_type> const*
@@ -69,10 +72,11 @@ struct MyProblemSpecificFunctions : public DefaultProblemSpecificFunctions
 #endif
     // find the centre of the domain
     const amrex::Real cen[AMREX_SPACEDIM] = {AMREX_D_DECL(
-      prob_lo[0] + 0.5 * Lx, prob_lo[1] + 0.5 * Ly, prob_lo[2] + 0.5 * Lz)};
+      prob_lo[0] + 0.5_rt * Lx, prob_lo[1] + 0.5_rt * Ly,
+      prob_lo[2] + 0.5_rt * Lz)};
     const amrex::Real x[AMREX_SPACEDIM] = {AMREX_D_DECL(
-      prob_lo[0] + (i + 0.5) * dx[0], prob_lo[1] + (j + 0.5) * dx[1],
-      prob_lo[2] + (k + 0.5) * dx[2])};
+      prob_lo[0] + (i + 0.5_rt) * dx[0], prob_lo[1] + (j + 0.5_rt) * dx[1],
+      prob_lo[2] + (k + 0.5_rt) * dx[2])};
     // radius from centre
     const amrex::Real r = std::sqrt(AMREX_D_TERM(
       (x[0] - cen[0]) * (x[0] - cen[0]), +(x[1] - cen[1]) * (x[1] - cen[1]),

Exec/RegTests/SootRadTest/pelelmex_prob.H

@@ -4,6 +4,7 @@
 #include <AMReX_Geometry.H>
 #include <AMReX_FArrayBox.H>
 #include <AMReX_ParmParse.H>
+#include <AMReX_REAL.H>
 
 #include <PeleLMeX_ProblemSpecificFunctions.H>
 #include <PMF.H>
@@ -13,6 +14,8 @@
 #include <PelePhysics.H>
 #include "SootModel.H"
 
+using namespace amrex::literals;
+
 struct MyProbParm : public ProbParmDefault
 {
   amrex::GpuArray<amrex::Real, NUM_SOOT_MOMENTS + 1> soot_vals;
@@ -47,7 +50,7 @@ struct MyProblemSpecificFunctions : public DefaultProblemSpecificFunctions
     state(i, j, k, TEMP) = pmf_vals[1];
     amrex::Real norm = 0.;
     for (int n = 0; n < NUM_SPECIES; n++) {
-      massfrac[n] = amrex::max(0., amrex::min(1., pmf_vals[3 + n]));
+      massfrac[n] = amrex::Clamp(pmf_vals[3 + n], 0.0_rt, 1.0_rt);
       norm += massfrac[n];
     }
     for (int n = 0; n < NUM_SPECIES; ++n) {

Exec/RegTests/TripleFlame/pelelmex_prob.H

@@ -52,7 +52,7 @@ void
 set_Y_from_Ksi(
   MyProbParm const& prob_parm, const amrex::Real& ksi_in, amrex::Real Y[])
 {
-  amrex::Real phi = ksi_in / (1.0 - amrex::min(ksi_in, 0.999999)) *
+  amrex::Real phi = ksi_in / (1.0 - amrex::min<amrex::Real>(ksi_in, 0.999999)) *
                     (1.0 - prob_parm.Zst) / prob_parm.Zst;
 
   set_Y_from_Phi(prob_parm, phi, Y);
@@ -100,7 +100,8 @@ struct MyProblemSpecificFunctions : public DefaultProblemSpecificFunctions
 
     amrex::Real rad = std::sqrt(
       (x - prob_parm.splitx) * (x - prob_parm.splitx) +
-      (amrex::min(y, y_lo) - y_lo) * (amrex::min(y, y_lo) - y_lo));
+      (amrex::min<amrex::Real>(y, y_lo) - y_lo) *
+        (amrex::min<amrex::Real>(y, y_lo) - y_lo));
 
     amrex::Real Gaussian_T = std::exp(
       -(rad * rad) / (2.0 * (Gauss_T_width * (1.0 + (y - y_lo) / 0.03)) *

Source/PeleLMeX_BC.cpp

@@ -124,7 +124,7 @@ PeleLM::setBoundaryConditions()
 
   // Initialize the BCRecs
   m_bcrec_state.resize(NVAR);
-  constexpr int sizeForceBC = amrex::max(AMREX_SPACEDIM, NUM_SPECIES + 2);
+  constexpr int sizeForceBC = amrex::max<int>(AMREX_SPACEDIM, NUM_SPECIES + 2);
   m_bcrec_force.resize(sizeForceBC);
   m_bcrec_aux.resize(m_nAux);