Understanding and reorganization of NEURON libraries

[iomaganaris]

This discussion is instantiated to place my findings regarding what libraries are built for NEURON in different building scenarios.
This way I hope that we can get a better overview of all the different ways NEURON is built and how we can reorganize the libraries in such way that we make NEURON building easier to understand, more streamlined and less error prone.

The main CMake options that influence what and how libraries and executables of NEURON are built are the following:

• NRN_ENABLE_SHARED
• NRN_ENABLE_PYTHON
• NRN_ENABLE_MPI
• NRN_ENABLE_CORENEURON
• NRN_ENABLE_PYTHON_DYNAMIC
• NRN_LINK_AGAINST_PYTHON
• NRN_PYTHON_DYNAMIC
• NRN_ENABLE_MPI_DYNAMIC
• NRN_MPI_DYNAMIC
• NRN_ENABLE_MUSIC

For different combinations of these options I will try to list the resulted libraries, executables and what purpose those options serve.

Some ideas for improvement:

• For the direct mode between NEURON and CoreNEURON we load in NEURON in nrncore_nrn the CoreNEURON library either libcorenrnmech.so which is build using nrnivmodl-core with external mod files (from x86_64) or libcorenrnmech_internal.so. Instead of doing this dynamically during program execution we could manually link together at build time nrniv with libcorenrnmech_internal.so and special with libcorenrnmech.so. This would simplify both the recipes for building special/special-core and the running code
• Consider building NEURON and CoreNEURON source files together in a single library. The MPI libraries of each can also be built together

[iomaganaris]

Some important information regarding the CMake targets/libraries build for NEURON and CoreNEURON.
In NEURON there is a single shared/static library built in all cases called libnrniv. This library includes all the NEURON source files plus some extra source files depending on the build options. Here is the related part from CMake:

set(NRN_NRNIV_LIB_SRC_FILES
    ${NRN_IVOC_SRC_FILES}
    ${NRN_NRNCVODE_SRC_FILES}
    ${NRN_NRNIV_SRC_FILES}
    ${NRN_NRNOC_SRC_FILES}
    ${NRN_OC_SRC_FILES}
    ${NRN_MESCH_SRC_FILES}
    ${NRN_MODFILE_CPP}
    ${NRN_SCOPMATH_SRC_FILES}
    ${NRN_SPARSE_SRC_FILES}
    ${NRN_SPARSE13_SRC_FILES}
    ${NRN_SUNDIALS_SRC_FILES})

if(NRN_ENABLE_MPI)
  list(APPEND NRN_NRNIV_LIB_SRC_FILES ${NRN_PARALLEL_SRC_FILES})
endif()

if(NRN_ENABLE_MPI_DYNAMIC)
  list(APPEND NRN_NRNIV_LIB_SRC_FILES ${PROJECT_SOURCE_DIR}/src/nrnmpi/nrnmpi_dynam.cpp)
else()
  list(APPEND NRN_NRNIV_LIB_SRC_FILES ${NRN_NRNMPI_SRC_FILES})
endif()

if(MINGW)
  list(APPEND NRN_NRNIV_LIB_SRC_FILES ${PROJECT_SOURCE_DIR}/src/mswin/dlfcn.c)
endif()

# NRN_ENABLE_MODULE_INSTALL will create a separate nrnpython lib
if(NRN_ENABLE_PYTHON AND NOT NRN_ENABLE_PYTHON_DYNAMIC)
  # Include nrnpython in nrniv - useful for single lib neuron and wheels
  list(APPEND NRN_NRNIV_LIB_SRC_FILES ${NRN_NRNPYTHON_SRC_FILES})
endif()

if(NOT NRN_ENABLE_INTERVIEWS)
  list(APPEND NRN_NRNIV_LIB_SRC_FILES ${NRN_IVOS_SRC_FILES})
endif()

if(NRN_ENABLE_MUSIC)
  if(NRN_ENABLE_MPI_DYNAMIC)
    list(APPEND NRN_NRNIV_LIB_SRC_FILES ${PROJECT_SOURCE_DIR}/src/neuronmusic/nrnmusic_dynam.cpp)
    set_property(
      SOURCE ${NRN_NRNIV_LIB_SRC_FILES} ${NRN_NRNMPI_SRC_FILES}
      APPEND
      PROPERTY COMPILE_DEFINITIONS NRNMUSIC_DYNAMIC=1)
  else()
    list(APPEND NRN_NRNIV_LIB_SRC_FILES ${NRN_MUSIC_SRC_FILES})
  endif()
endif()

Link

add_library(nrniv_lib ${NRN_LIBRARY_TYPE} ${NRN_NRNIV_LIB_SRC_FILES})
target_link_libraries(nrniv_lib nrngnu)
cpp_cc_configure_sanitizers(TARGET nrniv_lib)
# Source-directory .cpp needs to find generated .hpp.
target_include_directories(nrniv_lib PUBLIC "${NRN_OC_GEN}")
if(NRN_ENABLE_PYTHON AND NOT NRN_ENABLE_PYTHON_DYNAMIC)
  target_include_directories(nrniv_lib PUBLIC "${NRN_DEFAULT_PYTHON_INCLUDES}")
endif()
if(NRN_ENABLE_THREADS)
  target_link_libraries(nrniv_lib Threads::Threads)
endif()
if(NRN_WINDOWS_BUILD)
  target_link_libraries(nrniv_lib ${TERMCAP_LIBRARIES} ${Readline_LIBRARY})
else()
  if(READLINE_FOUND)
    target_link_libraries(nrniv_lib ${Readline_LIBRARY})
  else()
    target_link_libraries(nrniv_lib readline)
  endif()

  if(CURSES_FOUND)
    target_link_libraries(nrniv_lib ${CURSES_LIBRARIES})
  elseif(TERMCAP_FOUND)
    target_link_libraries(nrniv_lib ${TERMCAP_LIBRARIES})
  endif()
endif()

if(NRN_ENABLE_MUSIC AND NOT NRN_ENABLE_MPI_DYNAMIC)
  target_link_libraries(nrniv_lib ${MUSIC_LIBRARY})
endif()

if(NRN_ENABLE_PROFILING)
  target_link_libraries(nrniv_lib ${likwid_LIBRARIES} ${CALIPER_LIB})
endif()

set_property(TARGET nrniv_lib PROPERTY OUTPUT_NAME nrniv)

Link

In CoreNEURON we have a static libcoreneuron-core library that includes the main/core files of CoreNEURON. Since CoreNEURON comes with some extra mechanisms we also build a libcorenrnmech_internal.so library which includes the code for these mechanisms.

set(nrniv_core_prefix "${CMAKE_BINARY_DIR}/bin/${CMAKE_SYSTEM_PROCESSOR}")
set(corenrn_mech_library
    "${nrniv_core_prefix}/${CMAKE_${COMPILE_LIBRARY_TYPE}_LIBRARY_PREFIX}corenrnmech_internal${CMAKE_${COMPILE_LIBRARY_TYPE}_LIBRARY_SUFFIX}"
)
set(output_binaries "${nrniv_core_prefix}/special-core" "${corenrn_mech_library}")

add_custom_command(
  OUTPUT ${output_binaries}
  DEPENDS coreneuron-core ${NMODL_TARGET_TO_DEPEND} ${CORENEURON_BUILTIN_MODFILES}
  COMMAND
    ${CMAKE_COMMAND} -E env NMODLHOME=${NMODL_PROJECT_BINARY_DIR}
    ${CMAKE_BINARY_DIR}/bin/nrnivmodl-core -b ${COMPILE_LIBRARY_TYPE} -m ${CORENRN_NMODL_BINARY} -n
    "internal" -p 4 "."
  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bin
  COMMENT "Running nrnivmodl-core with internal mod files")
add_custom_target(nrniv-core ALL DEPENDS ${output_binaries})

# Build a target representing the libcorenrnmech_internal.so that is produced under bin/x86_64,
# which executables such as the unit tests must link against
add_library(builtin-libcorenrnmech SHARED IMPORTED)
add_dependencies(builtin-libcorenrnmech nrniv-core)
set_target_properties(builtin-libcorenrnmech PROPERTIES IMPORTED_LOCATION "${corenrn_mech_library}")

if(CORENRN_ENABLE_GPU)
  separate_arguments(CORENRN_ACC_FLAGS UNIX_COMMAND "${NVHPC_ACC_COMP_FLAGS}")
  target_compile_options(coreneuron-core PRIVATE ${CORENRN_ACC_FLAGS})
endif()

Link

file(
  GLOB
  CORENEURON_CODE_FILES
  "apps/main1.cpp"
  "apps/corenrn_parameters.cpp"
  "gpu/nrn_acc_manager.cpp"
  "io/*.cpp"
  "io/reports/*.cpp"
  "mechanism/*.cpp"
  "mpi/core/nrnmpi_def_cinc.cpp"
  "network/*.cpp"
  "permute/*.cpp"
  "sim/*.cpp"
  "sim/scopmath/abort.cpp"
  "sim/scopmath/newton_thread.cpp"
  "utils/*.cpp"
  "utils/*/*.c"
  "utils/*/*.cpp")
  set(MPI_LIB_FILES "mpi/lib/mpispike.cpp" "mpi/lib/nrnmpi.cpp")
if(NRN_ENABLE_MPI)
  # Building these requires -ldl, which is only added if MPI is enabled.
  list(APPEND CORENEURON_CODE_FILES "mpi/core/resolve.cpp")
endif()

list(APPEND CORENEURON_CODE_FILES ${PROJECT_BINARY_DIR}/coreneuron/config/config.cpp)
  if(EXISTS ${CORENRN_NMODL_INCLUDE}/partial_piv_lu/partial_piv_lu.cpp)
    list(APPEND CORENEURON_CODE_FILES ${CORENRN_NMODL_INCLUDE}/partial_piv_lu/partial_piv_lu.cpp)
endif()

# name of coreneuron mpi objects or dynamic library
set(CORENRN_MPI_LIB_NAME
    "corenrn_mpi"
    CACHE INTERNAL "")

# for non-dynamic mpi mode just build object files
if(NRN_ENABLE_MPI AND NOT NRN_ENABLE_MPI_DYNAMIC)
  add_library(${CORENRN_MPI_LIB_NAME} OBJECT ${MPI_LIB_FILES})
  target_include_directories(
    ${CORENRN_MPI_LIB_NAME} PRIVATE ${MPI_C_INCLUDE_DIRS} ${PROJECT_SOURCE_DIR}/src
                                    ${CMAKE_BINARY_DIR}/generated)
  target_link_libraries(${CORENRN_MPI_LIB_NAME} ${CORENRN_CALIPER_LIB})
  set_property(TARGET ${CORENRN_MPI_LIB_NAME} PROPERTY POSITION_INDEPENDENT_CODE ON)
  set(CORENRN_MPI_OBJ $<TARGET_OBJECTS:${CORENRN_MPI_LIB_NAME}>)
endif()

# Library containing the bulk of the non-mechanism CoreNEURON code. This is always created and
# installed as a static library, and then the nrnivmodl-core workflow extracts the object files from
# it and does one of the following:
#
# * shared build: creates libcorenrnmech_internal.so from these objects plus those from the
#   translated MOD files
# * static build: creates a (temporary, does not get installed) libcorenrnmech_internal.a from these
#   objects plus those from the translated MOD files, then statically links that into special-core
#   (nrniv-core)
#
# This scheme means that both core and mechanism .o files are linked in a single step, which is
# important for GPU linking. It does, however, mean that the core code is installed twice, once in
# libcoreneuron-core.a and once in libcorenrnmech_internal.so (shared) or nrniv-core (static). In a
# GPU build, libcoreneuron-cuda.{a,so} is also linked to provide the CUDA implementation of the
# Hines solver. This cannot be included in coreneuron-core because of this issue:
# https://forums.developer.nvidia.com/t/cannot-dynamically-load-a-shared-library-containing-both-openacc-and-cuda-code/210972
add_library(coreneuron-core STATIC ${CORENEURON_CODE_FILES} ${CORENRN_MPI_OBJ})
if(CORENRN_ENABLE_GPU)
  set(coreneuron_cuda_target coreneuron-cuda)
  add_library(coreneuron-cuda ${COMPILE_LIBRARY_TYPE} ${CORENEURON_CUDA_FILES})
  target_link_libraries(coreneuron-core PUBLIC coreneuron-cuda)
endif()

foreach(target coreneuron-core ${coreneuron_cuda_target})
  target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/src
                                               ${CMAKE_BINARY_DIR}/generated)
endforeach()

# we can link to MPI libraries in non-dynamic-mpi build
if(NRN_ENABLE_MPI AND NOT NRN_ENABLE_MPI_DYNAMIC)
  target_link_libraries(coreneuron-core PUBLIC ${MPI_CXX_LIBRARIES})
endif()

Link

All below tests were run on an Ubuntu 22.04 machine with Python 3.10.2.

• NRN_ENABLE_MPI_DYNAMIC=OFF, NRN_ENABLE_SHARED=ON, CORENRN_ENABLE_SHARED=ON

cmake .. -DCMAKE_INSTALL_PREFIX=./install -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_TESTS=ON -DNRN_ENABLE_MPI_DYNAMIC=OFF -DNRN_ENABLE_SHARED=ON -DCORENRN_ENABLE_SHARED=ON

├── libcodegen.a
├── libcoreneuron-core.a
├── libcorenrnmech_internal.so
├── libfmt.a
├── liblexer.a
├── libnrngnu.a
├── libnrniv.so
├── libpyembed.a
├── libpywrapper.a
├── librxdmath.so
├── libspdlog.a
├── libsymtab.a
├── libutil.a
├── libvisitor.a
├── nmodl
│   ├── _nmodl.cpython-310-x86_64-linux-gnu.so
├── nrnunits.lib.in
└── python
    └── neuron
        ├── hoc.cpython-310-x86_64-linux-gnu.so

• ⚠️ NRN_ENABLE_MPI_DYNAMIC=OFF, NRN_ENABLE_SHARED=OFF, CORENRN_ENABLE_SHARED=OFF

cmake .. -DCMAKE_INSTALL_PREFIX=./install -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_TESTS=ON -DNRN_ENABLE_MPI_DYNAMIC=OFF -DNRN_ENABLE_SHARED=OFF -DCORENRN_ENABLE_SHARED=OFF

[ 63%] Generating inithoc.cpp
Building python module with: /usr/bin/python3 setup.py build
      --cmake-build-dir;/home/magkanar/bbp_repos/nrn/build_non_dynamic_mpi_static;--disable-rx3d;--without-nrnpython;--build-lib=/home/magkanar/bbp_repos/nrn/build_non_dynamic_mpi_static/lib/python build_ext;--define=USE_PYTHON,NRN_ENABLE_THREADS
INFO:root:setup.py called with:setup.py build --cmake-build-dir /home/magkanar/bbp_repos/nrn/build_non_dynamic_mpi_static --disable-rx3d --without-nrnpython --build-lib=/home/magkanar/bbp_repos/nrn/build_non_dynamic_mpi_static/lib/python build_ext --define=USE_PYTHON,NRN_ENABLE_THREADS
Traceback (most recent call last):
  File "/home/magkanar/bbp_repos/nrn/setup.py", line 56, in <module>
    raise RuntimeError(
RuntimeError: CMake build directory does not contain libnrniv: /home/magkanar/bbp_repos/nrn/build_non_dynamic_mpi_static
gmake[2]: *** [src/nrnpython/CMakeFiles/hoc_module.dir/build.make:83: hoc_module] Error 1
gmake[1]: *** [CMakeFiles/Makefile2:3324: src/nrnpython/CMakeFiles/hoc_module.dir/all] Error 2
gmake[1]: *** Waiting for unfinished jobs....

• NRN_ENABLE_MPI_DYNAMIC=ON, NRN_ENABLE_SHARED=ON, CORENRN_ENABLE_SHARED=ON, `NRN_MPI_DYNAMIC="/usr/include/x86_64-linux-gnu/mpich;/usr/include/x86_64-linux-gnu/openmpi"

cmake .. -DCMAKE_INSTALL_PREFIX=./install -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_TESTS=ON -DNRN_ENABLE_MPI_DYNAMIC=ON -DNRN_MPI_DYNAMIC="/usr/include/x86_64-linux-gnu/mpich;/usr/include/x86_64-linux-gnu/openmpi"

install/lib
├── libcodegen.a
├── libcoreneuron-core.a
├── libcorenrnmech_internal.so
├── libcorenrnmpi_mpich.so
├── libcorenrnmpi_ompi.so
├── libfmt.a
├── liblexer.a
├── libnrngnu.a
├── libnrniv.so
├── libnrnmpi_mpich.so
├── libnrnmpi_ompi.so
├── libpyembed.a
├── libpywrapper.a
├── librxdmath.so
├── libspdlog.a
├── libsymtab.a
├── libutil.a
├── libvisitor.a
├── nmodl
│   ├── _nmodl.cpython-310-x86_64-linux-gnu.so
├── nrnunits.lib.in
└── python
    └── neuron
        ├── hoc.cpython-310-x86_64-linux-gnu.so

• ⚠️ NRN_ENABLE_MPI_DYNAMIC=ON, NRN_ENABLE_SHARED=OFF, CORENRN_ENABLE_SHARED=OFF, NRN_MPI_DYNAMIC="/usr/include/x86_64-linux-gnu/mpich;/usr/include/x86_64-linux-gnu/openmpi"

cmake .. -DCMAKE_INSTALL_PREFIX=./install -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_TESTS=ON -DNRN_ENABLE_MPI_DYNAMIC=ON -DNRN_MPI_DYNAMIC="/usr/include/x86_64-linux-gnu/mpich;/usr/include/x86_64-linux-gnu/openmpi" -DNRN_ENABLE_SHARED=OFF -DCORENRN_ENABLE_SHARED=OFF

[ 64%] Generating inithoc.cpp
Building python module with: /usr/bin/python3 setup.py build
      --cmake-build-dir;/home/magkanar/bbp_repos/nrn/build_dynamic_mpi_static;--disable-rx3d;--without-nrnpython;--build-lib=/home/magkanar/bbp_repos/nrn/build_dynamic_mpi_static/lib/python build_ext;--define=USE_PYTHON,NRN_ENABLE_THREADS
INFO:root:setup.py called with:setup.py build --cmake-build-dir /home/magkanar/bbp_repos/nrn/build_dynamic_mpi_static --disable-rx3d --without-nrnpython --build-lib=/home/magkanar/bbp_repos/nrn/build_dynamic_mpi_static/lib/python build_ext --define=USE_PYTHON,NRN_ENABLE_THREADS
Traceback (most recent call last):
  File "/home/magkanar/bbp_repos/nrn/setup.py", line 56, in <module>
    raise RuntimeError(
RuntimeError: CMake build directory does not contain libnrniv: /home/magkanar/bbp_repos/nrn/build_dynamic_mpi_static
gmake[2]: *** [src/nrnpython/CMakeFiles/hoc_module.dir/build.make:83: hoc_module] Error 1
gmake[1]: *** [CMakeFiles/Makefile2:3340: src/nrnpython/CMakeFiles/hoc_module.dir/all] Error 2
gmake[1]: *** Waiting for unfinished jobs....