backup work on new ariel fork impl

Author: plavin

config/sst_check_ariel_mpi.m4

@@ -48,7 +48,7 @@ LTDL_INIT([recursive])
 
 AC_CACHE_SAVE
 
-SST_CHECK_ARIEL_MPI([], [AC_MSG_ERROR([Can't enable Ariel MPI])])
+#SST_CHECK_ARIEL_MPI([], [AC_MSG_ERROR([Can't enable Ariel MPI])])
 
 SST_CORE_CHECK_INSTALL()
 

configure.ac

@@ -1,7 +1,4 @@
-SUBDIRS = api
-if SST_USE_ARIEL_MPI
-SUBDIRS += mpi
-endif
+SUBDIRS = api mpi
 
 AM_CPPFLAGS += \
 	-I$(top_srcdir)/src

src/sst/elements/ariel/Makefile.am

@@ -1,11 +1,3 @@
-if SST_USE_ARIEL_MPI
-CC_LOCAL = $(ARIEL_MPICC)
-CFLAGS_LOCAL = -fopenmp $(ARIEL_MPI_CFLAGS) -DENABLE_ARIEL_MPI=1
-else
-CC_LOCAL = $(CC)
-CFLAGS_LOCAL =
-endif
-
 AM_CPPFLAGS += \
 	$(MPI_CPPFLAGS) \
 	-I$(top_srcdir)/src

src/sst/elements/ariel/api/Makefile.am

@@ -99,7 +99,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required, int *provided) {
     omp_parallel_region();
     return PMPI_Init_thread(argc, argv, required, provided);
 #else
-    printf("Error: arielapi.c: MPI_Init_thread called in arielapi.c but this file was compiled without MPI. Please recompile the API with `CC=mpicc make`123123.\n");
+    printf("Error: arielapi.c: MPI_Init_thread called in arielapi.c but this file was compiled without MPI. Please recompile the API with `CC=mpicc make`.\n");
     exit(1);
 #endif
 }

src/sst/elements/ariel/api/arielapi.c

@@ -67,6 +67,7 @@ class ArielCPU : public SST::Component {
         {"mpilauncher", "Specify a launcher to be used for MPI executables in conjuction with <launcher>", STRINGIZE(MPILAUNCHER_EXECUTABLE)},
         {"mpiranks", "Number of ranks to be launched by <mpilauncher>. Only <mpitracerank> will be traced by <launcher>.", "1" },
         {"mpitracerank", "Rank to be traced by <launcher>.", "0" },
+        {"mpitraceall", "Rank to be traced by <launcher>.", "false" },
         {"envparamcount", "Number of environment parameters to supply to the Ariel executable, default=-1 (use SST environment)", "-1"},
         {"envparamname%(envparamcount)d", "Sets the environment parameter name", ""},
         {"envparamval%(envparamcount)d", "Sets the environment parameter value", ""},

src/sst/elements/ariel/arielcpu.h

@@ -13,11 +13,13 @@
 // information, see the LICENSE file in the top level directory of the
 // distribution.
 
-
 #include <sst_config.h>
-
 #include "pin3frontend.h"
 
+#ifdef SST_CONFIG_HAVE_MPI
+//TODO?
+#endif
+
 #include <signal.h>
 #if !defined(SST_COMPILE_MACOSX)
 #include <sys/prctl.h>
@@ -30,9 +32,7 @@
 #include <fcntl.h>
 #include <stdio.h>
 #include <fcntl.h>
-
 #include <time.h>
-
 #include <string.h>
 
 #define ARIEL_INNER_STRINGIZE(input) #input
@@ -111,7 +111,13 @@ Pin3Frontend::Pin3Frontend(ComponentId_t id, Params& params, uint32_t cores, uin
     output->verbose(CALL_INFO, 1, 0, "Base pipe name: %s\n", shmem_region_name.c_str());
 
     // MPI Launcher options
-    mpimode = params.find<int>("mpimode", 0);
+    //mpimode = params.find<int>("mpimode", 0);
+#ifdef USE_MPI
+    mpimode = 1;
+#else
+    mpimode = 0;
+#endif
+
     if (mpimode) {
         mpilauncher = params.find<std::string>("mpilauncher",  ARIEL_STRINGIZE(MPILAUNCHER_EXECUTABLE));
         mpiranks = params.find<int>("mpiranks", 1);
@@ -322,10 +328,16 @@ void Pin3Frontend::init(unsigned int phase)
         // Init the child_pid = 0, this prevents problems in emergencyShutdown()
         // if forkPINChild() calls fatal (i.e. the child_pid would not be set)
         child_pid = 0;
-        if (mpimode == 1) {
+        if (mpimode == 0) { // TODO CHANGE BACK
             // Ariel will fork the MPI launcher which will itself fork pin
-            child_pid = forkPINChild(mpilauncher.c_str(), execute_args, execute_env, redirect_info);
+            printf("--------------------\n");
+            printf(" --- MPI     PIN ---\n");
+            printf("--------------------\n");
+            child_pid = forkPINChildMPI(mpilauncher.c_str(), execute_args, execute_env, redirect_info);
         } else {
+            printf("--------------------\n");
+            printf(" --- Regular PIN ---\n");
+            printf("--------------------\n");
             child_pid = forkPINChild(appLauncher.c_str(), execute_args, execute_env, redirect_info);
         }
         output->verbose(CALL_INFO, 1, 0, "Returned from launching PIN.  Waiting for child to attach.\n");
@@ -348,6 +360,59 @@ ArielTunnel* Pin3Frontend::getTunnel() {
     return tunnel;
 }
 
+int Pin3Frontend::forkPINChildMPI(const char* app, char** args, std::map<std::string, std::string>& app_env, redirect_info_t redirect_info) {
+#ifdef USE_MPI
+    if(isSimulationRunModeInit())
+        return 0;
+#endif
+
+    // Convert app to non-const
+
+    char *app2 = (char*)malloc(std::strlen(app) + 1);
+    if (!app2) return 0; // TODO proper error condition
+    std::strcpy(app2, app);
+
+    // Find where the PIN arguments end
+    int app_idx = -1;
+    for (int i = 0; args[i] != NULL; i++) {
+        if (strcmp(args[i], "--") == 0) {
+            app_idx = i+1;
+            break;
+        }
+    }
+    if (app_idx == -1) {
+        // TODO: Error not found
+    }
+
+    for (int i = app_idx; args[i] != NULL; i++) {
+        printf(" app %d - %s\n", i, args[1]);
+    }
+
+    int count = 1;
+    char *array_of_commands[] = {args[0]};
+    char *argv[] = {"20", NULL};
+    char **array_of_argv[] = {args+1};
+    int array_of_maxprocs[] = {1};
+
+    /* Working for 1 rank
+    int count = 1;
+    char *array_of_commands[] = {args[0]};
+    char *argv[] = {"20", NULL};
+    char **array_of_argv[] = {args+1};
+    int array_of_maxprocs[] = {1};
+    */
+
+    int ret = SST::Core::Interprocess::SST_MPI_Comm_spawn_multiple(count,
+            array_of_commands,
+            array_of_argv,
+            array_of_maxprocs);
+
+    //TODO: Check ret and exit is non-zero
+    free(app2);
+
+    return 1;
+}
+
 int Pin3Frontend::forkPINChild(const char* app, char** args, std::map<std::string, std::string>& app_env, redirect_info_t redirect_info) {
     // If user only wants to init the simulation then we do NOT fork the binary
     if(isSimulationRunModeInit())

src/sst/elements/ariel/frontend/pin3/pin3frontend.cc

@@ -27,6 +27,7 @@
 #include <string>
 #include <map>
 
+
 #include "arielfrontend.h"
 #include "ariel_shmem.h"
 
@@ -93,6 +94,7 @@ class Pin3Frontend : public ArielFrontend {
     private:
 
         int forkPINChild(const char* app, char** args, std::map<std::string, std::string>& app_env, redirect_info_t redirect_info);
+        int forkPINChildMPI(const char* app, char** args, std::map<std::string, std::string>& app_env, redirect_info_t redirect_info);
 
         SST::Output* output;
 
@@ -112,6 +114,10 @@ class Pin3Frontend : public ArielFrontend {
         int mpitracerank;
         bool use_mpilauncher;
 
+#ifdef USE_MPI
+        MPI_Comm intercomm;
+#endif
+
 
         char **execute_args;
         std::map<std::string, std::string> execute_env;

src/sst/elements/ariel/frontend/pin3/pin3frontend.h

@@ -9,3 +9,7 @@ hello:
 	     mpicxx hello.cc -o hello -I$(API_DIR) -L$(API_DIR) -larielapi -fopenmp
 reduce:
 	     mpicxx reduce.cc -o reduce -I$(API_DIR) -L$(API_DIR) -larielapi -fopenmp
+
+reduce-noapi:
+	     mpicxx reduce-noapi.cc -o reduce-noapi -fopenmp
+

src/sst/elements/ariel/tests/testMPI/Makefile

@@ -1,6 +1,7 @@
 import sst
 import sys
 import os
+from pathlib import Path
 
 # Detect if we will use MPI mode or not
 mpi_mode = True
@@ -39,11 +40,13 @@
 ## Set component parameters and fill subcomponent slots
 #########################################################################
 # Core: 2.4GHz, 2 accesses/cycle, STREAM (triad) pattern generator with 1000 elements per array
+exe = "./reduce"
+full_exe = str(Path(exe).resolve())
 core.addParams({
     "clock" : "2.4GHz",
     "verbose" : 1,
     #"executable" : "./hello-nompi"
-    "executable" : "./reduce",
+    "executable" : full_exe,
     #"executable" : "/home/prlavin/projects/reference-paper-2024/apps/install/bin/amg",
     "arielmode" : 0,
     "corecount" : ncores,