[projmgr] Add RPC method DiscoverLayers (#1330)

* [projmgr] Add RPC method `DiscoverLayers`

* Update tools/projmgr/CMakeLists.txt

* Address review comments

Author: brondani

tools/projmgr/include/ProjMgr.h

@@ -70,6 +70,14 @@ class ProjMgr {
   */
   bool LoadSolution(const std::string& csolution, const std::string& activeTargetSet);
 
+  /**
+   * @brief setup contexts
+   * @param path to <solution>.csolution.yml file
+   * @param active target set in the format <target-type>[@<set>]
+   * @return processing status
+  */
+  bool SetupContexts(const std::string& csolution, const std::string& activeTargetSet);  
+
   /**
    * @brief convert solution and generate yml files
    * @param path to <solution>.csolution.yml file

tools/projmgr/include/ProjMgrWorker.h

@@ -367,6 +367,42 @@ struct DebuggerType {
   CustomItem custom;
 };
 
+/**
+ * @brief settings type containing
+ *        connection set
+*/
+struct SettingsType {
+  std::string set;
+};
+
+/**
+ * @brief layer variable type containing
+ *        variable name
+ *        clayer path
+ *        brief description
+ *        list of connection sets
+ *        directory path that contains the layer (from PDSC)
+ *        clayer path relative to directory path (from PDSC)
+ *        proposed destination directory for the layer (from PDSC)
+*/
+struct LayerVariable {
+  std::string name;
+  std::string clayer;
+  std::string description;
+  std::vector<SettingsType> settings;
+  std::string path;
+  std::string file;
+  std::string copyTo;
+};
+
+/**
+ * @brief variables configuration type containing
+ *        list of layer variables
+*/
+struct VariablesConfiguration {
+  std::vector<LayerVariable> variables;
+};
+
 /**
  * @brief project context item containing
  *        pointer to csolution,
@@ -431,6 +467,7 @@ struct DebuggerType {
  *        image only flag
  *        west options
  *        west on flag
+ *        layer variables configurations
 */
 struct ContextItem {
   CdefaultItem* cdefault = nullptr;
@@ -502,6 +539,7 @@ struct ContextItem {
   bool imageOnly = false;
   WestDesc west;
   bool westOn = false;
+  std::vector<VariablesConfiguration> variablesConfigurations;
 };
 
 /**
@@ -1051,6 +1089,18 @@ class ProjMgrWorker {
   */
   bool IsLibOnly(const std::vector<ContextItem*>& contexts);
 
+  /**
+   * @brief get processed contexts
+   * @return reference to vector with context items
+  */
+  const std::vector<ContextItem*>& GetProcessedContexts(void);
+
+  /**
+   * @brief elaborate possible variables configurations according to compatible layers
+   * @return true if there are configurations available
+  */
+  bool ElaborateVariablesConfigurations();
+
   /**
    * @brief clear worker members for reloading a solution
    * @return true if there is no error
@@ -1094,6 +1144,7 @@ class ProjMgrWorker {
   std::map<std::string, ContextItem> m_contexts;
   std::map<std::string, std::set<std::string>> m_contextErrMap;
   std::vector<std::string> m_selectedContexts;
+  std::vector<ContextItem*> m_processedContexts;
   std::string m_outputDir;
   std::string m_packRoot;
   std::string m_compilerRoot;

tools/projmgr/src/ProjMgr.cpp

@@ -1104,22 +1104,11 @@ bool ProjMgr::RunListLayers(void) {
   // Step4: Run only when --update-idx flag is used
   // Update the cbuild-idx.yml file with layer information
   if (m_updateIdx) {
-    map<string, ContextItem>* contexts = nullptr;
-    m_worker.GetContexts(contexts);
-
-    // Check if contexts were properly retrieved
-    if (contexts == nullptr) {
-      return false;
-    }
-
     // Generate Cbuild index
-    m_processedContexts.clear();
-    for (auto& contextName : m_worker.GetSelectedContexts()) {
-      auto& contextItem = (*contexts)[contextName];
-      m_processedContexts.push_back(&contextItem);
-    }
-    if (!m_processedContexts.empty()) {
-      if (!m_emitter.GenerateCbuildIndex(m_processedContexts,
+    const auto& processedContexts = m_worker.GetProcessedContexts();
+    if (!processedContexts.empty()) {
+      m_worker.ElaborateVariablesConfigurations();
+      if (!m_emitter.GenerateCbuildIndex(processedContexts,
         m_failedContext, map<string, ExecutesItem>())) {
         return false;
       }
@@ -1301,10 +1290,11 @@ void ProjMgr::InitSolution(const std::string& csolution, const std::string& acti
     m_contextSet = true;
   } else {
     m_activeTargetSet = activeTargetSet;
+    m_contextSet = false;
   }
 }
 
-bool ProjMgr::LoadSolution(const std::string& csolution, const std::string& activeTargetSet) {
+bool ProjMgr::SetupContexts(const std::string& csolution, const std::string& activeTargetSet) {
 
   InitSolution(csolution, activeTargetSet, false);
 
@@ -1314,6 +1304,14 @@ bool ProjMgr::LoadSolution(const std::string& csolution, const std::string& acti
   if (!ParseAndValidateContexts()) {
     return false;
   }
+  return true;
+}
+
+bool ProjMgr::LoadSolution(const std::string& csolution, const std::string& activeTargetSet) {
+
+  if (!SetupContexts(csolution, activeTargetSet)) {
+    return false;
+  }
   if (!ProcessContexts()) {
     return false;
   }

tools/projmgr/src/ProjMgrCbuildIdx.cpp

@@ -22,7 +22,7 @@ class ProjMgrCbuildIdx : public ProjMgrCbuildBase {
     const map<string, ExecutesItem>& executes);
 private:
   void SetExecutesNode(YAML::Node node, const map<string, ExecutesItem>& executes, const string& base, const string& ref);
-  void SetVariablesNode(YAML::Node node, ProjMgrParser* parser, const ContextItem* context, const map<string, map<string, set<const ConnectItem*>>>& layerTypes);
+  void SetVariablesNode(YAML::Node node, ProjMgrParser* parser, const ContextItem* context, const VariablesConfiguration& configuration);
 };
 
 ProjMgrCbuildIdx::ProjMgrCbuildIdx(YAML::Node node,
@@ -64,37 +64,17 @@ ProjMgrCbuildIdx::ProjMgrCbuildIdx(YAML::Node node,
     }
 
     // Collect layer connection info specific to each target
-    if (context->validConnections.size() > 0) {
+    if (!context->variablesConfigurations.empty()) {
       configTargets.push_back(context->type.target);
-      map<int, map<string, map<string, set<const ConnectItem*>>>> configurations;
-      int index = 0;
-      for (const auto& combination : context->validConnections) {
-        index++;
-        for (const auto& item : combination) {
-          if (item.type.empty())
-            continue;
-          for (const auto& [type, _] : context->compatibleLayers) {
-            // add all required layer types, including discarded optionals
-            configurations[index][type].insert({});
-          }
-          for (const auto& connect : item.connections) {
-            configurations[index][item.type][item.filename].insert(connect);
-          }
-        }
-      }
-
-      // Process connection info and generate nodes
-      if (configurations.size() > 0) {
-        YAML::Node targetTypeNode;
-        SetNodeValue(targetTypeNode[YAML_TARGETTYPE], context->type.target);
-        for (const auto& [index, types] : configurations) {
-          YAML::Node configurationsNode;
-          configurationsNode[YAML_CONFIGURATION] = YAML::Null;
-          SetVariablesNode(configurationsNode[YAML_VARIABLES], parser, context, types);
-          targetTypeNode[YAML_TARGET_CONFIGURATIONS].push_back(configurationsNode);
-        }
-        node[YAML_CONFIGURATIONS].push_back(targetTypeNode);
+      YAML::Node targetTypeNode;
+      SetNodeValue(targetTypeNode[YAML_TARGETTYPE], context->type.target);
+      for (const auto& configuration : context->variablesConfigurations) {
+        YAML::Node configurationsNode;
+        configurationsNode[YAML_CONFIGURATION] = YAML::Null;
+        SetVariablesNode(configurationsNode[YAML_VARIABLES], parser, context, configuration);
+        targetTypeNode[YAML_TARGET_CONFIGURATIONS].push_back(configurationsNode);
       }
+      node[YAML_CONFIGURATIONS].push_back(targetTypeNode);
     }
   }
 
@@ -206,53 +186,22 @@ ProjMgrCbuildIdx::ProjMgrCbuildIdx(YAML::Node node,
 }
 
 void ProjMgrCbuildIdx::SetVariablesNode(YAML::Node node, ProjMgrParser* parser, const ContextItem* context,
-  const map<string, map<string, set<const ConnectItem*>>>& layerTypes) {
-  for (const auto& [type, filenames] : layerTypes) {
-    if (type.empty()) {
-      continue;
-    }
+  const VariablesConfiguration& configuration) {
+  for (const auto& variable : configuration.variables) {
     YAML::Node layerNode;
-    string layerFile;
-    const string layerId = context->layerVariables.find(type) != context->layerVariables.end() ?
-      context->layerVariables.at(type) : type + "-Layer";
-    if (filenames.empty()) {
-      layerNode[layerId] = "";
+    if (variable.clayer.empty()) {
+      layerNode[variable.name] = "";
     }
-    for (const auto& [filename, options] : filenames) {
-      string packRoot = ProjMgrKernel::Get()->GetCmsisPackRoot();
-      layerFile = filename;
-      RteFsUtils::NormalizePath(layerFile);
-      layerFile = RteFsUtils::MakePathCanonical(layerFile);
-      // Replace with ${CMSIS_PACK_ROOT} or $SolutionDir()$ depending on the detected path
-      size_t index = layerFile.find(packRoot);
-      if (index != string::npos) {
-        layerFile.replace(index, packRoot.length(), "${CMSIS_PACK_ROOT}");
-      }
-      else {
-        string relPath = RteFsUtils::RelativePath(filename, context->csolution->directory);
-        if (!relPath.empty()) {
-          layerFile = "$" + string(RteConstants::AS_SOLUTION_DIR_BR) + "$/" + RteFsUtils::LexicallyNormal(relPath);
-        }
-      }
-      SetNodeValue(layerNode[layerId], layerFile);
-      if (parser->GetGenericClayers().find(filename) != parser->GetGenericClayers().end()) {
-        const auto& clayer = parser->GetGenericClayers().at(filename);
-        SetNodeValue(layerNode[YAML_DESCRIPTION], clayer.description);
-      }
-      for (const auto& connect : options) {
-        if (!connect->set.empty()) {
-          YAML::Node setNode;
-          SetNodeValue(setNode[YAML_SET], connect->set + " (" + connect->connect + (connect->info.empty() ? "" : " - " + connect->info) + ")");
-          layerNode[YAML_SETTINGS].push_back(setNode);
-        }
-      }
-      if (context->packLayers.find(filename) != context->packLayers.end()) {
-        const auto& clayer = context->packLayers.at(filename);
-        SetNodeValue(layerNode[YAML_PATH], clayer->GetOriginalAbsolutePath(clayer->GetPathString()));
-        SetNodeValue(layerNode[YAML_FILE], clayer->GetFileString());
-        SetNodeValue(layerNode[YAML_COPY_TO], clayer->GetCopyToString());
-      }
+    SetNodeValue(layerNode[variable.name], variable.clayer);
+    SetNodeValue(layerNode[YAML_DESCRIPTION], variable.description);
+    for (const auto& setting : variable.settings) {
+      YAML::Node setNode;
+      SetNodeValue(setNode[YAML_SET], setting.set);
+      layerNode[YAML_SETTINGS].push_back(setNode);
     }
+    SetNodeValue(layerNode[YAML_PATH], variable.path);
+    SetNodeValue(layerNode[YAML_FILE], variable.file);
+    SetNodeValue(layerNode[YAML_COPY_TO], variable.copyTo);
     node.push_back(layerNode);
   }
 }

tools/projmgr/src/ProjMgrRpcServer.cpp

@@ -97,6 +97,7 @@ class RpcHandler : public RpcMethods {
   RpcArgs::LogMessages GetLogMessages(void) override;
   RpcArgs::DraftProjectsInfo GetDraftProjects(const RpcArgs::DraftProjectsFilter& filter) override;
   RpcArgs::ConvertSolutionResult ConvertSolution(const string& solution, const string& activeTarget, const bool& updateRte) override;
+  RpcArgs::DiscoverLayersInfo DiscoverLayers(const string& solution, const string& activeTarget) override;
 
 protected:
   enum Exception
@@ -126,6 +127,7 @@ class RpcHandler : public RpcMethods {
   RteTarget* GetActiveTarget(const string& context) const;
   RteComponentAggregate* GetComponentAggregate(const string& context, const string& id) const;
   bool SelectVariantOrVersion(const string& context, const string& id, const string& value, bool bVariant);
+  bool CheckSolutionArg(string& solution, optional<string>& message) const;
 };
 
 bool ProjMgrRpcServer::Run(void) {
@@ -169,6 +171,19 @@ bool ProjMgrRpcServer::Run(void) {
   return true;
 }
 
+bool RpcHandler::CheckSolutionArg(string& solution, optional<string>& message) const {
+  if (!regex_match(solution, regex(".*\\.csolution\\.(yml|yaml)"))) {
+    message = solution + " is not a *.csolution.yml file";
+    return false;
+  }
+  solution = RteFsUtils::MakePathCanonical(solution);
+  if (!RteFsUtils::Exists(solution)) {
+    message = solution + " file does not exist";
+    return false;
+  }
+  return true;
+}
+
 const ContextItem& RpcHandler::GetContext(const string& context) const {
   if (!m_solutionLoaded) {
     throw JsonRpcException(SOLUTION_NOT_LOADED, "a valid solution must be loaded before proceeding");
@@ -690,9 +705,8 @@ RpcArgs::DraftProjectsInfo RpcHandler::GetDraftProjects(const RpcArgs::DraftProj
 
 RpcArgs::ConvertSolutionResult RpcHandler::ConvertSolution(const string& solution, const string& activeTarget, const bool& updateRte) {
   RpcArgs::ConvertSolutionResult result = {{ false }};
-  const auto csolutionFile = RteFsUtils::MakePathCanonical(solution);
-  if (!regex_match(csolutionFile, regex(".*\\.csolution\\.(yml|yaml)"))) {
-    result.message = solution + " is not a *.csolution.yml file";
+  string csolutionFile = solution;
+  if (!CheckSolutionArg(csolutionFile, result.message)) {
     return result;
   }
   if (!m_manager.RunConvert(csolutionFile, activeTarget, updateRte) || !ProjMgrLogger::Get().GetErrors().empty()) {
@@ -712,4 +726,67 @@ RpcArgs::ConvertSolutionResult RpcHandler::ConvertSolution(const string& solutio
   return result;
 }
 
+RpcArgs::DiscoverLayersInfo RpcHandler::DiscoverLayers(const string& solution, const string& activeTarget) {
+  RpcArgs::DiscoverLayersInfo result = {{ false }};
+  string csolutionFile = solution;
+  if (!CheckSolutionArg(csolutionFile, result.message)) {
+    return result;
+  }
+  if (!m_manager.SetupContexts(csolutionFile, activeTarget)) {
+    result.message = "Setup of solution contexts failed";
+    return result;
+  }
+  m_worker.SetUpCommand(true);
+  StrVec layers;
+  StrSet fails;
+  if (!m_worker.ListLayers(layers, "", fails) || !m_worker.ElaborateVariablesConfigurations()) {
+    result.message = "No compatible software layer found. Review required connections of the project";
+    return result;
+  } else {
+    // retrieve valid configurations
+    vector<RpcArgs::VariablesConfiguration> vcVec;
+    const auto& processedContexts = m_worker.GetProcessedContexts();
+    for (const auto& context : processedContexts) {
+      if (!context->variablesConfigurations.empty()) {
+        for (const auto& configuration : context->variablesConfigurations) {
+          RpcArgs::VariablesConfiguration vc;
+          vector<RpcArgs::LayerVariable> lvVec;
+          for (const auto& variable : configuration.variables) {
+            RpcArgs::LayerVariable lv = { variable.name, variable.clayer };
+            vector<RpcArgs::SettingsType> settings;
+            for (const auto& s : variable.settings) {
+              if (!s.set.empty()) {
+                settings.push_back(RpcArgs::SettingsType{s.set});
+              }
+            }
+            if (!settings.empty()) {
+              lv.settings = settings;
+            }
+            if (!variable.path.empty()) {
+              lv.path = variable.path;
+            }
+            if (!variable.file.empty()) {
+              lv.file = variable.file;
+            }
+            if (!variable.copyTo.empty()) {
+              lv.copyTo = variable.copyTo;
+            }
+            lvVec.push_back(lv);
+          }
+          vc.variables = lvVec;
+          vcVec.push_back(vc);
+        }
+        // break after first project context with possible variable configurations
+        // solutions with undefined layer variables over multiple projects are currently not supported
+        break;
+      }
+    }
+    if (!vcVec.empty()) {
+      result.configurations = vcVec;
+    }
+    result.success = true;
+    return result;
+  }
+}
+
 // end of ProkMgrRpcServer.cpp