MAINT: CLI end of prov lib refactor (#384)

Author: Oddant1

q2cli/core/usage.py

@@ -12,16 +12,15 @@
 import re
 import shlex
 import textwrap
-from typing import Any, Callable, Dict, List, Tuple
+from typing import Any, Callable, Dict, List, Tuple, Type
 
 from qiime2 import ResultCollection
 import qiime2.sdk.usage as usage
 from qiime2.sdk.usage import (
-    UsageVariable, Usage, UsageInputs, UsageOutputs, UsageOutputNames
+    UsageVariable, Usage, UsageInputs, UsageOutputs
 )
-from qiime2.sdk import Action
 from qiime2.core.archive.provenance_lib.usage_drivers import (
-    build_header, build_footer
+    build_header, build_footer, ReplayUsageAction
 )
 from qiime2.core.archive.provenance_lib import ProvDAG
 
@@ -439,6 +438,8 @@ def to_interface_name(self):
 
 
 class ReplayCLIUsage(CLIUsage):
+    UsageAction: Type[ReplayUsageAction] = ReplayUsageAction
+
     shebang = '#!/usr/bin/env bash'
     header_boundary = ('#' * 79)
     set_ex = [
@@ -506,11 +507,16 @@ def _append_action_line(self, signature, param_name: str, value):
                 'when the plugin version you have\n  # installed does not '
                 'match the version used in the original analysis.\n  # '
                 'Please see the docs and correct the parameter name '
-                'before running.\n')
-            cli_name = re.sub('_', '-', param_name)
-            line += self.INDENT + '--?-' + cli_name + ' ' + str(value)
-            line += ' \\'
+                'before running.')
             self.recorder.append(line)
+            self._append_unknown_param(param_name, value)
+
+    def _append_unknown_param(self, param_name, value):
+        line = ''
+        cli_name = re.sub('_', '-', param_name)
+        line += self.INDENT + '--?-' + cli_name + ' ' + str(value)
+        line += ' \\'
+        self.recorder.append(line)
 
     def _make_param(self, value: Any, state: Dict) -> List[Tuple]:
         '''
@@ -630,9 +636,9 @@ def comment(self, text):
 
     def action(
         self,
-        action: Action,
+        action: ReplayUsageAction,
         inputs: UsageInputs,
-        outputs: UsageOutputNames
+        outputs: UsageOutputs
     ) -> UsageOutputs:
         '''
         Overrides parent method to fill in missing outputlines from
@@ -641,18 +647,40 @@ def action(
 
         Parameters
         ----------
-        action : Action
-            The underlying sdk.Action object.
+        action : ReplayUsageAction
+            The object representing the Action we are replaying.
         inputs : UsageInputs
             Mapping of parameter names to arguments for the action.
-        outputs : UsageOutputNames
+        outputs : ReplayUsageOutputNames
             Mapping of registered output names to usage variable names.
 
         Returns
         -------
         UsageOutputs
             The results returned by the action.
         '''
+        plugin_name = q2cli.util.to_cli_name(action.plugin_id)
+        action_name = q2cli.util.to_cli_name(action.action_id)
+
+        if action.node._action_present_:
+            variables = self._action_found(
+                plugin_name, action_name, action, inputs, outputs
+            )
+        else:
+            variables = self._action_not_found(
+                plugin_name, action_name, action, inputs, outputs
+            )
+
+        return variables
+
+    def _action_found(
+        self, plugin_name: str, action_name: str,
+        action: ReplayUsageAction,
+        inputs: UsageInputs,
+        outputs: UsageOutputs
+    ):
+        self.recorder.append('qiime %s %s \\' % (plugin_name, action_name))
+
         variables = Usage.action(self, action, inputs, outputs)
         vars_dict = variables._asdict()
 
@@ -668,11 +696,6 @@ def action(
                 # Otherwise, we should add filler values to missing_outputs
                 missing_outputs[output] = f'XX_{output}'
 
-        plugin_name = q2cli.util.to_cli_name(action.plugin_id)
-        action_name = q2cli.util.to_cli_name(action.action_id)
-        self.recorder.append('qiime %s %s \\' % (plugin_name, action_name))
-
-        action_f = action.get_action()
         action_state = get_action_state(action_f)
 
         ins = inputs.map_variables(lambda v: v.to_interface_name())
@@ -699,6 +722,39 @@ def action(
         self.recorder.append('')
         return variables
 
+    def _action_not_found(
+        self, plugin_name: str, action_name: str,
+        action: ReplayUsageAction,
+        inputs: UsageInputs,
+        outputs: UsageOutputs
+    ):
+        variables = Usage.action_not_found(self, action, inputs, outputs)
+        vars_dict = variables._asdict()
+
+        ins = inputs.map_variables(lambda v: v.to_interface_name())
+
+        self.recorder.append(
+            '# FIXME: The following action was not found in your current '
+            'QIIME 2\n# environment. Please ensure the action and its '
+            'parameters are correct before\n# running.'
+        )
+        self.recorder.append('qiime %s %s \\' % (plugin_name, action_name))
+
+        for param_name, value in ins.items():
+            self._append_unknown_param(param_name, value)
+
+        dir_name = self._build_output_dir_name(plugin_name, action_name)
+        self.recorder.append(
+            self.INDENT + '--output-dir %s \\' % (dir_name)
+        )
+        self._rename_outputs(vars_dict, dir_name)
+
+        self.recorder[-1] = self.recorder[-1][:-2]  # remove trailing `\`
+
+        self.recorder.append('')
+
+        return variables
+
     def render(self, flush=False):
         '''
         Return a newline-seperated string of CLI script.

q2cli/tests/data/rarefied_table.qza

@@ -9,6 +9,8 @@
 import os
 import gc
 import re
+import yaml
+import pytest
 import shutil
 import unittest
 from unittest.mock import patch
@@ -1277,6 +1279,105 @@ def test_replay_supplement_zipfile(self):
 
             self.assertEqual(os.listdir(unzipped_path), ['supplement'])
 
+    # Leave me alone I know the checksums don't match
+    @pytest.mark.filterwarnings('ignore::UserWarning')
+    def test_replay_param_not_found(self):
+        qiime_cli = RootCommand()
+        command = qiime_cli.get_command(ctx=None, name='dummy-plugin')
+
+        in_fp = os.path.join(self.tempdir, 'concated_ints.qza')
+        left_path = os.path.join(self.tempdir, 'left.qza')
+        right_path = os.path.join(self.tempdir, 'right.qza')
+
+        result = self.runner.invoke(
+            command, ['split-ints', '--i-ints', in_fp,
+                      '--o-left', left_path, '--o-right', right_path,
+                      '--verbose'])
+
+        self.assertEqual(result.exit_code, 0)
+
+        left = Artifact.load(left_path)
+        action_fp = os.path.join(
+            left._archiver.provenance_dir, 'action', 'action.yaml'
+        )
+        with open(action_fp, 'r+') as action_fh:
+            action_string = action_fh.read()
+            action_yaml = yaml.safe_load(action_string)
+            action_yaml['action']['parameters'].append({'not': 'real'})
+
+            action_fh.write(yaml.dump(action_yaml))
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            out_fp = os.path.join(tmpdir, 'rendered.txt')
+            result = self.runner.invoke(
+                tools,
+                ['replay-provenance', '--in-fp', left_path, '--out-fp',
+                 out_fp, '--no-dump-recorded-metadata']
+            )
+            self.assertEqual(result.exit_code, 0)
+
+            with open(out_fp, 'r') as fh:
+                rendered = fh.read()
+
+        MISSING_PARAM = \
+"""
+  # FIXME: The following parameter name was not found in your current
+  # QIIME 2 environment. This may occur when the plugin version you have
+  # installed does not match the version used in the original analysis.
+  # Please see the docs and correct the parameter name before running.
+  --?-not real \\
+"""  # noqa: E128
+        self.assertIn(MISSING_PARAM, rendered)
+
+    def test_replay_action_not_found(self):
+        datadir = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), 'data'
+        )
+        artifact_fp = os.path.join(datadir, 'rarefied_table.qza')
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            out_fp = os.path.join(tmpdir, 'rendered.txt')
+            result = self.runner.invoke(
+                tools,
+                ['replay-provenance', '--in-fp', artifact_fp, '--out-fp',
+                 out_fp, '--no-dump-recorded-metadata']
+            )
+            self.assertEqual(result.exit_code, 0)
+
+            with open(out_fp, 'r') as fh:
+                rendered = fh.read()
+
+        imports = \
+"""
+qiime tools import \\
+  --type 'Phylogeny[Rooted]' \\
+  --input-path <your data here> \\
+  --output-path phylogeny-rooted-0.qza
+
+qiime tools import \\
+  --type 'FeatureTable[Frequency]' \\
+  --input-path <your data here> \\
+  --output-path feature-table-frequency-0.qza
+"""  # noqa: E128
+        self.assertIn(imports, rendered)
+
+        FIXME_action = \
+"""
+# FIXME: The following action was not found in your current QIIME 2
+# environment. Please ensure the action and its parameters are correct before
+# running.
+qiime diversity core-metrics-phylogenetic \\
+  --?-table feature-table-frequency-0.qza \\
+  --?-phylogeny phylogeny-rooted-0.qza \\
+  --?-sampling-depth 13 \\
+  --?-metadata <your metadata filepath>.tsv \\
+  --?-with-replacement False \\
+  --?-n-jobs-or-threads 1 \\
+  --?-ignore-missing-samples False \\
+  --output-dir diversity-core-metrics-phylogenetic
+"""  # noqa: E128
+        self.assertIn(FIXME_action, rendered)
+
 
 class TestAnnotations(unittest.TestCase):
     def setUp(self):