Add 'report_diff' key in result data in case of available diff data

In case that a 'diff' key with 'before' and 'after' data is available
create a unified diff from 'before' and 'after' and store it below the
'report_diff' key as a string for later consumption in config reports
on the Foreman server.
Remove the 'diff' key afterwards to save space as in case of a file the
content is probably big and it is stored twice (before and after).

* plugins/callback/foreman.py:
  Implement feature.

* tests/callback/three_hosts.yml:
  Add tasks to generate diffs when running in diff mode.

* tests/conftest.py:
  Allow to run the playbook in diff mode.

* tests/test_callback.py:
  - Run the playbook for the foreman report type in diff mode
  - Ignore further items when comparing the json data that depend on the
    local environment
  - Remove additional strings that are different in each run
  - Transform the json string into json data to improve data handling

* tests/fixtures/callback/dir_store/*:
  Adjust fixtures to new tests and changes in format.

Signed-off-by: Ruediger Pluem <ruediger.pluem@vodafone.com>

Author: rpluem-vf

plugins/callback/foreman.py

@@ -110,6 +110,7 @@
 from collections import defaultdict
 import json
 import time
+import re
 
 try:
     import requests
@@ -123,36 +124,6 @@
 from ansible.plugins.callback import CallbackBase
 
 
-def build_log_foreman(data_list):
-    """
-    Transform the internal log structure to one accepted by Foreman's
-    config_report API.
-    """
-    for data in data_list:
-        result = data.pop('result')
-        task = data.pop('task')
-        result['failed'] = data.get('failed')
-        result['module'] = task.get('action')
-        if data.get('failed'):
-            level = 'err'
-        elif result.get('changed'):
-            level = 'notice'
-        else:
-            level = 'info'
-
-        yield {
-            "log": {
-                'sources': {
-                    'source': task.get('name'),
-                },
-                'messages': {
-                    'message': json.dumps(result, sort_keys=True, cls=AnsibleNoVaultJSONEncoder),
-                },
-                'level': level,
-            }
-        }
-
-
 def get_time():
     """
     Return the time for measuring duration. Prefers monotonic time but
@@ -269,6 +240,46 @@ def _send_data(self, data_type, report_type, host, data):
                 self._display.warning(u'Sending data to Foreman at {url} failed for {host}: {err}'.format(
                     host=to_text(host), err=to_text(err), url=to_text(self.foreman_url)))
 
+    def _build_log_foreman(self, data_list):
+        """
+        Transform the internal log structure to one accepted by Foreman's
+        config_report API.
+        """
+        for data in data_list:
+            result = data.pop('result')
+            task = data.pop('task')
+            result['failed'] = data.get('failed')
+            result['module'] = task.get('action')
+            if data.get('failed'):
+                level = 'err'
+            elif result.get('changed'):
+                level = 'notice'
+            else:
+                level = 'info'
+
+            # Check if the 'diff' key is set and transform the state before
+            # and after the change into a unified diff string and store it
+            # below the 'report_diff' key. Remove the 'diff' key afterwards
+            # as in case of a file the content is probably big and it is
+            # stored twice (before and after).
+            if 'diff' in result:
+                diff = self._get_diff(result['diff'])
+                # Remove color escape sequences for terminal output
+                result['report_diff'] = re.sub(u'\033\\[0.*?m', '', diff)
+                del result['diff']
+
+            yield {
+                "log": {
+                    'sources': {
+                        'source': task.get('name'),
+                    },
+                    'messages': {
+                        'message': json.dumps(result, sort_keys=True, cls=AnsibleNoVaultJSONEncoder),
+                    },
+                    'level': level,
+                }
+            }
+
     def send_facts(self):
         """
         Sends facts to Foreman, to be parsed by foreman_ansible fact
@@ -336,7 +347,7 @@ def send_reports_foreman(self, stats):
                         "failed": total['failures'] + total['unreachable'],
                         "skipped": total['skipped'],
                     },
-                    "logs": list(build_log_foreman(self.items[host])),
+                    "logs": list(self._build_log_foreman(self.items[host])),
                     "reporter": "ansible",
                     "check_mode": self.check_mode,
                 }

tests/callback/three_hosts.yml

@@ -57,6 +57,18 @@
         unsafe: !unsafe |
           THIS IS {{ crypt }}
 
+    - name: Generate a diff
+      copy:
+        content: Test
+        dest: "/tmp/test_{{ lookup('env', 'FOREMAN_REPORT_TYPE') }}.txt"
+        remote_src: false
+        mode: '0644'
+
+    - name: Remove file
+      file:
+        path: "/tmp/test_{{ lookup('env', 'FOREMAN_REPORT_TYPE') }}.txt"
+        state: absent
+
   handlers:
     - name: Test handler 1
       command: echo foo

tests/conftest.py

@@ -47,7 +47,7 @@ def get_foreman_url():
     return server_yml_content['foreman_server_url']
 
 
-def run_playbook(module, extra_vars=None, limit=None, inventory=None, check_mode=False, extra_env=None):
+def run_playbook(module, extra_vars=None, limit=None, inventory=None, check_mode=False, diff_mode=False, extra_env=None):
     # Assemble parameters for playbook call
     os.environ['ANSIBLE_CONFIG'] = os.path.join(os.getcwd(), 'ansible.cfg')
     if extra_env is not None:
@@ -62,8 +62,13 @@ def run_playbook(module, extra_vars=None, limit=None, inventory=None, check_mode
         kwargs['extravars'] = extra_vars
     if limit:
         kwargs['limit'] = limit
+    cmdline = []
     if check_mode:
-        kwargs['cmdline'] = "--check"
+        cmdline.append("--check")
+    if diff_mode:
+        cmdline.append("--diff")
+    if len(cmdline) > 0:
+        kwargs['cmdline'] = ' '.join(cmdline)
     return ansible_runner.run(**kwargs)