From 8e44d035241ff73c746207427f39e434ff174fed Mon Sep 17 00:00:00 2001 From: Sidharth Surana Date: Mon, 26 Feb 2018 16:37:35 -0800 Subject: [PATCH 1/2] Add support for ansible > 2.4.0 fix the vault password read mechanism Based on the comments in https://github.com/ansible/ansible/issues/30824 Signed-off-by: Sidharth Surana --- column/api_runner.py | 51 +++++++++++++++++++++++--------------------- column/utils.py | 20 +++++++++++------ requirements.txt | 2 +- tests/test_utils.py | 31 +++++++++++++++++++-------- 4 files changed, 64 insertions(+), 40 deletions(-) diff --git a/column/api_runner.py b/column/api_runner.py index bfc3319..e41e0d0 100644 --- a/column/api_runner.py +++ b/column/api_runner.py @@ -3,22 +3,23 @@ import logging import os -import six +from ansible import cli from ansible import constants from ansible.executor import playbook_executor from ansible.executor import task_queue_manager -from ansible import inventory +from ansible.inventory.manager import InventoryManager from ansible.parsing import dataloader from ansible.parsing.splitter import parse_kv from ansible.playbook import play -from ansible import vars +from ansible.plugins.loader import get_all_plugin_loaders +from ansible.vars.manager import VariableManager +import six from column import callback from column import exceptions from column import runner - LOG = logging.getLogger(__name__) @@ -78,22 +79,19 @@ def run_playbook(self, playbook_file, inventory_file=None, **kwargs): options = self._build_opt_dict(inventory_file, **kwargs) - variable_manager = vars.VariableManager() loader = dataloader.DataLoader() + inventory = InventoryManager(loader=loader, sources=options.inventory) + + # create the variable manager, which will be shared throughout + # the code, ensuring a consistent view of global variables + variable_manager = VariableManager(loader=loader, inventory=inventory) options.extra_vars = {six.u(key): six.u(value) for key, value in options.extra_vars.items()} - variable_manager.extra_vars = options.extra_vars - - ansible_inv = inventory.Inventory(loader=loader, - variable_manager=variable_manager, - host_list=options.inventory) - ansible_inv.set_playbook_basedir(os.path.dirname(playbook_file)) - variable_manager.set_inventory(ansible_inv) - ansible_inv.subset(options.subset) - + variable_manager.extra_vars = cli.load_extra_vars(loader, options) + inventory.subset(options.subset) pbex = playbook_executor.PlaybookExecutor( playbooks=playbooks, - inventory=ansible_inv, + inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, @@ -130,16 +128,20 @@ def run_module(self, module_name='ping', module_args=None, hosts="all", passwords = {'conn_pass': conn_pass, 'become_pass': become_pass} options = self._build_opt_dict(inventory_file, **kwargs) + # dynamically load any plugins + get_all_plugin_loaders() - variable_manager = vars.VariableManager() loader = dataloader.DataLoader() - variable_manager.extra_vars = options.extra_vars + inventory = InventoryManager(loader=loader, sources=options.inventory) + + # create the variable manager, which will be shared throughout + # the code, ensuring a consistent view of global variables + variable_manager = VariableManager(loader=loader, inventory=inventory) + options.extra_vars = {six.u(key): six.u(value) for key, value in + options.extra_vars.items()} + variable_manager.extra_vars = cli.load_extra_vars(loader, options) - ansible_inv = inventory.Inventory(loader=loader, - variable_manager=variable_manager, - host_list=options.inventory) - variable_manager.set_inventory(ansible_inv) - ansible_inv.subset(options.subset) + inventory.subset(options.subset) play_ds = self._play_ds(hosts, module_name, module_args) play_obj = play.Play().load(play_ds, variable_manager=variable_manager, @@ -147,7 +149,7 @@ def run_module(self, module_name='ping', module_args=None, hosts="all", try: tqm = task_queue_manager.TaskQueueManager( - inventory=ansible_inv, + inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, @@ -190,7 +192,8 @@ def _build_opt_dict(self, inventory_file, **kwargs): 'extra_vars': {}, 'subset': constants.DEFAULT_SUBSET, 'tags': [], 'verbosity': 0, 'connection': constants.DEFAULT_TRANSPORT, - 'timeout': constants.DEFAULT_TIMEOUT + 'timeout': constants.DEFAULT_TIMEOUT, + 'diff': constants.DIFF_ALWAYS } args.update(self.custom_opts) args.update(kwargs) diff --git a/column/utils.py b/column/utils.py index 2869c52..e0a28be 100644 --- a/column/utils.py +++ b/column/utils.py @@ -4,6 +4,7 @@ import os from ansible import cli +from ansible import constants from ansible import errors from ansible.parsing import dataloader from ansible.parsing import vault @@ -24,10 +25,19 @@ def _get_vault_password_file(): return cfg.get('defaults', 'vault_password_file') +def _get_vault_lib(): + loader = dataloader.DataLoader() + vault_ids = constants.DEFAULT_VAULT_IDENTITY_LIST + + vault_secrets = cli.CLI.setup_vault_secrets(loader, vault_ids=vault_ids, + vault_password_files=[_get_vault_password_file()], + ask_vault_pass=False, + auto_prompt=False) + return vault.VaultLib(secrets=vault_secrets) + + def vault_decrypt(value): - vault_password = cli.CLI.read_vault_password_file( - _get_vault_password_file(), dataloader.DataLoader()) - this_vault = vault.VaultLib(vault_password) + this_vault = _get_vault_lib() try: return this_vault.decrypt(value) except errors.AnsibleError: @@ -35,7 +45,5 @@ def vault_decrypt(value): def vault_encrypt(value): - vault_password = cli.CLI.read_vault_password_file( - _get_vault_password_file(), dataloader.DataLoader()) - this_vault = vault.VaultLib(vault_password) + this_vault = _get_vault_lib() return this_vault.encrypt(value) diff --git a/requirements.txt b/requirements.txt index 676b404..c485ebe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ansible>=2.2.0.0,<2.4.0 +ansible>=2.4.0 flask flask_restful six diff --git a/tests/test_utils.py b/tests/test_utils.py index 649f17c..cb8414e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: GPL-3.0 from mock import patch - +import os +import tempfile from testtools import TestCase from column import utils @@ -15,8 +16,9 @@ def setUp(self): self.vault_password = 'h2RV4pEX2M2TXvLxYhuy' @patch('__builtin__.reload') - @patch('ansible.cli.CLI.read_vault_password_file') - def test_vault_decrypt(self, mock_read_vault_password, mock_reload): + @patch('column.utils._get_vault_password_file') + def test_vault_decrypt(self, mock_column_utils_get_vault_password_file, + mock_reload): encrypted_value = ( '$ANSIBLE_VAULT;1.1;AES256\n' '3139663233656537383737613633343638313934' @@ -29,14 +31,25 @@ def test_vault_decrypt(self, mock_read_vault_password, mock_reload): '3637636234666230333764323361643866643863\n' '3933\n' ) - mock_read_vault_password.return_value = self.vault_password + fd, tmp_file = tempfile.mkstemp() + with open(tmp_file, 'w') as f: + f.write(self.vault_password) + mock_column_utils_get_vault_password_file.return_value = tmp_file self.assertEqual('vmware', utils.vault_decrypt(encrypted_value)) - self.assertTrue(mock_read_vault_password.called) + self.assertTrue(mock_column_utils_get_vault_password_file.called) + os.close(fd) + os.remove(tmp_file) @patch('__builtin__.reload') - @patch('ansible.cli.CLI.read_vault_password_file') - def test_vault_encrypt(self, mock_read_vault_password, mock_reload): - mock_read_vault_password.return_value = self.vault_password + @patch('column.utils._get_vault_password_file') + def test_vault_encrypt(self, mock_column_utils_get_vault_password_file, + mock_reload): + fd, tmp_file = tempfile.mkstemp() + with open(tmp_file, 'w') as f: + f.write(self.vault_password) + mock_column_utils_get_vault_password_file.return_value = tmp_file encrypted = utils.vault_encrypt('vmware') self.assertTrue(encrypted.startswith('$ANSIBLE_VAULT;1.1;AES256')) - self.assertTrue(mock_read_vault_password.called) + self.assertTrue(mock_column_utils_get_vault_password_file.called) + os.close(fd) + os.remove(tmp_file) From 0d3a13a302422bd767a2389961dc9c43e70674da Mon Sep 17 00:00:00 2001 From: Xiangfei Zhu Date: Sun, 8 Apr 2018 08:07:35 +0000 Subject: [PATCH 2/2] Pass work_dir as extra_vars correctly Otherwise work_dir is not passed to ansible. --- column/api_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/column/api_runner.py b/column/api_runner.py index 9b7b6cd..f282b2d 100644 --- a/column/api_runner.py +++ b/column/api_runner.py @@ -89,7 +89,7 @@ def run_playbook(self, playbook_file, inventory_file=None, **kwargs): # create the variable manager, which will be shared throughout # the code, ensuring a consistent view of global variables variable_manager = VariableManager(loader=loader, inventory=inventory) - variable_manager.extra_vars = cli.load_extra_vars(loader, options) + variable_manager.extra_vars = options.extra_vars inventory.subset(options.subset) pbex = playbook_executor.PlaybookExecutor( playbooks=playbooks,