Merge pull request #1308 from chaoss/test: Release

deps worker updates, documentation updates, docker major enhancements, and dependency updates

Author: sgoggins

.gitignore

@@ -167,3 +167,6 @@ workers/clustering_worker/vocabulary_count
 
 # gzipped files
 *.gz
+
+#nohup logs
+nohup.out

augur/housekeeper.py

@@ -167,8 +167,8 @@ def shutdown_updates(self):
             process.terminate()
 
     def prep_jobs(self):
-        logger.info("Preparing housekeeper jobs")
-        for job in self.jobs:
+        for index, job in enumerate(self.jobs, start=1):
+            self.printProgressBar(index, len(self.jobs), 'Preparing housekeeper jobs:', 'Complete', 1, 50)
             if 'repo_group_id' in job or 'repo_ids' in job:
                 # If RG id is 0 then it just means to query all repos
                 where_and = 'AND' if job['model'] == 'issues' and 'repo_group_id' in job else 'WHERE'
@@ -427,3 +427,25 @@ def parseRepoName(repo_url):
         parts = path.split('/')
         return parts[1:]
 
+
+    def printProgressBar(self, iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█', printEnd = "\r"):
+        """
+        Call in a loop to create terminal progress bar
+        @params:
+            iteration   - Required  : current iteration (Int)
+            total       - Required  : total iterations (Int)
+            prefix      - Optional  : prefix string (Str)
+            suffix      - Optional  : suffix string (Str)
+            decimals    - Optional  : positive number of decimals in percent complete (Int)
+            length      - Optional  : character length of bar (Int)
+            fill        - Optional  : bar fill character (Str)
+            printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
+        """
+
+        percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
+        filledLength = int(length * iteration // total)
+        bar = fill * filledLength + '-' * (length - filledLength)
+        print(f'\r{prefix} |{bar}| {percent}% {suffix}', end='\r')
+        # Print New Line on Complete
+        if iteration == total:
+            print()

database-compose.yml

@@ -1,8 +1,9 @@
 #SPDX-License-Identifier: MIT
+#Format for ports: <outside>:<inside>
 version: '3'
 services:
    database:
-    image: augurlabs/augur:database
+    image: augurlabs/augur:${AUGUR_DB_TYPE}
     restart: unless-stopped
     ports:
      - 5434:5432

docker-compose.yml

@@ -1,17 +1,24 @@
 #SPDX-License-Identifier: MIT
+#Something having to do with the workers is preventing it from working.
 version: '3'
 services:
-      backend:
-       image: augurlabs/augur:backend
-       restart: unless-stopped
-       ports:
-         - 5000:5000
-         - 50000-52000:50000-52000
-       env_file: docker_env.txt
-
-      frontend:
-        image: augurlabs/augur:frontend
-        restart: unless-stopped
-        ports:
-          - 8080:8080
+  backend:
+    image: isaacmilarky/augur_backend
+    build: 
+      context: .
+      dockerfile: ./util/docker/backend/Dockerfile
+    restart: always
+    #DO NOT declare ports here in the range that the workers use for API calls. It breaks the docker compose
+    ports:
+      - 5000:5000
+    extra_hosts:
+      - "database:${AUGUR_DB_HOST}"
+    env_file: docker_env.txt
+#  frontend:
+#    image: augurlabs/augur:frontend
+#    restart: unless-stopped
+#    ports:
+#      - 8080:8080
+#    extra_hosts:
+#      - "localhost:127.0.0.1"
 

docker-setup.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+#automate the small things for setting up docker containers
+#This file sets up the backend and the frontend and optional database container.
+#NOTE: The frontend is currently under construction.
+#
+#The script is needed to handle:
+# -Environment variables for
+#   -Runtime values needing to be set and accurate for the backend's database credentials and github api key
+#   -Pre-runtime values needing to be set and accurate for the database hostname and type (whether it is test data or not). The ip address needs to be added in the extra_hosts argument of the yml markup.
+# -Setting up a network alias in order to let the docker container communicate with local hosts.
+# -Easily seeing console output and process statistics from one convienient window.
+# -Easily save console output to logs.
+#
+#This file uses two environment files
+# - One called docker_env.txt which holds the runtime enviroment variables that the container itself uses
+# - One called .env which holds the environment variables that docker-compose.yml uses and holds the database type.
+#TODO:
+  #Let users know how to configure the database to work for local connection because its not *that* clear right now. Its in the docs at least.
+  #Make container work with gitlab key
+  #Test this script on macOS
+#
+missingModules=""
+
+#Check everything that needs to be in the $PATH is in there.
+#Bash doesn't let this work if this is in an if statement for some reason it has to be chained
+type -P "docker" &>/dev/null && echo "docker found..." || missingModules="${missingModules} docker"
+type -P "docker-compose" &>/dev/null && echo "docker-compose found..." || missingModules="${missingModules} docker-compose"
+type -P "ifconfig" &>/dev/null && echo "ifconfig found..." || missingModules="${missingModules} ifconfig (part of net-tools)"
+type -P "psql" &>/dev/null && echo "psql found..." || missingModules="${missingModules} psql"
+type -P "watch" &>/dev/null && echo "watch found..." || missingModules="${missingModules} watch"
+
+if [ ! -z "$missingModules" ]
+then
+  echo "One or more modules required to run this script is missing or not in your \$PATH:"
+  echo "Note: OSX users will need to install watch with \"brew install watch\""
+  echo "Including:$missingModules"
+  exit 1
+fi
+unset $missingModules
+
+if [ "$EUID" -ne 0 ];
+  then echo "Please run as root"
+  exit 1
+fi
+
+#Always use a clean .env file because it is a subset of docker_env.txt so we can just generate it from that.
+if [[ -f ".env" ]]
+then
+  rm .env
+fi
+touch .env
+
+#This is differant for MacOS
+#Script uses an alias for localhost that is the below ip
+echo "Setting up network alias..."
+#Check kernel for OS, assumes either linux or macOS
+if [ "$(uname -s)" == "Linux" ]
+then
+  ifconfig lo:0 10.254.254.254
+  ifconfig lo:0
+  echo "Linux detected..."
+else
+  ifconfig lo0 alias 10.254.254.254
+  ifconfig lo0
+fi
+
+#Prompt for deploy type.
+echo "Types of docker deployment: "
+echo
+echo "1. Deploy the backend using docker connected to a non-docker database."
+echo "2. Deploy backend and database together in docker containers."
+echo "3. Deploy the backend and database together in docker containers using premade test data."
+echo
+read -p "Would you like to use : " deployChoice
+
+case $deployChoice in
+
+  1)
+    #Start script to set up just two containers
+    exec scripts/docker/docker-setup-external.sh
+    ;;
+  
+  2)
+    #Start script to set up all three containers.
+    #Set env variable to not use test data
+    echo "AUGUR_DB_TYPE=database" >> .env
+    exec scripts/docker/docker-setup-database.sh
+    ;;
+
+  3)
+    #Start script to set up all three containers
+    #Set env variable to use test data.
+    echo "AUGUR_DB_TYPE=test_data" >> .env
+    exec scripts/docker/docker-setup-database.sh
+    ;;
+
+  *)
+    echo "Invalid choice!"
+    exit 1
+    ;;
+esac

docs/source/docker/docker-compose.rst

@@ -1,17 +1,64 @@
-Docker Compose
+Docker Compose Deployment
 =========================
 
-This section of the documentation details how to use Augur's Docker Compose configuration to get the full stack up and running as fast as possible. This section assumes you have read and configured your Docker installation as detailed `here <toc.html#getting-started>`_.
+This section assumes you have read and configured your Docker installation as detailed `here <toc.html#getting-started>`_.
 
 The default ports for each service are\:
 
 - ``backend``: ``5000:50100-50800``
 - ``frontend``: ``8080``
 - ``database``: ``5434``
 
+.. note::
+
+    Make sure your database is configured to listen on all addresses in order to work with the containers. The most common error an improperly configured database throws is
+    ::
+
+        psql: could not connect to server: Connection refused
+        Is the server running on host 10.254.254.254 and accepting
+        TCP/IP connections on port 5432?
+  
+
+Docker Compose with the script (recommended)
+============================================
+This section details how to use Augur's docker-setup script in order to get a docker-compose deployment up and running as fast as possible.
+
 Running the containers
 -----------------------
 
+.. warning::
+
+    Don't forget to provide your external database credentials in the ``docker_env.txt`` file or generate it within the script. `More about the configuration file here <getting-started.html>`_
+
+To run Augur
+
+.. code-block:: bash
+
+    sudo ./docker-setup.sh
+
+Answer the prompts depending on your needs. If you are using a local database it is important to use 10.254.254.254 as a hostname or localhost if prompted. If you are using the container database or the test database press 2 or 3 for the prompt answer.
+
+The script should automatically generate the environment variables for the docker containers and compose files. Additionally it will set up a network alias so that the containers can communicate with localhost. Finally, it also takes care of whether or not to generate the schema to protect the integrity of any databases in use.
+
+
+.. warning::
+
+    It is also important to only generate the schema if you need to otherwise your database could become unusable later on.
+
+Stopping the containers
+-------------------------
+
+To stop the containers, do a keyboard inturrupt while the script is running ``Ctrl+C``. The script will then ask if you want to generate log files to look at later.
+
+Once you've got your container up and running, checkout out `how to use them <usage.html>`_ 
+
+
+Docker Compose without a script
+===============================
+
+This section of the documentation details how to use Augur's Docker Compose configuration to get the full stack up and running as fast as possible without the recommended helper script. 
+
+
 To run Augur **without** the database container:
 
 .. code-block:: bash
@@ -20,7 +67,7 @@ To run Augur **without** the database container:
 
 .. warning::
 
-    Don't forget to provide your external database credentials in the ``env.txt`` file.
+    Don't forget to provide your external database credentials in the ``docker_env.txt`` file. Additionally the ``.env`` file is needed for the ``*.yml`` files' environment variables. Don't forget to set the variables specified specified in these files namely ``AUGUR_DB_TYPE`` and ``AUGUR_DB_HOST``.
 
 To run Augur **with** the database container:
 
@@ -34,9 +81,11 @@ If you want to use the ``test_data`` image with the data preloaded, change the `
 
     image: augurlabs/augur:test_data
 
+Or you can set it dynamically in the .env file.
+
 Stopping the containers
 -------------------------
 
 To stop the containers, run ``docker-compose down --remove-orphans``. The flag is necessary to stop the database container if you used one; run the command again to delete them. 
 
-Once you've got your container up and running, checkout out `how to use them <usage.html>`_ 
+Once you've got your container up and running, checkout out `how to use them <usage.html>`_ 

docs/source/docker/docker.rst

@@ -5,7 +5,7 @@ Augur provides a separate Docker image for each layer of our application (databa
 
 .. note::
 
-    This page is primarily targeted at developers. If you're solely interested in collecting data, we recommend using `Docker Compose <docker-compose.html>`_.
+    This page is primarily targeted at developers. If you're solely interested in collecting data, we recommend using our `Docker Setup Script <docker-compose.html>`_.
 
 Building the images
 --------------------
@@ -28,12 +28,16 @@ To start a container, use the command below. ``<container_name>`` can be whateve
 
 .. note::
 
-    If you are running the ``backend`` service, then ``<docker_port>`` needs to be ``5000``; for ``frontend`` and ``database`` the ports are ``8080`` and ``5432``. You can set the ``<host_port>`` to any **available** port on your machine for any of the services.
+    If you are running the ``backend`` service, then ``<docker_port>`` needs to be ``5000``; for ``frontend`` and ``database`` the ports are ``8080`` and ``5434``. You can set the ``<host_port>`` to any **available** port on your machine for any of the services.
 
 .. note::
-    If you are running the backend service, you'll also need to add ``--env-file docker_env.txt`` to your command in order to make the container aware of your configuration file.
+    If you are running the backend service, you'll also need to add ``--env-file docker_env.txt`` to your command in order to make the container aware of your configuration file. You'll also need to add ``--add-host=database:<ip_of_database>`` to your command in order to make the container connect to your database. Make sure your database is configured to accept the container's connections by making sure that ``listen_addresses = '*'`` wherever the postgresql.conf is located on your machine and change the pg_hba.conf to accept hosts with a line similar to ``host	all  		all 		0.0.0.0/0 		md5``.
 
-.. code-block:: bash
+.. warning::
+    If you are using a local database, you need to set up an alias for your localhost. In a terminal, type ``sudo ifconfig lo:0 10.254.254.254`` if you are using linux or ``sudo ifconfig lo0 alias 10.254.254.254`` if you are using macOS. You will use this alias, ``10.254.254.254`` to connect to the local database. Make sure your database is properly configured by using this command ``psql -d "postgresql://augur:<your_password>@10.254.254.254/augur" -c "select now()"``. This command should have the same output if you replace the ip with ``localhost``.
+
+
+.. code-block::bash
 
     # in the root augur/ directory
     $ docker run -p <host_port>:<docker_port> --name <container_name> <tag_name>