Fix issues with CI build, update README

Author: opcode81

.github/workflows/ci.yaml

@@ -13,10 +13,12 @@ jobs:
         strategy:
             matrix:
                 # py_backwardscompat is not included because there is also an issue with numpy compatibility on Python 3.10
-                env_name: [py_pinned_dependencies, py_latest_dependencies]  # removed py_backwardscompat
+                env_name: [py_pinned_dependencies, py_latest_dependencies]
         steps:
             - name: Checkout Code
               uses: actions/checkout@v3
+              with:
+                  lfs: true
 
             - name: Setup Python
               uses: actions/setup-python@v4
@@ -26,27 +28,55 @@ jobs:
             - name: Install Dependencies
               run: |
                   python -m pip install --upgrade pip
-                  if [[ "${{ matrix.env_name }}" == "py" ]]; then
-                    pip install -v -r  requirements.txt -r requirements_torch.txt -r requirements_lightgbm.txt -r requirements_geoanalytics.txt -r requirements_xgboost.txt pytest pytest-cov pytest-xdist pytorch-lightning~=1.1.0;
+                  if [[ "${{ matrix.env_name }}" == "py_pinned_dependencies" ]]; then
+                    pip install -v \
+                      -r requirements.txt \
+                      -r requirements_torch.txt \
+                      -r requirements_lightgbm.txt \
+                      -r requirements_geoanalytics.txt \
+                      -r requirements_xgboost.txt \
+                      pytest \
+                      pytest-cov \
+                      pytest-xdist \
+                      pytorch-lightning~=1.1.0 \
+                      coverage \
+                      coverage-badge
                   elif [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then
-                    pip install pytest jupyter==1.0.0 nbconvert==6.5.0 clearml==0.17.1 pytorch-lightning>=1.1 ".[full]";
+                    pip install \
+                      pytest \
+                      jupyter==1.0.0 \
+                      nbconvert==6.5.0 \
+                      clearml==0.17.1 \
+                      "pytorch-lightning>=1.1" \
+                      ".[full]"
                   elif [[ "${{ matrix.env_name }}" == "py_backwardscompat" ]]; then
-                    pip install pytest scikit-learn==1.0.2 pytorch-lightning>=1.1 ".[torch]";
+                    pip install \
+                      pytest \
+                      "scikit-learn==1.0.2" \
+                      "numpy<1.21" \
+                      ".[torch]"
                   fi
                   pip install --no-deps -e .
 
             - name: Run Tests
               run: |
-                  if [[ "${{ matrix.env_name }}" == "py" ]]; then
+                  if [[ "${{ matrix.env_name }}" == "py_pinned_dependencies" ]]; then
                     coverage erase
                     pytest -n 4 --cov --cov-append --cov-report=term-missing tests;
+                    coverage html
+                    coverage-badge -o badges/coverage.svg -f
                   elif [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then
                     pytest
-                    pytest notebooks;
                   elif [[ "${{ matrix.env_name }}" == "py_backwardscompat" ]]; then
                     pytest tests/backwardscompat;
                   fi
 
+            - name: Run Notebook Tests
+              run: |
+                  if [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then
+                    pytest notebooks;
+                  fi
+
     # Documentation build job
     docs:
         runs-on: ubuntu-latest
@@ -62,31 +92,16 @@ jobs:
             - name: Install Documentation Dependencies
               run: |
                   python -m pip install --upgrade pip
-                  pip install sphinx==5.0.2 sphinxcontrib-websupport==1.2.4 sphinx-toolbox==3.7.0 sphinx_rtd_theme nbsphinx ipython ipywidgets jupyter-book==0.15.1
+                  pip install \ 
+                    sphinx==5.0.2 \
+                    sphinxcontrib-websupport==1.2.4 \
+                    sphinx-toolbox==3.7.0 \
+                    sphinx_rtd_theme \
+                    nbsphinx \
+                    ipython \
+                    ipywidgets \
+                    jupyter-book==0.15.1
 
             - name: Build Documentation
               run: sh build-docs.sh
 
-    # Coverage report generation
-    coverage:
-        needs: test
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout Code
-              uses: actions/checkout@v3
-
-            - name: Setup Python
-              uses: actions/setup-python@v4
-              with:
-                  python-version: "3.10"
-
-            - name: Install Coverage Dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  pip install coverage coverage-badge
-
-            - name: Generate Coverage Report
-              run: |
-                  coverage html
-                  coverage-badge -o badges/coverage.svg -f
-                  coverage erase

README-dev.md

@@ -18,24 +18,32 @@ Use conda to set up the Python environment:
 
 Solving the environment may take several minutes (but should ultimately work).
 
-NOTE: versions are mostly unpinned in the environment specification, because this facilitates conda dependency resolution. Also, sensAI is intended to be compatible with *all* (newer) versions of the dependencies. If it isn't, we need to specify  an upper version bound in `setup.py` (where it matters the most) as well as in `environment.yml`. Compatibility with old (pinned) versions and the latest versions is tested in the tox build (see below).
+NOTE: versions are mostly unpinned in the environment specification, because this facilitates conda dependency resolution. 
+Also, sensAI is intended to be compatible with *all* (newer) versions of the dependencies. 
+If it isn't, we need to specify  an upper version bound in `setup.py` (where it matters the most) as well as in `environment.yml`. 
+Compatibility with old (pinned) versions and the latest versions is tested in the GitHub build (see below).
 
 # Build and Test Pipeline
 
-The tests and docs build are executed via **tox** in several environments:
-* `py`: the "regular" test environment, where we test against the pinned dependencies (by explicitly including `requirements.txt` with the pinned versions; this is also the environment in which we test the execution of notebooks
-* `py_latest_dependencies`: the environment where we use the latest versions of all dependencies (except where we have identified an incompatibility; see `setup.py` definitions `DEPS_VERSION_LOWER_BOUND` and `DEPS_VERSION_UPPER_BOUND_EXCLUSIVE`); by not including `requirements.txt`, we depend on the latest admissible versions according to `setup.py`
-* `docs`: the environment in which docs are built via sphinx 
+The tests and docs build are executed in several environments:
+* `py_pinned_dependencies`: the "regular" test environment, where we test against the pinned dependencies 
+  (by explicitly including `requirements.txt` with the pinned versions; this is also the environment in which we test the 
+  execution of notebooks
+* `py_latest_dependencies`: the environment where we use the latest versions of all dependencies (except where we have  
+  identified an incompatibility; see `setup.py` definitions `DEPS_VERSION_LOWER_BOUND` and `DEPS_VERSION_UPPER_BOUND_EXCLUSIVE`); 
+  by not including `requirements.txt`, we depend on the latest admissible versions according to `setup.py`
+* `py_backwardscompa`: a special environment with old versions of some critical dependences where we can test backwards compatibility
+  with persisted models of very old sensAI versions (that used older versions of the dependencies, e.g. sklearn)
 
 ## Automated Tests
 
-The tests can be locally run without tox via
+The tests can be locally via
 
     sh run_pytest_tests.sh
 
 ## Docs Build
 
-Docs are automatically created during the GitHub build via tox.
+Docs are automatically created during the GitHub build.
 
 All .rst files are auto-generated (by `build_scripts/update_docs.py`), with the exception of the root index file  `index.rst`.
 
@@ -56,7 +64,7 @@ For changes in notebooks to be reflected in the docs build, the test needs to be
 
 ### Manually Running the Docs Build
 
-The docs build can be run without tox via 
+The docs build can be run via 
 
     sh build-docs.sh