Merge remote-tracking branch 'origin/develop' into rl4sem

Conflicts:
	src/sensai/data_transformation/dft.py
	src/sensai/torch/torch_base.py

Author: opcode81

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 1.4.0
 commit = False
 tag = False
 allow_dirty = False

.github/workflows/ci.yaml

@@ -0,0 +1,136 @@
+name: CI
+
+on:
+    push:
+        branches: [develop, master]
+    pull_request:
+        branches: [develop]
+
+jobs:
+    # Define matrix build for multiple test scenarios
+    test:
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                # py_backwardscompat is not included because there is also an issue with numpy compatibility on Python 3.10
+                env_name: [py_pinned_dependencies, py_latest_dependencies]
+        steps:
+            - name: Checkout Code
+              uses: actions/checkout@v3
+              with:
+                  lfs: true
+
+            - name: Setup Python
+              uses: actions/setup-python@v4
+              with:
+                  python-version: "3.10"
+
+            - name: Install Dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  if [[ "${{ matrix.env_name }}" == "py_pinned_dependencies" ]]; then
+                    pip install -v \
+                      -r requirements.txt \
+                      -r requirements_torch.txt \
+                      -r requirements_lightgbm.txt \
+                      -r requirements_geoanalytics.txt \
+                      -r requirements_xgboost.txt \
+                      pytest \
+                      pytest-cov \
+                      pytest-xdist \
+                      pytorch-lightning~=1.1.0 \
+                      coverage \
+                      coverage-badge
+                  elif [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then
+                    pip install \
+                      pytest \
+                      jupyter==1.0.0 \
+                      nbconvert==6.5.0 \
+                      clearml==0.17.1 \
+                      "pytorch-lightning>=1.1" \
+                      ".[full]"
+                  elif [[ "${{ matrix.env_name }}" == "py_backwardscompat" ]]; then
+                    pip install \
+                      pytest \
+                      "scikit-learn==1.0.2" \
+                      "numpy<1.21" \
+                      ".[torch]"
+                  fi
+                  pip install --no-deps -e .
+
+            - name: Run Tests
+              run: |
+                  if [[ "${{ matrix.env_name }}" == "py_pinned_dependencies" ]]; then
+                    coverage erase
+                    pytest -n 4 --cov --cov-append --cov-report=term-missing tests;
+                    coverage html
+                    coverage-badge -o badges/coverage.svg -f
+                  elif [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then
+                    pytest
+                  elif [[ "${{ matrix.env_name }}" == "py_backwardscompat" ]]; then
+                    pytest tests/backwardscompat;
+                  fi
+
+            - name: Run Notebook Tests
+              run: |
+                  if [[ "${{ matrix.env_name }}" == "py_latest_dependencies" ]]; then
+                    pytest notebooks;
+                  fi
+
+            - name: Upload Notebook Artifacts
+              if: matrix.env_name == 'py_latest_dependencies'
+              uses: actions/upload-artifact@v4
+              with:
+                  name: docs_outputs
+                  path: docs/
+
+    # Documentation build job
+    docs:
+        needs: test
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout Code
+              uses: actions/checkout@v3
+
+            - name: Download Notebook Artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  name: docs_outputs
+                  path: docs/
+
+            - name: Setup Python
+              uses: actions/setup-python@v4
+              with:
+                  python-version: "3.10"
+
+            - name: Install Documentation Dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install \
+                    sphinx==5.0.2 \
+                    sphinxcontrib-websupport==1.2.4 \
+                    sphinx-toolbox==3.7.0 \
+                    sphinx_rtd_theme \
+                    nbsphinx \
+                    ipython \
+                    ipywidgets \
+                    jupyter-book==0.15.1
+
+            - name: Build Documentation
+              run: sh build-docs.sh
+
+            - name: Prepare Pages
+              if: github.ref == 'refs/heads/develop'
+              run: |
+                  mv docs/build/* public/docs
+
+            - name: Deploy Pages
+              if: github.ref == 'refs/heads/develop'
+              uses: JamesIves/github-pages-deploy-action@3.7.1
+              with:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  BRANCH: gh-pages
+                  FOLDER: public
+                  TARGET_FOLDER: .
+                  CLEAN: true
+                  SINGLE_COMMIT: true

.github/workflows/release.yaml

@@ -32,7 +32,7 @@ jobs:
       - name: Set up Python for PyPI Release
         uses: actions/setup-python@v1
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - name: Install dependencies for PyPI Release
         run: |
           python -m pip install --upgrade pip

.github/workflows/tox.yaml

@@ -1,8 +1,48 @@
 # Changelog
 
-
 ## Unreleased
 
+### Breaking Changes
+
+* Dropped support for Python versions below 3.10
+* Dropped support for TensorFlow (removing `sensai.tensorflow`)
+
+### Improvements/Changes
+
+* `util`:
+  * `util.cache`:
+    * `cache_mysql.MySQLPersistentKeyValueCache`:
+      * Switch from MySQLdb to pymsql
+      * Add support for additional connection arguments
+      * Use autocommit and remove option of using deferred commits as it's the only way to guarantee
+        that no stale data is read due to transactions going on too long
+      * Handle duplicate key upon insertion due to race condition by providing a more informative Exception
+  * `util.logging`:
+      * `configure`: Allow the output stream to be configured
+  * `util.git`:
+      * `git_status`: Add option `log_error` 
+
+
+## 1.4.0 (2025-01-21)
+
+### Improvements/Changes
+
+* `evaluation`: For cases where the model projects the input data to a subset of rows,
+  the evaluator now projects the ground truth data accordingly.
+* `util`:
+  * `util.deprecation`: Annotations of class init functions now report the respective class as being deprecated
+  * `util.plot`: Add method `Plot.show` for convenience
+  * `util.string`:
+    * `pretty_string_repr` (and `ToStringMixin.pprint` and `.pprints`):
+      Handle content in curly braces, i.e. dictionaries, treating them like an object with content indented if it is too long
+    * `to_string` (and `ToStringMixin`):
+      for objects of type `str`, return a quoted string (as returned by `repr`) to avoid strings with line breaks distracting the output
+  * `util.helper`: Add `flatten_dict` to flatten a dictionary into a single-level dictionary with keys as dot-separated paths
+  * `util.tensorboard`: New module with utilities for loading/comparing series data from tensorboard logs
+  * `util.git`: New module with function `git_status` for retrieving the status of a git repository (which can be useful for logging)
+
+## v1.3.0 (2024-11-29)
+
 ### Improvements/Changes
 
 * `vector_model`:

CHANGELOG.md

@@ -18,30 +18,39 @@ Use conda to set up the Python environment:
 
 Solving the environment may take several minutes (but should ultimately work).
 
-NOTE: versions are mostly unpinned in the environment specification, because this facilitates conda dependency resolution. Also, sensAI is intended to be compatible with *all* (newer) versions of the dependencies. If it isn't, we need to specify  an upper version bound in `setup.py` (where it matters the most) as well as in `environment.yml`. Compatibility with old (pinned) versions and the latest versions is tested in the tox build (see below).
+NOTE: versions are mostly unpinned in the environment specification, because this facilitates conda dependency resolution. 
+Also, sensAI is intended to be compatible with *all* (newer) versions of the dependencies. 
+If it isn't, we need to specify  an upper version bound in `setup.py` (where it matters the most) as well as in `environment.yml`. 
+Compatibility with old (pinned) versions and the latest versions is tested in the GitHub build (see below).
 
 # Build and Test Pipeline
 
-The tests and docs build are executed via **tox** in several environments:
-* `py`: the "regular" test environment, where we test against the pinned dependencies (by explicitly including `requirements.txt` with the pinned versions; this is also the environment in which we test the execution of notebooks
-* `py_latest_dependencies`: the environment where we use the latest versions of all dependencies (except where we have identified an incompatibility; see `setup.py` definitions `DEPS_VERSION_LOWER_BOUND` and `DEPS_VERSION_UPPER_BOUND_EXCLUSIVE`); by not including `requirements.txt`, we depend on the latest admissible versions according to `setup.py`
-* `docs`: the environment in which docs are built via sphinx 
+The tests and docs build are executed in several environments:
+* `py_pinned_dependencies`: the "regular" test environment, where we test against the pinned dependencies 
+  (by explicitly including `requirements.txt` with the pinned versions; this is also the environment in which we test the 
+  execution of notebooks
+* `py_latest_dependencies`: the environment where we use the latest versions of all dependencies (except where we have  
+  identified an incompatibility; see `setup.py` definitions `DEPS_VERSION_LOWER_BOUND` and `DEPS_VERSION_UPPER_BOUND_EXCLUSIVE`); 
+  by not including `requirements.txt`, we depend on the latest admissible versions according to `setup.py`
+* `py_backwardscompa`: a special environment with old versions of some critical dependences where we can test backwards compatibility
+  with persisted models of very old sensAI versions (that used older versions of the dependencies, e.g. sklearn)
 
 ## Automated Tests
 
-The tests can be locally run without tox via
+The tests can be locally via
 
     sh run_pytest_tests.sh
 
 ## Docs Build
 
-Docs are automatically created during the GitHub build via tox.
+Docs are automatically created during the GitHub build.
 
 All .rst files are auto-generated (by `build_scripts/update_docs.py`), with the exception of the root index file  `index.rst`.
 
-### Declaring Optional Dependencies
+### Declaring Mock Imports for Dependencies
 
-**Attention**: Make sure that any optional sensAI dependencies (which are not included in the `docs` tox environment) are added to `docs/conf.py` under `autodoc_mock_imports`. Otherwise the tox build will fail.
+**Attention**: Make sure that any sensAI dependencies are added to `docs/_config.yml` under `autodoc_mock_imports`. 
+Otherwise, the docs build will fail.
 
 ### Notebooks
 
@@ -55,7 +64,7 @@ For changes in notebooks to be reflected in the docs build, the test needs to be
 
 ### Manually Running the Docs Build
 
-The docs build can be run without tox via 
+The docs build can be run via 
 
     sh build-docs.sh
 

README-dev.md

@@ -7,7 +7,7 @@
   <div align="center" style="text-align:center">
   <a href="https://pypi.org/project/sensai/" style="text-decoration:none"><img src="https://img.shields.io/pypi/v/sensai.svg" alt="PyPI"></a>
   <a href="https://raw.githubusercontent.com/jambit/sensAI/master/LICENSE" style="text-decoration:none"><img alt="License" src="https://img.shields.io/pypi/l/sensai"></a>
-  <a href="https://github.com/jambit/sensAI/actions/workflows/tox.yaml" style="text-decoration:none"><img src="https://github.com/jambit/sensAI/actions/workflows/tox.yaml/badge.svg" alt="Build status"></a>
+  <a href="https://github.com/jambit/sensAI/actions/workflows/ci.yaml" style="text-decoration:none"><img src="https://github.com/jambit/sensAI/actions/workflows/ci.yaml/badge.svg" alt="Build status"></a>
   </div>
 </p>