[DAPS-1770] - release: v4.0.0

* [DAPS-1585] - update: dependencies, upgrade ssl dependency. 3.2.5 (#1646)
* [DAPS-1605] - fix: scripts, install_foxx.sh by splitting ssl_args (#1623)
* [DAPS-1651] - refactor: scripts, compose, univify treatment of env variables in compose env generator (#1656) (#1658)
* [DAPS-1675] - feature: foxx, adding the logger functions for future PR's (#1675)
* [DAPS-1659] - refactor: scripts, remove dependencies install scripts (#1660)
* [DAPS-1670] - feature: common, core, repo, python_client, web, allow passing repo types in protobuf messages (#1670)
* [DAPS-1671] - feature: foxx, add repository and execution strategy types (#1672)
* [DAPS-1661] - refactor: compose, scripts, remove remaining occurences of zeromq system secret. (#1661)
* [DAPS-1522] - refactor: foxx, user router logging improvements, remove non helpful logs from tasks.js (#1629)
* [DAPS-1691] - refactor: foxx, adjust validation.js swap g_lib with error_code require rem… (#1691)
* [DAPS-1692] - tests: ci, End-to-end web tests, fix flaky test (#1693)
* [DAPS-1694] - refactor: foxx, move permissions functions from support.js to lib/permissions (#1695)
* [DAPS-1685] - feature: compose, enable arangodb ssl (#1687)
* [DAPS-1700] - fix: ci, limit arangodb job output to last 3 hours. (#1701)
* [DAPS-1676] - feature: foxx, arango add factory for repositories for metadata and globus (#1697)
* [DAPS-1718] - feature: web, core, python client, Protobuf ExecutionMethod enum, add RepoAllocationCreateResponse (#1719)
* [DAPS-1713] - refactor: core, web, python client, protobuf, allow optional fields when creating repo to support metadat… (#1714)
* [DAPS-1715] - refactor: core, make path, pub_key, address, endpoint optional in repoCreateRequest (#1716)
* [DAPS-1705] - feature: foxx, integrate metadata globus factory repo router create (#1706)
* [DAPS-1688] - update: dependencies, core, repo, authz, gcs, Crypto libssl switched to version 3 globus_sdk version pinned (#1689)
* [DAPS-1729] - fix: ci, downstream datafed dependencies pipelines are building the container image from incorrect sha (#1732)
* [DAPS-1711] - refactor: foxx standardize repo response schema (#1712)
* [DAPS-1725] - refactor: remove confusing apache conf file. (#1728)
* [DAPS-1707] - update: dependencies, web, update web dependencies before install (#1709)
* [DAPS-1522] - refactor: foxx, task router logging improvements (#1648)
* [DAPS-1522] - refactor: foxx, query router logging improvements (#1627)
* [DAPS-1735] - bug: foxx, remove duplicate user_router test (#1736)
* [DAPS-1731] - reature: scripts, compose, add scripts to generate globus credentials for web service (#1731)
* [DAPS-1725] - refactor: tests, mock core server centralized (#1726)
* [DAPS-1741] - update: scripts, native client id in intialize_globus_endpoint and globus_clea… (#1741)
* [DAPS-1745]  - fix: scripts, account for nested client credentials. (#1746)
* [DAPS-1725-2] - fix; tests, centralized mock core service libraries fixed (part 2) (#1747)
* [DAPS-1742] - refactor script replace os.path.join with urllib.parse.urljoin (#1744)
* [DAPS-1749] - refactor: cmake, set cmake policy to silence noisy warning. (#1750)
* [DAPS-1522] - refactor: foxx, feature tag router logging improvements (#1734)
* [DAPS-1378] - fix: web, mapping of multiple globus accounts. (#1753)
* [DAPS-1756] - fix: scripts, foxx, add retries and connection check to install_foxx.sh script. (#1757)
* [DAPS-1522] - refactor: foxx, Version Router Logging Improvements (#1758)
* [DAPS-1737] - refactor: compose, cleanup arango ssl env variables (#1765)
* [DAPS-1766] - fix: ci, python client provisioning job
* [DAPS-1663] - feature: core Service, adding Correlation ID to Logging (#1704)

Co-authored-by: Aaron Perez <perezam@ornl.gov>
Co-authored-by: AronPerez <aperez0295@gmail.com>
Co-authored-by: Blake Nedved <nedvedba@ornl.gov>
Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com>
Co-authored-by: nedvedba <145805866+nedvedba@users.noreply.github.com>
Co-authored-by: Austin Hampton <amh107@latech.edu>
Co-authored-by: Austin Hampton <44103380+megatnt1122@users.noreply.github.com>
Co-authored-by: Blake Nedved <blakeanedved@gmail.com>
Co-authored-by: Polina Shpilker <infinite.loopholes@gmail.com>

Author: 7 people

.github/workflows/build-docs.yml

@@ -40,7 +40,7 @@ jobs:
                   ./scripts/generate_datafed.sh
             - name: Install documentation build dependencies
               run: |
-                  sudo ./scripts/install_docs_dependencies.sh
+                  sudo ./external/DataFedDependencies/scripts/install_docs_dependencies.sh
             - name: Build documentation
               run: |
                   cmake -S. -B build -DBUILD_AUTHZ=OFF -DBUILD_CORE_SERVER=OFF -DBUILD_COMMON=OFF -DBUILD_DOCS=ON -DBUILD_FOXX=OFF -DBUILD_REPO_SERVER=OFF -DBUILD_PYTHON_CLIENT=ON -DBUILD_TESTS=OFF -DBUILD_WEB_SERVER=OFF -DENABLE_UNIT_TESTS=OFF

.github/workflows/unit-tests.yml

@@ -10,9 +10,12 @@ jobs:
             - name: Update debian
               run: apt update
             - name: Install dependencies
+              with:
+                  submodules: recursive
+                  fetch-depth: 0
               run: |
                   ./scripts/generate_datafed.sh
-                  ./scripts/install_core_dependencies.sh
+                  ./external/DataFedDependencies/scripts/install_core_dependencies.sh
             - name: Build
               run: |
                   /opt/datafed/dependencies/bin/cmake -S. -B build -DCMAKE_BUILD_TYPE=Debug -DBUILD_WEB_SERVER=OFF

.gitignore

@@ -53,7 +53,8 @@ scripts/admin_datafed_backup.sh
 scripts/admin_refresh_certs.sh
 scripts/globus/__pycache__
 services/
-tmp/
+tests/mock_core/Version.hpp
+tmp
 web/SDMS.proto
 web/SDMS_Anon.proto
 web/SDMS_Auth.proto
@@ -65,6 +66,8 @@ web/package-lock.json
 web/static/datafed-core-key.pub
 *.swp
 *.swo
+*.tar.gz
+*.tgz
 
 # for web tests
 tests/end-to-end/web-UI/node_modules/

.gitlab-ci.yml

@@ -18,6 +18,7 @@ include:
     - local: .gitlab/stage_pipeline_serialize.yml
     - local: .gitlab/infrastructure.yml
     - local: .gitlab/stage_clear_cache.yml
+    - local: .gitlab/stage_base_image_check.yml
     - local: .gitlab/stage_build_base.yml
     - local: .gitlab/stage_provision_client.yml
     - local: .gitlab/stage_image_check.yml
@@ -32,6 +33,7 @@ stages:
     - trigger-infrastructure
     - signal
     - clear-docker-cache
+    - base-image-check
     - build-base
     - provision-client
     - image-check
@@ -56,3 +58,4 @@ variables:
     REGISTRY: "camden.ornl.gov"
     DATAFED_DEPENDENCIES_INSTALL_PATH: "/shared/install"
     DOCKER_TLS_CERTDIR: "" # Required for running docker in docker
+    GIT_SUBMODULE_STRATEGY: recursive

.gitlab/build/build_core_image.yml

@@ -1,53 +1,53 @@
 ---
 stages:
-  - build
+    - build
 
 include:
-  - local: .gitlab/common.yml
+    - local: .gitlab/common.yml
 
 build-core:
-  extends: .docker_build_script
-  stage: build
-  variables:
-    PROJECT: "datafed"
-    COMPONENT: "core"
-    GIT_STRATEGY: clone
-    DOCKER_FILE_PATH: "core/docker/Dockerfile"
-    DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
-    BUILD_INTERMEDIATE: "FALSE"
-  tags:
-    - ci-datafed-core
-    - docker
-  rules:
-    - changes:
-        - docker/**/*
-        - scripts/**/*
-        - core/**/*
-        - common/**/*
-        - CMakeLists.txt
-        - cmake/**/*
-        - .gitlab-ci.yml
-      when: on_success
+    extends: .docker_build_script
+    stage: build
+    variables:
+        PROJECT: "datafed"
+        COMPONENT: "core"
+        GIT_STRATEGY: clone
+        DOCKER_FILE_PATH: "core/docker/Dockerfile"
+        DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+        BUILD_INTERMEDIATE: "FALSE"
+    tags:
+        - ci-datafed-core
+        - docker
+    rules:
+        - changes:
+              - docker/**/*
+              - scripts/**/*
+              - core/**/*
+              - common/**/*
+              - CMakeLists.txt
+              - cmake/**/*
+              - .gitlab-ci.yml
+          when: on_success
 
 retag-image:
-  extends: .docker_retag_image
-  stage: build
-  variables:
-    PROJECT: "datafed"
-    COMPONENT: "core"
-    GIT_STRATEGY: clone
-    DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
-    BUILD_INTERMEDIATE: "FALSE"
-  tags:
-    - docker
-  rules:
-    - changes:
-        - docker/**/*
-        - scripts/**/*
-        - core/**/*
-        - common/**/*
-        - CMakeLists.txt
-        - cmake/**/*
-        - .gitlab-ci.yml
-      when: never
-    - when: on_success
+    extends: .docker_retag_image
+    stage: build
+    variables:
+        PROJECT: "datafed"
+        COMPONENT: "core"
+        GIT_STRATEGY: clone
+        DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+        BUILD_INTERMEDIATE: "FALSE"
+    tags:
+        - docker
+    rules:
+        - changes:
+              - docker/**/*
+              - scripts/**/*
+              - core/**/*
+              - common/**/*
+              - CMakeLists.txt
+              - cmake/**/*
+              - .gitlab-ci.yml
+          when: never
+        - when: on_success

.gitlab/build/build_foxx_image.yml

@@ -1,56 +1,56 @@
 ---
 stages:
-  - build
+    - build
 
 include:
-  - local: .gitlab/common.yml
+    - local: .gitlab/common.yml
 
 build-foxx:
-  extends: .docker_build_script
-  stage: build
-  variables:
-    PROJECT: "datafed"
-    COMPONENT: "foxx"
-    GIT_STRATEGY: clone
-    DOCKER_FILE_PATH: "docker/Dockerfile.foxx"
-    DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
-    BUILD_INTERMEDIATE: "FALSE"
-  tags:
-    - docker
-  rules:
-    - changes:
-        - docker/**/*
-        - scripts/**/*
-        - cmake/**/*
-        - core/database/**/*
-        - core/CMakeLists.txt
-        - common/proto/**/*
-        - .gitlab-ci.yml
-        - .gitlab/**/*
-        - CMakeLists.txt
-      when: on_success
+    extends: .docker_build_script
+    stage: build
+    variables:
+        PROJECT: "datafed"
+        COMPONENT: "foxx"
+        GIT_STRATEGY: clone
+        DOCKER_FILE_PATH: "docker/Dockerfile.foxx"
+        DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+        BUILD_INTERMEDIATE: "FALSE"
+    tags:
+        - docker
+    rules:
+        - changes:
+              - docker/**/*
+              - scripts/**/*
+              - cmake/**/*
+              - core/database/**/*
+              - core/CMakeLists.txt
+              - common/proto/**/*
+              - .gitlab-ci.yml
+              - .gitlab/**/*
+              - CMakeLists.txt
+          when: on_success
 
 retag-image:
-  extends: .docker_retag_image
-  stage: build
-  variables:
-    PROJECT: "datafed"
-    COMPONENT: "foxx"
-    GIT_STRATEGY: clone
-    DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
-    BUILD_INTERMEDIATE: "FALSE"
-  tags:
-    - docker
-  rules:
-    - changes:
-        - docker/**/*
-        - scripts/**/*
-        - cmake/**/*
-        - core/database/**/*
-        - core/CMakeLists.txt
-        - common/proto/**/*
-        - .gitlab-ci.yml
-        - .gitlab/**/*
-        - CMakeLists.txt
-      when: never
-    - when: on_success
+    extends: .docker_retag_image
+    stage: build
+    variables:
+        PROJECT: "datafed"
+        COMPONENT: "foxx"
+        GIT_STRATEGY: clone
+        DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+        BUILD_INTERMEDIATE: "FALSE"
+    tags:
+        - docker
+    rules:
+        - changes:
+              - docker/**/*
+              - scripts/**/*
+              - cmake/**/*
+              - core/database/**/*
+              - core/CMakeLists.txt
+              - common/proto/**/*
+              - .gitlab-ci.yml
+              - .gitlab/**/*
+              - CMakeLists.txt
+          when: never
+        - when: on_success