diff --git a/.github/scripts/main/main.sh b/.github/scripts/main/main.sh index 1207e1a54bb..ba867508860 100644 --- a/.github/scripts/main/main.sh +++ b/.github/scripts/main/main.sh @@ -12,11 +12,10 @@ unset-dev-version () { export OCAMLRUNPARAM=b (set +x ; echo -en "::group::build opam\r") 2>/dev/null -if [[ "$OPAM_TEST" -eq 1 ]] || [[ "$OPAM_DOC" -eq 1 ]] ; then +if [[ "$OPAM_TEST" -eq 1 ]] || [[ "$OPAM_DOC" -eq 1 ]] || [[ "$OPAM_DEPENDS" -eq 1 ]] ; then export OPAMROOT=$OPAMBSROOT # If the cached root is newer, regenerate a binary compatible root opam env || { rm -rf $OPAMBSROOT; init-bootstrap; } - eval $(opam env) fi case "$1" in @@ -49,7 +48,7 @@ export PATH="$PREFIX/bin:$PATH" opam --version if [[ "$OPAM_DOC" -eq 1 ]]; then - make -C doc html man-html pages + opam exec -- make -C doc html man-html pages if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then . .github/scripts/common/hygiene-preamble.sh @@ -89,6 +88,49 @@ if [[ "$OPAM_DOC" -eq 1 ]]; then echo '::endgroup::checking generated files' fi +prepare_project () { + # warning, perform a cd + url=$1 + project=$2 + + (set +x; echo -en "::group::prepare-$project\r") 2>/dev/null + dir="$CACHE/$project" + if [ ! -d "$CACHE/$project" ]; then + git clone "$url" "$dir" + fi + cd "$dir" + git fetch origin + if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && git ls-remote --exit-code origin "$GITHUB_PR_USER/$BRANCH" ; then + BRANCH=$GITHUB_PR_USER/$BRANCH + fi + if git ls-remote --exit-code origin "$BRANCH"; then + PR_BRANCH=$BRANCH + elif [ "$GITHUB_EVENT_NAME" = pull_request ] && git ls-remote --exit-code origin "$GITHUB_BASE_REF"; then + PR_BRANCH=$GITHUB_BASE_REF + elif git ls-remote --exit-code origin main; then + PR_BRANCH=main + elif git ls-remote --exit-code origin master; then + PR_BRANCH=master + elif git ls-remote --exit-code origin trunk; then + PR_BRANCH=trunk + else + echo "No valid default branch found for $project on $url" + return 1 + fi + + # Checkout or create tracking branch + if git branch | grep -q "$PR_BRANCH"; then + git checkout "$PR_BRANCH" + git reset --hard "origin/$PR_BRANCH" + else + git checkout -b "$PR_BRANCH" "origin/$PR_BRANCH" + fi + + test -d _opam || opam switch create . --no-install --formula '"ocaml-system"' + opam pin "$GITHUB_WORKSPACE" -yn + (set +x ; echo -en "::endgroup::prepare-$project\r") 2>/dev/null +} + if [ "$OPAM_TEST" = "1" ]; then # test if an upgrade is needed rcode=0 @@ -116,35 +158,87 @@ if [ "$OPAM_TEST" = "1" ]; then # Compile and run opam-rt (set +x ; echo -en "::group::opam-rt\r") 2>/dev/null - opamrt_url="https://github.com/ocaml-opam/opam-rt" - if [ ! -d $CACHE/opam-rt ]; then - git clone $opamrt_url $CACHE/opam-rt - fi - cd $CACHE/opam-rt - git fetch origin - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && git ls-remote --exit-code origin "$GITHUB_PR_USER/$BRANCH" ; then - BRANCH=$GITHUB_PR_USER/$BRANCH - fi - if git ls-remote --exit-code origin "$BRANCH"; then - OPAM_RT_BRANCH=$BRANCH - elif [ "$GITHUB_EVENT_NAME" = pull_request ] && git ls-remote --exit-code origin "$GITHUB_BASE_REF"; then - OPAM_RT_BRANCH=$GITHUB_BASE_REF - else - OPAM_RT_BRANCH=master - fi - if git branch | grep -q "$OPAM_RT_BRANCH"; then - git checkout "$OPAM_RT_BRANCH" - git reset --hard "origin/$OPAM_RT_BRANCH" - else - git checkout -b "$OPAM_RT_BRANCH" "origin/$OPAM_RT_BRANCH" - fi + prepare_project "https://github.com/ocaml-opam/opam-rt" "opam-rt" - test -d _opam || opam switch create . --no-install --formula '"ocaml-system"' - eval $(opam env) - opam pin $GITHUB_WORKSPACE -yn --with-version to-test # opam lib pins defined in opam-rt are ignored as there is a local pin opam pin . -yn --ignore-pin-depends - opam install opam-rt --deps-only opam-devel.to-test - make || { opam reinstall opam-client -y; make; } + opam install opam-rt --deps-only opam-devel + opam exec -- make || { opam reinstall opam-client -y; opam exec -- make; } (set +x ; echo -en "::endgroup::opam-rt\r") 2>/dev/null fi + +test_project () { + url=$1 + project=$2 + + (set +x; echo -en "::group::depends-$project\r") 2>/dev/null + opam pin "$url" --kind git -yn + for pkg_name in $(opam show . -f name); do + echo "Installing dependencies for $pkg_name" + deps_code=0 + opam install "$pkg_name" --deps-only || deps_code=$? + if [ $deps_code -ne 0 ]; then + echo "Dependency installation failed for $pkg_name" + DEPENDS_ERRORS="$DEPENDS_ERRORS $pkg_name" + else + echo "Installing opam-client and $pkg_name" + opam install opam-client + code=0 + opam install "$pkg_name" || code=$? + if [ $code -ne 0 ]; then + LIB_ERRORS="$LIB_ERRORS $project" + echo -e "\e[31mErrors while installing $pkg_name\e[0m"; + fi + fi + done + (set +x ; echo -en "::endgroup::depends-$project\r") 2>/dev/null +} + +if [ "$OPAM_DEPENDS" = "1" ]; then + DEPENDS_ERRORS="" + LIB_ERRORS="" + OCAMLVER=$(ocamlc -version) + + (set +x; echo -en "::group::depends\r") 2>/dev/null + VERSION="2.4.1" + opam_libs=$(opam show . -f name 2>/dev/null) + depends_on=$(echo "$opam_libs" | sed "s/\$/.${VERSION}/" | paste -sd, -) + packages=$(opam list --or --depends-on "$depends_on" --columns name | tail -n +3) + set +x + for exclude in $opam_libs; do + packages=$(echo "$packages" | grep -vF "$exclude") + done + set -x + + for pkg in $packages; do + dev_repo=$(opam show "$pkg" -f dev-repo 2>/dev/null) + dev_repo=$(echo "$dev_repo" | sed -E 's/^"//;s/"$//;s/^git\+//;s/\.git$//') + + if [[ -n "$dev_repo" ]]; then + prepare_project "$dev_repo" "$pkg" + test_project "$dev_repo" "$pkg" + fi + done + + if [ -n "$DEPENDS_ERRORS" ]; then + echo -e "\e[31mErrors detected in dependencies of $DEPENDS_ERRORS\e[0m"; + fi + + if [ -n "$LIB_ERRORS" ]; then + FAIL=() + set +x + for critical in $FAIL_IF_DEPENDENT; do + if echo "$LIB_ERRORS" | grep -Fq "$critical"; then + FAIL+=("$critical") + fi + done + set -x + echo "Packages tested: $packages" + if [ -n "${FAIL[*]}" ]; then + echo "::error ::${FAIL[*]} is broken" + exit 1 + fi + fi + + (set +x ; echo -en "::endgroup::depends\r") 2>/dev/null +fi diff --git a/.github/scripts/main/preamble.sh b/.github/scripts/main/preamble.sh index ba09f1ca12b..4d3b9062bf9 100644 --- a/.github/scripts/main/preamble.sh +++ b/.github/scripts/main/preamble.sh @@ -19,6 +19,7 @@ PATH=$OPAM_LOCAL/bin:$OCAML_LOCAL/bin:$PATH; export PATH OPAM_COLD=${OPAM_COLD:-0} OPAM_TEST=${OPAM_TEST:-0} OPAM_DOC=${OPAM_DOC:-0} +OPAM_DEPENDS=${OPAM_DEPENDS:-0} OPAM_UPGRADE=${OPAM_UPGRADE:-0} OPAM_REPO_MAIN=https://github.com/ocaml/opam-repository.git @@ -42,15 +43,22 @@ else OPAM_REPO_CACHE=$OPAM_REPO_MAIN fi -# used only for TEST jobs +# used only for TEST and DOC jobs init-bootstrap () { - if [ "$OPAM_TEST" = "1" ] || [ "$OPAM_DOC" = "1" ] || [ -n "$SOLVER" ]; then + if [ "$OPAM_TEST" = "1" ] || [ "$OPAM_DOC" = "1" ] || [ "$OPAM_DEPENDS" = "1" ] || [ -n "$SOLVER" ]; then export OPAMROOT=$OPAMBSROOT # The system compiler will be picked up + + if [ "$OPAM_DEPENDS" = "1" ]; then + REPO_SHA=$OPAM_TEST_REPO_SHA + else + REPO_SHA=$OPAM_REPO_SHA + fi + if [ "${OPAM_REPO%.git}" != "${OPAM_REPO_MAIN%.git}" ]; then - opam init --no-setup git+$OPAM_REPO_MAIN#$OPAM_REPO_SHA + opam init --no-setup git+$OPAM_REPO_MAIN#$REPO_SHA else - opam init --no-setup git+$OPAM_REPO_CACHE#$OPAM_REPO_SHA + opam init --no-setup git+$OPAM_REPO_CACHE#$REPO_SHA fi cat >> $OPAMROOT/config < upgrade_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ~section:"Upgrade from 1.2 to current" Linux @@ fun _ -> upgrade_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job MacOS @@ fun _ -> hygiene_job ~analyse_job (Specific (Linux, "22.04")) + @@ fun _ -> depends_job ~analyse_job ~build_linux_job Linux @@ fun _ -> end_workflow let () = diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 10605798826..f4548b4f850 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ env: OPAMBSROOT: ~/.cache/.opam.cached OPAM12CACHE: ~/.cache/opam1.2/cache OPAM_REPO: https://github.com/ocaml/opam-repository.git - OPAM_TEST_REPO_SHA: e9ce8525130a382fac004612302b2f2268f4188c + OPAM_TEST_REPO_SHA: 3dab8c734b15bf2b5c1d8b99bb134f51361a6bee OPAM_REPO_SHA: e9ce8525130a382fac004612302b2f2268f4188c SOLVER: CYGWIN_MIRROR: http://mirrors.kernel.org/sourceware/cygwin/ @@ -657,12 +657,24 @@ jobs: runs-on: ubuntu-22.04 needs: Analyse steps: - - name: Install system's dune and ocaml packages - run: sudo apt install dune ocaml - name: Checkout tree uses: actions/checkout@v5 + - name: Get changed files + id: files + if: github.event_name == 'pull_request' + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + configure.ac + shell/install.sh + src_ext/* + .github/workflows + - name: Install system's dune and ocaml packages + if: steps.files.outputs.all != '' + run: sudo apt install dune ocaml - name: src_ext/archives and opam-repository Cache id: archives + if: steps.files.outputs.all != '' uses: actions/cache@v4 with: path: | @@ -670,27 +682,75 @@ jobs: ~/opam-repository key: ${{ needs.Analyse.outputs.archives }} enableCrossOsArchive: true + - name: Hygiene + env: + BASE_REF_SHA: ${{ github.event.pull_request.base.sha }} + PR_REF_SHA: ${{ github.event.pull_request.head.sha }} + if: steps.files.outputs.all != '' + run: bash -exu .github/scripts/main/hygiene.sh + + Depends-Linux: + runs-on: ubuntu-latest + needs: [ Analyse, Build-Linux ] + strategy: + matrix: + ocamlv: [ 4.14.2, 5.3.0 ] + fail-fast: false + env: + OPAM_DEPENDS: 1 + steps: + - name: Checkout tree + uses: actions/checkout@v5 - name: Get changed files id: files if: github.event_name == 'pull_request' uses: Ana06/get-changed-files@v2.3.0 - - name: Changed files list - run: | - for changed_file in ${{ steps.files.outputs.modified }}; do - echo "M ${changed_file}." - done - for changed_file in ${{ steps.files.outputs.removed }}; do - echo "D ${changed_file}." - done - for changed_file in ${{ steps.files.outputs.added }}; do - echo "A ${changed_file}." - done - for changed_file in ${{ steps.files.outputs.renamed }}; do - echo "AD ${changed_file}." - done - - name: Hygiene + with: + filter: '*.mli' + - name: src_ext/archives and opam-repository Cache + id: archives + if: steps.files.outputs.all != '' + uses: actions/cache@v4 + with: + path: | + src_ext/archives + ~/opam-repository + key: ${{ needs.Analyse.outputs.archives }} + enableCrossOsArchive: true + - name: Install bubblewrap + if: steps.files.outputs.all != '' + run: sudo apt install bubblewrap + - name: Disable AppArmor + if: steps.files.outputs.all != '' + run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns + - name: OCaml ${{ matrix.ocamlv }} Cache + id: ocaml-cache + if: steps.files.outputs.all != '' + uses: actions/cache@v4 + with: + path: ~/.cache/ocaml-local/** + key: ${{ runner.os }}-ocaml-${{ matrix.ocamlv }}-${{ needs.Analyse.outputs.ocaml-cache }} + - name: Create OCaml ${{ matrix.ocamlv }} cache + if: steps.files.outputs.all != '' && steps.ocaml-cache.outputs.cache-hit != 'true' + run: bash -exu .github/scripts/main/ocaml-cache.sh ${{ runner.os }} ${{ matrix.ocamlv }} + - name: opam bootstrap Cache + id: opam-bootstrap + if: steps.files.outputs.all != '' + uses: actions/cache@v4 + with: + path: | + ${{ env.OPAMBSROOT }}/** + ~/.cache/opam-local/bin/** + key: opamdepends-${{ runner.os }}-${{ env.OPAMBSVERSION }}-${{ matrix.ocamlv }}-${{ env.OPAM_REPO_SHA }}-${{ needs.Analyse.outputs.opam-bs-cache }} + - name: Create opam bootstrap cache + if: steps.files.outputs.all != '' && steps.opam-bootstrap.outputs.cache-hit != 'true' + run: bash -exu .github/scripts/main/opam-bs-cache.sh + - name: Compile env: BASE_REF_SHA: ${{ github.event.pull_request.base.sha }} PR_REF_SHA: ${{ github.event.pull_request.head.sha }} - if: contains(steps.files.outputs.modified, 'configure.ac') || contains(steps.files.outputs.modified, 'shell/install.sh') || contains(steps.files.outputs.all, 'src_ext') || contains(steps.files.outputs.all, '.github/workflows') - run: bash -exu .github/scripts/main/hygiene.sh + GITHUB_PR_USER: ${{ github.event.pull_request.user.login }} + JOB_URL: ${{ steps.get-job-id.outputs.job_url }} + FAIL_IF_DEPENDENT: opam-publish opam-rt opam-build opam-test + if: steps.files.outputs.all != '' + run: bash -exu .github/scripts/main/main.sh x86_64-pc-linux-gnu diff --git a/master_changes.md b/master_changes.md index be366568dff..2323455a97b 100644 --- a/master_changes.md +++ b/master_changes.md @@ -126,6 +126,8 @@ users) * Fix the nixos depexts tests (git is now already installed in the nix docker image) [#6652 @kit-ty-kate] * Ensure every part of the scripts are run with `set -ue` [#6648 @kit-ty-kate] * Only run the `get-changed-files` action when in a PR [#6582 @kit-ty-kate] + * Add a CI job to test reverse dependencies of opam. Track and report dependency and build failures, hard-failing only on maintained packages. [#6394 @rjbou @arozovyk] + * Enhance changed files job dependant handling [#6394 @rjbou] ## Doc * Update the installation documentation with the release of opam 2.4.1 [#6620 @kit-ty-kate] @@ -134,8 +136,8 @@ users) * Fix URL to Software Heritage [#6650 @gahr] * Clarify conditions in subsection titles in the Packaging page [#6653 @jmid] * Upgrade the deprecated md5 `checksum` example to sha256 [#6653 @jmid] - -* Add mention of `opam admin compare-versions` in the Manual. [#6596 @mbarbin] + * Add mention of `opam admin compare-versions` in the Manual. [#6596 @mbarbin] + * Update release documentation to add a step updating test repository hash and version number in reverse dependecies test script [#6364 @arozovyk] ## Security fixes diff --git a/release/readme.md b/release/readme.md index dbfb3ed10b7..2d6f4138d4d 100644 --- a/release/readme.md +++ b/release/readme.md @@ -37,6 +37,7 @@ * finalise the release (publish) * add hashes in `install.sh` and `install.ps1` (and check signatures) +* update `OPAM_TEST_REPO_SHA` in `ci.ml` and update version in `main.sh` for dependencies job * bring the changes to the changelog (CHANGES) from the branch of the release to the `master` branch * Update doc/pages/Install.md * publish opam packages in opam-repository (use `opam publish --pre-release` if this is not a stable version)