Gha dependencies

.github/scripts/main/main.sh

@@ -12,11 +12,10 @@ unset-dev-version () {
 export OCAMLRUNPARAM=b
 
 (set +x ; echo -en "::group::build opam\r") 2>/dev/null
-if [[ "$OPAM_TEST" -eq 1 ]] || [[ "$OPAM_DOC" -eq 1 ]] ; then
+if [[ "$OPAM_TEST" -eq 1 ]] || [[ "$OPAM_DOC" -eq 1 ]] || [[ "$OPAM_DEPENDS" -eq 1 ]] ; then
   export OPAMROOT=$OPAMBSROOT
   # If the cached root is newer, regenerate a binary compatible root
   opam env || { rm -rf $OPAMBSROOT; init-bootstrap; }
-  eval $(opam env)
 fi
 
 case "$1" in
@@ -49,7 +48,7 @@ export PATH="$PREFIX/bin:$PATH"
 opam --version
 
 if [[ "$OPAM_DOC" -eq 1 ]]; then
-  make -C doc html man-html pages
+  opam exec -- make -C doc html man-html pages
 
   if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
     . .github/scripts/common/hygiene-preamble.sh
@@ -89,6 +88,49 @@ if [[ "$OPAM_DOC" -eq 1 ]]; then
   echo '::endgroup::checking generated files'
 fi
 
+prepare_project () {
+  # warning, perform a cd
+  url=$1
+  project=$2
+
+  (set +x; echo -en "::group::prepare-$project\r") 2>/dev/null
+  dir="$CACHE/$project"
+  if [ ! -d "$CACHE/$project" ]; then
+    git clone "$url" "$dir"
+  fi
+  cd "$dir"
+  git fetch origin
+  if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && git ls-remote --exit-code origin "$GITHUB_PR_USER/$BRANCH" ; then
+    BRANCH=$GITHUB_PR_USER/$BRANCH
+  fi
+  if git ls-remote --exit-code origin "$BRANCH"; then
+    PR_BRANCH=$BRANCH
+  elif [ "$GITHUB_EVENT_NAME" = pull_request ] && git ls-remote --exit-code origin "$GITHUB_BASE_REF"; then
+    PR_BRANCH=$GITHUB_BASE_REF
+  elif git ls-remote --exit-code origin main; then
+    PR_BRANCH=main
+  elif git ls-remote --exit-code origin master; then
+    PR_BRANCH=master
+  elif git ls-remote --exit-code origin trunk; then
+    PR_BRANCH=trunk
+  else
+    echo "No valid default branch found for $project on $url"
+    return 1
+  fi
+
+  # Checkout or create tracking branch
+  if git branch | grep -q "$PR_BRANCH"; then
+    git checkout "$PR_BRANCH"
+    git reset --hard "origin/$PR_BRANCH"
+  else
+    git checkout -b "$PR_BRANCH" "origin/$PR_BRANCH"
+  fi
+
+  test -d _opam || opam switch create . --no-install --formula '"ocaml-system"'
+  opam pin "$GITHUB_WORKSPACE" -yn
+  (set +x ; echo -en "::endgroup::prepare-$project\r") 2>/dev/null
+}
+
 if [ "$OPAM_TEST" = "1" ]; then
   # test if an upgrade is needed
   rcode=0
@@ -116,35 +158,87 @@ if [ "$OPAM_TEST" = "1" ]; then
 
   # Compile and run opam-rt
   (set +x ; echo -en "::group::opam-rt\r") 2>/dev/null
-  opamrt_url="https://github.com/ocaml-opam/opam-rt"
-  if [ ! -d $CACHE/opam-rt ]; then
-    git clone $opamrt_url  $CACHE/opam-rt
-  fi
-  cd $CACHE/opam-rt
-  git fetch origin
-  if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && git ls-remote --exit-code origin "$GITHUB_PR_USER/$BRANCH" ; then
-    BRANCH=$GITHUB_PR_USER/$BRANCH
-  fi
-  if git ls-remote --exit-code origin "$BRANCH"; then
-    OPAM_RT_BRANCH=$BRANCH
-  elif [ "$GITHUB_EVENT_NAME" = pull_request ] && git ls-remote --exit-code origin "$GITHUB_BASE_REF"; then
-    OPAM_RT_BRANCH=$GITHUB_BASE_REF
-  else
-    OPAM_RT_BRANCH=master
-  fi
-  if git branch | grep -q "$OPAM_RT_BRANCH"; then
-    git checkout "$OPAM_RT_BRANCH"
-    git reset --hard "origin/$OPAM_RT_BRANCH"
-  else
-    git checkout -b "$OPAM_RT_BRANCH" "origin/$OPAM_RT_BRANCH"
-  fi
+  prepare_project "https://github.com/ocaml-opam/opam-rt" "opam-rt"
 
-  test -d _opam || opam switch create . --no-install --formula '"ocaml-system"'
-  eval $(opam env)
-  opam pin $GITHUB_WORKSPACE -yn --with-version to-test
   # opam lib pins defined in opam-rt are ignored as there is a local pin
   opam pin . -yn --ignore-pin-depends
-  opam install opam-rt --deps-only opam-devel.to-test
-  make || { opam reinstall opam-client -y; make; }
+  opam install opam-rt --deps-only opam-devel
+  opam exec -- make || { opam reinstall opam-client -y; opam exec -- make; }
   (set +x ; echo -en "::endgroup::opam-rt\r") 2>/dev/null
 fi
+
+test_project () {
+  url=$1
+  project=$2
+
+  (set +x; echo -en "::group::depends-$project\r") 2>/dev/null
+  opam pin "$url" --kind git -yn
+  for pkg_name in $(opam show . -f name); do
+    echo "Installing dependencies for $pkg_name"
+    deps_code=0
+    opam install "$pkg_name" --deps-only || deps_code=$?
+    if [ $deps_code -ne 0 ]; then
+      echo "Dependency installation failed for $pkg_name"
+      DEPENDS_ERRORS="$DEPENDS_ERRORS $pkg_name"
+    else
+      echo "Installing opam-client and $pkg_name"
+      opam install opam-client
+      code=0
+      opam install "$pkg_name" || code=$?
+      if [ $code -ne 0 ]; then
+        LIB_ERRORS="$LIB_ERRORS $project"
+        echo -e "\e[31mErrors while installing $pkg_name\e[0m";
+      fi
+    fi
+  done
+  (set +x ; echo -en "::endgroup::depends-$project\r") 2>/dev/null
+}
+
+if [ "$OPAM_DEPENDS" = "1" ]; then
+  DEPENDS_ERRORS=""
+  LIB_ERRORS=""
+  OCAMLVER=$(ocamlc -version)
+
+  (set +x; echo -en "::group::depends\r") 2>/dev/null
+  VERSION="2.4.1"
+  opam_libs=$(opam show . -f name 2>/dev/null)
+  depends_on=$(echo "$opam_libs" | sed "s/\$/.${VERSION}/" | paste -sd, -)
+  packages=$(opam list --or --depends-on "$depends_on" --columns name | tail -n +3)
+  set +x
+  for exclude in $opam_libs; do
+    packages=$(echo "$packages" | grep -vF "$exclude")
+  done
+  set -x
+
+  for pkg in $packages; do
+    dev_repo=$(opam show "$pkg" -f dev-repo 2>/dev/null)
+    dev_repo=$(echo "$dev_repo" | sed -E 's/^"//;s/"$//;s/^git\+//;s/\.git$//')
+
+    if [[ -n "$dev_repo" ]]; then
+      prepare_project "$dev_repo" "$pkg"
+      test_project "$dev_repo" "$pkg"
+    fi
+  done
+
+  if [ -n "$DEPENDS_ERRORS" ]; then
+    echo -e "\e[31mErrors detected in dependencies of $DEPENDS_ERRORS\e[0m";
+  fi
+
+  if [ -n "$LIB_ERRORS" ]; then
+    FAIL=()
+    set +x
+    for critical in $FAIL_IF_DEPENDENT; do
+      if echo "$LIB_ERRORS" | grep -Fq "$critical"; then
+        FAIL+=("$critical")
+      fi
+    done
+    set -x
+    echo "Packages tested: $packages"
+    if [ -n "${FAIL[*]}" ]; then
+      echo "::error ::${FAIL[*]} is broken"
+      exit 1
+    fi
+  fi
+
+  (set +x ; echo -en "::endgroup::depends\r") 2>/dev/null
+fi

.github/scripts/main/preamble.sh

@@ -19,6 +19,7 @@ PATH=$OPAM_LOCAL/bin:$OCAML_LOCAL/bin:$PATH; export PATH
 OPAM_COLD=${OPAM_COLD:-0}
 OPAM_TEST=${OPAM_TEST:-0}
 OPAM_DOC=${OPAM_DOC:-0}
+OPAM_DEPENDS=${OPAM_DEPENDS:-0}
 OPAM_UPGRADE=${OPAM_UPGRADE:-0}
 
 OPAM_REPO_MAIN=https://github.com/ocaml/opam-repository.git
@@ -42,15 +43,22 @@ else
   OPAM_REPO_CACHE=$OPAM_REPO_MAIN
 fi
 
-# used only for TEST jobs
+# used only for TEST and DOC jobs
 init-bootstrap () {
-  if [ "$OPAM_TEST" = "1" ] || [ "$OPAM_DOC" = "1" ] || [ -n "$SOLVER" ]; then
+  if [ "$OPAM_TEST" = "1" ] || [ "$OPAM_DOC" = "1" ] || [ "$OPAM_DEPENDS" = "1" ] || [ -n "$SOLVER" ]; then
     export OPAMROOT=$OPAMBSROOT
     # The system compiler will be picked up
+
+    if [ "$OPAM_DEPENDS" = "1" ]; then
+      REPO_SHA=$OPAM_TEST_REPO_SHA
+    else
+      REPO_SHA=$OPAM_REPO_SHA
+    fi
+
     if [ "${OPAM_REPO%.git}" != "${OPAM_REPO_MAIN%.git}" ]; then
-      opam init --no-setup git+$OPAM_REPO_MAIN#$OPAM_REPO_SHA
+      opam init --no-setup git+$OPAM_REPO_MAIN#$REPO_SHA
     else
-      opam init --no-setup git+$OPAM_REPO_CACHE#$OPAM_REPO_SHA
+      opam init --no-setup git+$OPAM_REPO_CACHE#$REPO_SHA
     fi
 
     cat >> $OPAMROOT/config <<EOF

.github/workflows/ci.ml

@@ -233,6 +233,19 @@ let install_sys_packages packages ~descr ?cond platforms =
 let install_sys_opam ?cond = install_sys_packages ["opam"] ~descr:"Install system's opam package" ?cond
 let install_sys_dune ?cond = install_sys_packages ["dune"; "ocaml"] ~descr:"Install system's dune and ocaml packages" ?cond
 
+(* The clean way to do it is to have the changed file in their proper job and
+   condition the other job to its output, but it will lead to extra steps
+   (checkout). The choice here is to add for all further steps the condition
+   on the output of 'files' step (modified files). *)
+let changed_files ?withs () =
+  uses "Get changed files" ~id:"files"
+    ~cond:(Predicate(true, Compare("github.event_name", "pull_request")))
+    ?withs
+    (* ~continue_on_error:true see https://github.com/jitterbit/get-changed-files/issues/19 *)
+    "Ana06/get-changed-files@v2.3.0" (* see https://github.com/jitterbit/get-changed-files/issues/55 ; Ana06'fork contains #19 and #55 fixes *)
+
+(** Jobs *)
+
 let analyse_job ~oc ~workflow ~platforms ~keys f =
   let oses = List.map os_of_platform platforms in
   let outputs =
@@ -470,34 +483,48 @@ let upgrade_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_jo
     ++ run "Test (upgrade)" ["bash -exu .github/scripts/main/upgrade.sh"]
     ++ end_job f
 
+let depends_job ~analyse_job ~build_linux_job ?section runner ~oc ~workflow f =
+  let platform = os_of_platform runner in
+  let host = host_of_platform platform in
+  let only_on target = only_on platform target in
+  let needs = [analyse_job; build_linux_job ] in
+  let env = [("OPAM_DEPENDS", "1")] in
+  let matrix = platform_ocaml_matrix ~fail_fast:false start_latests_ocaml in
+  let ocamlv = "${{ matrix.ocamlv }}" in
+  let fail_if_dependent = ["opam-publish"; "opam-rt"; "opam-build"; "opam-test"] in
+  let cond = Predicate(false, Compare("steps.files.outputs.all", "")) in
+  job ~oc ~workflow ?section ~runs_on:(Runner [platform]) ~env ~needs ~matrix
+    ("Depends-" ^ name_of_platform platform)
+  ++ checkout ()
+  ++ changed_files ~withs:[ "filter", Literal ["'*.mli'"] ] ()
+  ++ cache ~cond Archives
+  ++ only_on Linux (run ~cond "Install bubblewrap" ["sudo apt install bubblewrap"])
+  ++ only_on Linux (run ~cond "Disable AppArmor" ["echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns"])
+  ++ cache ~cond OCaml platform ocamlv host
+  ++ build_cache ~cond OCaml platform ocamlv host
+  ++ cache ~cond OpamBS ocamlv "depends"
+  ++ build_cache ~cond OpamBS ocamlv "depends"
+  ++ run ~cond "Compile" ~env:[("BASE_REF_SHA", "${{ github.event.pull_request.base.sha }}");
+                               ("PR_REF_SHA", "${{ github.event.pull_request.head.sha }}");
+                               ("GITHUB_PR_USER", "${{ github.event.pull_request.user.login }}");
+                               ("JOB_URL", "${{ steps.get-job-id.outputs.job_url }}");
+                               ("FAIL_IF_DEPENDENT", String.concat " " fail_if_dependent)]
+    ["bash -exu .github/scripts/main/main.sh " ^ host]
+  ++ end_job f
+
 let hygiene_job (type a) ~analyse_job (platform : a platform) ~oc ~workflow f =
+  let cond = Predicate(false, Compare("steps.files.outputs.all", "")) in
+  let withs = [ "filter", Literal [ "configure.ac"; "shell/install.sh"; "src_ext/*"; ".github/workflows"] ] in
   job ~oc ~workflow ~section:"Around opam tests" ~runs_on:(Runner [platform]) ~needs:[analyse_job] "Hygiene"
-    ++ install_sys_dune [os_of_platform platform]
-    ++ checkout ()
-    ++ cache Archives
-    ++ uses "Get changed files" ~id:"files" ~cond:(Predicate(true, Compare("github.event_name", "pull_request"))) (* ~continue_on_error:true see https://github.com/jitterbit/get-changed-files/issues/19 *) "Ana06/get-changed-files@v2.3.0" (* see https://github.com/jitterbit/get-changed-files/issues/55 ; Ana06'fork contains #19 and #55 fixes *)
-    ++ run "Changed files list" [
-         "for changed_file in ${{ steps.files.outputs.modified }}; do";
-         "  echo \"M  ${changed_file}.\"";
-         "done";
-         "for changed_file in ${{ steps.files.outputs.removed }}; do";
-         "  echo \"D  ${changed_file}.\"";
-         "done";
-         "for changed_file in ${{ steps.files.outputs.added }}; do";
-         "  echo \"A  ${changed_file}.\"";
-         "done";
-         "for changed_file in ${{ steps.files.outputs.renamed }}; do";
-         "  echo \"AD ${changed_file}.\"";
-         "done";
-       ]
-    ++ run "Hygiene" ~cond:(Or[Predicate(true, Contains("steps.files.outputs.modified", "configure.ac"));
-                               Predicate(true, Contains("steps.files.outputs.modified", "shell/install.sh"));
-                               Predicate(true, Contains("steps.files.outputs.all", "src_ext"));
-                               Predicate(true, Contains("steps.files.outputs.all", ".github/workflows"))])
-                     ~env:[("BASE_REF_SHA", "${{ github.event.pull_request.base.sha }}");
-                           ("PR_REF_SHA", "${{ github.event.pull_request.head.sha }}")]
-                     ["bash -exu .github/scripts/main/hygiene.sh"]
-    ++ end_job f
+  ++ checkout ()
+  ++ changed_files ~withs ()
+  ++ install_sys_dune ~cond [os_of_platform platform]
+  ++ cache ~cond Archives
+  ++ run "Hygiene" ~cond
+    ~env:[("BASE_REF_SHA", "${{ github.event.pull_request.base.sha }}");
+          ("PR_REF_SHA", "${{ github.event.pull_request.head.sha }}")]
+    ["bash -exu .github/scripts/main/hygiene.sh"]
+  ++ end_job f
 
 let empty_job ~oc ~workflow f
 (*    ~analyse_job:_ *)
@@ -517,7 +544,7 @@ let main oc : unit =
     ("OPAM12CACHE", "~/.cache/opam1.2/cache");
     (* These should be identical to the values in appveyor.yml *)
     ("OPAM_REPO", "https://github.com/ocaml/opam-repository.git");
-    ("OPAM_TEST_REPO_SHA", "e9ce8525130a382fac004612302b2f2268f4188c");
+    ("OPAM_TEST_REPO_SHA", "3dab8c734b15bf2b5c1d8b99bb134f51361a6bee");
     ("OPAM_REPO_SHA", "e9ce8525130a382fac004612302b2f2268f4188c");
     ("SOLVER", "");
     (* Cygwin configuration *)
@@ -547,6 +574,7 @@ let main oc : unit =
   @@ fun _ -> upgrade_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ~section:"Upgrade from 1.2 to current" Linux
   @@ fun _ -> upgrade_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job MacOS
   @@ fun _ -> hygiene_job ~analyse_job (Specific (Linux, "22.04"))
+  @@ fun _ -> depends_job ~analyse_job ~build_linux_job Linux
   @@ fun _ -> end_workflow
 
 let () =