HBASE-29829 Pagination support for Hide Old Comments in PRs

Author: ndimiduk

dev-support/gh_hide_old_comments.sh

@@ -40,25 +40,79 @@ declare CURL="${CURL:-curl}"
 function fetch_comments {
     local pr="$1"
     local comments_file
+    local page_file
+    local headers_file
     local -a curl_args
-    curl_args=(
-        --fail
-        "${GITHUB_AUTH[@]}"
-        --header 'Accept: application/vnd.github+json'
-        --header 'X-GitHub-Api-Version: 2022-11-28'
-        --request GET
-        --url "${GITHUB_API_URL}/repos/${REPO}/issues/${pr}/comments?per_page=500"
-    )
-    if [ "${DEBUG}" = true ] ; then
-        curl_args+=(--verbose)
-    else
-        curl_args+=(--silent)
-    fi
+    local page=1
+    local next_url
 
     comments_file="$(mktemp "comments_${pr}" 2>/dev/null || mktemp -t "comments_${pr}.XXXXXXXXXX")" || \
         { >&2 echo 'cannot create temp file';  exit 1 ;}
-    "${CURL}" "${curl_args[@]}" > "${comments_file}"
+    page_file="$(mktemp "page_${pr}" 2>/dev/null || mktemp -t "page_${pr}.XXXXXXXXXX")" || \
+        { >&2 echo 'cannot create temp file';  exit 1 ;}
+    headers_file="$(mktemp "headers_${pr}" 2>/dev/null || mktemp -t "headers_${pr}.XXXXXXXXXX")" || \
+        { >&2 echo 'cannot create temp file';  exit 1 ;}
+
+    # cleanup temp files on error
+    trap 'rm -f "${page_file}" "${headers_file}"; exit 1' ERR
+
+    next_url="${GITHUB_API_URL}/repos/${REPO}/issues/${pr}/comments?per_page=100"
+
+    # start with empty JSON array
+    echo '[]' > "${comments_file}"
+
+    while [ -n "${next_url}" ] ; do
+        curl_args=(
+            --fail
+            --max-time 30
+            "${GITHUB_AUTH[@]}"
+            --header 'Accept: application/vnd.github+json'
+            --header 'X-GitHub-Api-Version: 2022-11-28'
+            --dump-header "${headers_file}"
+            --request GET
+            --url "${next_url}"
+        )
+        if [ "${DEBUG}" = true ] ; then
+            curl_args+=(--verbose)
+            >&2 echo "Fetching page ${page}: ${next_url}"
+        else
+            curl_args+=(--silent)
+        fi
+
+        if ! "${CURL}" "${curl_args[@]}" > "${page_file}"; then
+            >&2 echo "Failed to fetch page ${page}: ${next_url}"
+            rm -f "${page_file}" "${headers_file}"
+            exit 1
+        fi
+
+        if [ "${DEBUG}" = 'true' ] ; then
+            >&2 echo "Page ${page} returned $(jq length "${page_file}") comments"
+        fi
+
+        # merge this page into the accumulated results
+        if ! jq -s '.[0] + .[1]' "${comments_file}" "${page_file}" > "${comments_file}.tmp"; then
+            >&2 echo "Failed to merge comments from page ${page}"
+            rm -f "${page_file}" "${headers_file}" "${comments_file}.tmp"
+            exit 1
+        fi
+        mv "${comments_file}.tmp" "${comments_file}"
+
+        # check for next page in Link header
+        # Link header format: <url>; rel="next", <url>; rel="last"
+        # Extract URL associated with rel="next" regardless of position
+        next_url=""
+        if grep -qi '^link:' "${headers_file}" ; then
+            next_url=$(grep -i '^link:' "${headers_file}" | tr ',' '\n' | grep 'rel="next"' | sed -n 's/.*<\([^>]*\)>.*/\1/p' || true)
+        fi
+
+        page=$((page + 1))
+    done
+
+    rm -f "${page_file}" "${headers_file}"
+    trap - ERR
+
     if [ "${DEBUG}" = 'true' ] ; then
+        >&2 echo "Total comments fetched: $(jq length "${comments_file}")"
         >&2 cat "${comments_file}"
     fi
     echo "${comments_file}"