Skip to content

Detect a node that dies when nothing else is left to report it #867

Detect a node that dies when nothing else is left to report it

Detect a node that dies when nothing else is left to report it #867

Workflow file for this run

name: Quality
on:
pull_request:
branches: [main]
push:
branches: [main]
concurrency:
group: quality-${{ github.head_ref || github.sha }}
cancel-in-progress: true
permissions:
contents: read
jobs:
format-lint:
runs-on: ubuntu-latest
container:
image: ubuntu:noble
timeout-minutes: 45
defaults:
run:
shell: bash
steps:
- name: Install Git
run: |
apt-get update
apt-get install -y git
- name: Checkout repository
uses: actions/checkout@v4
# Source-tree half of the coverage-scope gate. Reads only CMakeLists.txt,
# needs neither ROS nor a build, so it sits directly after checkout: a
# package that compiles C++ without opting into coverage instrumentation
# fails here in under a second instead of at the end of the coverage job.
- name: Every C++ package opts into coverage instrumentation
run: ./scripts/check_coverage_packages.sh --static-only
- name: Pre-install ROS 2 apt source
uses: ./.github/actions/ros-apt-source
- name: Set up ROS 2 Jazzy
uses: ros-tooling/setup-ros@v0.7
with:
required-ros-distributions: jazzy
- name: Install dependencies
run: |
apt-get update
apt-get install -y clang-format ccache
source /opt/ros/jazzy/setup.bash
rosdep update
rosdep install --from-paths src --ignore-src -y
- name: Cache ccache
uses: actions/cache@v4
with:
path: /root/.cache/ccache
key: ccache-jazzy-lint-${{ github.sha }}
restore-keys: |
ccache-jazzy-lint-
- name: Build (minimal, for ament test infrastructure)
env:
CCACHE_DIR: /root/.cache/ccache
CCACHE_MAXSIZE: 500M
CCACHE_SLOPPINESS: pch_defines,time_macros
run: |
source /opt/ros/jazzy/setup.bash
ccache -z
colcon build --symlink-install \
--cmake-args -DCMAKE_BUILD_TYPE=Release \
--event-handlers console_direct+
ccache -s
./scripts/ccache_report.sh jazzy-lint
- name: Run format linters
run: |
source /opt/ros/jazzy/setup.bash
source install/setup.bash
colcon test --return-code-on-test-failure \
--ctest-args -L linter -E clang_tidy \
--event-handlers console_direct+
- name: No naked rclcpp subscription APIs (issue #375 regression gate)
run: ./scripts/check_no_naked_subscriptions.sh
- name: Handlers read query params via typed query<T>() (zero-query-params regression gate)
run: ./src/ros2_medkit_gateway/scripts/check_handlers_typed_query.sh
- name: ccache report warns on the right states
run: ./scripts/test_ccache_report.sh
- name: Stale result sweep removes everything colcon would count
run: ./scripts/test_drop_stale_results.sh
- name: Show results
if: always()
run: colcon test-result --verbose
clang-tidy:
runs-on: ubuntu-latest
container:
image: ubuntu:noble
timeout-minutes: 90
defaults:
run:
shell: bash
steps:
- name: Install Git
run: |
apt-get update
apt-get install -y git
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Fix git safe directory
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Pre-install ROS 2 apt source
uses: ./.github/actions/ros-apt-source
- name: Set up ROS 2 Jazzy
uses: ros-tooling/setup-ros@v0.7
with:
required-ros-distributions: jazzy
- name: Install ccache and clang-tidy
run: apt-get install -y ccache clang-tidy
- name: Cache ccache
uses: actions/cache@v4
with:
path: /root/.cache/ccache
key: ccache-jazzy-tidy-${{ github.sha }}
restore-keys: |
ccache-jazzy-tidy-
- name: Install clang-tidy-cache (ctcache)
run: |
apt-get install -y python3-pip
pip3 install --break-system-packages git+https://github.com/matus-chochlik/ctcache.git@3880541496c7ce0d2e2830d2f525375078fe0b1f
printf '#!/bin/sh\nexec clang-tidy-cache /usr/bin/clang-tidy "$@"\n' > /usr/local/bin/ctcache-wrapper
chmod +x /usr/local/bin/ctcache-wrapper
- name: Cache ctcache results
uses: actions/cache@v4
with:
path: /root/.ctcache
key: ctcache-jazzy-${{ hashFiles('src/ros2_medkit_cmake/cmake/.clang-tidy') }}-${{ github.sha }}
restore-keys: |
ctcache-jazzy-${{ hashFiles('src/ros2_medkit_cmake/cmake/.clang-tidy') }}-
- name: Install dependencies
run: |
apt-get update
apt-get install -y ros-jazzy-test-msgs
source /opt/ros/jazzy/setup.bash
rosdep update
rosdep install --from-paths src --ignore-src -y
- name: Build (compile database only, no clang-tidy)
env:
CCACHE_DIR: /root/.cache/ccache
CCACHE_MAXSIZE: 500M
CCACHE_SLOPPINESS: pch_defines,time_macros
run: |
source /opt/ros/jazzy/setup.bash
ccache -z
colcon build --symlink-install \
--cmake-args -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DENABLE_CLANG_TIDY=OFF \
--event-handlers console_direct+
ccache -s
./scripts/ccache_report.sh jazzy-tidy
- name: Merge compile databases
run: |
# Simple merge: combine per-package databases into build/compile_commands.json
mkdir -p build
python3 -c "
import json, glob, sys
entries = []
for db in glob.glob('build/*/compile_commands.json'):
with open(db) as f:
entries.extend(json.load(f))
with open('build/compile_commands.json', 'w') as f:
json.dump(entries, f, indent=2)
print(f'Merged {len(entries)} entries from {len(glob.glob(\"build/*/compile_commands.json\"))} packages')
"
- name: Determine files to analyze
id: files
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
# Incremental: only changed C++ files
changed=$(git diff --name-only --diff-filter=ACMR origin/main...HEAD -- '*.cpp' '*.hpp' | tr '\n' ' ')
if [ -z "$changed" ]; then
echo "No C++ files changed - skipping clang-tidy"
echo "skip=true" >> "$GITHUB_OUTPUT"
else
echo "Analyzing changed files: $changed"
echo "skip=false" >> "$GITHUB_OUTPUT"
echo "files=$changed" >> "$GITHUB_OUTPUT"
fi
else
# Full: all source files
echo "Full analysis on push to main"
echo "skip=false" >> "$GITHUB_OUTPUT"
echo "files=" >> "$GITHUB_OUTPUT"
fi
- name: Run clang-tidy
if: steps.files.outputs.skip != 'true'
env:
CTCACHE_DIR: /root/.ctcache
CHANGED_FILES: ${{ steps.files.outputs.files }}
run: |
mkdir -p /root/.ctcache
source /opt/ros/jazzy/setup.bash
source install/setup.bash
if [ -n "$CHANGED_FILES" ]; then
# Incremental: specific files
run-clang-tidy -p build -clang-tidy-binary ctcache-wrapper \
-header-filter='.*ros2_medkit.*' \
$CHANGED_FILES
else
# Full: analyze all source files under src/ros2_medkit_*/src/
# Intentionally excludes test/ directories (tested via sanitizer jobs)
run-clang-tidy -p build -clang-tidy-binary ctcache-wrapper \
-header-filter='.*ros2_medkit.*' \
'src/ros2_medkit_.*/src/.*\.cpp$'
fi
sanitizer-asan:
runs-on: ubuntu-latest
container:
image: ubuntu:noble
# Bind-mount the runner's large temp disk. The ASan+UBSan build tree
# measured ~27 GB before the sanitizer module started passing -g1; the
# post -g1 size has not been measured on a real run yet, so treat that
# figure as an upper bound. GitHub-hosted runners ship either a 72 GB /
# (only ~20 GB free, too small) or a 145 GB / disk, so the job used to
# pass or fail purely on which runner it landed. /mnt always has ~60+ GB
# free, which keeps that true either way.
volumes:
- "/mnt:/mnt"
# Cold-cache ASan builds are ~33 min; with the test budget a full run is
# ~55 min, so the previous 60 min still cancelled on slower runners before
# the post step could save the ccache. 90 min gives a cold-cache cycle real
# headroom so a slow runner does not cancel the job mid-run.
timeout-minutes: 90
defaults:
run:
shell: bash
steps:
- name: Install Git
run: |
apt-get update
apt-get install -y git
- name: Checkout repository
uses: actions/checkout@v4
- name: Pre-install ROS 2 apt source
uses: ./.github/actions/ros-apt-source
- name: Set up ROS 2 Jazzy
uses: ros-tooling/setup-ros@v0.7
with:
required-ros-distributions: jazzy
- name: Install ccache
run: apt-get install -y ccache
- name: Cache ccache
uses: actions/cache@v4
with:
path: /root/.cache/ccache
key: ccache-jazzy-asan-${{ github.sha }}
restore-keys: |
ccache-jazzy-asan-
- name: Install dependencies
run: |
apt-get update
apt-get install -y ros-jazzy-test-msgs
source /opt/ros/jazzy/setup.bash
rosdep update
rosdep install --from-paths src --ignore-src -y
- name: Redirect heavy build output to /mnt
run: |
# The ASan+UBSan RelWithDebInfo build tree measured ~27 GB before -g1
# and does not fit the container overlay on small (72 GB) GitHub
# runners even at a fraction of that size. Point the
# colcon build base and the compiler's scratch files at the large
# /mnt temp disk (bind-mounted above). install/ stays on the overlay:
# with --symlink-install it is only symlinks plus small setup files.
mkdir -p /mnt/asan/build /mnt/asan/tmp
ln -sfn /mnt/asan/build build
df -h / /mnt
- name: Build with ASan + UBSan
env:
CCACHE_DIR: /root/.cache/ccache
# 2G, not the 500M the other jobs use. At 500M this job's ccache was
# measured at 101-104% of its ceiling with 551-1019 cleanups DURING a
# single build - ccache evicting the objects that same build was still
# producing - and it never exceeded 18% hits. The instrumented object
# set does not fit in 500M and cannot be made to; the -g1 change in
# ROS2MedkitSanitizers.cmake shrinks it, this gives it somewhere to
# land. Sized against the repo's 10 GB Actions cache quota, which the
# docker-publish mode=min change frees room in.
CCACHE_MAXSIZE: 2G
CCACHE_SLOPPINESS: pch_defines,time_macros
TMPDIR: /mnt/asan/tmp
run: |
source /opt/ros/jazzy/setup.bash
# RelWithDebInfo (not Debug): Debug's -O0 roughly doubles binary size
# vs -O1, and ASan/UBSan instrumentation inflates it further. The
# sanitizer cmake module forces -O1 regardless, so the build type only
# affects debug-info size - and the module now also forces -g1, which
# is what cut that. The build tree lands on /mnt (redirect step above)
# so the instrumented objects + DWARF fit whatever the current size.
ccache -z
colcon build --symlink-install \
--cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSANITIZER=asan,ubsan \
--event-handlers console_direct+
ccache -s
./scripts/ccache_report.sh jazzy-asan
df -h / /mnt
- name: Extend test timeouts for ASan overhead
run: |
# Multiply EVERY declared CTest timeout by three, whatever it is. The
# x3 factor is the one the two literal rewrites this replaced encoded
# (60 -> 180, 120 -> 360), so no test's effective budget changes here;
# what changes is that a test declared at any OTHER value is no longer
# silently left on its native budget under a sanitizer that makes it
# several times slower. A fixed pair of literals goes stale the moment
# a budget is raised for good reasons, and the failure it produces is
# the worst kind: ctest kills the process and the test's own output
# dies with it, so the run reports a timeout and no test name.
find build/ -name "CTestTestfile.cmake" -exec \
perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} +
# Print what the tests will actually run with, so a sanitizer job that
# is killed on time can be read against its real budgets. `cat` first
# so grep sees one stream: its exit status then means "the rewrite had
# nothing to act on", which under pipefail correctly fails the step,
# rather than "one -exec batch happened to hold no timeouts".
find build/ -name "CTestTestfile.cmake" -exec cat {} + \
| grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c
- name: Run unit + integration tests with ASan + UBSan
timeout-minutes: 45
env:
# detect_leaks=0: FastDDS allocator leaks on shutdown (not our code)
# new_delete_type_mismatch=0: ROS 2 DDS scalar/array new/delete mismatch
ASAN_OPTIONS: halt_on_error=1:detect_leaks=0:new_delete_type_mismatch=0
UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
# Same factor as the ctest TIMEOUT rewrite above, for the wall-clock
# budgets tests assert internally - ctest's clock cannot reach those.
MEDKIT_TEST_TIME_SCALE: 3
# Lets tests size instrumented-only-expensive resources down (e.g. a
# 256-thread executor whose teardown outlives launch_testing's grace
# period under instrumentation). Detection also falls back to the
# sanitizer's own *SAN_OPTIONS, so this is belt and braces.
MEDKIT_TEST_SANITIZED: 1
run: |
source /opt/ros/jazzy/setup.bash
source install/setup.bash
failed=0
for pkg_dir in build/ros2_medkit_*/; do
pkg=$(basename "$pkg_dir")
# graph_watchdog is tested in sanitizer-graph-watchdog. Its
# end-to-end suite is 24 minutes under instrumentation, which is
# more than the headroom left here, and a package that overruns
# this step takes every package after it down with it.
if [ "$pkg" = "ros2_medkit_graph_watchdog" ]; then continue; fi
echo "::group::Testing $pkg"
(cd "$pkg_dir" && ctest -LE "linter" --output-on-failure) || failed=1
echo "::endgroup::"
done
exit $failed
- name: Show test results
if: always()
run: |
for pkg_dir in build/ros2_medkit_*/; do
if [ "$(basename "$pkg_dir")" = "ros2_medkit_graph_watchdog" ]; then continue; fi
colcon test-result --test-result-base "$pkg_dir" --verbose 2>/dev/null || true
done
sanitizer-tsan:
runs-on: ubuntu-latest
container:
image: ubuntu:noble
# See sanitizer-asan: the instrumented RelWithDebInfo build tree does not
# fit the container overlay on small GitHub runners, so build on /mnt.
volumes:
- "/mnt:/mnt"
# Successful runs reach 59 min, and a timeout here reads as "TSan found
# something" rather than "the job ran out of clock". Keep the headroom.
timeout-minutes: 90
defaults:
run:
shell: bash
steps:
- name: Install Git
run: |
apt-get update
apt-get install -y git
- name: Checkout repository
uses: actions/checkout@v4
- name: Pre-install ROS 2 apt source
uses: ./.github/actions/ros-apt-source
- name: Set up ROS 2 Jazzy
uses: ros-tooling/setup-ros@v0.7
with:
required-ros-distributions: jazzy
- name: Install ccache
run: apt-get install -y ccache
- name: Cache ccache
uses: actions/cache@v4
with:
path: /root/.cache/ccache
key: ccache-jazzy-tsan-${{ github.sha }}
restore-keys: |
ccache-jazzy-tsan-
- name: Install dependencies
run: |
apt-get update
apt-get install -y ros-jazzy-test-msgs
source /opt/ros/jazzy/setup.bash
rosdep update
rosdep install --from-paths src --ignore-src -y
- name: Redirect heavy build output to /mnt
run: |
# TSan RelWithDebInfo build tree is large too; keep it off the cramped
# container overlay on small GitHub runners (see sanitizer-asan).
mkdir -p /mnt/tsan/build /mnt/tsan/tmp
ln -sfn /mnt/tsan/build build
df -h / /mnt
- name: Build with TSan
env:
CCACHE_DIR: /root/.cache/ccache
# 1.5G - same reasoning as the ASan job, one sanitizer instead of two.
# Measured at 99.5-99.9% of the old 500M ceiling with 697-1017
# cleanups inside one build.
CCACHE_MAXSIZE: 1.5G
CCACHE_SLOPPINESS: pch_defines,time_macros
TMPDIR: /mnt/tsan/tmp
run: |
source /opt/ros/jazzy/setup.bash
ccache -z
colcon build --symlink-install \
--cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSANITIZER=tsan \
--event-handlers console_direct+
ccache -s
./scripts/ccache_report.sh jazzy-tsan
df -h / /mnt
- name: Extend test timeouts for TSan overhead
run: |
# Same generic x3 rewrite as the ASan job - see the comment there for
# why it is not a list of literal values.
find build/ -name "CTestTestfile.cmake" -exec \
perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} +
find build/ -name "CTestTestfile.cmake" -exec cat {} + \
| grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c
- name: Run unit + integration tests with TSan
timeout-minutes: 45
env:
# Same factor as the ctest TIMEOUT rewrite above, for the wall-clock
# budgets tests assert internally - ctest's clock cannot reach those.
MEDKIT_TEST_TIME_SCALE: 3
# Lets tests size instrumented-only-expensive resources down (e.g. a
# 256-thread executor whose teardown outlives launch_testing's grace
# period under instrumentation). Detection also falls back to the
# sanitizer's own *SAN_OPTIONS, so this is belt and braces.
MEDKIT_TEST_SANITIZED: 1
run: |
export TSAN_OPTIONS="halt_on_error=0:history_size=4:suppressions=$(pwd)/tsan_suppressions.txt"
source /opt/ros/jazzy/setup.bash
source install/setup.bash
failed=0
for pkg_dir in build/ros2_medkit_*/; do
pkg=$(basename "$pkg_dir")
# graph_watchdog is tested in sanitizer-graph-watchdog. Its
# end-to-end suite is 24 minutes under instrumentation, which is
# more than the headroom left here, and a package that overruns
# this step takes every package after it down with it.
if [ "$pkg" = "ros2_medkit_graph_watchdog" ]; then continue; fi
echo "::group::Testing $pkg"
(cd "$pkg_dir" && ctest -j1 -LE "linter" --output-on-failure) || failed=1
echo "::endgroup::"
done
exit $failed
- name: Show test results
if: always()
run: |
for pkg_dir in build/ros2_medkit_*/; do
if [ "$(basename "$pkg_dir")" = "ros2_medkit_graph_watchdog" ]; then continue; fi
colcon test-result --test-result-base "$pkg_dir" --verbose 2>/dev/null || true
done
# ros2_medkit_graph_watchdog's end-to-end suite runs 24 minutes under
# instrumentation, against a 45-minute test budget the rest of the workspace
# already spends 22 to 38 of. Testing it here instead of in the workspace
# sweeps gives it a budget of its own, and stops a suite that grows with every
# new detector from deciding whether the packages behind it get to run at all.
# It is not path-filtered: the plugin drives the gateway, the fault manager and
# discovery, so the changes most likely to break it are not in its own tree.
sanitizer-graph-watchdog:
name: Sanitizer ${{ matrix.name }} (graph_watchdog)
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- sanitizer: asan
name: ASan + UBSan
cmake_sanitizer: asan,ubsan
# Both sizes are the ones the matching workspace job argued for.
# This job restores that job's cache and must not evict what it
# restored, so it cannot be smaller here.
ccache_size: 2G
- sanitizer: tsan
name: TSan
cmake_sanitizer: tsan
ccache_size: 1.5G
container:
image: ubuntu:noble
# See sanitizer-asan: the instrumented build tree does not fit the
# container overlay on small GitHub runners, so build on /mnt.
volumes:
- "/mnt:/mnt"
# A cold cache builds the chain from scratch, which is the ~20 min the TSan
# workspace job measures, before the 45-minute test budget starts.
timeout-minutes: 90
defaults:
run:
shell: bash
steps:
- name: Install Git
run: |
apt-get update
apt-get install -y git
- name: Checkout repository
uses: actions/checkout@v4
- name: Pre-install ROS 2 apt source
uses: ./.github/actions/ros-apt-source
- name: Set up ROS 2 Jazzy
uses: ros-tooling/setup-ros@v0.7
with:
required-ros-distributions: jazzy
- name: Install ccache
run: apt-get install -y ccache
- name: Restore ccache
# Restore, never save. This job compiles a subset of the sources
# sanitizer-${{ matrix.sanitizer }} compiles, with the same flags, so
# that job's cache is a superset of what this one needs and serves its
# misses. A second copy would buy nothing and would spend the
# repository's 10 GB Actions cache quota twice for one set of objects.
uses: actions/cache/restore@v4
with:
path: /root/.cache/ccache
key: ccache-jazzy-${{ matrix.sanitizer }}-${{ github.sha }}
restore-keys: |
ccache-jazzy-${{ matrix.sanitizer }}-
- name: Install dependencies
run: |
apt-get update
apt-get install -y ros-jazzy-test-msgs
source /opt/ros/jazzy/setup.bash
rosdep update
rosdep install --from-paths src --ignore-src -y
- name: Redirect heavy build output to /mnt
run: |
# Same reasoning as the workspace sanitizer jobs, and the same need:
# the chain built here still includes the gateway.
mkdir -p /mnt/gw/build /mnt/gw/tmp
ln -sfn /mnt/gw/build build
df -h / /mnt
- name: Build with ${{ matrix.name }}
env:
CCACHE_DIR: /root/.cache/ccache
CCACHE_MAXSIZE: ${{ matrix.ccache_size }}
CCACHE_SLOPPINESS: pch_defines,time_macros
TMPDIR: /mnt/gw/tmp
run: |
source /opt/ros/jazzy/setup.bash
ccache -z
# --packages-up-to rather than the workspace: the chain reaches the
# gateway, the fault manager and ros2_medkit_integration_tests, which
# is a test dependency and is where the launch helpers and the demo
# nodes the end-to-end scenarios start actually live.
colcon build --symlink-install \
--packages-up-to ros2_medkit_graph_watchdog \
--cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DSANITIZER=${{ matrix.cmake_sanitizer }} \
--event-handlers console_direct+
ccache -s
./scripts/ccache_report.sh jazzy-${{ matrix.sanitizer }}-graph-watchdog
df -h / /mnt
- name: Extend test timeouts for sanitizer overhead
run: |
# The same generic x3 rewrite the workspace sanitizer jobs apply - see
# the comment in sanitizer-asan for why it is not a list of literals.
find build/ -name "CTestTestfile.cmake" -exec \
perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} +
find build/ -name "CTestTestfile.cmake" -exec cat {} + \
| grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c
- name: Run graph_watchdog tests with ${{ matrix.name }}
timeout-minutes: 45
env:
# Same factor as the ctest TIMEOUT rewrite above, for the wall-clock
# budgets tests assert internally - ctest's clock cannot reach those.
MEDKIT_TEST_TIME_SCALE: 3
# Lets tests size instrumented-only-expensive resources down.
MEDKIT_TEST_SANITIZED: 1
run: |
if [ "${{ matrix.sanitizer }}" = "tsan" ]; then
export TSAN_OPTIONS="halt_on_error=0:history_size=4:suppressions=$(pwd)/tsan_suppressions.txt"
else
# detect_leaks=0: FastDDS allocator leaks on shutdown (not our code)
# new_delete_type_mismatch=0: ROS 2 DDS scalar/array new/delete mismatch
export ASAN_OPTIONS=halt_on_error=1:detect_leaks=0:new_delete_type_mismatch=0
export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1
fi
source /opt/ros/jazzy/setup.bash
source install/setup.bash
cd build/ros2_medkit_graph_watchdog
ctest -j1 -LE "linter" --output-on-failure
- name: Show test results
if: always()
run: |
colcon test-result --test-result-base build/ros2_medkit_graph_watchdog \
--verbose 2>/dev/null || true