fix(fault_manager): keep the near-miss series when a fault is cleared #1891

Workflow file for this run

	name: CI

	on:
	pull_request:
	branches: [main]
	push:
	branches: [main]

	jobs:
	build-and-test:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	include:
	- ros_distro: humble
	os_image: ubuntu:jammy
	- ros_distro: lyrical
	os_image: ubuntu:resolute
	container:
	image: ${{ matrix.os_image }}
	# Must stay above the build plus the test step's own budget below, or the job cap kills the
	# run before that step's cap can - which loses the "which step ran long" answer. The build
	# side of this job is around 18 minutes.
	timeout-minutes: 90
	defaults:
	run:
	shell: bash

	steps:
	- name: Install Git
	run: \|
	apt-get update
	apt-get install -y git

	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Pre-install ROS 2 apt source
	uses: ./.github/actions/ros-apt-source

	- name: Set up ROS 2 ${{ matrix.ros_distro }}
	uses: ros-tooling/setup-ros@v0.7
	with:
	required-ros-distributions: ${{ matrix.ros_distro }}

	- name: Install ccache
	run: apt-get install -y ccache

	- name: Cache ccache
	uses: actions/cache@v4
	with:
	path: /root/.cache/ccache
	key: ccache-${{ matrix.ros_distro }}-${{ github.sha }}
	restore-keys: \|
	ccache-${{ matrix.ros_distro }}-

	- name: Install dependencies
	run: \|
	apt-get update
	apt-get install -y ros-${{ matrix.ros_distro }}-test-msgs
	if [ "${{ matrix.ros_distro }}" = "humble" ]; then
	apt-get install -y ros-humble-rmw-cyclonedds-cpp
	fi
	source /opt/ros/${{ matrix.ros_distro }}/setup.bash
	rosdep update
	# Linters (clang-tidy / clang-format) only run in the Jazzy quality
	# job (quality.yml). Multiple package.xml files still list them as
	# <test_depend>, so we skip the rosdep keys here to keep build-and-
	# test runners from pulling in linter packages they never use. Also
	# defensive against transient binary-deb gaps on newly-released
	# distros.
	rosdep install --from-paths src --ignore-src -y \
	--skip-keys "ament_cmake_clang_tidy ament_cmake_clang_format"

	- name: Build packages
	env:
	CCACHE_DIR: /root/.cache/ccache
	CCACHE_MAXSIZE: 500M
	CCACHE_SLOPPINESS: pch_defines,time_macros
	run: \|
	source /opt/ros/${{ matrix.ros_distro }}/setup.bash
	ccache -z
	colcon build --symlink-install \
	--packages-skip ros2_medkit_opcua \
	--cmake-args -DCMAKE_BUILD_TYPE=Release \
	--event-handlers console_direct+
	ccache -s
	./scripts/ccache_report.sh "${{ matrix.ros_distro }}"

	- name: Run unit and integration tests
	# The suite keeps outgrowing this budget as packages land, and every overrun so far has
	# been real work finishing rather than a hang: 15 minutes was raised to 25 when lyrical
	# reached 14m13s, and 25 was reached again once the graph watchdog's end-to-end scenarios
	# arrived (lyrical 22m50s before them, jazzy 28m15s after). The jazzy job below caps the
	# whole job instead of this step, so only these two distros can be killed mid-package -
	# which reads as a test failure and hides every test the kill cut off.
	timeout-minutes: 45
	env:
	# FastRTPS 2.6 on Humble has a known use-after-free in the
	# discovery-teardown path (EDP::unpairWriterProxy) that segfaults
	# peer nodes when the gateway shuts down, so we force CycloneDDS
	# there. Lyrical ships a newer FastRTPS without that bug and hits
	# a separate iceoryx-shared-memory crash under CycloneDDS during
	# shutdown ("string capacity was zero for allocated data"), so on
	# Lyrical we keep the default FastRTPS. (Jazzy still uses
	# CycloneDDS - see jazzy-test job below for its rationale.)
	RMW_IMPLEMENTATION: ${{ matrix.ros_distro == 'humble' && 'rmw_cyclonedds_cpp' \|\| '' }}
	run: \|
	source /opt/ros/${{ matrix.ros_distro }}/setup.bash
	colcon test --return-code-on-test-failure \
	--packages-skip ros2_medkit_opcua \
	--ctest-args -LE linter \
	--event-handlers console_direct+

	- name: Show test results
	if: always()
	run: colcon test-result --verbose

	- name: Upload test results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: test-results-${{ matrix.ros_distro }}
	path: \|
	log/
	build/*/test_results/

	# Builds AND tests Jazzy. These were two jobs (jazzy-build -> jazzy-test)
	# passing a tarred build/ + install/ tree between them. That split dates from
	# when lint ran off the same artifact in parallel; lint has since moved to
	# quality.yml, where format-lint and clang-tidy each do their own build, so
	# the artifact was left with exactly one consumer and the split bought no
	# parallelism at all. It cost 4.24 min of duplicated container bring-up and
	# ROS setup in the second job plus 0.50 min of tar/upload in the first.
	#
	# The job id stays jazzy-test because notify-demos gates on it by name.
	jazzy-test:
	runs-on: ubuntu-latest
	container:
	image: ubuntu:noble
	# The more generous of the two timeouts this job replaces (build had 60,
	# test had 45). One job now covers both phases, and a cold ccache makes the
	# build phase the variable one, so keep the headroom rather than the sum.
	timeout-minutes: 60
	defaults:
	run:
	shell: bash

	steps:
	- name: Install Git
	run: \|
	apt-get update
	apt-get install -y git

	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Pre-install ROS 2 apt source
	uses: ./.github/actions/ros-apt-source

	- name: Set up ROS 2 Jazzy
	uses: ros-tooling/setup-ros@v0.7
	with:
	required-ros-distributions: jazzy

	- name: Install ccache
	run: apt-get install -y ccache

	- name: Cache ccache
	uses: actions/cache@v4
	with:
	path: /root/.cache/ccache
	# ccache-jazzy-test-, not ccache-jazzy-. The shorter prefix also
	# matches ccache-jazzy-asan-, -tsan-, -lint- and -tidy-, and
	# restore-keys takes the most recently created match, so this job kept
	# restoring whichever other Jazzy job finished last. It was measured
	# loading the ASan cache: 2 GB of instrumented objects it cannot use,
	# 10% hits, and then 219 cleanups as its own Release objects fought
	# for the 500M ceiling on top of them.
	key: ccache-jazzy-test-${{ github.sha }}
	restore-keys: \|
	ccache-jazzy-test-

	- name: Install dependencies
	run: \|
	apt-get update
	apt-get install -y \
	ros-jazzy-test-msgs \
	ros-jazzy-rmw-cyclonedds-cpp
	source /opt/ros/jazzy/setup.bash
	rosdep update
	rosdep install --from-paths src --ignore-src -y

	- name: Build packages
	env:
	CCACHE_DIR: /root/.cache/ccache
	CCACHE_MAXSIZE: 500M
	CCACHE_SLOPPINESS: pch_defines,time_macros
	run: \|
	source /opt/ros/jazzy/setup.bash
	ccache -z
	colcon build --symlink-install \
	--packages-skip ros2_medkit_opcua \
	--cmake-args -DCMAKE_BUILD_TYPE=Release \
	--event-handlers console_direct+
	ccache -s
	./scripts/ccache_report.sh jazzy-test

	- name: Run unit and integration tests
	env:
	# FastRTPS has a known use-after-free in discovery-teardown that
	# can segfault peer nodes when another node (typically the gateway)
	# shuts down. CycloneDDS avoids it.
	RMW_IMPLEMENTATION: rmw_cyclonedds_cpp
	run: \|
	source /opt/ros/jazzy/setup.bash
	colcon test --return-code-on-test-failure \
	--packages-skip ros2_medkit_opcua \
	--ctest-args -LE linter \
	--event-handlers console_direct+

	- name: Show test results
	if: always()
	run: colcon test-result --verbose

	- name: Upload test results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: test-results-jazzy-test
	path: \|
	log/
	build/*/test_results/

	coverage:
	# Push to main only. This job owned CI's critical path in 79% of PR runs at
	# ~40 min, and what a PR loses by not running it is narrow: its Codecov
	# upload was already gated to push (see the last step), and the source-tree
	# half of the coverage-scope gate - the half that catches a new package
	# compiling C++ without opting into instrumentation - runs on every PR in
	# quality.yml's format-lint job via check_coverage_packages.sh --static-only.
	# What moves to main-only is the runtime half, which needs .gcda files and
	# therefore needs this job's build and test run.
	#
	# This was also the only PR job building without -DNDEBUG, so on its own it
	# would have taken every assert() out of pull request builds - the other
	# jobs are Release or RelWithDebInfo. ROS2MedkitSanitizers.cmake now passes
	# -UNDEBUG, which puts the asserts back in the two sanitizer jobs. Those run
	# ctest -LE linter per package, a superset of the selection here - they do
	# not skip ros2_medkit_opcua - so the asserts are still exercised against
	# unit and integration tests on every PR.
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	runs-on: ubuntu-latest
	container:
	image: ubuntu:noble
	timeout-minutes: 90
	defaults:
	run:
	shell: bash

	steps:
	- name: Install Git
	run: \|
	apt-get update
	apt-get install -y git

	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Pre-install ROS 2 apt source
	uses: ./.github/actions/ros-apt-source

	- name: Set up ROS 2 Jazzy
	uses: ros-tooling/setup-ros@v0.7
	with:
	required-ros-distributions: jazzy

	- name: Install ccache
	run: apt-get install -y ccache

	- name: Cache ccache
	uses: actions/cache@v4
	with:
	path: /root/.cache/ccache
	key: ccache-coverage-${{ github.sha }}
	restore-keys: \|
	ccache-coverage-

	- name: Install dependencies
	run: \|
	apt-get update
	# gpg is required by codecov/codecov-action@v5 dependency check
	apt-get install -y lcov ros-jazzy-test-msgs gpg
	source /opt/ros/jazzy/setup.bash
	rosdep update
	rosdep install --from-paths src --ignore-src -r -y

	- name: Build packages with coverage
	env:
	CCACHE_DIR: /root/.cache/ccache
	CCACHE_MAXSIZE: 500M
	CCACHE_SLOPPINESS: pch_defines,time_macros
	run: \|
	source /opt/ros/jazzy/setup.bash
	ccache -z
	colcon build --symlink-install \
	--packages-skip ros2_medkit_opcua \
	--cmake-args -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON \
	--event-handlers console_direct+
	ccache -s
	./scripts/ccache_report.sh coverage

	- name: Run unit and integration tests for coverage
	run: \|
	source /opt/ros/jazzy/setup.bash
	# --return-code-on-test-failure, as in the two other colcon test
	# invocations in this file. Without it colcon exits 0 whatever the
	# tests did, so this job ran the full Jazzy suite under a coverage
	# build and could not fail CI on a failure - it read as a gate
	# without being one.
	colcon test --return-code-on-test-failure \
	--packages-skip ros2_medkit_opcua \
	--ctest-args -LE linter \
	--event-handlers console_direct+

	- name: Generate coverage report
	run: \|
	# --parallel: this single lcov --capture was measured at >99% of the
	# whole step (13.43 min of 13.50, 17.87 of 17.95) and is otherwise
	# single-threaded, while --extract, --remove, --list and genhtml
	# together finish in under six seconds. It forks per .gcda directory,
	# so the output is the same records from the same gcov runs, only
	# gathered concurrently.
	lcov --capture --directory build --output-file coverage.raw.info \
	--parallel "$(nproc)" \
	--ignore-errors mismatch,negative,empty,gcov

	if [ ! -s coverage.raw.info ] \|\| ! grep -q 'SF:' coverage.raw.info; then
	echo "::error::No valid coverage data found in coverage.raw.info"
	exit 1
	fi

	# $PWD, not pwd -P: gcov records the path the compiler was given, which
	# is the logical one. Resolving symlinks here would stop matching.
	WS="${PWD}"
	lcov --extract coverage.raw.info \
	"${WS}/src//src/" \
	"${WS}/src//include/" \
	--output-file coverage.extracted.info \
	--ignore-errors unused,empty

	lcov --remove coverage.extracted.info \
	'/vendored/' \
	--output-file coverage.info \
	--ignore-errors unused,empty

	if [ ! -s coverage.info ]; then
	echo "::error::Filtered coverage.info is empty - no source files matched"
	exit 1
	fi

	lcov --list coverage.info
	genhtml coverage.info --output-directory coverage_html --ignore-errors source

	# A package that never gets --coverage emits no .gcda and drops out of the
	# report entirely - out of the numerator and the denominator both, so the
	# percentage silently stops describing the workspace. Assert the report
	# still covers every package the source tree says holds production C++.
	- name: Verify every C++ package reached the coverage report
	run: \|
	./scripts/check_coverage_packages.sh coverage.info \
	--skip ros2_medkit_opcua

	- name: Upload coverage HTML report as artifact
	uses: actions/upload-artifact@v4
	with:
	name: coverage-report
	path: coverage_html/

	- name: Upload coverage to Codecov
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	uses: codecov/codecov-action@v7
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	files: coverage.info
	flags: unittests,integration
	name: ros2_medkit-coverage
	fail_ci_if_error: true
	verbose: true

	notify-demos:
	needs: [jazzy-test]
	if: >-
	github.event_name == 'push' &&
	github.ref == 'refs/heads/main' &&
	needs.jazzy-test.result == 'success'
	runs-on: ubuntu-latest
	# Only Jazzy gates the dispatch. Humble failures do not block demos CI - intentional tradeoff.
	permissions: {}
	steps:
	- name: Trigger selfpatch_demos CI
	uses: peter-evans/repository-dispatch@v4
	with:
	token: ${{ secrets.DEMOS_DISPATCH_TOKEN }}
	repository: selfpatch/selfpatch_demos
	event-type: ros2_medkit_updated
	client-payload: '{"sha":"${{ github.sha }}","run_url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}'

Navigation Menu

Uh oh!

fix(fault_manager): keep the near-miss series when a fault is cleared #1891

Workflow file

fix(fault_manager): keep the near-miss series when a fault is cleared #1891

Uh oh!

Workflow file for this run