From 3fd7398272586169d6db47c5ae7810c555572f42 Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Thu, 2 Apr 2026 20:39:36 +0200 Subject: [PATCH] test: Fix flaky full-stack tests by randomizing base ports Motivation: Multiple full-stack tests were failing intermittently when run in parallel due to "Address already in use" errors. They all defaulted to the same base port (9526), causing conflicts in concurrent test environments. Design Choices: - Introduced `setup_random_base_port` helper in `OpenQA::Test::Utils` to centralize randomization logic. - Applied `setup_random_base_port` to `t/05-scheduler-full.t`, `t/25-cache-service.t`, `t/33-developer_mode.t`, `t/43-scheduling-and-worker-scalability.t`, and `t/full-stack.t`. - Uses `20000 + int rand 10000` to pick a likely-free port range. - The helper uses `||=` to allow overriding via environment variables. - Removed duplicate randomization logic and redundant comments across multiple test files. - Cleaned up `BEGIN` blocks and imports in full-stack tests. Benefits: - Eliminates port conflicts during parallel test execution. - Improves test stability and reliability for CI/CD. - Centralizes logic for easier future maintenance and consistent port selection strategy. - Cleaner and more maintainable test code. --- t/05-scheduler-full.t | 5 +++-- t/25-cache-service.t | 13 +++++++++---- t/33-developer_mode.t | 9 +++++++-- t/43-scheduling-and-worker-scalability.t | 11 ++++++++++- t/full-stack.t | 4 +++- t/lib/OpenQA/Test/Utils.pm | 14 ++++---------- 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/t/05-scheduler-full.t b/t/05-scheduler-full.t index c0cc9d42036..58e8f6cd91b 100644 --- a/t/05-scheduler-full.t +++ b/t/05-scheduler-full.t @@ -27,7 +27,7 @@ use OpenQA::Scheduler::Client; use OpenQA::Scheduler::Model::Jobs; use OpenQA::WebAPI; use OpenQA::Worker::WebUIConnection; -use OpenQA::Utils; +use OpenQA::Utils qw(service_port); require OpenQA::Test::Database; use OpenQA::Test::Utils qw( setup_mojo_app_with_default_worker_timeout @@ -35,8 +35,9 @@ use OpenQA::Test::Utils qw( create_webapi setup_share_dir create_websocket_server stop_service unstable_worker unresponsive_worker broken_worker rejective_worker - wait_for simulate_load + wait_for simulate_load setup_random_base_port ); +setup_random_base_port; use OpenQA::Test::TimeLimit '150'; # treat this test like the fullstack test diff --git a/t/25-cache-service.t b/t/25-cache-service.t index ad520c260c7..8d57fc7aad6 100644 --- a/t/25-cache-service.t +++ b/t/25-cache-service.t @@ -10,15 +10,20 @@ $ENV{MOJO_LOG_LEVEL} = 'info'; my $tempdir; BEGIN { - use Mojo::File qw(path tempdir); - $ENV{OPENQA_CACHE_SERVICE_QUIET} = $ENV{HARNESS_IS_VERBOSE} ? 0 : 1; $ENV{OPENQA_CACHE_ATTEMPTS} = 3; $ENV{OPENQA_CACHE_ATTEMPT_SLEEP_TIME} = 0; $ENV{OPENQA_RSYNC_RETRY_PERIOD} = 0; $ENV{OPENQA_RSYNC_RETRIES} = 1; $ENV{OPENQA_METRICS_DOWNLOAD_SIZE} = 1024; - $ENV{OPENQA_BASE_PORT} = 20000 + int rand 10000; +} + +use OpenQA::Test::Utils + qw(fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for setup_random_base_port); +setup_random_base_port; + +BEGIN { + use Mojo::File qw(path tempdir); $ENV{OPENQA_TEST_WAIT_INTERVAL} = 0.05; $tempdir = tempdir; @@ -47,7 +52,7 @@ use POSIX '_exit'; use Mojo::IOLoop::ReadWriteProcess qw(queue process); use Mojo::IOLoop::ReadWriteProcess::Session 'session'; use OpenQA::Test::Utils - qw(fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for); + qw(fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for setup_random_base_port); use OpenQA::Test::TimeLimit '90'; use Mojo::Util qw(md5_sum); use OpenQA::CacheService; diff --git a/t/33-developer_mode.t b/t/33-developer_mode.t index 5afaa05454c..20fb82d1a21 100644 --- a/t/33-developer_mode.t +++ b/t/33-developer_mode.t @@ -34,8 +34,9 @@ use Module::Load::Conditional 'can_load'; use OpenQA::Utils qw(service_port); use OpenQA::Test::Utils qw( create_websocket_server create_scheduler create_live_view_handler setup_share_dir setup_fullstack_temp_dir - start_worker stop_service + start_worker stop_service setup_random_base_port ); +setup_random_base_port; use OpenQA::Test::FullstackUtils; use OpenQA::SeleniumTest; @@ -48,9 +49,13 @@ my $livehandler; my $scheduler; sub turn_down_stack { - stop_service($_) for ($worker, $ws, $livehandler, $scheduler); + stop_service $scheduler; + stop_service $livehandler; + stop_service $ws; + stop_service $worker; } + driver_missing unless check_driver_modules; # setup directories diff --git a/t/43-scheduling-and-worker-scalability.t b/t/43-scheduling-and-worker-scalability.t index 15b8d404d2f..b471ba19d16 100644 --- a/t/43-scheduling-and-worker-scalability.t +++ b/t/43-scheduling-and-worker-scalability.t @@ -25,7 +25,8 @@ use OpenQA::Log qw(setup_log); use OpenQA::Test::Utils qw( setup_mojo_app_with_default_worker_timeout create_user_for_workers create_webapi create_websocket_server - stop_service setup_fullstack_temp_dir simulate_load); + stop_service setup_fullstack_temp_dir simulate_load setup_random_base_port); +setup_random_base_port; use OpenQA::Test::TimeLimit '20'; use OpenQA::Utils 'testcasedir'; @@ -35,6 +36,14 @@ BEGIN { $ENV{SCALABILITY_TEST_WITH_OFFLINE_WEBUI_HOST} //= 1; } +use OpenQA::Test::Utils qw( + setup_mojo_app_with_default_worker_timeout + create_user_for_workers create_webapi create_websocket_server + stop_service setup_fullstack_temp_dir simulate_load setup_random_base_port); +setup_random_base_port; +use OpenQA::Test::TimeLimit '20'; +use OpenQA::Utils 'testcasedir'; + setup_mojo_app_with_default_worker_timeout; OpenQA::Setup::read_config(my $app = OpenQA::App->singleton); diff --git a/t/full-stack.t b/t/full-stack.t index 85c0f5de552..ccd086e65fb 100644 --- a/t/full-stack.t +++ b/t/full-stack.t @@ -49,9 +49,11 @@ use Module::Load::Conditional 'can_load'; use OpenQA::Test::Utils qw(create_websocket_server create_live_view_handler setup_share_dir), qw(cache_minion_worker cache_worker_service setup_fullstack_temp_dir), - qw(start_worker stop_service wait_for_or_bail_out); + qw(start_worker stop_service wait_for_or_bail_out setup_random_base_port); +setup_random_base_port; use OpenQA::Test::FullstackUtils; + plan skip_all => 'set FULLSTACK=1 (be careful)' unless $ENV{FULLSTACK}; my $worker; diff --git a/t/lib/OpenQA/Test/Utils.pm b/t/lib/OpenQA/Test/Utils.pm index 65bf478a148..7f0a4f77882 100644 --- a/t/lib/OpenQA/Test/Utils.pm +++ b/t/lib/OpenQA/Test/Utils.pm @@ -50,6 +50,7 @@ BEGIN { our (@EXPORT, @EXPORT_OK); @EXPORT_OK = qw( + setup_random_base_port setup_mojo_app_with_default_worker_timeout create_user_for_workers create_webapi create_websocket_server create_scheduler create_live_view_handler unresponsive_worker broken_worker rejective_worker setup_share_dir setup_fullstack_temp_dir run_gru_job @@ -60,16 +61,9 @@ our (@EXPORT, @EXPORT_OK); simulate_load ); -# The function OpenQA::Utils::service_port method hardcodes ports in a -# sequential range starting with OPENQA_BASE_PORT. This can cause problems -# especially in repeated testing if any of the ports in that range is already -# occupied. So we inject random, free ports for the services here. -# -# Potential point for later improvement: In -# Mojo::IOLoop::Server::generate_port keep the sock object on the port and -# reuse it in listen to prevent race condition -# -# Potentially this approach can also be used in production code. +sub setup_random_base_port { + $ENV{OPENQA_BASE_PORT} ||= 20000 + int rand 10000; +} sub setup_mojo_app_with_default_worker_timeout ($class = 'Mojolicious') { my $app = $class->new(config => {global => {worker_timeout => DEFAULT_WORKER_TIMEOUT}}, log => undef);