Skip to content

test: Fix flaky full-stack tests by randomizing base ports #7244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
okurz wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

test: Fix flaky full-stack tests by randomizing base ports #7244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions t/05-scheduler-full.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,17 @@ use OpenQA::Scheduler::Client;
use OpenQA::Scheduler::Model::Jobs;
use OpenQA::WebAPI;
use OpenQA::Worker::WebUIConnection;
use OpenQA::Utils;
use OpenQA::Utils qw(service_port);
require OpenQA::Test::Database;
use OpenQA::Test::Utils qw(
setup_mojo_app_with_default_worker_timeout
setup_fullstack_temp_dir create_user_for_workers
create_webapi setup_share_dir create_websocket_server
stop_service unstable_worker
unresponsive_worker broken_worker rejective_worker
wait_for simulate_load
wait_for simulate_load setup_random_base_port
);
setup_random_base_port;
use OpenQA::Test::TimeLimit '150';

# treat this test like the fullstack test
Expand Down
13 changes: 9 additions & 4 deletions t/25-cache-service.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,20 @@ $ENV{MOJO_LOG_LEVEL} = 'info';
my $tempdir;

BEGIN {
use Mojo::File qw(path tempdir);

$ENV{OPENQA_CACHE_SERVICE_QUIET} = $ENV{HARNESS_IS_VERBOSE} ? 0 : 1;
$ENV{OPENQA_CACHE_ATTEMPTS} = 3;
$ENV{OPENQA_CACHE_ATTEMPT_SLEEP_TIME} = 0;
$ENV{OPENQA_RSYNC_RETRY_PERIOD} = 0;
$ENV{OPENQA_RSYNC_RETRIES} = 1;
$ENV{OPENQA_METRICS_DOWNLOAD_SIZE} = 1024;
$ENV{OPENQA_BASE_PORT} = 20000 + int rand 10000;
}

use OpenQA::Test::Utils
qw(fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for setup_random_base_port);
setup_random_base_port;

BEGIN {
use Mojo::File qw(path tempdir);
$ENV{OPENQA_TEST_WAIT_INTERVAL} = 0.05;

$tempdir = tempdir;
Expand Down Expand Up @@ -47,7 +52,7 @@ use POSIX '_exit';
use Mojo::IOLoop::ReadWriteProcess qw(queue process);
use Mojo::IOLoop::ReadWriteProcess::Session 'session';
use OpenQA::Test::Utils
qw(fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for);
qw(fake_asset_server cache_minion_worker cache_worker_service wait_for_or_bail_out perform_minion_jobs wait_for setup_random_base_port);
use OpenQA::Test::TimeLimit '90';
use Mojo::Util qw(md5_sum);
use OpenQA::CacheService;
Expand Down
9 changes: 7 additions & 2 deletions t/33-developer_mode.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ use Module::Load::Conditional 'can_load';
use OpenQA::Utils qw(service_port);
use OpenQA::Test::Utils qw(
create_websocket_server create_scheduler create_live_view_handler setup_share_dir setup_fullstack_temp_dir
start_worker stop_service
start_worker stop_service setup_random_base_port
);
setup_random_base_port;
use OpenQA::Test::FullstackUtils;
use OpenQA::SeleniumTest;

Expand All @@ -48,9 +49,13 @@ my $livehandler;
my $scheduler;

sub turn_down_stack {
stop_service($_) for ($worker, $ws, $livehandler, $scheduler);
stop_service $scheduler;
stop_service $livehandler;
stop_service $ws;
stop_service $worker;
}


driver_missing unless check_driver_modules;

# setup directories
Expand Down
11 changes: 10 additions & 1 deletion t/43-scheduling-and-worker-scalability.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ use OpenQA::Log qw(setup_log);
use OpenQA::Test::Utils qw(
setup_mojo_app_with_default_worker_timeout
create_user_for_workers create_webapi create_websocket_server
stop_service setup_fullstack_temp_dir simulate_load);
stop_service setup_fullstack_temp_dir simulate_load setup_random_base_port);
setup_random_base_port;
use OpenQA::Test::TimeLimit '20';
use OpenQA::Utils 'testcasedir';

Expand All @@ -35,6 +36,14 @@ BEGIN {
$ENV{SCALABILITY_TEST_WITH_OFFLINE_WEBUI_HOST} //= 1;
}

use OpenQA::Test::Utils qw(
setup_mojo_app_with_default_worker_timeout
create_user_for_workers create_webapi create_websocket_server
stop_service setup_fullstack_temp_dir simulate_load setup_random_base_port);
setup_random_base_port;
use OpenQA::Test::TimeLimit '20';
use OpenQA::Utils 'testcasedir';

setup_mojo_app_with_default_worker_timeout;
OpenQA::Setup::read_config(my $app = OpenQA::App->singleton);

Expand Down
4 changes: 3 additions & 1 deletion t/full-stack.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,11 @@ use Module::Load::Conditional 'can_load';
use OpenQA::Test::Utils
qw(create_websocket_server create_live_view_handler setup_share_dir),
qw(cache_minion_worker cache_worker_service setup_fullstack_temp_dir),
qw(start_worker stop_service wait_for_or_bail_out);
qw(start_worker stop_service wait_for_or_bail_out setup_random_base_port);
setup_random_base_port;
use OpenQA::Test::FullstackUtils;


plan skip_all => 'set FULLSTACK=1 (be careful)' unless $ENV{FULLSTACK};

my $worker;
Expand Down
14 changes: 4 additions & 10 deletions t/lib/OpenQA/Test/Utils.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ BEGIN {

our (@EXPORT, @EXPORT_OK);
@EXPORT_OK = qw(
setup_random_base_port
setup_mojo_app_with_default_worker_timeout create_user_for_workers
create_webapi create_websocket_server create_scheduler create_live_view_handler
unresponsive_worker broken_worker rejective_worker setup_share_dir setup_fullstack_temp_dir run_gru_job
Expand All @@ -60,16 +61,9 @@ our (@EXPORT, @EXPORT_OK);
simulate_load
);

# The function OpenQA::Utils::service_port method hardcodes ports in a
# sequential range starting with OPENQA_BASE_PORT. This can cause problems
# especially in repeated testing if any of the ports in that range is already
# occupied. So we inject random, free ports for the services here.
#
# Potential point for later improvement: In
# Mojo::IOLoop::Server::generate_port keep the sock object on the port and
# reuse it in listen to prevent race condition
#
# Potentially this approach can also be used in production code.
sub setup_random_base_port {
Copy link
Copy Markdown
Contributor

@perlpunk perlpunk Apr 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now I'm wondering why mock_service_ports was removed: 8d7b9a5
I guess the according comment block was overlooked.

Copy link
Copy Markdown
Member Author

@okurz okurz Apr 2, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

huh. That's why this comment block was a bit dangling :D

$ENV{OPENQA_BASE_PORT} ||= 20000 + int rand 10000;
}

sub setup_mojo_app_with_default_worker_timeout ($class = 'Mojolicious') {
my $app = $class->new(config => {global => {worker_timeout => DEFAULT_WORKER_TIMEOUT}}, log => undef);
Expand Down
Loading