perf: handle huge job groups gracefully

- Optimize compute_build_results by using database-level aggregation
  instead of fetching and iterating all job objects.
- Move job deduplication (per scenario) to the database level.
- Implement a safety limit of 5,000 jobs per build to prevent timeouts.
- Optimize comment/review tracking by only checking failed jobs.
- Add t/61-job_group_aggregation.t to verify aggregation and limit enforcement.
- Extract category mapping into _get_job_result_category helper.
- Reuse helper in both count_job and _count_job_aggregated.
- Ensure consistent result categorization across legacy and optimized paths.
- Add job_group_overview_max_jobs to misc_limits in openqa.ini.
- Pass this limit from web and API controllers to compute_build_results.
- De-duplicate common job data in t/61-job_group_aggregation.t.
- Add controller and API tests for limit enforcement.

Related progress issue: https://progress.opensuse.org/issues/196913

Author: okurz

lib/OpenQA/BuildResults.pm

@@ -13,6 +13,9 @@ use OpenQA::Log qw(log_error);
 use Date::Format;
 use Sort::Versions;
 use Time::Seconds;
+use List::Util 'any';
+
+use constant DEFAULT_MAX_JOBS_PER_BUILD => 5000;
 
 sub init_job_figures ($job_result) {
 
@@ -30,41 +33,41 @@ sub init_job_figures ($job_result) {
     $job_result->{total} = 0;
 }
 
+sub _get_job_result_category ($state, $result) {
+    if ($state eq OpenQA::Jobs::Constants::DONE) {
+        return 'passed' if $result eq OpenQA::Jobs::Constants::PASSED;
+        return 'softfailed' if $result eq OpenQA::Jobs::Constants::SOFTFAILED;
+        return 'skipped' if any { $result eq $_ } OpenQA::Jobs::Constants::ABORTED_RESULTS;
+        return 'failed' if any { $result eq $_ } OpenQA::Jobs::Constants::NOT_OK_RESULTS;
+    }
+    return 'skipped' if $state eq OpenQA::Jobs::Constants::CANCELLED;
+    return 'unfinished';
+}
+
+sub _count_job_aggregated ($stat, $jr, $count) {
+    $jr->{total} += $count;
+    my $category = _get_job_result_category($stat->state, $stat->result);
+    $jr->{$category} += $count;
+}
+
 sub count_job ($job, $jr, $labels) {
 
     $jr->{total}++;
-    if ($job->state eq OpenQA::Jobs::Constants::DONE) {
-        if ($job->result eq OpenQA::Jobs::Constants::PASSED) {
-            $jr->{passed}++;
-            return;
-        }
-        if ($job->result eq OpenQA::Jobs::Constants::SOFTFAILED) {
-            $jr->{softfailed}++;
-            return;
-        }
-        if (grep { $job->result eq $_ } OpenQA::Jobs::Constants::ABORTED_RESULTS) {
-            $jr->{skipped}++;
-            return;
-        }
-        if (grep { $job->result eq $_ } OpenQA::Jobs::Constants::NOT_OK_RESULTS) {
-            my $comment_data = $labels->{$job->id};
-            $jr->{failed}++;
-            if ($comment_data) {
-                $jr->{labeled}++ if $comment_data->{reviewed};
-                $jr->{comments}++ if $comment_data->{comments} || $comment_data->{reviewed};
-            }
-            return;
+    my ($state, $result) = ($job->state, $job->result);
+    my $category = _get_job_result_category($state, $result);
+    $jr->{$category}++;
+
+    if ($category eq 'failed') {
+        my $comment_data = $labels->{$job->id};
+        if ($comment_data) {
+            $jr->{labeled}++ if $comment_data->{reviewed};
+            $jr->{comments}++ if $comment_data->{comments} || $comment_data->{reviewed};
         }
-        # note: Incompletes and timeouts are accounted to both categories - failed and skipped.
     }
-    if ($job->state eq OpenQA::Jobs::Constants::CANCELLED) {
-        $jr->{skipped}++;
-        return;
+    elsif ($category eq 'unfinished') {
+        log_error('Encountered not-implemented state:' . $state . ' result:' . $result)
+          unless any { $state eq $_ } (OpenQA::Jobs::Constants::PENDING_STATES);
     }
-    my $state = $job->state;
-    log_error('Encountered not-implemented state:' . $job->state . ' result:' . $job->result)
-      unless grep { /$state/ } (OpenQA::Jobs::Constants::PENDING_STATES);
-    $jr->{unfinished}++;
     return;
 }
 
@@ -111,13 +114,17 @@ sub find_child_groups ($group, $subgroup_filter) {
     return filter_subgroups($group, $subgroup_filter);
 }
 
-sub compute_build_results ($group, $limit, $time_limit_days, $tags, $subgroup_filter, $show_tags) {
+sub compute_build_results ($group, $limit, $time_limit_days, $tags, $subgroup_filter, $show_tags,
+    $max_jobs_per_build = undef)
+{
 
     # find relevant child groups taking filter into account
     my $child_groups = find_child_groups($group, $subgroup_filter);
     my $group_ids = $child_groups->{group_ids};
     my $children = $child_groups->{children};
 
+    $max_jobs_per_build //= DEFAULT_MAX_JOBS_PER_BUILD;
+
     my @sorted_results;
     my %result = (
         build_results => \@sorted_results,
@@ -185,14 +192,24 @@ sub compute_build_results ($group, $limit, $time_limit_days, $tags, $subgroup_fi
         last if defined($limit) && (--$limit < 0);
 
         my ($version, $buildnr) = ($build->VERSION, $build->BUILD);
-        my $jobs = $jobs_resultset->search(
+        my $jobs_search_filter = {
+            VERSION => $version,
+            BUILD => $buildnr,
+            group_id => {in => $group_ids},
+            clone_id => undef,
+        };
+        my $latest_job_ids_rs = $jobs_resultset->search($jobs_search_filter,
+            {select => [{max => 'id'}], as => [qw(id)], group_by => [qw(TEST ARCH FLAVOR MACHINE)]});
+        my @latest_ids = grep { defined } map { $_->get_column('id') } $latest_job_ids_rs->all;
+        next unless @latest_ids;
+
+        my $stats_rs = $jobs_resultset->search(
+            {id => {-in => \@latest_ids}},
             {
-                VERSION => $version,
-                BUILD => $buildnr,
-                group_id => {in => $group_ids},
-                clone_id => undef,
-            },
-            {order_by => 'me.id DESC'});
+                select => [qw(state result group_id DISTRI), {count => '*'}],
+                as => [qw(state result group_id DISTRI count)],
+                group_by => [qw(state result group_id DISTRI)],
+            });
         my %jr = (
             key => $build->{key},
             build => $buildnr,
@@ -201,34 +218,80 @@ sub compute_build_results ($group, $limit, $time_limit_days, $tags, $subgroup_fi
         );
         init_job_figures(\%jr);
         for my $child (@$children) {
-            init_job_figures($jr{children}->{$child->id} = {});
+            init_job_figures($jr{children}->{$child->id} = {version => $version});
         }
-
-        my %seen;
-        my @jobs = map {
-            my $key = $_->TEST . '-' . $_->ARCH . '-' . $_->FLAVOR . '-' . ($_->MACHINE // '');
-            $seen{$key}++ ? () : $_;
-        } $jobs->all;
-        next unless @jobs;
-        my $comment_data = $group->result_source->schema->resultset('Comments')->comment_data_for_jobs(\@jobs);
-        for my $job (@jobs) {
-            $jr{distris}->{$job->DISTRI} = 1;
+        my $total_for_build = 0;
+        while (my $stat = $stats_rs->next) {
+            my $count = $stat->get_column('count');
+            $total_for_build += $count;
+            _count_job_aggregated($stat, \%jr, $count);
+            my $distri = $stat->get_column('DISTRI');
+            $jr{distris}->{$distri} = 1;
+            if ($jr{children}) {
+                my $child = $jr{children}->{$stat->group_id};
+                _count_job_aggregated($stat, $child, $count);
+                $child->{distris}->{$distri} = 1;
+            }
+        }
+        if ($total_for_build > $max_jobs_per_build) {
+            die 'Build '
+              . $buildnr . ' has '
+              . $total_for_build
+              . ' jobs, which exceeds the limit of '
+              . $max_jobs_per_build
+              . '. Please contact your openQA administrator.';
+        }
+        next unless $total_for_build;
+        my $extra_info_rs = $jobs_resultset->search(
+            {id => {-in => \@latest_ids}},
+            {
+                select => [{($newest ? 'max' : 'min') => 't_created'}],
+                as => [qw(t_created)],
+            });
+        while (my $info = $extra_info_rs->next) {
+            my $t_created = $info->get_column('t_created');
+            if ($t_created && !ref $t_created) {
+                require DateTime::Format::Pg;
+                $t_created = DateTime::Format::Pg->parse_datetime($t_created);
+            }
             if ($newest) {
-                $jr{oldest_newest} //= $job->t_created;
+                $jr{oldest_newest} //= $t_created;
             }
             else {
-                $jr{oldest_newest} = $job->t_created;
+                $jr{oldest_newest} = $t_created;
             }
-            count_job($job, \%jr, $comment_data);
-            if ($jr{children}) {
-                my $child = $jr{children}->{$job->group_id};
-                $child->{distris}->{$job->DISTRI} = 1;
-                $child->{version} //= $job->VERSION;
-                $child->{build} //= $job->BUILD;
-                count_job($job, $child, $comment_data);
-                add_review_badge($child);
+        }
+        my $not_ok_rs = $jobs_resultset->search(
+            {
+                id => {-in => \@latest_ids},
+                state => OpenQA::Jobs::Constants::DONE,
+                result => {in => [OpenQA::Jobs::Constants::NOT_OK_RESULTS]},
+            },
+            {select => [qw(id group_id)]});
+        my @not_ok_jobs = $not_ok_rs->all;
+        my $comment_data = $group->result_source->schema->resultset('Comments')->comment_data_for_jobs(\@not_ok_jobs);
+        for my $job (@not_ok_jobs) {
+            my $cd = $comment_data->{$job->id};
+            next unless $cd;
+            if ($cd->{reviewed}) {
+                $jr{labeled}++;
+                if ($jr{children} && (my $child = $jr{children}->{$job->group_id})) {
+                    $child->{labeled}++;
+                }
+            }
+            if ($cd->{comments} || $cd->{reviewed}) {
+                $jr{comments}++;
+                if ($jr{children} && (my $child = $jr{children}->{$job->group_id})) {
+                    $child->{comments}++;
+                }
             }
         }
+        if ($jr{children}) {
+            for my $child_id (keys %{$jr{children}}) {
+                add_review_badge($jr{children}->{$child_id});
+            }
+        }
+
         $jr{date} = delete $jr{oldest_newest};
         $jr{escaped_version} = $jr{version};
         $jr{escaped_version} =~ s/\W/_/g;

lib/OpenQA/Setup.pm

@@ -242,6 +242,7 @@ sub read_config ($app) {
             generic_default_limit => 10000,
             generic_max_limit => 100000,
             tests_overview_max_jobs => 2000,
+            job_group_overview_max_jobs => 5000,
             all_tests_default_finished_jobs => 500,
             all_tests_max_finished_jobs => 5000,
             list_templates_default_limit => 5000,

lib/OpenQA/WebAPI/Controller/API/V1/JobGroup.pm

@@ -400,10 +400,18 @@ sub build_results ($self) {
     my $show_tags = $validation->param('show_tags') // $only_tagged;
 
     my $tags = $show_tags ? $group->tags : undef;
-    my $cbr
-      = OpenQA::BuildResults::compute_build_results($group, $limit_builds,
-        $time_limit_days, $only_tagged ? $tags : undef,
-        [], $tags);
+    my $max_jobs_per_build = $self->app->config->{misc_limits}->{job_group_overview_max_jobs};
+    my $cbr;
+    try {
+        $cbr
+          = OpenQA::BuildResults::compute_build_results($group, $limit_builds,
+            $time_limit_days, $only_tagged ? $tags : undef,
+            [], $tags, $max_jobs_per_build);
+    }
+    catch ($e) {
+        die $e unless $e =~ qr/^(invalid regex: |Build .* has .* jobs, which exceeds the limit)/;
+        return $self->render(json => {error => "$e"}, status => 400);
+    }
     $self->render(json => $cbr);
 }
 

lib/OpenQA/WebAPI/Controller/Main.pm

@@ -134,14 +134,15 @@ sub _group_overview ($self, $resultset, $template) {
 
     my $tags = $group->tags;
     my $cbr;
+    my $max_jobs_per_build = $self->app->config->{misc_limits}->{job_group_overview_max_jobs};
     try {
         $cbr
           = OpenQA::BuildResults::compute_build_results($group, $limit_builds,
             $time_limit_days, $only_tagged ? $tags : undef,
-            $group_params, $tags);
+            $group_params, $tags, $max_jobs_per_build);
     }
     catch ($e) {
-        die $e unless $e =~ qr/^invalid regex: /;
+        die $e unless $e =~ qr/^(invalid regex: |Build .* has .* jobs, which exceeds the limit)/;
         return $self->_respond_error_for_group_overview($e);
     }
     my $build_results = $cbr->{build_results};