Implement metrics for Fedora CI (#3007)

Implement metrics for Fedora CI

Fixes #2975
Assisted-by: Cursor(Claude)

Reviewed-by: gemini-code-assist[bot]
Reviewed-by: Nikola Forró
Reviewed-by: Laura Barcziová

Author: centosinfra-prod-github-app[bot]

packit_service/worker/handlers/distgit.py

@@ -921,6 +921,7 @@ def _run(self) -> TaskResults:
                 description="RPM build was submitted ...",
                 url=url,
             )
+            self.pushgateway.fedora_ci_koji_builds_queued.inc()
         except Exception as ex:
             sentry_integration.send_to_sentry(ex)
             self.report(

packit_service/worker/handlers/koji.py

@@ -7,7 +7,7 @@
 
 import logging
 from abc import ABC, abstractmethod
-from datetime import datetime
+from datetime import datetime, timezone
 from os import getenv
 from typing import Optional
 
@@ -42,6 +42,7 @@
 from packit_service.utils import (
     dump_job_config,
     dump_package_config,
+    elapsed_seconds,
 )
 from packit_service.worker.checker.abstract import Checker
 from packit_service.worker.checker.koji import (
@@ -162,6 +163,9 @@ def notify_about_failure_if_configured(
     def trigger_log_detective_if_configured(self):
         pass
 
+    @abstractmethod
+    def push_metrics(self) -> None: ...
+
     @property
     def build(self) -> Optional[KojiBuildTargetModel]:
         if not self._build:
@@ -242,6 +246,7 @@ def _run(self) -> TaskResults:
             )
 
         else:
+            self.push_metrics()
             self.build.set_status(new_commit_status.value)
             self.report(description, new_commit_status, dashboard_url, koji_web_url)
             koji_build_logs = self.koji_task_event.get_koji_build_rpm_tasks_logs_urls(
@@ -319,6 +324,9 @@ def notify_about_failure_if_configured(
     def trigger_log_detective_if_configured(self):
         pass
 
+    def push_metrics(self) -> None:
+        pass
+
 
 @reacts_to_as_fedora_ci(event=koji.result.Task)
 class KojiTaskReportDownstreamHandler(AbstractKojiTaskReportHandler, FedoraCIJobHandler):
@@ -376,6 +384,21 @@ def trigger_log_detective_if_configured(self):
         if not trigger_success:
             logger.error("Log Detective was not properly triggered for a failed Koji build")
 
+    def push_metrics(self) -> None:
+        """Track Fedora CI Koji build metrics."""
+        state = self.koji_task_event.state
+
+        if state == KojiTaskState.open:
+            self.pushgateway.fedora_ci_koji_builds_started.inc()
+        elif state in (KojiTaskState.closed, KojiTaskState.canceled, KojiTaskState.failed):
+            self.pushgateway.fedora_ci_koji_builds_finished.inc()
+            if self.build.submitted_time:
+                koji_build_time = elapsed_seconds(
+                    begin=self.build.submitted_time,
+                    end=datetime.now(timezone.utc),
+                )
+                self.pushgateway.fedora_ci_koji_build_finished_time.observe(koji_build_time)
+
 
 @configured_as(job_type=JobType.koji_build)
 @configured_as(job_type=JobType.bodhi_update)

packit_service/worker/handlers/testing_farm.py

@@ -458,7 +458,7 @@ def run_for_fedora_ci_test(
         failed: dict,
     ):
         if self.celery_task.retries == 0:
-            self.pushgateway.test_runs_queued.inc()
+            self.pushgateway.fedora_ci_test_runs_queued.inc()
         result = self.downstream_testing_farm_job_helper.run_testing_farm(test_run)
         if not result["success"]:
             failed[test_run.data["fedora_ci_test"]] = result.get("details")
@@ -697,14 +697,14 @@ def _run(self) -> TaskResults:
             summary = self.summary or "Error ..."
 
         if self.result == TestingFarmResult.running:
-            self.pushgateway.test_runs_started.inc()
+            self.pushgateway.fedora_ci_test_runs_started.inc()
         else:
-            self.pushgateway.test_runs_finished.inc()
+            self.pushgateway.fedora_ci_test_runs_finished.inc()
             test_run_time = elapsed_seconds(
                 begin=test_run_model.submitted_time,
                 end=datetime.now(timezone.utc),
             )
-            self.pushgateway.test_run_finished_time.observe(test_run_time)
+            self.pushgateway.fedora_ci_test_run_finished_time.observe(test_run_time)
 
         test_run_model.set_web_url(self.log_url)
         # For Fedora CI, try to link directly to Testing Farm results instead of dashboard

packit_service/worker/jobs.py

@@ -507,6 +507,7 @@ def report_task_accepted_for_fedora_ci(self, handler_kls: type[FedoraCIJobHandle
             target_branch=self.event.pull_request_object.target_branch,
         )
 
+        first_status_reported = False
         for check_name in handler_kls.get_check_names(
             self.service_config, self.event.project, metadata
         ):
@@ -516,6 +517,18 @@ def report_task_accepted_for_fedora_ci(self, handler_kls: type[FedoraCIJobHandle
                 url="",
                 check_name=check_name,
             )
+            # Track the time for the first status report
+            if not first_status_reported:
+                first_status_reported = True
+                response_time = elapsed_seconds(
+                    begin=self.event.created_at,
+                    end=datetime.now(),
+                )
+                logger.debug(
+                    "Reporting Fedora CI first initial status "
+                    f"check time: {response_time} seconds.",
+                )
+                self.pushgateway.fedora_ci_first_initial_status_time.observe(response_time)
 
     def search_distgit_config_in_issue(self) -> Optional[tuple[str, PackageConfig]]:
         """Get a tuple (dist-git repo url, package config loaded from dist-git yaml file).

packit_service/worker/monitoring.py

@@ -191,6 +191,80 @@ def __init__(self):
             registry=self.registry,
         )
 
+        # Fedora CI metrics
+        self.fedora_ci_koji_builds_queued = Counter(
+            "fedora_ci_koji_builds_queued",
+            "Number of Fedora CI Koji scratch builds queued",
+            registry=self.registry,
+        )
+
+        self.fedora_ci_koji_builds_started = Counter(
+            "fedora_ci_koji_builds_started",
+            "Number of Fedora CI Koji scratch builds started",
+            registry=self.registry,
+        )
+
+        self.fedora_ci_koji_builds_finished = Counter(
+            "fedora_ci_koji_builds_finished",
+            "Number of Fedora CI Koji scratch builds finished",
+            registry=self.registry,
+        )
+
+        self.fedora_ci_koji_build_finished_time = Histogram(
+            "fedora_ci_koji_build_finished_time",
+            "Time it takes from submitting Fedora CI Koji scratch build to finished",
+            registry=self.registry,
+            buckets=(
+                1800,
+                3600,
+                3 * 3600,
+                6 * 3600,
+                12 * 3600,
+                24 * 3600,
+                float("inf"),
+            ),
+        )
+
+        self.fedora_ci_test_runs_queued = Counter(
+            "fedora_ci_test_runs_queued",
+            "Number of Fedora CI test runs queued",
+            registry=self.registry,
+        )
+
+        self.fedora_ci_test_runs_started = Counter(
+            "fedora_ci_test_runs_started",
+            "Number of Fedora CI test runs started",
+            registry=self.registry,
+        )
+
+        self.fedora_ci_test_runs_finished = Counter(
+            "fedora_ci_test_runs_finished",
+            "Number of Fedora CI test runs finished",
+            registry=self.registry,
+        )
+
+        self.fedora_ci_test_run_finished_time = Histogram(
+            "fedora_ci_test_run_finished_time",
+            "Time it takes from submitting Fedora CI test run to finished",
+            registry=self.registry,
+            buckets=(
+                1800,
+                3600,
+                3 * 3600,
+                6 * 3600,
+                12 * 3600,
+                24 * 3600,
+                float("inf"),
+            ),
+        )
+
+        self.fedora_ci_first_initial_status_time = Histogram(
+            "fedora_ci_first_initial_status_time",
+            "Time it takes to set the initial status for the first Fedora CI check",
+            registry=self.registry,
+            buckets=(5, 15, 20, 25, 30, 40, 60, float("inf")),
+        )
+
     def push(self):
         if not (self.pushgateway_address and self.worker_name):
             logger.debug("Pushgateway address or worker name not defined.")

tests/conftest.py

@@ -360,6 +360,7 @@ def koji_build_pr_downstream():
         status="some-status",
         group_of_targets=koji_group,
         runs=runs,
+        submitted_time=None,
     )
     koji_build_model._srpm_build_for_mocking = srpm_build
     koji_build_model.get_project_event_object = lambda: pr_model

tests/integration/test_logdetective_koji.py

@@ -91,8 +91,14 @@ def test_logdetective_koji_build_scratch_downstream(
     )
     flexmock(LogDetectiveRunModel).should_receive("create").times(ld_calls)
 
-    pushgateway = flexmock(log_detective_runs_started=flexmock())
+    pushgateway = flexmock(
+        log_detective_runs_started=flexmock(),
+        fedora_ci_koji_builds_started=flexmock(),
+        fedora_ci_koji_builds_finished=flexmock(),
+        fedora_ci_koji_build_finished_time=flexmock(),
+    )
     pushgateway.log_detective_runs_started.should_receive("inc").times(ld_calls).and_return()
+    pushgateway.fedora_ci_koji_builds_finished.should_receive("inc").once().and_return()
     pushgateway.should_receive("push").and_return()
     flexmock(Pushgateway).new_instances(pushgateway)
 

tests_openshift/service/conftest.py

@@ -90,14 +90,25 @@ def mock_metrics_counters():
     # Specific metrics for Log Detective
     mock_pushgateway.log_detective_runs_finished = mock_counter
     mock_pushgateway.log_detective_run_finished = mock_histogram
-    mock_pushgateway.log_detective_runs_started = mock_counter  # Use mock_counter for this too
+    mock_pushgateway.log_detective_runs_started = mock_counter
 
     # General metrics used by SteveJobs
     mock_pushgateway.events_processed = mock_counter
     mock_pushgateway.events_not_handled = mock_counter
     mock_pushgateway.events_pre_check_failed = mock_counter
     mock_pushgateway.rate_limited_tasks_enqueued = mock_counter
 
+    # Fedora CI metrics
+    mock_pushgateway.fedora_ci_koji_builds_queued = mock_counter
+    mock_pushgateway.fedora_ci_koji_builds_started = mock_counter
+    mock_pushgateway.fedora_ci_koji_builds_finished = mock_counter
+    mock_pushgateway.fedora_ci_koji_build_finished_time = mock_histogram
+    mock_pushgateway.fedora_ci_test_runs_queued = mock_counter
+    mock_pushgateway.fedora_ci_test_runs_started = mock_counter
+    mock_pushgateway.fedora_ci_test_runs_finished = mock_counter
+    mock_pushgateway.fedora_ci_test_run_finished_time = mock_histogram
+    mock_pushgateway.fedora_ci_first_initial_status_time = mock_histogram
+
     # Inject the mock instance
     flexmock(Pushgateway).new_instances(mock_pushgateway)