Merge pull request #73 from tofu-rocketry/various-fixes

tofu-rocketry · web-flow · commit 4f17c53e16f0 · 2025-12-15T16:28:13.000Z
Various fixes
diff --git a/README.md b/README.md
@@ -1,2 +1,2 @@
-# monitoring
-Monitoring system for APEL servers
+# APEL Data Validation System
+Monitoring system for APEL servers. Built using Django and Django REST framework.
diff --git a/monitoring/benchmarks/templates/benchmarks_by_submithost.html b/monitoring/benchmarks/templates/benchmarks_by_submithost.html
@@ -11,7 +11,7 @@
 </head>
 
 <body>
-  <h2>Sites publishing benchmark records in last 3 months</h2>
+  <h2>Sites publishing benchmark records in the last two months</h2>
   <p>Page last updated: {{ last_fetched|date:"Y-m-d H:i O" }}</p>
   <table>
     <tr>
diff --git a/monitoring/benchmarks/views.py b/monitoring/benchmarks/views.py
@@ -22,8 +22,6 @@ class BenchmarksViewSet(viewsets.ReadOnlyModelViewSet):
 
     def list(self, request):
         last_fetched = BenchmarksBySubmithost.objects.aggregate(Max('fetched'))['fetched__max']
-        if last_fetched is not None:
-            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
 
         response = super(BenchmarksViewSet, self).list(request)
 
diff --git a/monitoring/db_update_sqlite.py b/monitoring/db_update_sqlite.py
@@ -156,7 +156,7 @@ def refresh_gridsite():
                 Site,
                 max(LatestEndTime) AS LatestPublish
             FROM VSuperSummaries
-            WHERE LatestEndTime > DATE_SUB(NOW(), INTERVAL 1 YEAR)
+            WHERE LatestEndTime >= NOW() - INTERVAL 1 YEAR
             GROUP BY 1;
         """
         fetchset = VSuperSummaries.objects.using('grid').raw(sql_query)
@@ -176,31 +176,30 @@ def refresh_gridsite():
 def refresh_cloudsite():
     try:
         sql_query = """
-            SELECT
-                b.SiteName,
-                COUNT(DISTINCT VMUUID) as VMs,
-                CloudType,
-                b.UpdateTime
-            FROM(
+            WITH ranked AS (
                 SELECT
                     SiteName,
-                    MAX(UpdateTime) AS latest
+                    CloudType,
+                    UpdateTime,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY SiteName
+                        ORDER BY UpdateTime DESC
+                    ) AS rn
                 FROM VAnonCloudRecords
-                WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 1 YEAR)
-                GROUP BY SiteName
+                WHERE UpdateTime >= NOW() - INTERVAL 6 MONTH
             )
-            AS a
-            INNER JOIN VAnonCloudRecords
-            AS b
-            ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest
-            GROUP BY SiteName;
+            SELECT
+                SiteName,
+                CloudType,
+                UpdateTime
+            FROM ranked
+            WHERE rn = 1;
         """
         fetchset = VAnonCloudRecord.objects.using('cloud').raw(sql_query)
 
         for f in fetchset:
             CloudSite.objects.update_or_create(
                 defaults={
-                    'Vms': f.VMs,
                     'Script': f.CloudType,
                     'updated': f.UpdateTime
                 },
@@ -305,53 +304,82 @@ def refresh_BenchmarksBySubmitHost_from_view(view_name):
     try:
         if view_name == 'VSummaries':
             sql_query = f"""
-            SELECT DISTINCT v.Site, v.SubmitHost, v.ServiceLevelType, v.ServiceLevel, v.UpdateTime AS LatestPublish
-            FROM {view_name} AS v
-            JOIN (
-                SELECT Site, SubmitHost, MAX(UpdateTime) AS LatestPublish
-                FROM {view_name}
-                WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH)
-                GROUP BY Site, SubmitHost, ServiceLevelType, ServiceLevel
-            ) AS latest
-            ON v.Site = latest.Site
-               AND v.SubmitHost = latest.SubmitHost
-               AND v.UpdateTime = latest.LatestPublish
-            WHERE v.UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH);
-        """
+                WITH latest AS (
+                    SELECT
+                        Site,
+                        SubmitHost,
+                        ServiceLevelType,
+                        ServiceLevel,
+                        UpdateTime,
+                        ROW_NUMBER() OVER (
+                            PARTITION BY Site, SubmitHost
+                            ORDER BY Year DESC, Month DESC, UpdateTime DESC
+                        ) AS rn
+                    FROM VSummaries
+                    WHERE UpdateTime >= NOW() - INTERVAL 2 MONTH
+                )
+                SELECT
+                    Site,
+                    SubmitHost,
+                    ServiceLevelType,
+                    ServiceLevel,
+                    UpdateTime AS LatestPublish
+                FROM latest
+                WHERE rn = 1;
+            """
         elif view_name == 'VJobRecords':
             sql_query = f"""
-            SELECT DISTINCT v.Site, v.SubmitHost, v.ServiceLevelType, v.ServiceLevel, v.UpdateTime AS LatestPublish
-            FROM {view_name} AS v
-            JOIN (
-                SELECT Site, SubmitHost, MAX(UpdateTime) AS LatestPublish
-                FROM {view_name}
-                WHERE EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH)
-                      AND UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH)
-                GROUP BY Site, SubmitHost, ServiceLevelType, ServiceLevel
-            ) AS latest
-            ON v.Site = latest.Site
-               AND v.SubmitHost = latest.SubmitHost
-               AND v.UpdateTime = latest.LatestPublish
-            WHERE v.EndTime > DATE_SUB(NOW(), INTERVAL 3 MONTH)
-                  AND v.UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH);
-        """
+                WITH latest AS (
+                    SELECT
+                        Site,
+                        SubmitHost,
+                        ServiceLevelType,
+                        ServiceLevel,
+                        UpdateTime,
+                        ROW_NUMBER() OVER (
+                            PARTITION BY Site, SubmitHost
+                            ORDER BY EndTime DESC, UpdateTime DESC
+                        ) AS rn
+                    FROM VJobRecords
+                    WHERE EndTime >= NOW() - INTERVAL 2 MONTH
+                        AND UpdateTime >= NOW() - INTERVAL 2 MONTH
+                )
+                SELECT
+                    Site,
+                    SubmitHost,
+                    ServiceLevelType,
+                    ServiceLevel,
+                    UpdateTime AS LatestPublish
+                FROM latest
+                WHERE rn = 1;
+            """
         elif view_name == 'VNormalisedSummaries':
             sql_query = f"""
-            SELECT DISTINCT v.Site, v.SubmitHost, v.ServiceLevelType, ROUND(v.NormalisedWallDuration / v.WallDuration, 3) AS ServiceLevel, v.UpdateTime AS LatestPublish
-            FROM {view_name} AS v
-            JOIN (
-                SELECT Site, SubmitHost, MAX(UpdateTime) AS LatestPublish, ROUND(NormalisedWallDuration / WallDuration, 3) AS ServiceLevel
-                FROM {view_name}
-                WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH)
-                    AND WallDuration > 0
-                GROUP BY Site, SubmitHost, ServiceLevelType, ServiceLevel
-            ) AS latest
-            ON v.Site = latest.Site
-               AND v.SubmitHost = latest.SubmitHost
-               AND v.UpdateTime = latest.LatestPublish
-            WHERE v.UpdateTime > DATE_SUB(NOW(), INTERVAL 3 MONTH)
-                  AND v.WallDuration > 0;
-        """
+                WITH latest AS (
+                    SELECT
+                        Site,
+                        SubmitHost,
+                        ServiceLevelType,
+                        NormalisedWallDuration,
+                        WallDuration,
+                        UpdateTime,
+                        ROW_NUMBER() OVER (
+                            PARTITION BY Site, SubmitHost
+                            ORDER BY Year DESC, Month DESC, UpdateTime DESC
+                        ) AS rn
+                    FROM VNormalisedSummaries
+                    WHERE UpdateTime >= NOW() - INTERVAL 2 MONTH
+                        AND WallDuration > 0
+                )
+                SELECT
+                    Site,
+                    SubmitHost,
+                    ServiceLevelType,
+                    ROUND(NormalisedWallDuration / WallDuration, 3) AS ServiceLevel,
+                    UpdateTime AS LatestPublish
+                FROM latest
+                WHERE rn = 1;
+            """
         else:
             log.warning(f"Unknown view name: {view_name}")
             return
diff --git a/monitoring/iris/templates/iris_cloud_and_grid.html b/monitoring/iris/templates/iris_cloud_and_grid.html
@@ -11,7 +11,7 @@
 </head>
 
 <body>
-  <h2>Sites publishing cloud and grid accounting records to IRIS Accounting in the last year</h2>
+  <h2>Sites publishing cloud and grid accounting records to IRIS Accounting in the last six months</h2>
   <p>Page last updated: {{ last_fetched|date:"Y-m-d H:i O" }}</p>
 
   <br/>
diff --git a/monitoring/iris_db_update_sqlite.py b/monitoring/iris_db_update_sqlite.py
@@ -65,7 +65,7 @@ def refresh_iris_cloud_and_grid():
                 Site,
                 max(LatestEndTime) AS LatestPublish
             FROM VSuperSummaries
-            WHERE LatestEndTime > DATE_SUB(NOW(), INTERVAL 1 YEAR)
+            WHERE LatestEndTime >= NOW() - INTERVAL 6 MONTH
             AND (Site LIKE 'UK%%' OR Site LIKE 'RAL-LCG2')
             GROUP BY 1;
         """
@@ -81,20 +81,10 @@ def refresh_iris_cloud_and_grid():
 
         sql_query = """
             SELECT
-                b.SiteName,
-                b.UpdateTime AS LatestPublish
-            FROM(
-                SELECT
-                    SiteName,
-                    MAX(UpdateTime) AS latest
-                FROM VAnonCloudRecords
-                WHERE UpdateTime > DATE_SUB(NOW(), INTERVAL 1 YEAR)
-                GROUP BY SiteName
-            )
-            AS a
-            INNER JOIN VAnonCloudRecords
-            AS b
-            ON b.SiteName = a.SiteName AND b.UpdateTime = a.latest
+                SiteName,
+                MAX(UpdateTime) AS LatestPublish
+            FROM VAnonCloudRecords
+            WHERE UpdateTime >= NOW() - INTERVAL 6 MONTH
             GROUP BY SiteName;
         """
         fetchset = VAnonCloudRecords.objects.using('iris_cloud').raw(sql_query)
diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html
@@ -10,20 +10,18 @@
 </head>
 
 <body>
-  <h2>Sites publishing cloud accounting records in the last year</h2>
+  <h2>Sites publishing cloud accounting records in the last six months</h2>
   <p>Data last fetched: {{ last_fetched|date:"Y-m-d H:i O" }}</p>
   <table>
     <tr>
       <th class='tableheader'>Site</th>
-      <th class='tableheader'>VMsInLastUpdate</th>
       <th class='tableheader'>CloudType</th>
       <th class='tableheader'>LastUpdated</th>
       <th class='tableheader'>Publication Status</th>
     </tr>
     {% for site in sites %}
     <tr onmouseover="this.className='highlight-row'" onmouseout="this.className='tabletext'" class="tabletext">
       <td>{{ site.SiteName }}</td>
-      <td>{{ site.Vms }}</td>
       <td>{{ site.Script }}</td>
       <td>{{ site.updated|date:"Y-m-d H:i:s" }}</td>
       <td>{{ site.stdout }}</td>
diff --git a/monitoring/publishing/views.py b/monitoring/publishing/views.py
@@ -26,11 +26,11 @@
     GridSiteSyncSubmitHSerializer
 )
 
-def update_dict_stdout_and_returncode(single_dict, date, days=7):
+def update_dict_stdout_and_returncode(single_dict, date, warn_days=7, crit_days=31):
     today = datetime.today()
 
-    # Handle future dates
-    if date > today:
+    # Handle null values
+    if date is None:
         single_dict.update({
             'returncode': 3,
             'stdout': "UNKNOWN"
@@ -40,12 +40,15 @@ def update_dict_stdout_and_returncode(single_dict, date, days=7):
     diff_days = (today - date).days
     formatted_date = date.strftime("%Y-%m-%d")
 
-    if diff_days <= days:
-        status = "OK"
-        returncode = 0
-    else:
+    if diff_days > crit_days:
+        status = "CRITICAL"
+        returncode = 2
+    elif diff_days > warn_days:
         status = "WARNING"
         returncode = 1
+    else:
+        status = "OK"
+        returncode = 0
 
     single_dict.update({
         'returncode': returncode,
@@ -63,8 +66,6 @@ class GridSiteViewSet(viewsets.ReadOnlyModelViewSet):
 
     def list(self, request):
         last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max']
-        if last_fetched is not None:
-            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
 
         final_response = []
         response = super(GridSiteViewSet, self).list(request)
@@ -85,8 +86,6 @@ def list(self, request):
     def retrieve(self, request, SiteName=None):
         last_fetched = GridSite.objects.aggregate(Max('fetched'))['fetched__max']
         # If there's no data then last_fetched is None.
-        if last_fetched is not None:
-            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
 
         response = super(GridSiteViewSet, self).retrieve(request)
         date = response.data['updated'].replace(tzinfo=None)
@@ -124,9 +123,6 @@ def get_template_names(self):
     def list(self, request):
         last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max']
 
-        if last_fetched is not None:
-            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
-
         response = super(GridSiteSyncViewSet, self).list(request)
         response.data = {
             'records': response.data,
@@ -137,9 +133,6 @@ def list(self, request):
     def retrieve(self, request, SiteName=None):
         last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max']
 
-        if last_fetched is not None:
-            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
-
         sites_list_qs = GridSiteSync.objects.filter(SiteName=SiteName)
         sites_list_serializer = self.get_serializer(sites_list_qs, many=True)
 
@@ -186,9 +179,6 @@ def list(self, request):
     def retrieve(self, request, SiteName=None, YearMonth=None):
         last_fetched = GridSiteSyncSubmitH.objects.aggregate(Max('fetched'))['fetched__max']
 
-        if last_fetched is not None:
-            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
-
         site_and_year_queryset = GridSiteSyncSubmitH.objects.filter(
             SiteName=SiteName,
             YearMonth=YearMonth
@@ -212,8 +202,6 @@ class CloudSiteViewSet(viewsets.ReadOnlyModelViewSet):
 
     def list(self, request):
         last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max']
-        if last_fetched is not None:
-            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
 
         final_response = []
         response = super(CloudSiteViewSet, self).list(request)
@@ -233,7 +221,6 @@ def list(self, request):
 
     def retrieve(self, request, SiteName=None):
         last_fetched = CloudSite.objects.aggregate(Max('fetched'))['fetched__max']
-        print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
 
         response = super(CloudSiteViewSet, self).retrieve(request)
         # Wrap data in a dict so that it can display in template.
