fixup! Fix: Many false positives for waf_blocked alarm in idle deployments (#7642)

Author: dsotirho-ucsc

environment.py

@@ -919,5 +919,13 @@ def env() -> Mapping[str, str | None]:
         # $1 per one million requests above ten million requests. The blocking
         # only applies to URLs disallowed via robots.txt.
         #
-        'azul_waf_bot_control': '0'
+        'azul_waf_bot_control': '0',
+
+        # The maximum allowed percentage of blocked requests (number of blocked
+        # requests, divided by the number of all requests, times 100) for a
+        # configured period before a metric alarm is tripped.
+        #
+        # If None, a default value set in azul.Config will be used.
+        #
+        'azul_blocked_alarm_threshold': None
     }

src/azul/__init__.py

@@ -1855,6 +1855,15 @@ def __attrs_post_init__(self):
     def waf_bot_control(self) -> bool:
         return self._boolean(self.environ['azul_waf_bot_control'])
 
+    @property
+    def blocked_alarm_threshold(self) -> int:
+        value = self.environ['azul_blocked_alarm_threshold']
+        if value is None:
+            return 25 if self.deployment_stage == 'prod' else 50
+        else:
+            return int(value)
+
+
     @property
     def vpc_cidr(self) -> str:
         return self.environ['azul_vpc_cidr']

terraform/api_gateway.tf.json.template.py

@@ -181,8 +181,9 @@ def waf_match_path(path_regex: str) -> JSON:
 def add_waf_blocked_alarm(resources: JSON) -> JSON:
     """
     Add a metric alarm that trips if the ratio between blocked and overall
-    requests goes above the set threshold. Note that requests blocked by rules
-    listed in :py:attr:`Config.waf_rules_not_logged` are not considered.
+    requests goes above a deployment-specific threshold. Note that requests
+    blocked by rules listed in :py:attr:`Config.waf_rules_not_logged` are not
+    considered.
     """
     if not config.enable_monitoring:
         return resources
@@ -208,7 +209,6 @@ def add_waf_blocked_alarm(resources: JSON) -> JSON:
         ]
         m_sum = '+'.join(f'm{i}' for i in range(1, len(metrics)))
         expression = f'({m_sum})/(m0+{m_sum})*100'
-        threshold = 25 if config.deployment_stage == 'prod' else 50
 
         assert 'aws_cloudwatch_metric_alarm' not in resources
         return resources | {
@@ -241,7 +241,7 @@ def add_waf_blocked_alarm(resources: JSON) -> JSON:
                         }
                     ],
                     'comparison_operator': 'GreaterThanThreshold',
-                    'threshold': threshold,
+                    'threshold': config.blocked_alarm_threshold,
                     'evaluation_periods': 1,
                     'datapoints_to_alarm': 1,
                     'alarm_actions': ['${data.aws_sns_topic.monitoring.arn}'],