Fix: Many false positives for waf_blocked alarm in idle deployments (#7642)

Author: dsotirho-ucsc

terraform/api_gateway.tf.json.template.py

@@ -181,8 +181,8 @@ def waf_match_path(path_regex: str) -> JSON:
 def add_waf_blocked_alarm(resources: JSON) -> JSON:
     """
     Add a metric alarm that trips if the ratio between blocked and overall
-    requests goes above 25%. Note that requests blocked by rules listed in
-    :py:attr:`Config.waf_rules_not_logged` are not considered.
+    requests goes above the set threshold. Note that requests blocked by rules
+    listed in :py:attr:`Config.waf_rules_not_logged` are not considered.
     """
     if not config.enable_monitoring:
         return resources
@@ -208,6 +208,7 @@ def add_waf_blocked_alarm(resources: JSON) -> JSON:
         ]
         m_sum = '+'.join(f'm{i}' for i in range(1, len(metrics)))
         expression = f'({m_sum})/(m0+{m_sum})*100'
+        threshold = 25 if config.deployment_stage == 'prod' else 50
 
         assert 'aws_cloudwatch_metric_alarm' not in resources
         return resources | {
@@ -240,7 +241,7 @@ def add_waf_blocked_alarm(resources: JSON) -> JSON:
                         }
                     ],
                     'comparison_operator': 'GreaterThanThreshold',
-                    'threshold': 25,  # percent blocked of total requests in a period
+                    'threshold': threshold,
                     'evaluation_periods': 1,
                     'datapoints_to_alarm': 1,
                     'alarm_actions': ['${data.aws_sns_topic.monitoring.arn}'],