Move the Celery broker to SQS

backend/.env.example

@@ -13,6 +13,7 @@ APP_ALLOW_FIRST_TIME_REGISTER=
 
 # -- BACKEND --
 BACKEND_URL=http://localhost:5000
+CELERY_BROKER=redis
 
 # -- FRONTEND --
 FRONTEND_URL=http://localhost:8090

backend/.env.test

@@ -7,6 +7,7 @@ LOG_USE_STREAM=1
 
 # -- BACKEND --
 BACKEND_URL=http://localhost:5000
+CELERY_BROKER=redis
 
 # -- FRONTEND --
 FRONTEND_URL=http://localhost:8090

backend/requirements.txt

@@ -3,7 +3,7 @@ argon2-cffi==25.*
 argon2-cffi-bindings==25.*
 Babel==2.*
 caldav==1.6.0
-celery[redis]==5.*
+celery[sqs]==5.*
 cryptography==46.0.6
 dnspython==2.*
 fastapi[standard]==0.*

backend/src/appointment/celery_app.py

@@ -20,17 +20,23 @@ def _base_redis_url() -> str:
 def create_celery_app() -> Celery:
     load_dotenv()
 
-    redis_url = _base_redis_url()
-
-    broker_url = '/'.join(filter(None, [os.getenv('CELERY_BROKER') or redis_url, os.getenv('REDIS_CELERY_DB')]))
-    result_backend = '/'.join(
-        filter(None, [os.getenv('CELERY_BACKEND') or redis_url, os.getenv('REDIS_CELERY_RESULTS_DB')])
-    )
-
-    if broker_url.startswith('rediss://'):
-        broker_url = f'{broker_url}?ssl_cert_reqs=CERT_REQUIRED'
-    if result_backend.startswith('rediss://'):
-        result_backend = f'{result_backend}?ssl_cert_reqs=CERT_REQUIRED'
+    _celery_broker = os.getenv('CELERY_BROKER', 'redis')
+    if _celery_broker == 'redis':
+        redis_url = _base_redis_url()
+
+        broker_url = '/'.join(filter(None, [os.getenv('CELERY_BROKER') or redis_url, os.getenv('REDIS_CELERY_DB')]))
+        result_backend = '/'.join(
+            filter(None, [os.getenv('CELERY_BACKEND') or redis_url, os.getenv('REDIS_CELERY_RESULTS_DB')])
+        )
+
+        if broker_url.startswith('rediss://'):
+            broker_url = f'{broker_url}?ssl_cert_reqs=CERT_REQUIRED'
+        if result_backend.startswith('rediss://'):
+            result_backend = f'{result_backend}?ssl_cert_reqs=CERT_REQUIRED'
+    elif _celery_broker == 'sqs':
+
+    else:
+        raise ValueError(f'"{_celery_broker}" is not a valid value for CELERY_BROKER. Choose "redis" or "sqs".')
 
     result_expires = 3600
     task_always_eager = os.getenv('CELERY_EAGER', 'False') == 'True'

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - ./backend/.env:/app/.env
       - ./backend/src/appointment:/app/appointment
     environment:
+      - CELERY_BROKER=redis
       - CONTAINER_ROLE=api
       - IS_LOCAL_DEV=yes
       - RELEASE_VERSION=localdev
@@ -23,6 +24,7 @@ services:
     <<: *backend
     ports: []
     environment:
+      - CELERY_BROKER=redis
       - CONTAINER_ROLE=worker
     depends_on:
       - backend
@@ -32,6 +34,7 @@ services:
     <<: *backend
     ports: []
     environment:
+      - CELERY_BROKER=redis
       - CONTAINER_ROLE=beat
     depends_on:
       - backend
@@ -42,6 +45,7 @@ services:
     ports:
       - 5556:5555
     environment:
+      - CELERY_BROKER=redis
       - CONTAINER_ROLE=flower
       - FLOWER_UNAUTHENTICATED_API=true
     depends_on:

pulumi/__main__.py

@@ -13,6 +13,7 @@
 from fargate import fargate
 from redis import redis_cache
 from security_groups import security_groups
+from sqs import sqs
 
 #: Environments in which we should build a proxy for Posthog calls
 POSTHOG_PROXY_STACKS = ['prod']
@@ -91,6 +92,13 @@
     for afc_name, afc_config in resources.get('tb:fargate:AutoscalingFargateCluster', {}).items()
 }
 
+# Celery does not currently support clustered Redis brokers. Therefore, we build an SQS queue to route tasks through.
+sqs(
+    project=project,
+    api_exec_role=fargate_clusters['backend'].resources['task_role'].name,
+    celery_exec_role=afcs['appointment'].resources['execution_roles']['celery'].name,
+)
+
 # CloudFront function to handle request rewrites headed to the backend
 rewrite_function = cloudfront.rewrite_function(project=project)
 project.resources['cf_rewrite_function'] = rewrite_function

pulumi/config.dev.yaml

@@ -3,21 +3,25 @@
 ### Special variables used throughout this file
 
 # Update this value to update all containers based on the thunderbird/appointment image
-.apmt_image: &APMT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/appointment:85f15a97bf5ee889bb3ca56cea3f4dac7f5c00d2
+.apmt_image: &APMT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/appointment:f0b6b209385f77dd2d522ded8e4826b66eb045df
 
 # These variables are common to Accounts application environments. Some tasks will require additional configuration.
 .app_env: &VAR_APP_ENV {name: "APP_ENV", value: "dev"}
 .auth_scheme: &VAR_AUTH_SCHEME {name: "AUTH_SCHEME", value: "oidc"}
+.celery_broken: &VAR_CELERY_BROKER {name: "CELERY_BROKER", value: "sqs"}
 .database_engine: &VAR_DATABASE_ENGINE {name: "DATABASE_ENGINE", value: "postgresql"}
 .database_port: &VAR_DATABASE_PORT {name: "DATABASE_PORT", value: "5432"}
+.frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment-dev.tb.pro"}
+.fxa_callback: &VAR_FXA_CALLBACK {name: "FXA_CALLBACK", value: "https://appointment-dev.tb.pro/fxa"}
+.google_auth_callback: &VAR_GOOGLE_AUTH_CALLBACK {name: "GOOGLE_AUTH_CALLBACK", value: "https://appointment-dev.tb.pro/api/v1/google/callback"}
 .jwt_algo: &VAR_JWT_ALGO {name: "JWT_ALGO", value: "HS256"}
 .jwt_expire_in_mins: &VAR_JWT_EXPIRE_IN_MINS {name: "JWT_EXPIRE_IN_MINS", value: "10000"}
 .log_level: &VAR_LOG_LEVEL {name: "LOG_LEVEL", value: "ERROR"}
 .log_use_stream: &VAR_LOG_USE_STREAM {name: "LOG_USE_STREAM", value: "True"}
-.oids_exp_grace_period: &VAR_OIDC_EXP_GRACE_PERIOD {name: "OIDC_EXP_GRACE_PERIOD", value: "60"}
+.oidc_exp_grace_period: &VAR_OIDC_EXP_GRACE_PERIOD {name: "OIDC_EXP_GRACE_PERIOD", value: "60"}
 .oidc_fallback_match_by_email: &VAR_OIDC_FALLBACK_MATCH_BY_EMAIL {name: "OIDC_FALLBACK_MATCH_BY_EMAIL", value: "True"}
 .oidc_token_introspection_url: &VAR_OIDC_TOKEN_INTROSPECTION_URL {name: "OIDC_TOKEN_INTROSPECTION_URL", value: "https://auth-dev.tb.pro/realms/tbpro/protocol/openid-connect/token/introspect"}
-.frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment-dev.tb.pro"}
+.posthog_host: &VAR_POSTHOG_HOST {name: "POSTHOG_HOST", value: "https://us.i.posthog.com"}
 .redis_db: &VAR_REDIS_DB {name: "REDIS_DB", value: "0"}
 .redis_port: &VAR_REDIS_PORT {name: "REDIS_PORT", value: "6379"}
 .redis_use_ssl: &VAR_REDIS_USE_SSL {name: "REDIS_USE_SSL", value: "True"}
@@ -26,20 +30,43 @@
 .sentry_dsn: &VAR_SENTRY_DSN {name: "SENTRY_DSN", value: "https://5dddca3ecc964284bb8008bc2beef808@o4505428107853824.ingest.sentry.io/4505428124827648"}
 .service_email: &VAR_SERVICE_EMAIL {name: "SERVICE_EMAIL", value: "no-reply@appointment-dev.tb.pro"}
 .short_base_url: &VAR_SHORT_BASE_URL {name: "SHORT_BASE_URL", value: "https://dev.apt.mt"}
+.smtp_port: &VAR_SMTP_PORT {name: "SMTP_PORT", value: "587"}
+.smtp_security: &VAR_SMTP_SECURITY {name: "SMTP_SECURITY", value: "STARTTLS"}
 .tb_accounts_caldav_url: &VAR_TB_ACCOUNTS_CALDAV_URL {name: "TB_ACCOUNTS_CALDAV_URL", value: "https://dev-thundermail.com"}
+.tb_accounts_host: &VAR_TB_ACCOUNTS_HOST {name: "TB_ACCOUNTS_HOST", value: "https://accounts-dev.tb.pro"}
 .tier_basic_calendar_limit: &VAR_TIER_BASIC_CALENDAR_LIMIT {name: "TIER_BASIC_CALENDAR_LIMIT", value: "3"}
 .tier_plus_calendar_limit: &VAR_TIER_PLUS_CALENDAR_LIMIT {name: "TIER_PLUS_CALENDAR_LIMIT", value: "5"}
 .tier_pro_calendar_limit: &VAR_TIER_PRO_CALENDAR_LIMIT {name: "TIER_PRO_CALENDAR_LIMIT", value: "10"}
 .zoom_api_enabled: &VAR_ZOOM_API_ENABLED {name: "ZOOM_API_ENABLED", value: "True"}
+.zoom_api_new_app: &VAR_ZOOM_API_NEW_APP {name: "ZOOM_API_NEW_APP", value: "False"}
 .zoom_auth_callback: &VAR_ZOOM_AUTH_CALLBACK {name: "ZOOM_AUTH_CALLBACK", value: "https://appointment-dev.tb.pro/api/v1/zoom/callback"}
 
 # These variables are also common to our environments, but are pulled from secret stores instead
+.app_admin_allow_list: &SECRET_APP_ADMIN_ALLOW_LIST {name: "APP_ADMIN_ALLOW_LIST", valueFrom: ""}
+.appointment_caldav_secret: &SECRET_APPOINTMENT_CALDAV_SECRET {name: "APPOINTMENT_CALDAV_SECRET", valueFrom: ""}
 .database_host: &SECRET_DATABASE_HOST {name: "DATABASE_HOST", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-host-pnM2d9"}
 .database_name: &SECRET_DATABASE_NAME {name: "DATABASE_NAME", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-name-slQJtj"}
-.database_username: &SECRET_DATABASE_USERNAME {name: "DATABASE_USERNAME", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-user-9EgJWM"}
 .database_password: &SECRET_DATABASE_PASSWORD {name: "DATABASE_PASSWORD", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-password-yaGZdw"}
+.database_username: &SECRET_DATABASE_USERNAME {name: "DATABASE_USERNAME", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/database-user-9EgJWM"}
+.db_secret: &SECRET_DB_SECRET {name: "DB_SECRET", valueFrom: ""}
+.fxa_allow_list: &SECRET_FXA_ALLOW_LIST {name: "FXA_ALLOW_LIST", valueFrom: ""}
+.fxa_client_id: &SECRET_FXA_CLIENT_ID {name: "FXA_CLIENT_ID", valueFrom: ""}
+.fxa_open_id_config: &SECRET_FXA_OPEN_ID_CONFIG {name: "FXA_OPEN_ID_CONFIG", valueFrom: ""}
+.fxa_secret: &SECRET_FXA_SECRET {name: "FXA_SECRET", valueFrom: ""}
+.google_auth_client_id: &SECRET_GOOGLE_AUTH_CLIENT_ID {name: "GOOGLE_AUTH_CLIENT_ID", valueFrom: ""}
+.google_auth_project_id: &SECRET_GOOGLE_AUTH_PROJECT_ID {name: "GOOGLE_AUTH_PROJECT_ID", valueFrom: ""}
+.google_auth_secret: &SECRET_GOOGLE_AUTH_SECRET {name: "GOOGLE_AUTH_SECRET", valueFrom: ""}
+.jwt_secret: &SECRET_JWT_SECRET {name: "JWT_SECRET", valueFrom: ""}
 .oidc_client_id: &SECRET_OIDC_CLIENT_ID {name: "OIDC_CLIENT_ID", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/oidc-client-id-sbebkz"}
 .oidc_client_secret: &SECRET_OIDC_CLIENT_SECRET {name: "OIDC_CLIENT_SECRET", valueFrom: "arn:aws:secretsmanager:eu-central-1:768512802988:secret:appointment/dev/oidc-client-secret-I9BTf2"}
+.posthog_project_key: &SECRET_POSTHOG_PROJECT_KEY {name: "POSTHOG_PROJECT_KEY", valueFrom: ""}
+.session_secret: &SECRET_SESSION_SECRET {name: "SESSION_SECRET", valueFrom: ""}
+.signed_secret: &SECRET_SIGNED_SECRET {name: "SIGNED_SECRET", valueFrom: ""}
+.smtp_pass: &SECRET_SMTP_PASS {name: "SMTP_PASS", valueFrom: ""}
+.smtp_url: &SECRET_SMTP_URL {name: "SMTP_URL", valueFrom: ""}
+.smtp_user: &SECRET_SMTP_USER {name: "SMTP_USER", valueFrom: ""}
+.zoom_api_secret: &SECRET_ZOOM_API_SECRET {name: "ZOOM_API_SECRET", valueFrom: ""}
+.zoom_auth_client_id: &SECRET_ZOOM_AUTH_CLIENT_ID {name: "ZOOM_AUTH_CLIENT_ID", valueFrom: ""}
 
 # This forms the base of Appointment-based task definitions. The blank fields commented here must be set on any
 # inheriting task definition:
@@ -502,8 +529,8 @@ resources:
 
       autoscalers:
         celery:
-          min_capacity: 1
-          max_capacity: 1
+          min_capacity: 0
+          max_capacity: 0
         flower:
           min_capacity: 1
           max_capacity: 1

pulumi/config.prod.yaml

@@ -8,6 +8,7 @@
 # These variables are common to Accounts application environments. Some tasks will require additional configuration.
 .app_env: &VAR_APP_ENV {name: "APP_ENV", value: "prod"}
 .auth_scheme: &VAR_AUTH_SCHEME {name: "AUTH_SCHEME", value: "oidc"}
+.celery_broker: &VAR_CELERY_BROKER {name: "CELERY_BROKER", value: "sqs"}
 .database_engine: &VAR_DATABASE_ENGINE {name: "DATABASE_ENGINE", value: "postgresql"}
 .database_port: &VAR_DATABASE_PORT {name: "DATABASE_PORT", value: "5432"}
 .frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment.tb.pro"}

pulumi/config.stage.yaml

@@ -8,6 +8,7 @@
 # These variables are common to Accounts application environments. Some tasks will require additional configuration.
 .app_env: &VAR_APP_ENV {name: "APP_ENV", value: "stage"}
 .auth_scheme: &VAR_AUTH_SCHEME {name: "AUTH_SCHEME", value: "oidc"}
+.celery_broken: &VAR_CELERY_BROKER {name: "CELERY_BROKER", value: "sqs"}
 .database_engine: &VAR_DATABASE_ENGINE {name: "DATABASE_ENGINE", value: "postgresql"}
 .database_port: &VAR_DATABASE_PORT {name: "DATABASE_PORT", value: "5432"}
 .frontend_url: &VAR_FRONTEND_URL {name: "FRONTEND_URL", value: "https://appointment-stage.tb.pro"}