#8 add Terraform plan debugging and timeout

Author: jayhill

.github/workflows/pr-checks.yml

@@ -67,8 +67,39 @@ jobs:
 
             - name: Terraform Plan
               working-directory: ./infra/terraform/dev
+              timeout-minutes: 5
               run: |
-                  terraform plan -no-color > plan_full.txt || { exit_code=$?; cat plan_full.txt; echo "plan<<EOF" >> $GITHUB_OUTPUT; cat plan_full.txt >> $GITHUB_OUTPUT; echo "EOF" >> $GITHUB_OUTPUT; echo "has_changes=true" >> $GITHUB_OUTPUT; exit $exit_code; }
+                  echo "Starting Terraform plan at $(date)"
+                  echo "Current working directory: $(pwd)"
+                  echo "Terraform version: $(terraform version)"
+                  echo "AWS region: $AWS_REGION"
+                  echo "Checking state lock status..."
+
+                  # Add debugging for state backend
+                  terraform show -json 2>/dev/null | jq -r '.backend // "No backend info"' || echo "Could not query backend info"
+
+                  # Enable verbose logging
+                  export TF_LOG=DEBUG
+                  export TF_LOG_PATH=/tmp/terraform.log
+
+                  echo "Running terraform plan with timeout and debugging..."
+                  timeout 240 terraform plan -no-color -lock-timeout=60s > plan_full.txt 2>&1 || {
+                    exit_code=$?
+                    echo "Terraform plan failed with exit code: $exit_code"
+                    echo "=== Plan output ==="
+                    cat plan_full.txt
+                    echo "=== Terraform debug logs (last 50 lines) ==="
+                    tail -n 50 /tmp/terraform.log 2>/dev/null || echo "No debug logs available"
+                    echo "=== Current processes ==="
+                    ps aux | grep -E "(terraform|aws)" | grep -v grep || echo "No terraform/aws processes found"
+                    echo "plan<<EOF" >> $GITHUB_OUTPUT
+                    cat plan_full.txt >> $GITHUB_OUTPUT
+                    echo "EOF" >> $GITHUB_OUTPUT
+                    echo "has_changes=true" >> $GITHUB_OUTPUT
+                    exit $exit_code
+                  }
+
+                  echo "Terraform plan completed successfully at $(date)"
                   cat plan_full.txt
 
                   # Check if there are any changes planned
@@ -133,14 +164,67 @@ jobs:
               working-directory: ./infra/terraform/prod
               run: terraform init
 
+            - name: Check and Clear State Locks
+              working-directory: ./infra/terraform/prod
+              run: |
+                  echo "Checking for existing state locks in prod account..."
+                  # Try to acquire a lock briefly to see if one exists
+                  if ! timeout 10 terraform plan -detailed-exitcode >/dev/null 2>&1; then
+                    echo "Terraform operation failed, checking if it's a lock issue..."
+
+                    # Check the state file directly for lock info
+                    if aws s3 cp s3://jaildata-tf-state/terraform.tfstate - 2>/dev/null | jq -e '.lineage' >/dev/null 2>&1; then
+                      echo "State file exists and is readable"
+                    else
+                      echo "State file may have issues"
+                    fi
+
+                    # Try to force unlock any existing locks
+                    echo "Attempting to clear any existing locks..."
+                    terraform force-unlock -force $(aws s3 cp s3://jaildata-tf-state/.terraform/terraform.tfstate.lock.info - 2>/dev/null | jq -r '.ID // empty') 2>/dev/null || echo "No lock file found or unable to unlock"
+                  else
+                    echo "No lock detected"
+                  fi
+
             - name: Set Terraform environment variables
               run: |
                   echo "TF_VAR_alert_email=${{ vars.ALERT_EMAIL }}" >> $GITHUB_ENV
 
             - name: Terraform Plan
               working-directory: ./infra/terraform/prod
+              timeout-minutes: 5
               run: |
-                  terraform plan -no-color > plan_full.txt || { exit_code=$?; cat plan_full.txt; echo "plan<<EOF" >> $GITHUB_OUTPUT; cat plan_full.txt >> $GITHUB_OUTPUT; echo "EOF" >> $GITHUB_OUTPUT; echo "has_changes=true" >> $GITHUB_OUTPUT; exit $exit_code; }
+                  echo "Starting Terraform plan at $(date)"
+                  echo "Current working directory: $(pwd)"
+                  echo "Terraform version: $(terraform version)"
+                  echo "AWS region: $AWS_REGION"
+                  echo "Checking state lock status..."
+
+                  # Add debugging for state backend
+                  terraform show -json 2>/dev/null | jq -r '.backend // "No backend info"' || echo "Could not query backend info"
+
+                  # Enable verbose logging
+                  export TF_LOG=DEBUG
+                  export TF_LOG_PATH=/tmp/terraform.log
+
+                  echo "Running terraform plan with timeout and debugging..."
+                  timeout 240 terraform plan -no-color -lock-timeout=60s > plan_full.txt 2>&1 || {
+                    exit_code=$?
+                    echo "Terraform plan failed with exit code: $exit_code"
+                    echo "=== Plan output ==="
+                    cat plan_full.txt
+                    echo "=== Terraform debug logs (last 50 lines) ==="
+                    tail -n 50 /tmp/terraform.log 2>/dev/null || echo "No debug logs available"
+                    echo "=== Current processes ==="
+                    ps aux | grep -E "(terraform|aws)" | grep -v grep || echo "No terraform/aws processes found"
+                    echo "plan<<EOF" >> $GITHUB_OUTPUT
+                    cat plan_full.txt >> $GITHUB_OUTPUT
+                    echo "EOF" >> $GITHUB_OUTPUT
+                    echo "has_changes=true" >> $GITHUB_OUTPUT
+                    exit $exit_code
+                  }
+
+                  echo "Terraform plan completed successfully at $(date)"
                   cat plan_full.txt
 
                   # Check if there are any changes planned