Merge pull request #3 from jacksonlester/staging

jacksonlester · web-flow · commit 949cbc7c8126 · 2025-10-07T22:53:30.000-07:00
Add staging environment support to data pipeline
diff --git a/.github/workflows/rebuild-cache.yml b/.github/workflows/rebuild-cache.yml
@@ -3,11 +3,21 @@ name: Rebuild Cache
 on:
   # Trigger manually from GitHub UI
   workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to deploy to'
+        required: true
+        default: 'staging'
+        type: choice
+        options:
+          - staging
+          - production
 
   # Trigger when events.csv or geometries are updated
   push:
     branches:
       - main
+      - staging
     paths:
       - 'events.csv'
       - 'geometries/**'
@@ -20,6 +30,20 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Determine environment
+        id: env
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "STAGING=${{ github.event.inputs.environment == 'staging' && 'true' || 'false' }}" >> $GITHUB_OUTPUT
+            echo "ENV_NAME=${{ github.event.inputs.environment }}" >> $GITHUB_OUTPUT
+          elif [ "${{ github.ref }}" == "refs/heads/staging" ]; then
+            echo "STAGING=true" >> $GITHUB_OUTPUT
+            echo "ENV_NAME=staging" >> $GITHUB_OUTPUT
+          else
+            echo "STAGING=false" >> $GITHUB_OUTPUT
+            echo "ENV_NAME=production" >> $GITHUB_OUTPUT
+          fi
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
@@ -32,4 +56,7 @@ jobs:
         env:
           SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
           SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
-        run: node rebuild-cache.js
+          STAGING: ${{ steps.env.outputs.STAGING }}
+        run: |
+          echo "Rebuilding ${{ steps.env.outputs.ENV_NAME }} cache..."
+          node rebuild-cache.js
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -2,6 +2,17 @@
 
 This dataset tracks autonomous vehicle deployments and powers [avmap.io](https://avmap.io). While currently focused on US services, we accept data from anywhere.
 
+## ⚠️ Important: Submit PRs to `staging` branch
+
+**All pull requests should target the `staging` branch, not `main`.** This allows us to test your changes on the staging site before promoting to production.
+
+1. Fork this repository
+2. Create a feature branch from `staging`
+3. Make your changes
+4. Submit a PR to the `staging` branch
+5. Your changes will be tested on the staging environment
+6. Once verified, they'll be promoted to production
+
 ## Event sourcing basics
 
 Each row in `events.csv` represents one change to a service. Service creation events include all attributes. **Update events must include the complete new state for the field being updated** (not deltas) and always include the `company` field to identify which service is being updated.
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@ Open dataset tracking autonomous vehicle deployments in the United States. Power
 
 ## What's here
 
-**events.csv** - Timeline of AV service changes since 2017 (launches, expansions, shutdowns, policy changes)  
+**events.csv** - Timeline of AV service changes since 2017 (launches, expansions, shutdowns, policy changes)
 **geometries/** - GeoJSON service area boundaries showing how coverage evolved
 
 This uses event sourcing: create events capture full service details, updates only record what changed. Every change has a source.
@@ -27,12 +27,76 @@ Load the GeoJSON files into any mapping tool. Parse the CSV for historical analy
 
 ## Contributing
 
-Add events to `events.csv`, run `python3 scripts/validate.py`, submit a PR.
+**Important: Submit all PRs to the `staging` branch, not `main`.**
+
+1. Fork this repository
+2. Create a feature branch from `staging`
+3. Add events to `events.csv` or update geometries
+4. Run `python3 scripts/validate.py` to verify your changes
+5. Submit a PR to the `staging` branch
+6. Your changes will be tested on the staging site before being promoted to production
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for the data format spec and examples.
 
 Most useful contributions: new service launches, boundary updates, fleet size changes, policy announcements.
 
+## Staging & Production Workflow
+
+This repository uses a staging environment to test data updates before they go live:
+
+- **`staging` branch** → Deploys to staging environment (test your changes here)
+- **`main` branch** → Deploys to production (avmap.io)
+
+### Testing Your Changes
+
+After submitting a PR to `staging`:
+1. Wait for the GitHub Action to rebuild the staging cache
+2. Check the staging site to verify your data looks correct
+3. Once approved, your PR will be merged to `staging`
+4. Later, staging will be promoted to `main` (production)
+
+## For Maintainers
+
+### Promoting Staging to Production
+
+Once staging data has been verified and looks good:
+
+```bash
+# Create a PR from staging → main
+gh pr create --base main --head staging --title "Promote staging to production"
+
+# Or via GitHub UI:
+# 1. Go to Pull Requests → New PR
+# 2. Base: main ← Compare: staging
+# 3. Review changes, then merge
+# 4. GitHub Action will automatically rebuild production cache
+```
+
+### Syncing Production Data to Staging
+
+To refresh staging with the latest production data:
+
+```bash
+# Sync database tables
+node sync-prod-to-staging.js
+
+# Copy geometry files
+node copy-geometries.js
+
+# Rebuild staging cache
+STAGING=true node rebuild-cache.js
+```
+
+### Manual Cache Rebuilds
+
+```bash
+# Rebuild staging cache
+STAGING=true node rebuild-cache.js
+
+# Rebuild production cache
+node rebuild-cache.js
+```
+
 ## License
 
 MIT - use it however you want.
diff --git a/copy-geometries.js b/copy-geometries.js
@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+
+/**
+ * Copy Geometry Files from Production to Staging
+ *
+ * This script copies all GeoJSON files from the production bucket to staging:
+ * - service-area-boundaries → staging-service-area-boundaries
+ *
+ * Run this after sync-prod-to-staging.js to ensure staging has all geometry files.
+ *
+ * Usage: node copy-geometries.js
+ */
+
+import { createClient } from '@supabase/supabase-js'
+import { config } from 'dotenv'
+
+// Load .env file
+if (!process.env.GITHUB_ACTIONS) {
+  config()
+}
+
+const supabaseUrl = process.env.SUPABASE_URL
+const supabaseServiceKey = process.env.SUPABASE_SERVICE_KEY || process.env.SUPABASE_ANON_KEY
+
+if (!supabaseUrl || !supabaseServiceKey) {
+  console.error('❌ Missing required environment variables!')
+  console.error('Please set SUPABASE_URL and SUPABASE_SERVICE_KEY in .env')
+  process.exit(1)
+}
+
+const supabase = createClient(supabaseUrl, supabaseServiceKey)
+
+const PROD_BUCKET = 'service-area-boundaries'
+const STAGING_BUCKET = 'staging-service-area-boundaries'
+
+async function copyGeometries() {
+  console.log('🔄 Copying geometry files from production to staging...\n')
+
+  try {
+    // Step 1: List all files in production bucket
+    console.log(`📡 Listing files in ${PROD_BUCKET}...`)
+    const { data: files, error: listError } = await supabase.storage
+      .from(PROD_BUCKET)
+      .list('', { limit: 1000 })
+
+    if (listError) throw listError
+
+    const geojsonFiles = files.filter(f => f.name.endsWith('.geojson'))
+    console.log(`   Found ${geojsonFiles.length} geometry files\n`)
+
+    if (geojsonFiles.length === 0) {
+      console.log('✅ No files to copy')
+      return
+    }
+
+    // Step 2: Copy each file
+    console.log(`📥 Copying files to ${STAGING_BUCKET}...`)
+    let successCount = 0
+    let failCount = 0
+
+    for (const file of geojsonFiles) {
+      try {
+        // Download from production
+        const { data: fileData, error: downloadError } = await supabase.storage
+          .from(PROD_BUCKET)
+          .download(file.name)
+
+        if (downloadError) throw downloadError
+
+        // Upload to staging
+        const { error: uploadError } = await supabase.storage
+          .from(STAGING_BUCKET)
+          .upload(file.name, fileData, {
+            contentType: 'application/json',
+            upsert: true  // Overwrite if exists
+          })
+
+        if (uploadError) throw uploadError
+
+        console.log(`   ✅ ${file.name}`)
+        successCount++
+
+      } catch (error) {
+        console.error(`   ❌ ${file.name}: ${error.message}`)
+        failCount++
+      }
+    }
+
+    console.log(`\n📊 Copy Summary:`)
+    console.log(`   ✅ Successful: ${successCount}`)
+    console.log(`   ❌ Failed: ${failCount}`)
+    console.log(`   📁 Total: ${geojsonFiles.length}`)
+
+    if (failCount === 0) {
+      console.log('\n✅ All geometry files copied successfully!')
+      console.log('\n📝 Next steps:')
+      console.log('   Run: STAGING=true node rebuild-cache.js')
+    } else {
+      console.error('\n⚠️  Some files failed to copy. Check errors above.')
+      process.exit(1)
+    }
+
+  } catch (error) {
+    console.error('❌ Copy failed:', error)
+    process.exit(1)
+  }
+}
+
+copyGeometries()
diff --git a/rebuild-cache.js b/rebuild-cache.js
@@ -1,19 +1,24 @@
 #!/usr/bin/env node
 
 /**
- * Production cache rebuild script
+ * Cache rebuild script - supports both staging and production environments
  *
  * This script:
  * 1. Fetches all data from Supabase database
  * 2. Loads all geometry files from storage
  * 3. Processes service areas using timeline logic
  * 4. Uploads combined JSON blob to storage
  *
+ * Environment Detection:
+ * - STAGING=true env var → Uses staging tables/buckets
+ * - Default → Uses production tables/buckets
+ *
  * Run this whenever you update av_events or service_area_geometries data
  */
 
 import { createClient } from '@supabase/supabase-js'
 import { config } from 'dotenv'
+import { execSync } from 'child_process'
 
 // Load .env file for local development (GitHub Actions sets env vars directly)
 if (!process.env.GITHUB_ACTIONS) {
@@ -34,6 +39,26 @@ if (!supabaseUrl || !supabaseServiceKey) {
   process.exit(1)
 }
 
+// Detect environment: staging or production
+const isStaging = process.env.STAGING === 'true'
+const environment = isStaging ? 'staging' : 'production'
+
+// Environment-specific configuration
+const config_env = {
+  eventsTable: isStaging ? 'av_events_staging' : 'av_events',
+  geometriesTable: isStaging ? 'service_area_geometries_staging' : 'service_area_geometries',
+  dataCacheBucket: isStaging ? 'staging-data-cache' : 'data-cache',
+  geometriesBucket: isStaging ? 'staging-service-area-boundaries' : 'service-area-boundaries'
+}
+
+console.log(`🌍 Environment: ${environment.toUpperCase()}`)
+console.log(`📋 Configuration:`)
+console.log(`   Events table: ${config_env.eventsTable}`)
+console.log(`   Geometries table: ${config_env.geometriesTable}`)
+console.log(`   Data cache bucket: ${config_env.dataCacheBucket}`)
+console.log(`   Geometries bucket: ${config_env.geometriesBucket}`)
+console.log('')
+
 const supabase = createClient(supabaseUrl, supabaseServiceKey)
 
 async function rebuildCache() {
@@ -45,13 +70,13 @@ async function rebuildCache() {
     console.log('📡 Fetching database data...')
     const [eventsResult, geometriesResult] = await Promise.all([
       supabase
-        .from('av_events')
+        .from(config_env.eventsTable)
         .select('*')
         .eq('aggregate_type', 'service_area')
         .order('event_date', { ascending: true }),
 
       supabase
-        .from('service_area_geometries')
+        .from(config_env.geometriesTable)
         .select('*')
         .order('created_at', { ascending: false })
     ])
@@ -149,35 +174,41 @@ async function rebuildCache() {
     const jsonData = JSON.stringify(cacheData)
     const sizeMB = (jsonData.length / 1024 / 1024).toFixed(2)
 
-    // Use timestamped filename to avoid caching issues
-    const timestamp = Date.now();
-    const filename = `all-data-${timestamp}.json`;
+    // Use timestamped filename to avoid caching issues (production only)
+    if (!isStaging) {
+      const timestamp = Date.now();
+      const filename = `all-data-${timestamp}.json`;
 
-    const { error: uploadError } = await supabase.storage
-      .from('data-cache')
-      .upload(filename, jsonData, {
-        contentType: 'application/json',
-        cacheControl: 'max-age=0, no-cache'
-      })
+      const { error: uploadError } = await supabase.storage
+        .from(config_env.dataCacheBucket)
+        .upload(filename, jsonData, {
+          contentType: 'application/json',
+          cacheControl: 'max-age=0, no-cache'
+        })
 
-    if (uploadError) throw uploadError
+      if (uploadError) throw uploadError
+      console.log(`   Backup saved: ${filename}`)
+    }
 
-    // Also upload as all-data.json for backwards compatibility
-    await supabase.storage.from('data-cache').remove(['all-data.json'])
+    // Upload as all-data.json
+    await supabase.storage.from(config_env.dataCacheBucket).remove(['all-data.json'])
     const { error: mainUploadError } = await supabase.storage
-      .from('data-cache')
+      .from(config_env.dataCacheBucket)
       .upload('all-data.json', jsonData, {
         contentType: 'application/json',
         cacheControl: 'max-age=0, no-cache'
       })
 
-    if (uploadError) throw uploadError
+    if (mainUploadError) throw mainUploadError
 
     const totalTime = Date.now() - startTime
+    const publicUrl = `${supabaseUrl}/storage/v1/object/public/${config_env.dataCacheBucket}/all-data.json`
+
     console.log(`🎉 Cache rebuild complete!`)
+    console.log(`   Environment: ${environment}`)
     console.log(`   Size: ${sizeMB}MB`)
     console.log(`   Time: ${totalTime}ms`)
-    console.log(`   URL: https://vbqijqcveavjycsfoszy.supabase.co/storage/v1/object/public/data-cache/all-data.json`)
+    console.log(`   URL: ${publicUrl}`)
 
     if (failed.length > 0) {
       console.log(`⚠️ ${failed.length} geometries failed to load`)
diff --git a/sync-prod-to-staging.js b/sync-prod-to-staging.js
