Improve Deckard integration with Redis Clusters

docker/docker-compose.yml

@@ -9,4 +9,87 @@ services:
   mongodb:
     image: mongo:6.0.4
     ports:
-      - "$MONGODB_PORT:$MONGODB_PORT"
+      - "$MONGODB_PORT:$MONGODB_PORT"
+
+  # Redis Cluster setup for testing
+  redis-cluster-node-1:
+    image: "redis:7.0"
+    command: redis-server --port 7000 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    ports:
+      - "7000:7000"
+    volumes:
+      - redis-cluster-1-data:/data
+
+  redis-cluster-node-2:
+    image: "redis:7.0"
+    command: redis-server --port 7001 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    ports:
+      - "7001:7001"
+    volumes:
+      - redis-cluster-2-data:/data
+
+  redis-cluster-node-3:
+    image: "redis:7.0"
+    command: redis-server --port 7002 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    ports:
+      - "7002:7002"
+    volumes:
+      - redis-cluster-3-data:/data
+
+  redis-cluster-node-4:
+    image: "redis:7.0"
+    command: redis-server --port 7003 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    ports:
+      - "7003:7003"
+    volumes:
+      - redis-cluster-4-data:/data
+
+  redis-cluster-node-5:
+    image: "redis:7.0"
+    command: redis-server --port 7004 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    ports:
+      - "7004:7004"
+    volumes:
+      - redis-cluster-5-data:/data
+
+  redis-cluster-node-6:
+    image: "redis:7.0"
+    command: redis-server --port 7005 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    ports:
+      - "7005:7005"
+    volumes:
+      - redis-cluster-6-data:/data
+
+  # Helper service to initialize the cluster
+  redis-cluster-init:
+    image: "redis:7.0"
+    depends_on:
+      - redis-cluster-node-1
+      - redis-cluster-node-2
+      - redis-cluster-node-3
+      - redis-cluster-node-4
+      - redis-cluster-node-5
+      - redis-cluster-node-6
+    command: >
+      sh -c "
+        sleep 10
+        redis-cli --cluster create 
+        redis-cluster-node-1:7000 
+        redis-cluster-node-2:7001 
+        redis-cluster-node-3:7002 
+        redis-cluster-node-4:7003 
+        redis-cluster-node-5:7004 
+        redis-cluster-node-6:7005 
+        --cluster-replicas 1 
+        --cluster-yes
+      "
+    networks:
+      - default
+
+volumes:
+  redis-cluster-1-data:
+  redis-cluster-2-data:
+  redis-cluster-3-data:
+  redis-cluster-4-data:
+  redis-cluster-5-data:
+  redis-cluster-6-data:

docs/redis-cluster-configuration.md

@@ -0,0 +1,169 @@
+# Redis Cluster Configuration
+
+This document explains how to configure Deckard to work with Redis Clusters.
+
+## Overview
+
+Deckard now supports both single-node Redis and Redis Cluster deployments. When Redis Cluster mode is enabled, Deckard uses hash tags in key naming to ensure all queue-related keys land on the same cluster slot, enabling atomic operations via Lua scripts.
+
+## Configuration
+
+### Single Node Redis (Default)
+
+This is the traditional configuration that works with a single Redis instance:
+
+```yaml
+cache:
+  type: REDIS
+redis:
+  address: localhost
+  port: 6379
+  password: ""  # optional
+  db: 0         # optional
+```
+
+Environment variables:
+```bash
+DECKARD_CACHE_TYPE=REDIS
+DECKARD_REDIS_ADDRESS=localhost
+DECKARD_REDIS_PORT=6379
+DECKARD_REDIS_PASSWORD=mypassword  # optional
+DECKARD_REDIS_DB=0                 # optional
+```
+
+### Redis Cluster
+
+To enable Redis Cluster support:
+
+```yaml
+cache:
+  type: REDIS
+redis:
+  cluster:
+    mode: true
+    addresses:
+      - redis-node-1:6379
+      - redis-node-2:6379
+      - redis-node-3:6379
+```
+
+Environment variables:
+```bash
+DECKARD_CACHE_TYPE=REDIS
+DECKARD_REDIS_CLUSTER_MODE=true
+DECKARD_REDIS_CLUSTER_ADDRESSES=redis-node-1:6379,redis-node-2:6379,redis-node-3:6379
+DECKARD_REDIS_PASSWORD=mypassword  # optional, if cluster requires auth
+```
+
+## Key Naming Differences
+
+### Single Node Redis
+Keys are named without hash tags:
+- `deckard:queue:myqueue`
+- `deckard:queue:myqueue:tmp`
+- `deckard:queue:myqueue:lock_ack`
+
+### Redis Cluster  
+Keys use hash tags to ensure co-location:
+- `deckard:queue:{myqueue}`
+- `deckard:queue:{myqueue}:tmp`
+- `deckard:queue:{myqueue}:lock_ack`
+
+The `{myqueue}` hash tag ensures all keys for the same queue hash to the same cluster slot.
+
+## Migration from Single Node to Cluster
+
+### Option 1: Fresh Deployment
+1. Deploy new Deckard instance with cluster configuration
+2. Migrate data using your preferred method
+3. Switch traffic to new deployment
+
+### Option 2: In-place Migration (Advanced)
+⚠️ **Warning**: This approach requires careful planning and testing.
+
+1. Ensure your Redis cluster is properly configured
+2. Update Deckard configuration to enable cluster mode
+3. Restart Deckard instance
+
+Note: Existing keys will remain in the old format and will need manual migration if cross-key atomicity is required.
+
+## Docker Compose Example
+
+```yaml
+version: "3.8"
+
+services:
+  deckard:
+    image: blipai/deckard:latest
+    environment:
+      - DECKARD_CACHE_TYPE=REDIS
+      - DECKARD_REDIS_CLUSTER_MODE=true
+      - DECKARD_REDIS_CLUSTER_ADDRESSES=redis-node-1:6379,redis-node-2:6379,redis-node-3:6379
+    depends_on:
+      - redis-node-1
+      - redis-node-2
+      - redis-node-3
+
+  redis-node-1:
+    image: redis:7.0
+    command: redis-server --port 6379 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000
+    ports:
+      - "6379:6379"
+
+  redis-node-2:
+    image: redis:7.0
+    command: redis-server --port 6379 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000
+    ports:
+      - "6380:6379"
+
+  redis-node-3:
+    image: redis:7.0
+    command: redis-server --port 6379 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000
+    ports:
+      - "6381:6379"
+```
+
+## Testing
+
+### Local Development with Docker
+Use the provided script to set up a local Redis cluster:
+
+```bash
+./scripts/setup-redis-cluster.sh
+```
+
+### Running Cluster Tests
+```bash
+export REDIS_CLUSTER_TEST=1
+go test -v ./internal/queue/cache/ -run Cluster
+```
+
+## Troubleshooting
+
+### Common Issues
+
+**1. "redis.cluster.addresses must be specified" error**
+- Ensure `DECKARD_REDIS_CLUSTER_ADDRESSES` is set when cluster mode is enabled
+
+**2. "CROSSSLOT Keys in request don't hash to the same slot" error**
+- This indicates the hash tag implementation may have an issue
+- Verify all keys for a queue use the same hash tag format
+
+**3. Connection timeouts to cluster**
+- Ensure all cluster nodes are accessible from Deckard
+- Check network connectivity and firewall rules
+- Verify cluster is properly initialized
+
+### Performance Considerations
+
+- Redis Cluster may have slightly higher latency than single-node due to network hops
+- Hash tags ensure optimal key distribution while maintaining atomicity
+- Monitor cluster node resource usage and scale as needed
+
+### Best Practices
+
+1. **Always use odd number of master nodes** (3, 5, 7) for proper quorum
+2. **Set up Redis Cluster with replicas** for high availability
+3. **Monitor cluster health** using Redis Cluster commands
+4. **Test failover scenarios** in staging environments
+5. **Use consistent hashing strategy** across all deployments

docs/redis-cluster-integration.md

@@ -0,0 +1,120 @@
+# Redis Cluster Integration Plan
+
+## Overview
+
+This document outlines the plan to improve Deckard's integration with Redis Clusters while maintaining backward compatibility with single-node Redis deployments.
+
+## Problem Statement
+
+Deckard currently uses Lua scripts for atomic operations on Redis, but these scripts access multiple keys that may hash to different slots in a Redis Cluster, causing the scripts to fail. Redis Cluster requires all keys accessed by a Lua script to be on the same hash slot.
+
+## Current Key Naming Pattern
+
+For a queue named `myqueue`, Deckard creates these keys:
+- `deckard:queue:myqueue` (active pool)
+- `deckard:queue:myqueue:tmp` (processing pool)
+- `deckard:queue:myqueue:ack` (ack lock pool)  
+- `deckard:queue:myqueue:nack` (nack lock pool)
+- `deckard:queue:myqueue:ack:score` (ack score pool)
+- `deckard:queue:myqueue:nack:score` (nack score pool)
+
+## Redis Cluster Constraints
+
+1. All keys in a Lua script must hash to the same slot
+2. Keys must use hash tags to control slot assignment
+3. Scripts can only access keys explicitly provided as KEYS arguments
+4. No programmatically-generated key names in scripts
+
+## Solution: Hash Tags
+
+Modify key naming to use Redis hash tags, ensuring all queue-related keys hash to the same slot:
+
+### New Key Naming Pattern (Cluster-Compatible)
+- `deckard:queue:{myqueue}` (active pool)
+- `deckard:queue:{myqueue}:tmp` (processing pool) 
+- `deckard:queue:{myqueue}:ack` (ack lock pool)
+- `deckard:queue:{myqueue}:nack` (nack lock pool)
+- `deckard:queue:{myqueue}:ack:score` (ack score pool)
+- `deckard:queue:{myqueue}:nack:score` (nack score pool)
+
+The `{myqueue}` hash tag ensures all keys for the same queue land on the same Redis Cluster slot.
+
+## Implementation Plan
+
+### Phase 1: Configuration Support
+- [ ] Add Redis Cluster configuration option
+- [ ] Add cluster-aware key naming toggle
+- [ ] Support both legacy and cluster-compatible modes
+
+### Phase 2: Key Naming Strategy
+- [ ] Modify key generation functions to use hash tags when cluster mode is enabled
+- [ ] Ensure backward compatibility with existing deployments
+- [ ] Update all key generation methods in RedisCache
+
+### Phase 3: Integration Testing
+- [ ] Create Redis Cluster docker-compose setup
+- [ ] Add integration tests for cluster functionality
+- [ ] Validate all Lua scripts work with clustered Redis
+- [ ] Test migration scenarios
+
+### Phase 4: Documentation & Deployment
+- [ ] Update deployment documentation
+- [ ] Add Redis Cluster configuration examples
+- [ ] Document migration path from single-node to cluster
+
+## Backward Compatibility
+
+The implementation will:
+1. Default to legacy key naming for existing deployments
+2. Require explicit configuration to enable cluster-compatible mode  
+3. Provide migration tools/documentation for upgrading existing deployments
+4. Maintain API compatibility
+
+## Testing Strategy
+
+1. **Unit Tests**: Validate key naming functions
+2. **Integration Tests**: Test against both single-node and cluster Redis
+3. **Migration Tests**: Verify smooth transition scenarios
+4. **Performance Tests**: Ensure no degradation in performance
+
+## Risks & Mitigations
+
+### Risk: Breaking existing deployments
+**Mitigation**: Feature flag and backward compatibility mode
+
+### Risk: Performance impact from hash tags
+**Mitigation**: Benchmark testing and optional enablement
+
+### Risk: Complex migration path
+**Mitigation**: Clear documentation and migration tools
+
+## Configuration Examples
+
+### Single Node (Legacy - Default)
+```yaml
+cache:
+  type: REDIS
+  redis_cluster_mode: false
+redis:
+  address: localhost
+  port: 6379
+```
+
+### Redis Cluster
+```yaml  
+cache:
+  type: REDIS
+  redis_cluster_mode: true
+  redis_cluster_addresses:
+    - redis-node-1:6379
+    - redis-node-2:6379  
+    - redis-node-3:6379
+```
+
+## Success Criteria
+
+1. All existing Lua scripts work with Redis Cluster
+2. No breaking changes to existing single-node deployments
+3. Comprehensive test coverage for both modes
+4. Clear documentation for cluster deployment
+5. Performance parity with single-node deployments

internal/config/cache.go

@@ -46,3 +46,15 @@ var RedisDB = Create(&ViperConfigKey{
 	Key:     "redis.db",
 	Default: 0,
 })
+
+// Redis Cluster Configurations
+
+var RedisClusterMode = Create(&ViperConfigKey{
+	Key:     "redis.cluster.mode",
+	Default: false,
+})
+
+var RedisClusterAddresses = Create(&ViperConfigKey{
+	Key:     "redis.cluster.addresses",
+	Default: nil,
+})