feat: enable shard-awareness with Proxy Protocol v2 in PrivateLink mode

Adds `proxyProtocol` support to `ClientRoutesConfig` so users can declare
that the NLB uses Proxy Protocol v2 (PP2) when forwarding connections to
ScyllaDB. With PP2, the NLB prepends a binary header carrying the original
client source IP and port to each connection it opens to ScyllaDB; ScyllaDB
reads the header and routes using the original source port, restoring
shard-aware routing end-to-end through the NLB.

Changes:
- `ClientRoutesConfig`: add `proxyProtocol` field + `withProxyProtocol()`
  builder method; update equals/hashCode/toString
- `DefaultDriverOption`: add `CLIENT_ROUTES_PROXY_PROTOCOL` enum constant
- `TypedDriverOption` / `OptionsMap`: add typed wrapper and default value
- `reference.conf`: add `advanced.client-routes.proxy-protocol = false`
- `DefaultDriverContext`: parse `proxy-protocol` from HOCON;
  warn (not fail) when `proxyProtocol=true` but shard awareness is disabled
- `TcpProxy` / `RoundRobinProxy`: add PP2 header injection mode —
  after connecting to the target, write the 28-byte TCP4 PP2 binary header
  carrying the original client IP and source port
- `NlbSimulator`: add `proxyProtocol` constructor parameter; pass flag to
  both per-node (`TcpProxy`) and discovery (`RoundRobinProxy`) proxies
- `ClientRoutesIT`: add `should_use_shard_awareness_through_pp2_nlb` test
- `PRIVATELINK.md`: document the PP2 mechanism, header format, configuration,
  and infrastructure requirements

Jira ID: DRIVER-391

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: nikagra

core/src/main/java/com/datastax/oss/driver/api/core/config/ClientRoutesConfig.java

@@ -73,8 +73,10 @@ public final class ClientRoutesConfig {
 
   private final List<ClientRouteProxy> endpoints;
   private final String tableName;
+  private final boolean proxyProtocol;
 
-  private ClientRoutesConfig(List<ClientRouteProxy> endpoints, String tableName) {
+  private ClientRoutesConfig(
+      List<ClientRouteProxy> endpoints, String tableName, boolean proxyProtocol) {
     if (endpoints == null || endpoints.isEmpty()) {
       throw new IllegalArgumentException("At least one endpoint must be specified");
     }
@@ -86,6 +88,7 @@ private ClientRoutesConfig(List<ClientRouteProxy> endpoints, String tableName) {
     }
     this.endpoints = Collections.unmodifiableList(new ArrayList<>(endpoints));
     this.tableName = tableName;
+    this.proxyProtocol = proxyProtocol;
   }
 
   /**
@@ -108,6 +111,21 @@ public String getTableName() {
     return tableName;
   }
 
+  /**
+   * Returns whether Proxy Protocol v2 is in use between the NLB and ScyllaDB nodes.
+   *
+   * <p>When {@code true}, the driver assumes the NLB prepends a PP2 binary header to each
+   * connection it opens to ScyllaDB. The header carries the original client IP and source port,
+   * which ScyllaDB uses for shard-aware routing. This restores shard-awareness end-to-end through
+   * the NLB. Requires both the NLB and ScyllaDB to be configured for Proxy Protocol v2, and {@code
+   * advanced-shard-awareness.enabled} must be {@code true} (the default).
+   *
+   * @return {@code true} if Proxy Protocol v2 is enabled.
+   */
+  public boolean isProxyProtocol() {
+    return proxyProtocol;
+  }
+
   /**
    * Creates a new builder for constructing a {@link ClientRoutesConfig}.
    *
@@ -127,12 +145,14 @@ public boolean equals(Object o) {
       return false;
     }
     ClientRoutesConfig that = (ClientRoutesConfig) o;
-    return endpoints.equals(that.endpoints) && tableName.equals(that.tableName);
+    return proxyProtocol == that.proxyProtocol
+        && endpoints.equals(that.endpoints)
+        && tableName.equals(that.tableName);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(endpoints, tableName);
+    return Objects.hash(endpoints, tableName, proxyProtocol);
   }
 
   @Override
@@ -143,6 +163,8 @@ public String toString() {
         + ", tableName='"
         + tableName
         + '\''
+        + ", proxyProtocol="
+        + proxyProtocol
         + '}';
   }
 
@@ -151,6 +173,7 @@ public static final class Builder {
     private final List<ClientRouteProxy> endpoints = new ArrayList<>();
     private final Set<String> seenConnectionIds = new HashSet<>();
     private String tableName = DEFAULT_TABLE_NAME;
+    private boolean proxyProtocol = false;
 
     /**
      * Adds an endpoint to the configuration.
@@ -208,6 +231,26 @@ Builder withTableName(@NonNull String tableName) {
       return this;
     }
 
+    /**
+     * Sets whether Proxy Protocol v2 (PP2) is in use between the NLB and ScyllaDB nodes.
+     *
+     * <p>When {@code true}, the driver assumes the NLB prepends a PP2 binary header to each
+     * connection it opens to ScyllaDB. The header carries the original client source IP and port,
+     * which ScyllaDB uses to route the connection to the correct shard. This restores
+     * shard-awareness end-to-end through the NLB.
+     *
+     * <p>Requires: (1) the NLB configured with PP2, (2) ScyllaDB configured to accept PP2, (3)
+     * {@code advanced-shard-awareness.enabled = true} (the default).
+     *
+     * @param proxyProtocol {@code true} to enable PP2 mode.
+     * @return this builder.
+     */
+    @NonNull
+    public Builder withProxyProtocol(boolean proxyProtocol) {
+      this.proxyProtocol = proxyProtocol;
+      return this;
+    }
+
     /**
      * Builds the {@link ClientRoutesConfig} with the configured endpoints and table name.
      *
@@ -216,7 +259,7 @@ Builder withTableName(@NonNull String tableName) {
      */
     @NonNull
     public ClientRoutesConfig build() {
-      return new ClientRoutesConfig(endpoints, tableName);
+      return new ClientRoutesConfig(endpoints, tableName, proxyProtocol);
     }
   }
 }

core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java

@@ -1097,7 +1097,19 @@ public enum DefaultDriverOption implements DriverOption {
    *
    * <p>Value type: list of HOCON objects
    */
-  CLIENT_ROUTES_ENDPOINTS("advanced.client-routes.endpoints");
+  CLIENT_ROUTES_ENDPOINTS("advanced.client-routes.endpoints"),
+  /**
+   * Whether Proxy Protocol v2 (PP2) is in use between the NLB and ScyllaDB nodes.
+   *
+   * <p>When {@code true}, the driver assumes the NLB prepends a PP2 binary header to each
+   * connection it opens to ScyllaDB. The header carries the original client source IP and port,
+   * enabling shard-aware routing through the NLB. Requires: (1) NLB configured with PP2, (2)
+   * ScyllaDB configured to accept PP2, (3) {@code advanced-shard-awareness.enabled = true} (the
+   * default).
+   *
+   * <p>Value type: boolean
+   */
+  CLIENT_ROUTES_PROXY_PROTOCOL("advanced.client-routes.proxy-protocol");
 
   private final String path;
 

core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java

@@ -398,6 +398,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
         "PRESERVE_REPLICA_ORDER");
     // CLIENT_ROUTES_ENDPOINTS is intentionally omitted: it is a list-of-objects (compound HOCON
     // values) with no sensible scalar default, analogous to how CONFIG_RELOAD_INTERVAL is omitted.
+    map.put(TypedDriverOption.CLIENT_ROUTES_PROXY_PROTOCOL, false);
   }
 
   @Immutable

core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java

@@ -944,6 +944,11 @@ public String toString() {
   // DriverExecutionProfile API); it is excluded from the TypedDriverOptionTest consistency check
   // accordingly.
 
+  /** Whether Proxy Protocol v2 is in use between the NLB and ScyllaDB nodes. */
+  public static final TypedDriverOption<Boolean> CLIENT_ROUTES_PROXY_PROTOCOL =
+      new TypedDriverOption<>(
+          DefaultDriverOption.CLIENT_ROUTES_PROXY_PROTOCOL, GenericType.BOOLEAN);
+
   private static Iterable<TypedDriverOption<?>> introspectBuiltInValues() {
     try {
       ImmutableList.Builder<TypedDriverOption<?>> result = ImmutableList.builder();

core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java

@@ -514,6 +514,11 @@ ClientRoutesConfig buildClientRoutesConfigFromFile() {
 
     ClientRoutesConfig.Builder builder = ClientRoutesConfig.builder();
 
+    if (defaultProfile.isDefined(DefaultDriverOption.CLIENT_ROUTES_PROXY_PROTOCOL)) {
+      builder.withProxyProtocol(
+          defaultProfile.getBoolean(DefaultDriverOption.CLIENT_ROUTES_PROXY_PROTOCOL));
+    }
+
     for (int i = 0; i < endpointsList.size(); i++) {
       if (!(endpointsList.get(i) instanceof ConfigObject)) {
         throw new IllegalArgumentException(
@@ -644,6 +649,29 @@ private void validateClientRoutesConfiguration(ClientRoutesConfig clientRoutesCo
               + "They are mutually exclusive. Please use either a secure connect bundle OR "
               + "client routes configuration, but not both.");
     }
+    if (clientRoutesConfig.isProxyProtocol()) {
+      boolean shardAwarenessEnabled =
+          getConfig()
+              .getDefaultProfile()
+              .getBoolean(DefaultDriverOption.CONNECTION_ADVANCED_SHARD_AWARENESS_ENABLED);
+      if (!shardAwarenessEnabled) {
+        // proxyProtocol=true signals that the NLB will forward the driver's original source port
+        // to ScyllaDB via a PP2 header, enabling shard-aware connection routing through the proxy.
+        // However, this only has effect when advanced shard awareness is also enabled — that is the
+        // mechanism that binds a shard-specific local port in the first place. With shard awareness
+        // disabled the driver uses a random local port, so PP2 forwards a port that carries no
+        // shard intent. The setting is therefore ignored. If shard-aware routing through the NLB
+        // is desired, enable advanced-shard-awareness (it is on by default).
+        LOG.warn(
+            "[{}] ClientRoutesConfig has proxyProtocol=true but {} is false. "
+                + "Proxy Protocol v2 has no effect without advanced shard awareness — "
+                + "the proxyProtocol flag will be ignored. To enable shard-aware routing "
+                + "through the NLB, set advanced-shard-awareness.enabled=true (the default).",
+            getSessionName(),
+            DefaultDriverOption.CONNECTION_ADVANCED_SHARD_AWARENESS_ENABLED.getPath());
+      }
+    }
+
     DriverExecutionProfile defaultProfile = getConfig().getDefaultProfile();
     if (defaultProfile.isDefined(DefaultDriverOption.ADDRESS_TRANSLATOR_CLASS)) {
       String className = defaultProfile.getString(DefaultDriverOption.ADDRESS_TRANSLATOR_CLASS);

core/src/main/resources/reference.conf

@@ -1146,6 +1146,25 @@ datastax-java-driver {
     # Required: yes (if client routes are to be used)
     # endpoints = []
 
+    # Whether Proxy Protocol v2 (PP2) is in use between the NLB and ScyllaDB nodes.
+    #
+    # When true, the driver assumes the NLB prepends a PP2 binary header to each connection it opens
+    # to ScyllaDB. The header carries the original client source IP and port, which ScyllaDB uses to
+    # route the connection to the correct shard. This restores shard-awareness end-to-end through the
+    # NLB. The driver's own role is to continue binding shard-specific local ports as usual
+    # (controlled by advanced-shard-awareness.enabled, which is true by default); the NLB then
+    # forwards those ports to ScyllaDB via the PP2 header.
+    #
+    # Requires: (1) NLB configured with PP2, (2) ScyllaDB configured to accept PP2,
+    #           (3) advanced.connection.advanced-shard-awareness.enabled = true (the default).
+    #
+    # If this is true but advanced-shard-awareness.enabled is false, the driver logs a warning and
+    # ignores the setting — there is no shard-specific port to forward.
+    #
+    # Required: no
+    # Default: false
+    proxy-protocol = false
+
   }
 
   # Whether to resolve the addresses passed to `basic.contact-points`.

integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java

@@ -820,6 +820,59 @@ public void should_select_tls_port_when_ssl_configured() throws Exception {
     }
   }
 
+  @Test
+  public void should_use_shard_awareness_through_pp2_nlb() throws Exception {
+    // Verify server supports client_routes
+    try (CqlSession admin = openAdminSession()) {
+      requireSystemClientRoutesTable(admin);
+    }
+    try (CcmBridge ccm = CcmBridge.builder().withNodes(1).build()) {
+      ccm.create();
+      ccm.start();
+
+      // Create PP2-enabled NLB
+      NlbSimulator nlb =
+          new NlbSimulator(ccm, NLB_ADDRESS, NLB_BASE_PORT, /* proxyProtocol= */ true);
+      try {
+        nlb.addNode(1);
+        Map<Integer, UUID> hostIds = collectHostIds(ccm, 1);
+        postClientRoutes(ccm, hostIds, nlb);
+        waitForRoutesVisibleOnAllNodes(ccm, hostIds.keySet(), hostIds.size());
+
+        ClientRoutesConfig config =
+            ClientRoutesConfig.builder()
+                .addEndpoint(new ClientRouteProxy(CONNECTION_ID, NLB_ADDRESS))
+                .withProxyProtocol(true)
+                .build();
+
+        try (CqlSession session = openNlbSession(config, nlb)) {
+          assertQueryWorks(session);
+
+          // With PP2, the NLB forwards the driver's original source port to ScyllaDB.
+          // The driver binds shard-specific local ports (advanced shard awareness, on by default),
+          // so ScyllaDB can route each connection to the correct shard. Verify the pool works
+          // correctly by running queries and waiting for the full pool to be established.
+          Node node = session.getMetadata().getNodes().values().iterator().next();
+          int shardCount =
+              node.getShardingInfo() != null ? node.getShardingInfo().getShardsCount() : 1;
+
+          // Wait for full shard-aware pool to be established (one connection per shard)
+          await()
+              .atMost(30, TimeUnit.SECONDS)
+              .pollInterval(1, TimeUnit.SECONDS)
+              .until(() -> node.getOpenConnections() >= shardCount);
+
+          // Run queries to verify shard-aware routing works end-to-end through the PP2 NLB
+          for (int i = 0; i < 20; i++) {
+            assertQueryWorks(session);
+          }
+        }
+      } finally {
+        nlb.close();
+      }
+    }
+  }
+
   @Test
   public void should_work_with_mixed_proxy_and_direct_nodes() throws Exception {
     try (CqlSession admin = openAdminSession()) {

integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/NlbSimulator.java

@@ -57,6 +57,7 @@ public class NlbSimulator implements Closeable {
   private final CcmBridge ccmBridge;
   private final String bindAddress;
   private final int basePort;
+  private final boolean proxyProtocol;
 
   // Discovery port proxy: round-robins across all nodes
   private volatile RoundRobinProxy discoveryProxy;
@@ -74,9 +75,24 @@ public class NlbSimulator implements Closeable {
    * @param basePort the base port for NLB (discovery on basePort, per-node on basePort+nodeId)
    */
   public NlbSimulator(CcmBridge ccmBridge, String bindAddress, int basePort) {
+    this(ccmBridge, bindAddress, basePort, false);
+  }
+
+  /**
+   * Creates an NLB simulator with optional Proxy Protocol v2 support.
+   *
+   * @param ccmBridge the CCM bridge to get node addresses from
+   * @param bindAddress the IP address to bind proxy listeners on (e.g. "127.254.254.254")
+   * @param basePort the base port for NLB (discovery on basePort, per-node on basePort+nodeId)
+   * @param proxyProtocol when {@code true}, all proxied connections will include a PP2 binary
+   *     header carrying the original client IP and source port
+   */
+  public NlbSimulator(
+      CcmBridge ccmBridge, String bindAddress, int basePort, boolean proxyProtocol) {
     this.ccmBridge = ccmBridge;
     this.bindAddress = bindAddress;
     this.basePort = basePort;
+    this.proxyProtocol = proxyProtocol;
   }
 
   /** Returns the bind address used by this NLB simulator. */
@@ -114,7 +130,7 @@ public synchronized void addNode(int nodeId) throws IOException {
 
     // Create per-node proxy first, before updating state
     int nodePort = getNodePort(nodeId);
-    TcpProxy proxy = new TcpProxy(bindAddress, nodePort, nodeAddr);
+    TcpProxy proxy = new TcpProxy(bindAddress, nodePort, nodeAddr, proxyProtocol);
 
     // Update state and rebuild discovery proxy; if rebuild fails, close the new proxy
     activeNodes.add(nodeId);
@@ -164,7 +180,7 @@ private void rebuildDiscoveryProxy() throws IOException {
     }
 
     // Create new proxy before closing old one to avoid inconsistent state on failure
-    discoveryProxy = new RoundRobinProxy(bindAddress, basePort, targets);
+    discoveryProxy = new RoundRobinProxy(bindAddress, basePort, targets, proxyProtocol);
     if (oldProxy != null) {
       oldProxy.close();
     }