From d37373ef48c274682f0d571ec4ae6ee06f177f63 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 19:08:45 -0700 Subject: [PATCH 01/24] proto: sync PulsarApi.proto from upstream (PIP-420 support) Sync the protobuf definition from apache/pulsar master to pick up: - External = 22 in Schema.Type enum - optional bytes schema_id field in MessageMetadata Also fix compilation errors from new fields added to CommandCloseProducer, CommandCloseConsumer, and CommandUnsubscribe by using ..Default::default() / .. in struct literals and patterns. Co-Authored-By: Claude Opus 4.6 --- PulsarApi.proto | 1504 ++++++++++++++++++++++++--------------------- src/connection.rs | 3 + src/message.rs | 1 + 3 files changed, 801 insertions(+), 707 deletions(-) diff --git a/PulsarApi.proto b/PulsarApi.proto index 836c1e9a..ed49f7bc 100644 --- a/PulsarApi.proto +++ b/PulsarApi.proto @@ -23,643 +23,690 @@ option java_package = "org.apache.pulsar.common.api.proto"; option optimize_for = LITE_RUNTIME; message Schema { - enum Type { - None = 0; - String = 1; - Json = 2; - Protobuf = 3; - Avro = 4; - Bool = 5; - Int8 = 6; - Int16 = 7; - Int32 = 8; - Int64 = 9; - Float = 10; - Double = 11; - Date = 12; - Time = 13; - Timestamp = 14; - KeyValue = 15; - Instant = 16; - LocalDate = 17; - LocalTime = 18; - LocalDateTime = 19; - ProtobufNative = 20; - } - - required string name = 1; - required bytes schema_data = 3; - required Type type = 4; - repeated KeyValue properties = 5; + enum Type { + None = 0; + String = 1; + Json = 2; + Protobuf = 3; + Avro = 4; + Bool = 5; + Int8 = 6; + Int16 = 7; + Int32 = 8; + Int64 = 9; + Float = 10; + Double = 11; + Date = 12; + Time = 13; + Timestamp = 14; + KeyValue = 15; + Instant = 16; + LocalDate = 17; + LocalTime = 18; + LocalDateTime = 19; + ProtobufNative = 20; + AutoConsume = 21; + External = 22; + } + + required string name = 1; + required bytes schema_data = 3; + required Type type = 4; + repeated KeyValue properties = 5; } message MessageIdData { - required uint64 ledgerId = 1; - required uint64 entryId = 2; - optional int32 partition = 3 [default = -1]; - optional int32 batch_index = 4 [default = -1]; - repeated int64 ack_set = 5; - optional int32 batch_size = 6; + required uint64 ledgerId = 1; + required uint64 entryId = 2; + optional int32 partition = 3 [default = -1]; + optional int32 batch_index = 4 [default = -1]; + repeated int64 ack_set = 5; + optional int32 batch_size = 6; - // For the chunk message id, we need to specify the first chunk message id. - optional MessageIdData first_chunk_message_id = 7; + // For the chunk message id, we need to specify the first chunk message id. + optional MessageIdData first_chunk_message_id = 7; } message KeyValue { - required string key = 1; - required string value = 2; + required string key = 1; + required string value = 2; } message KeyLongValue { - required string key = 1; - required uint64 value = 2; + required string key = 1; + required uint64 value = 2; } message IntRange { - required int32 start = 1; - required int32 end = 2; + required int32 start = 1; + required int32 end = 2; } message EncryptionKeys { - required string key = 1; - required bytes value = 2; - repeated KeyValue metadata = 3; + required string key = 1; + required bytes value = 2; + repeated KeyValue metadata = 3; } enum CompressionType { - NONE = 0; - LZ4 = 1; - ZLIB = 2; - ZSTD = 3; - SNAPPY = 4; + NONE = 0; + LZ4 = 1; + ZLIB = 2; + ZSTD = 3; + SNAPPY = 4; } enum ProducerAccessMode { - Shared = 0; // By default multiple producers can publish on a topic - Exclusive = 1; // Require exclusive access for producer. Fail immediately if there's already a producer connected. - WaitForExclusive = 2; // Producer creation is pending until it can acquire exclusive access - ExclusiveWithFencing = 3; // Require exclusive access for producer. Fence out old producer. + Shared = 0; // By default multiple producers can publish on a topic + Exclusive = 1; // Require exclusive access for producer. Fail immediately if there's already a producer connected. + WaitForExclusive = 2; // Producer creation is pending until it can acquire exclusive access + ExclusiveWithFencing = 3; // Require exclusive access for producer. Fence out old producer. } message MessageMetadata { - required string producer_name = 1; - required uint64 sequence_id = 2; - required uint64 publish_time = 3; - repeated KeyValue properties = 4; - - // Property set on replicated message, - // includes the source cluster name - optional string replicated_from = 5; - //key to decide partition for the msg - optional string partition_key = 6; - // Override namespace's replication - repeated string replicate_to = 7; - optional CompressionType compression = 8 [default = NONE]; - optional uint32 uncompressed_size = 9 [default = 0]; - // Removed below checksum field from Metadata as - // it should be part of send-command which keeps checksum of header + payload - //optional sfixed64 checksum = 10; - // differentiate single and batch message metadata - optional int32 num_messages_in_batch = 11 [default = 1]; - - // the timestamp that this event occurs. it is typically set by applications. - // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. - optional uint64 event_time = 12 [default = 0]; - // Contains encryption key name, encrypted key and metadata to describe the key - repeated EncryptionKeys encryption_keys = 13; - // Algorithm used to encrypt data key - optional string encryption_algo = 14; - // Additional parameters required by encryption - optional bytes encryption_param = 15; - optional bytes schema_version = 16; - - optional bool partition_key_b64_encoded = 17 [default = false]; - // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. - optional bytes ordering_key = 18; - - // Mark the message to be delivered at or after the specified timestamp - optional int64 deliver_at_time = 19; - - // Identify whether a message is a "marker" message used for - // internal metadata instead of application published data. - // Markers will generally not be propagated back to clients - optional int32 marker_type = 20; - - // transaction related message info - optional uint64 txnid_least_bits = 22; - optional uint64 txnid_most_bits = 23; - - /// Add highest sequence id to support batch message with external sequence id - optional uint64 highest_sequence_id = 24 [default = 0]; - - // Indicate if the message payload value is set - optional bool null_value = 25 [default = false]; - optional string uuid = 26; - optional int32 num_chunks_from_msg = 27; - optional int32 total_chunk_msg_size = 28; - optional int32 chunk_id = 29; - - // Indicate if the message partition key is set - optional bool null_partition_key = 30 [default = false]; + required string producer_name = 1; + required uint64 sequence_id = 2; + required uint64 publish_time = 3; + repeated KeyValue properties = 4; + + // Property set on replicated message, + // includes the source cluster name + optional string replicated_from = 5; + //key to decide partition for the msg + optional string partition_key = 6; + // Override namespace's replication + repeated string replicate_to = 7; + optional CompressionType compression = 8 [default = NONE]; + optional uint32 uncompressed_size = 9 [default = 0]; + // Removed below checksum field from Metadata as + // it should be part of send-command which keeps checksum of header + payload + //optional sfixed64 checksum = 10; + // differentiate single and batch message metadata + optional int32 num_messages_in_batch = 11 [default = 1]; + + // the timestamp that this event occurs. it is typically set by applications. + // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. + optional uint64 event_time = 12 [default = 0]; + // Contains encryption key name, encrypted key and metadata to describe the key + repeated EncryptionKeys encryption_keys = 13; + // Algorithm used to encrypt data key + optional string encryption_algo = 14; + // Additional parameters required by encryption + optional bytes encryption_param = 15; + optional bytes schema_version = 16; + + optional bool partition_key_b64_encoded = 17 [ default = false ]; + // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. + optional bytes ordering_key = 18; + + // Mark the message to be delivered at or after the specified timestamp + optional int64 deliver_at_time = 19; + + // Identify whether a message is a "marker" message used for + // internal metadata instead of application published data. + // Markers will generally not be propagated back to clients + optional int32 marker_type = 20; + + // transaction related message info + optional uint64 txnid_least_bits = 22; + optional uint64 txnid_most_bits = 23; + + /// Add highest sequence id to support batch message with external sequence id + optional uint64 highest_sequence_id = 24 [default = 0]; + + // Indicate if the message payload value is set + optional bool null_value = 25 [default = false]; + optional string uuid = 26; + optional int32 num_chunks_from_msg = 27; + optional int32 total_chunk_msg_size = 28; + optional int32 chunk_id = 29; + + // Indicate if the message partition key is set + optional bool null_partition_key = 30 [default = false]; + + // Indicates the indexes of messages retained in the batch after compaction. When a batch is compacted, + // some messages may be removed (compacted out). For example, if the original batch contains: + // `k0 => v0, k1 => v1, k2 => v2, k1 => null`, the compacted batch will retain only `k0 => v0` and `k2 => v2`. + // In this case, this field will be set to `[0, 2]`, and the payload buffer will only include the retained messages. + // + // Note: Batches compacted by older versions of the compaction service do not include this field. For such batches, + // the `compacted_out` field in `SingleMessageMetadata` must be checked to identify and filter out compacted + // messages (e.g., `k1 => v1` and `k1 => null` in the example above). + repeated int32 compacted_batch_indexes = 31; + optional bytes schema_id = 32; } message SingleMessageMetadata { - repeated KeyValue properties = 1; - optional string partition_key = 2; - required int32 payload_size = 3; - optional bool compacted_out = 4 [default = false]; - - // the timestamp that this event occurs. it is typically set by applications. - // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. - optional uint64 event_time = 5 [default = 0]; - optional bool partition_key_b64_encoded = 6 [default = false]; - // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. - optional bytes ordering_key = 7; - // Allows consumer retrieve the sequence id that the producer set. - optional uint64 sequence_id = 8; - // Indicate if the message payload value is set - optional bool null_value = 9 [default = false]; - // Indicate if the message partition key is set - optional bool null_partition_key = 10 [default = false]; + repeated KeyValue properties = 1; + optional string partition_key = 2; + required int32 payload_size = 3; + optional bool compacted_out = 4 [default = false]; + + // the timestamp that this event occurs. it is typically set by applications. + // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. + optional uint64 event_time = 5 [default = 0]; + optional bool partition_key_b64_encoded = 6 [ default = false ]; + // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. + optional bytes ordering_key = 7; + // Allows consumer retrieve the sequence id that the producer set. + optional uint64 sequence_id = 8; + // Indicate if the message payload value is set + optional bool null_value = 9 [ default = false ]; + // Indicate if the message partition key is set + optional bool null_partition_key = 10 [ default = false]; } // metadata added for entry from broker message BrokerEntryMetadata { - optional uint64 broker_timestamp = 1; - optional uint64 index = 2; + optional uint64 broker_timestamp = 1; + optional uint64 index = 2; } enum ServerError { - UnknownError = 0; - MetadataError = 1; // Error with ZK/metadata - PersistenceError = 2; // Error writing reading from BK - AuthenticationError = 3; // Non valid authentication - AuthorizationError = 4; // Not authorized to use resource - - ConsumerBusy = 5; // Unable to subscribe/unsubscribe because - // other consumers are connected - ServiceNotReady = 6; // Any error that requires client retry operation with a fresh lookup - ProducerBlockedQuotaExceededError = 7; // Unable to create producer because backlog quota exceeded - ProducerBlockedQuotaExceededException = 8; // Exception while creating producer because quota exceeded - ChecksumError = 9; // Error while verifying message checksum - UnsupportedVersionError = 10; // Error when an older client/version doesn't support a required feature - TopicNotFound = 11; // Topic not found - SubscriptionNotFound = 12; // Subscription not found - ConsumerNotFound = 13; // Consumer not found - TooManyRequests = 14; // Error with too many simultaneously request - TopicTerminatedError = 15; // The topic has been terminated - - ProducerBusy = 16; // Producer with same name is already connected - InvalidTopicName = 17; // The topic name is not valid - - IncompatibleSchema = 18; // Specified schema was incompatible with topic schema - ConsumerAssignError = 19; // Dispatcher assign consumer error - - TransactionCoordinatorNotFound = 20; // Transaction coordinator not found error - InvalidTxnStatus = 21; // Invalid txn status error - NotAllowedError = 22; // Not allowed error - - TransactionConflict = 23; // Ack with transaction conflict - TransactionNotFound = 24; // Transaction not found - - ProducerFenced = 25; // When a producer asks and fail to get exclusive producer access, - // or loses the exclusive status after a reconnection, the broker will - // use this error to indicate that this producer is now permanently - // fenced. Applications are now supposed to close it and create a - // new producer + UnknownError = 0; + MetadataError = 1; // Error with ZK/metadata + PersistenceError = 2; // Error writing reading from BK + AuthenticationError = 3; // Non valid authentication + AuthorizationError = 4; // Not authorized to use resource + + ConsumerBusy = 5; // Unable to subscribe/unsubscribe because + // other consumers are connected + ServiceNotReady = 6; // Any error that requires client retry operation with a fresh lookup + ProducerBlockedQuotaExceededError = 7; // Unable to create producer because backlog quota exceeded + ProducerBlockedQuotaExceededException = 8; // Exception while creating producer because quota exceeded + ChecksumError = 9; // Error while verifying message checksum + UnsupportedVersionError = 10; // Error when an older client/version doesn't support a required feature + TopicNotFound = 11; // Topic not found + SubscriptionNotFound = 12; // Subscription not found + ConsumerNotFound = 13; // Consumer not found + TooManyRequests = 14; // Error with too many simultaneously request + TopicTerminatedError = 15; // The topic has been terminated + + ProducerBusy = 16; // Producer with same name is already connected + InvalidTopicName = 17; // The topic name is not valid + + IncompatibleSchema = 18; // Specified schema was incompatible with topic schema + ConsumerAssignError = 19; // Dispatcher assign consumer error + + TransactionCoordinatorNotFound = 20; // Transaction coordinator not found error + InvalidTxnStatus = 21; // Invalid txn status error + NotAllowedError = 22; // Not allowed error + + TransactionConflict = 23; // Ack with transaction conflict + TransactionNotFound = 24; // Transaction not found + + ProducerFenced = 25; // When a producer asks and fail to get exclusive producer access, + // or loses the eclusive status after a reconnection, the broker will + // use this error to indicate that this producer is now permanently + // fenced. Applications are now supposed to close it and create a + // new producer } enum AuthMethod { - AuthMethodNone = 0; - AuthMethodYcaV1 = 1; - AuthMethodAthens = 2; + AuthMethodNone = 0; + AuthMethodYcaV1 = 1; + AuthMethodAthens = 2; } // Each protocol version identify new features that are // incrementally added to the protocol enum ProtocolVersion { - v0 = 0; // Initial versioning - v1 = 1; // Added application keep-alive - v2 = 2; // Added RedeliverUnacknowledgedMessages Command - v3 = 3; // Added compression with LZ4 and ZLib - v4 = 4; // Added batch message support - v5 = 5; // Added disconnect client w/o closing connection - v6 = 6; // Added checksum computation for metadata + payload - v7 = 7; // Added CommandLookupTopic - Binary Lookup - v8 = 8; // Added CommandConsumerStats - Client fetches broker side consumer stats - v9 = 9; // Added end of topic notification - v10 = 10;// Added proxy to broker - v11 = 11;// C++ consumers before this version are not correctly handling the checksum field - v12 = 12;// Added get topic's last messageId from broker - // Added CommandActiveConsumerChange - // Added CommandGetTopicsOfNamespace - v13 = 13; // Schema-registry : added avro schema format for json - v14 = 14; // Add CommandAuthChallenge and CommandAuthResponse for mutual auth - // Added Key_Shared subscription - v15 = 15; // Add CommandGetOrCreateSchema and CommandGetOrCreateSchemaResponse - v16 = 16; // Add support for broker entry metadata - v17 = 17; // Added support ack receipt - v18 = 18; // Add client support for broker entry metadata - v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse + v0 = 0; // Initial versioning + v1 = 1; // Added application keep-alive + v2 = 2; // Added RedeliverUnacknowledgedMessages Command + v3 = 3; // Added compression with LZ4 and ZLib + v4 = 4; // Added batch message support + v5 = 5; // Added disconnect client w/o closing connection + v6 = 6; // Added checksum computation for metadata + payload + v7 = 7; // Added CommandLookupTopic - Binary Lookup + v8 = 8; // Added CommandConsumerStats - Client fetches broker side consumer stats + v9 = 9; // Added end of topic notification + v10 = 10;// Added proxy to broker + v11 = 11;// C++ consumers before this version are not correctly handling the checksum field + v12 = 12;// Added get topic's last messageId from broker + // Added CommandActiveConsumerChange + // Added CommandGetTopicsOfNamespace + v13 = 13; // Schema-registry : added avro schema format for json + v14 = 14; // Add CommandAuthChallenge and CommandAuthResponse for mutual auth + // Added Key_Shared subscription + v15 = 15; // Add CommandGetOrCreateSchema and CommandGetOrCreateSchemaResponse + v16 = 16; // Add support for broker entry metadata + v17 = 17; // Added support ack receipt + v18 = 18; // Add client support for broker entry metadata + v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse + v20 = 20; // Add client support for topic migration redirection CommandTopicMigrated + v21 = 21; // Carry the AUTO_CONSUME schema to the Broker after this version } message CommandConnect { - required string client_version = 1; - optional AuthMethod auth_method = 2; // Deprecated. Use "auth_method_name" instead. - optional string auth_method_name = 5; - optional bytes auth_data = 3; - optional int32 protocol_version = 4 [default = 0]; + required string client_version = 1; // The version of the client. Proxy should forward client's client_version. + optional AuthMethod auth_method = 2; // Deprecated. Use "auth_method_name" instead. + optional string auth_method_name = 5; + optional bytes auth_data = 3; + optional int32 protocol_version = 4 [default = 0]; - // Client can ask to be proxyied to a specific broker - // This is only honored by a Pulsar proxy - optional string proxy_to_broker_url = 6; + // Client can ask to be proxyied to a specific broker + // This is only honored by a Pulsar proxy + optional string proxy_to_broker_url = 6; - // Original principal that was verified by - // a Pulsar proxy. In this case the auth info above - // will be the auth of the proxy itself - optional string original_principal = 7; + // Original principal that was verified by + // a Pulsar proxy. In this case the auth info above + // will be the auth of the proxy itself + optional string original_principal = 7; - // Original auth role and auth Method that was passed - // to the proxy. In this case the auth info above - // will be the auth of the proxy itself - optional string original_auth_data = 8; - optional string original_auth_method = 9; + // Original auth role and auth Method that was passed + // to the proxy. In this case the auth info above + // will be the auth of the proxy itself + optional string original_auth_data = 8; + optional string original_auth_method = 9; - // Feature flags - optional FeatureFlags feature_flags = 10; + // Feature flags + optional FeatureFlags feature_flags = 10; + + optional string proxy_version = 11; // Version of the proxy. Should only be forwarded by a proxy. } +// Please also add a new enum for the class "PulsarClientException.FailedFeatureCheck" when adding a new feature flag. message FeatureFlags { optional bool supports_auth_refresh = 1 [default = false]; optional bool supports_broker_entry_metadata = 2 [default = false]; optional bool supports_partial_producer = 3 [default = false]; + optional bool supports_topic_watchers = 4 [default = false]; + optional bool supports_get_partitioned_metadata_without_auto_creation = 5 [default = false]; + optional bool supports_repl_dedup_by_lid_and_eid = 6 [default = false]; + optional bool supports_topic_watcher_reconcile = 7 [default = false]; } message CommandConnected { - required string server_version = 1; - optional int32 protocol_version = 2 [default = 0]; - optional int32 max_message_size = 3; + required string server_version = 1; + optional int32 protocol_version = 2 [default = 0]; + optional int32 max_message_size = 3; + optional FeatureFlags feature_flags = 4; } message CommandAuthResponse { - optional string client_version = 1; - optional AuthData response = 2; - optional int32 protocol_version = 3 [default = 0]; + optional string client_version = 1; // The version of the client. Proxy should forward client's client_version. + optional AuthData response = 2; + optional int32 protocol_version = 3 [default = 0]; } message CommandAuthChallenge { - optional string server_version = 1; - optional AuthData challenge = 2; - optional int32 protocol_version = 3 [default = 0]; + optional string server_version = 1; + optional AuthData challenge = 2; + optional int32 protocol_version = 3 [default = 0]; } // To support mutual authentication type, such as Sasl, reuse this command to mutual auth. message AuthData { - optional string auth_method_name = 1; - optional bytes auth_data = 2; + optional string auth_method_name = 1; + optional bytes auth_data = 2; } enum KeySharedMode { - AUTO_SPLIT = 0; - STICKY = 1; + AUTO_SPLIT = 0; + STICKY = 1; } message KeySharedMeta { - required KeySharedMode keySharedMode = 1; - repeated IntRange hashRanges = 3; - optional bool allowOutOfOrderDelivery = 4 [default = false]; + required KeySharedMode keySharedMode = 1; + repeated IntRange hashRanges = 3; + optional bool allowOutOfOrderDelivery = 4 [default = false]; } message CommandSubscribe { - enum SubType { - Exclusive = 0; - Shared = 1; - Failover = 2; - Key_Shared = 3; - } - required string topic = 1; - required string subscription = 2; - required SubType subType = 3; - - required uint64 consumer_id = 4; - required uint64 request_id = 5; - optional string consumer_name = 6; - optional int32 priority_level = 7; - - // Signal wether the subscription should be backed by a - // durable cursor or not - optional bool durable = 8 [default = true]; - - // If specified, the subscription will position the cursor - // markd-delete position on the particular message id and - // will send messages from that point - optional MessageIdData start_message_id = 9; - - /// Add optional metadata key=value to this consumer - repeated KeyValue metadata = 10; - - optional bool read_compacted = 11; - - optional Schema schema = 12; - enum InitialPosition { - Latest = 0; - Earliest = 1; - } - // Signal whether the subscription will initialize on latest - // or not -- earliest - optional InitialPosition initialPosition = 13 [default = Latest]; - - // Mark the subscription as "replicated". Pulsar will make sure - // to periodically sync the state of replicated subscriptions - // across different clusters (when using geo-replication). - optional bool replicate_subscription_state = 14; - - // If true, the subscribe operation will cause a topic to be - // created if it does not exist already (and if topic auto-creation - // is allowed by broker. - // If false, the subscribe operation will fail if the topic - // does not exist. - optional bool force_topic_creation = 15 [default = true]; - - // If specified, the subscription will reset cursor's position back - // to specified seconds and will send messages from that point - optional uint64 start_message_rollback_duration_sec = 16 [default = 0]; - - optional KeySharedMeta keySharedMeta = 17; - - repeated KeyValue subscription_properties = 18; - - // The consumer epoch, when exclusive and failover consumer redeliver unack message will increase the epoch - optional uint64 consumer_epoch = 19; + enum SubType { + Exclusive = 0; + Shared = 1; + Failover = 2; + Key_Shared = 3; + } + required string topic = 1; + required string subscription = 2; + required SubType subType = 3; + + required uint64 consumer_id = 4; + required uint64 request_id = 5; + optional string consumer_name = 6; + optional int32 priority_level = 7; + + // Signal wether the subscription should be backed by a + // durable cursor or not + optional bool durable = 8 [default = true]; + + // If specified, the subscription will position the cursor + // markd-delete position on the particular message id and + // will send messages from that point + optional MessageIdData start_message_id = 9; + + /// Add optional metadata key=value to this consumer + repeated KeyValue metadata = 10; + + optional bool read_compacted = 11; + + optional Schema schema = 12; + enum InitialPosition { + Latest = 0; + Earliest = 1; + } + // Signal whether the subscription will initialize on latest + // or not -- earliest + optional InitialPosition initialPosition = 13 [default = Latest]; + + // Mark the subscription as "replicated". Pulsar will make sure + // to periodically sync the state of replicated subscriptions + // across different clusters (when using geo-replication). + optional bool replicate_subscription_state = 14; + + // If true, the subscribe operation will cause a topic to be + // created if it does not exist already (and if topic auto-creation + // is allowed by broker. + // If false, the subscribe operation will fail if the topic + // does not exist. + optional bool force_topic_creation = 15 [default = true]; + + // If specified, the subscription will reset cursor's position back + // to specified seconds and will send messages from that point + optional uint64 start_message_rollback_duration_sec = 16 [default = 0]; + + optional KeySharedMeta keySharedMeta = 17; + + repeated KeyValue subscription_properties = 18; + + // The consumer epoch, when exclusive and failover consumer redeliver unack message will increase the epoch + optional uint64 consumer_epoch = 19; } message CommandPartitionedTopicMetadata { - required string topic = 1; - required uint64 request_id = 2; - // TODO - Remove original_principal, original_auth_data, original_auth_method - // Original principal that was verified by - // a Pulsar proxy. - optional string original_principal = 3; + required string topic = 1; + required uint64 request_id = 2; + // TODO - Remove original_principal, original_auth_data, original_auth_method + // Original principal that was verified by + // a Pulsar proxy. + optional string original_principal = 3; - // Original auth role and auth Method that was passed - // to the proxy. - optional string original_auth_data = 4; - optional string original_auth_method = 5; + // Original auth role and auth Method that was passed + // to the proxy. + optional string original_auth_data = 4; + optional string original_auth_method = 5; + optional bool metadata_auto_creation_enabled = 6 [default = true]; } message CommandPartitionedTopicMetadataResponse { - enum LookupType { - Success = 0; - Failed = 1; - } - optional uint32 partitions = 1; // Optional in case of error - required uint64 request_id = 2; - optional LookupType response = 3; - optional ServerError error = 4; - optional string message = 5; + enum LookupType { + Success = 0; + Failed = 1; + } + optional uint32 partitions = 1; // Optional in case of error + required uint64 request_id = 2; + optional LookupType response = 3; + optional ServerError error = 4; + optional string message = 5; } message CommandLookupTopic { - required string topic = 1; - required uint64 request_id = 2; - optional bool authoritative = 3 [default = false]; + required string topic = 1; + required uint64 request_id = 2; + optional bool authoritative = 3 [default = false]; - // TODO - Remove original_principal, original_auth_data, original_auth_method - // Original principal that was verified by - // a Pulsar proxy. - optional string original_principal = 4; + // TODO - Remove original_principal, original_auth_data, original_auth_method + // Original principal that was verified by + // a Pulsar proxy. + optional string original_principal = 4; - // Original auth role and auth Method that was passed - // to the proxy. - optional string original_auth_data = 5; - optional string original_auth_method = 6; - // - optional string advertised_listener_name = 7; + // Original auth role and auth Method that was passed + // to the proxy. + optional string original_auth_data = 5; + optional string original_auth_method = 6; + // + optional string advertised_listener_name = 7; + // The properties used for topic lookup + repeated KeyValue properties = 8; } message CommandLookupTopicResponse { - enum LookupType { - Redirect = 0; - Connect = 1; - Failed = 2; - } - - optional string brokerServiceUrl = 1; // Optional in case of error - optional string brokerServiceUrlTls = 2; - optional LookupType response = 3; - required uint64 request_id = 4; - optional bool authoritative = 5 [default = false]; - optional ServerError error = 6; - optional string message = 7; - - // If it's true, indicates to the client that it must - // always connect through the service url after the - // lookup has been completed. - optional bool proxy_through_service_url = 8 [default = false]; + enum LookupType { + Redirect = 0; + Connect = 1; + Failed = 2; + } + + optional string brokerServiceUrl = 1; // Optional in case of error + optional string brokerServiceUrlTls = 2; + optional LookupType response = 3; + required uint64 request_id = 4; + optional bool authoritative = 5 [default = false]; + optional ServerError error = 6; + optional string message = 7; + + // If it's true, indicates to the client that it must + // always connect through the service url after the + // lookup has been completed. + optional bool proxy_through_service_url = 8 [default = false]; } /// Create a new Producer on a topic, assigning the given producer_id, /// all messages sent with this producer_id will be persisted on the topic message CommandProducer { - required string topic = 1; - required uint64 producer_id = 2; - required uint64 request_id = 3; + required string topic = 1; + required uint64 producer_id = 2; + required uint64 request_id = 3; - /// If a producer name is specified, the name will be used, - /// otherwise the broker will generate a unique name - optional string producer_name = 4; + /// If a producer name is specified, the name will be used, + /// otherwise the broker will generate a unique name + optional string producer_name = 4; - optional bool encrypted = 5 [default = false]; + optional bool encrypted = 5 [default = false]; - /// Add optional metadata key=value to this producer - repeated KeyValue metadata = 6; + /// Add optional metadata key=value to this producer + repeated KeyValue metadata = 6; - optional Schema schema = 7; + optional Schema schema = 7; - // If producer reconnect to broker, the epoch of this producer will +1 - optional uint64 epoch = 8 [default = 0]; + // If producer reconnect to broker, the epoch of this producer will +1 + optional uint64 epoch = 8 [default = 0]; - // Indicate the name of the producer is generated or user provided - // Use default true here is in order to be forward compatible with the client - optional bool user_provided_producer_name = 9 [default = true]; + // Indicate the name of the producer is generated or user provided + // Use default true here is in order to be forward compatible with the client + optional bool user_provided_producer_name = 9 [default = true]; - // Require that this producers will be the only producer allowed on the topic - optional ProducerAccessMode producer_access_mode = 10 [default = Shared]; + // Require that this producers will be the only producer allowed on the topic + optional ProducerAccessMode producer_access_mode = 10 [default = Shared]; - // Topic epoch is used to fence off producers that reconnects after a new - // exclusive producer has already taken over. This id is assigned by the - // broker on the CommandProducerSuccess. The first time, the client will - // leave it empty and then it will always carry the same epoch number on - // the subsequent reconnections. - optional uint64 topic_epoch = 11; + // Topic epoch is used to fence off producers that reconnects after a new + // exclusive producer has already taken over. This id is assigned by the + // broker on the CommandProducerSuccess. The first time, the client will + // leave it empty and then it will always carry the same epoch number on + // the subsequent reconnections. + optional uint64 topic_epoch = 11; - optional bool txn_enabled = 12 [default = false]; + optional bool txn_enabled = 12 [default = false]; - // Name of the initial subscription of the topic. - // If this field is not set, the initial subscription will not be created. - // If this field is set but the broker's `allowAutoSubscriptionCreation` - // is disabled, the producer will fail to be created. - optional string initial_subscription_name = 13; + // Name of the initial subscription of the topic. + // If this field is not set, the initial subscription will not be created. + // If this field is set but the broker's `allowAutoSubscriptionCreation` + // is disabled, the producer will fail to be created. + optional string initial_subscription_name = 13; } message CommandSend { - required uint64 producer_id = 1; - required uint64 sequence_id = 2; - optional int32 num_messages = 3 [default = 1]; - optional uint64 txnid_least_bits = 4 [default = 0]; - optional uint64 txnid_most_bits = 5 [default = 0]; + required uint64 producer_id = 1; + required uint64 sequence_id = 2; + optional int32 num_messages = 3 [default = 1]; + optional uint64 txnid_least_bits = 4 [default = 0]; + optional uint64 txnid_most_bits = 5 [default = 0]; + + /// Add highest sequence id to support batch message with external sequence id + optional uint64 highest_sequence_id = 6 [default = 0]; + optional bool is_chunk =7 [default = false]; - /// Add highest sequence id to support batch message with external sequence id - optional uint64 highest_sequence_id = 6 [default = 0]; - optional bool is_chunk = 7 [default = false]; + // Specify if the message being published is a Pulsar marker or not + optional bool marker = 8 [default = false]; - // Specify if the message being published is a Pulsar marker or not - optional bool marker = 8 [default = false]; + // Message id of this message, currently is used in replicator for shadow topic. + optional MessageIdData message_id = 9; } message CommandSendReceipt { - required uint64 producer_id = 1; - required uint64 sequence_id = 2; - optional MessageIdData message_id = 3; - optional uint64 highest_sequence_id = 4 [default = 0]; + required uint64 producer_id = 1; + required uint64 sequence_id = 2; + optional MessageIdData message_id = 3; + optional uint64 highest_sequence_id = 4 [default = 0]; } message CommandSendError { - required uint64 producer_id = 1; - required uint64 sequence_id = 2; - required ServerError error = 3; - required string message = 4; + required uint64 producer_id = 1; + required uint64 sequence_id = 2; + required ServerError error = 3; + required string message = 4; } message CommandMessage { - required uint64 consumer_id = 1; - required MessageIdData message_id = 2; - optional uint32 redelivery_count = 3 [default = 0]; - repeated int64 ack_set = 4; - optional uint64 consumer_epoch = 5; + required uint64 consumer_id = 1; + required MessageIdData message_id = 2; + optional uint32 redelivery_count = 3 [default = 0]; + repeated int64 ack_set = 4; + optional uint64 consumer_epoch = 5; } message CommandAck { - enum AckType { - Individual = 0; - Cumulative = 1; - } + enum AckType { + Individual = 0; + Cumulative = 1; + } - required uint64 consumer_id = 1; - required AckType ack_type = 2; + required uint64 consumer_id = 1; + required AckType ack_type = 2; - // In case of individual acks, the client can pass a list of message ids - repeated MessageIdData message_id = 3; + // In case of individual acks, the client can pass a list of message ids + repeated MessageIdData message_id = 3; - // Acks can contain a flag to indicate the consumer - // received an invalid message that got discarded - // before being passed on to the application. - enum ValidationError { - UncompressedSizeCorruption = 0; - DecompressionError = 1; - ChecksumMismatch = 2; - BatchDeSerializeError = 3; - DecryptionError = 4; - } + // Acks can contain a flag to indicate the consumer + // received an invalid message that got discarded + // before being passed on to the application. + enum ValidationError { + UncompressedSizeCorruption = 0; + DecompressionError = 1; + ChecksumMismatch = 2; + BatchDeSerializeError = 3; + DecryptionError = 4; + } - optional ValidationError validation_error = 4; - repeated KeyLongValue properties = 5; + optional ValidationError validation_error = 4; + repeated KeyLongValue properties = 5; - optional uint64 txnid_least_bits = 6 [default = 0]; - optional uint64 txnid_most_bits = 7 [default = 0]; - optional uint64 request_id = 8; + optional uint64 txnid_least_bits = 6 [default = 0]; + optional uint64 txnid_most_bits = 7 [default = 0]; + optional uint64 request_id = 8; } message CommandAckResponse { - required uint64 consumer_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; - optional uint64 request_id = 6; + required uint64 consumer_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; + optional uint64 request_id = 6; } // changes on active consumer message CommandActiveConsumerChange { - required uint64 consumer_id = 1; - optional bool is_active = 2 [default = false]; + required uint64 consumer_id = 1; + optional bool is_active = 2 [default = false]; } message CommandFlow { - required uint64 consumer_id = 1; + required uint64 consumer_id = 1; - // Max number of messages to prefetch, in addition - // of any number previously specified - required uint32 messagePermits = 2; + // Max number of messages to prefetch, in addition + // of any number previously specified + required uint32 messagePermits = 2; } message CommandUnsubscribe { - required uint64 consumer_id = 1; - required uint64 request_id = 2; + required uint64 consumer_id = 1; + required uint64 request_id = 2; + optional bool force = 3 [default = false]; } // Reset an existing consumer to a particular message id message CommandSeek { - required uint64 consumer_id = 1; - required uint64 request_id = 2; + required uint64 consumer_id = 1; + required uint64 request_id = 2; - optional MessageIdData message_id = 3; - optional uint64 message_publish_time = 4; + optional MessageIdData message_id = 3; + optional uint64 message_publish_time = 4; } // Message sent by broker to client when a topic // has been forcefully terminated and there are no more // messages left to consume message CommandReachedEndOfTopic { - required uint64 consumer_id = 1; + required uint64 consumer_id = 1; +} + +message CommandTopicMigrated { + enum ResourceType { + Producer = 0; + Consumer = 1; + } + required uint64 resource_id = 1; + required ResourceType resource_type = 2; + optional string brokerServiceUrl = 3; + optional string brokerServiceUrlTls = 4; + } + message CommandCloseProducer { - required uint64 producer_id = 1; - required uint64 request_id = 2; + required uint64 producer_id = 1; + required uint64 request_id = 2; + optional string assignedBrokerServiceUrl = 3; + optional string assignedBrokerServiceUrlTls = 4; } message CommandCloseConsumer { - required uint64 consumer_id = 1; - required uint64 request_id = 2; + required uint64 consumer_id = 1; + required uint64 request_id = 2; + optional string assignedBrokerServiceUrl = 3; + optional string assignedBrokerServiceUrlTls = 4; } message CommandRedeliverUnacknowledgedMessages { - required uint64 consumer_id = 1; - repeated MessageIdData message_ids = 2; - optional uint64 consumer_epoch = 3; + required uint64 consumer_id = 1; + repeated MessageIdData message_ids = 2; + optional uint64 consumer_epoch = 3; } message CommandSuccess { - required uint64 request_id = 1; - optional Schema schema = 2; + required uint64 request_id = 1; + optional Schema schema = 2; } /// Response from CommandProducer message CommandProducerSuccess { - required uint64 request_id = 1; - required string producer_name = 2; + required uint64 request_id = 1; + required string producer_name = 2; - // The last sequence id that was stored by this producer in the previous session - // This will only be meaningful if deduplication has been enabled. - optional int64 last_sequence_id = 3 [default = -1]; - optional bytes schema_version = 4; + // The last sequence id that was stored by this producer in the previous session + // This will only be meaningful if deduplication has been enabled. + optional int64 last_sequence_id = 3 [default = -1]; + optional bytes schema_version = 4; - // The topic epoch assigned by the broker. This field will only be set if we - // were requiring exclusive access when creating the producer. - optional uint64 topic_epoch = 5; + // The topic epoch assigned by the broker. This field will only be set if we + // were requiring exclusive access when creating the producer. + optional uint64 topic_epoch = 5; - // If producer is not "ready", the client will avoid to timeout the request - // for creating the producer. Instead it will wait indefinitely until it gets - // a subsequent `CommandProducerSuccess` with `producer_ready==true`. - optional bool producer_ready = 6 [default = true]; + // If producer is not "ready", the client will avoid to timeout the request + // for creating the producer. Instead it will wait indefinitely until it gets + // a subsequent `CommandProducerSuccess` with `producer_ready==true`. + optional bool producer_ready = 6 [default = true]; } message CommandError { - required uint64 request_id = 1; - required ServerError error = 2; - required string message = 3; + required uint64 request_id = 1; + required ServerError error = 2; + required string message = 3; } // Commands to probe the state of connection. @@ -671,396 +718,439 @@ message CommandPong { } message CommandConsumerStats { - required uint64 request_id = 1; - // required string topic_name = 2; - // required string subscription_name = 3; - required uint64 consumer_id = 4; + required uint64 request_id = 1; + // required string topic_name = 2; + // required string subscription_name = 3; + required uint64 consumer_id = 4; } message CommandConsumerStatsResponse { - required uint64 request_id = 1; - optional ServerError error_code = 2; - optional string error_message = 3; + required uint64 request_id = 1; + optional ServerError error_code = 2; + optional string error_message = 3; - /// Total rate of messages delivered to the consumer. msg/s - optional double msgRateOut = 4; + /// Total rate of messages delivered to the consumer. msg/s + optional double msgRateOut = 4; - /// Total throughput delivered to the consumer. bytes/s - optional double msgThroughputOut = 5; + /// Total throughput delivered to the consumer. bytes/s + optional double msgThroughputOut = 5; - /// Total rate of messages redelivered by this consumer. msg/s - optional double msgRateRedeliver = 6; + /// Total rate of messages redelivered by this consumer. msg/s + optional double msgRateRedeliver = 6; - /// Name of the consumer - optional string consumerName = 7; + /// Name of the consumer + optional string consumerName = 7; - /// Number of available message permits for the consumer - optional uint64 availablePermits = 8; + /// Number of available message permits for the consumer + optional uint64 availablePermits = 8; - /// Number of unacknowledged messages for the consumer - optional uint64 unackedMessages = 9; + /// Number of unacknowledged messages for the consumer + optional uint64 unackedMessages = 9; - /// Flag to verify if consumer is blocked due to reaching threshold of unacked messages - optional bool blockedConsumerOnUnackedMsgs = 10; + /// Flag to verify if consumer is blocked due to reaching threshold of unacked messages + optional bool blockedConsumerOnUnackedMsgs = 10; - /// Address of this consumer - optional string address = 11; + /// Address of this consumer + optional string address = 11; - /// Timestamp of connection - optional string connectedSince = 12; + /// Timestamp of connection + optional string connectedSince = 12; - /// Whether this subscription is Exclusive or Shared or Failover - optional string type = 13; + /// Whether this subscription is Exclusive or Shared or Failover + optional string type = 13; - /// Total rate of messages expired on this subscription. msg/s - optional double msgRateExpired = 14; + /// Total rate of messages expired on this subscription. msg/s + optional double msgRateExpired = 14; - /// Number of messages in the subscription backlog - optional uint64 msgBacklog = 15; + /// Number of messages in the subscription backlog + optional uint64 msgBacklog = 15; - /// Total rate of messages ack. msg/s - optional double messageAckRate = 16; + /// Total rate of messages ack. msg/s + optional double messageAckRate = 16; } message CommandGetLastMessageId { - required uint64 consumer_id = 1; - required uint64 request_id = 2; + required uint64 consumer_id = 1; + required uint64 request_id = 2; } message CommandGetLastMessageIdResponse { - required MessageIdData last_message_id = 1; - required uint64 request_id = 2; - optional MessageIdData consumer_mark_delete_position = 3; + required MessageIdData last_message_id = 1; + required uint64 request_id = 2; + optional MessageIdData consumer_mark_delete_position = 3; } message CommandGetTopicsOfNamespace { - enum Mode { - PERSISTENT = 0; - NON_PERSISTENT = 1; - ALL = 2; - } - required uint64 request_id = 1; - required string namespace = 2; - optional Mode mode = 3 [default = PERSISTENT]; - optional string topics_pattern = 4; - optional string topics_hash = 5; + enum Mode { + PERSISTENT = 0; + NON_PERSISTENT = 1; + ALL = 2; + } + required uint64 request_id = 1; + required string namespace = 2; + optional Mode mode = 3 [default = PERSISTENT]; + optional string topics_pattern = 4; + optional string topics_hash = 5; + // Context properties from the client + repeated KeyValue properties = 6; } message CommandGetTopicsOfNamespaceResponse { - required uint64 request_id = 1; - repeated string topics = 2; - // true iff the topic list was filtered by the pattern supplied by the client - optional bool filtered = 3 [default = false]; - // hash computed from the names of matching topics - optional string topics_hash = 4; - // if false, topics is empty and the list of matching topics has not changed - optional bool changed = 5 [default = true]; + required uint64 request_id = 1; + repeated string topics = 2; + // true iff the topic list was filtered by the pattern supplied by the client + optional bool filtered = 3 [default = false]; + // hash computed from the names of matching topics + optional string topics_hash = 4; + // if false, topics is empty and the list of matching topics has not changed + optional bool changed = 5 [default = true]; +} + +message CommandWatchTopicList { + required uint64 request_id = 1; + required uint64 watcher_id = 2; + required string namespace = 3; + required string topics_pattern = 4; + // Only present when the client reconnects: + optional string topics_hash = 5; +} + +message CommandWatchTopicListSuccess { + required uint64 request_id = 1; + required uint64 watcher_id = 2; + repeated string topic = 3; + required string topics_hash = 4; +} + +message CommandWatchTopicUpdate { + required uint64 watcher_id = 1; + repeated string new_topics = 2; + repeated string deleted_topics = 3; + required string topics_hash = 4; +} + +message CommandWatchTopicListClose { + required uint64 request_id = 1; + required uint64 watcher_id = 2; } message CommandGetSchema { - required uint64 request_id = 1; - required string topic = 2; + required uint64 request_id = 1; + required string topic = 2; - optional bytes schema_version = 3; + optional bytes schema_version = 3; } message CommandGetSchemaResponse { - required uint64 request_id = 1; - optional ServerError error_code = 2; - optional string error_message = 3; + required uint64 request_id = 1; + optional ServerError error_code = 2; + optional string error_message = 3; - optional Schema schema = 4; - optional bytes schema_version = 5; + optional Schema schema = 4; + optional bytes schema_version = 5; } message CommandGetOrCreateSchema { - required uint64 request_id = 1; - required string topic = 2; - required Schema schema = 3; + required uint64 request_id = 1; + required string topic = 2; + required Schema schema = 3; } message CommandGetOrCreateSchemaResponse { - required uint64 request_id = 1; - optional ServerError error_code = 2; - optional string error_message = 3; + required uint64 request_id = 1; + optional ServerError error_code = 2; + optional string error_message = 3; - optional bytes schema_version = 4; + optional bytes schema_version = 4; } /// --- transaction related --- enum TxnAction { - COMMIT = 0; - ABORT = 1; + COMMIT = 0; + ABORT = 1; } message CommandTcClientConnectRequest { - required uint64 request_id = 1; - required uint64 tc_id = 2 [default = 0]; + required uint64 request_id = 1; + required uint64 tc_id = 2 [default = 0]; } message CommandTcClientConnectResponse { - required uint64 request_id = 1; - optional ServerError error = 2; - optional string message = 3; + required uint64 request_id = 1; + optional ServerError error = 2; + optional string message = 3; } message CommandNewTxn { - required uint64 request_id = 1; - optional uint64 txn_ttl_seconds = 2 [default = 0]; - optional uint64 tc_id = 3 [default = 0]; + required uint64 request_id = 1; + optional uint64 txn_ttl_seconds = 2 [default = 0]; + optional uint64 tc_id = 3 [default = 0]; } message CommandNewTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandAddPartitionToTxn { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - repeated string partitions = 4; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + repeated string partitions = 4; } message CommandAddPartitionToTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message Subscription { - required string topic = 1; - required string subscription = 2; + required string topic = 1; + required string subscription = 2; } message CommandAddSubscriptionToTxn { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - repeated Subscription subscription = 4; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + repeated Subscription subscription = 4; } message CommandAddSubscriptionToTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandEndTxn { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional TxnAction txn_action = 4; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional TxnAction txn_action = 4; } message CommandEndTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandEndTxnOnPartition { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional string topic = 4; - optional TxnAction txn_action = 5; - optional uint64 txnid_least_bits_of_low_watermark = 6; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional string topic = 4; + optional TxnAction txn_action = 5; + optional uint64 txnid_least_bits_of_low_watermark = 6; } message CommandEndTxnOnPartitionResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandEndTxnOnSubscription { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional Subscription subscription = 4; - optional TxnAction txn_action = 5; - optional uint64 txnid_least_bits_of_low_watermark = 6; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional Subscription subscription= 4; + optional TxnAction txn_action = 5; + optional uint64 txnid_least_bits_of_low_watermark = 6; } message CommandEndTxnOnSubscriptionResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message BaseCommand { - enum Type { - CONNECT = 2; - CONNECTED = 3; - SUBSCRIBE = 4; + enum Type { + CONNECT = 2; + CONNECTED = 3; + SUBSCRIBE = 4; - PRODUCER = 5; + PRODUCER = 5; - SEND = 6; - SEND_RECEIPT = 7; - SEND_ERROR = 8; + SEND = 6; + SEND_RECEIPT= 7; + SEND_ERROR = 8; - MESSAGE = 9; - ACK = 10; - FLOW = 11; + MESSAGE = 9; + ACK = 10; + FLOW = 11; - UNSUBSCRIBE = 12; + UNSUBSCRIBE = 12; - SUCCESS = 13; - ERROR = 14; + SUCCESS = 13; + ERROR = 14; - CLOSE_PRODUCER = 15; - CLOSE_CONSUMER = 16; + CLOSE_PRODUCER = 15; + CLOSE_CONSUMER = 16; - PRODUCER_SUCCESS = 17; + PRODUCER_SUCCESS = 17; - PING = 18; - PONG = 19; + PING = 18; + PONG = 19; - REDELIVER_UNACKNOWLEDGED_MESSAGES = 20; + REDELIVER_UNACKNOWLEDGED_MESSAGES = 20; - PARTITIONED_METADATA = 21; - PARTITIONED_METADATA_RESPONSE = 22; + PARTITIONED_METADATA = 21; + PARTITIONED_METADATA_RESPONSE = 22; - LOOKUP = 23; - LOOKUP_RESPONSE = 24; + LOOKUP = 23; + LOOKUP_RESPONSE = 24; - CONSUMER_STATS = 25; - CONSUMER_STATS_RESPONSE = 26; + CONSUMER_STATS = 25; + CONSUMER_STATS_RESPONSE = 26; - REACHED_END_OF_TOPIC = 27; + REACHED_END_OF_TOPIC = 27; - SEEK = 28; + SEEK = 28; - GET_LAST_MESSAGE_ID = 29; - GET_LAST_MESSAGE_ID_RESPONSE = 30; + GET_LAST_MESSAGE_ID = 29; + GET_LAST_MESSAGE_ID_RESPONSE = 30; - ACTIVE_CONSUMER_CHANGE = 31; + ACTIVE_CONSUMER_CHANGE = 31; - GET_TOPICS_OF_NAMESPACE = 32; - GET_TOPICS_OF_NAMESPACE_RESPONSE = 33; + GET_TOPICS_OF_NAMESPACE = 32; + GET_TOPICS_OF_NAMESPACE_RESPONSE = 33; - GET_SCHEMA = 34; - GET_SCHEMA_RESPONSE = 35; + GET_SCHEMA = 34; + GET_SCHEMA_RESPONSE = 35; - AUTH_CHALLENGE = 36; - AUTH_RESPONSE = 37; + AUTH_CHALLENGE = 36; + AUTH_RESPONSE = 37; - ACK_RESPONSE = 38; + ACK_RESPONSE = 38; - GET_OR_CREATE_SCHEMA = 39; - GET_OR_CREATE_SCHEMA_RESPONSE = 40; + GET_OR_CREATE_SCHEMA = 39; + GET_OR_CREATE_SCHEMA_RESPONSE = 40; - // transaction related - NEW_TXN = 50; - NEW_TXN_RESPONSE = 51; + // transaction related + NEW_TXN = 50; + NEW_TXN_RESPONSE = 51; - ADD_PARTITION_TO_TXN = 52; - ADD_PARTITION_TO_TXN_RESPONSE = 53; + ADD_PARTITION_TO_TXN = 52; + ADD_PARTITION_TO_TXN_RESPONSE = 53; - ADD_SUBSCRIPTION_TO_TXN = 54; - ADD_SUBSCRIPTION_TO_TXN_RESPONSE = 55; + ADD_SUBSCRIPTION_TO_TXN = 54; + ADD_SUBSCRIPTION_TO_TXN_RESPONSE = 55; - END_TXN = 56; - END_TXN_RESPONSE = 57; + END_TXN = 56; + END_TXN_RESPONSE = 57; - END_TXN_ON_PARTITION = 58; - END_TXN_ON_PARTITION_RESPONSE = 59; + END_TXN_ON_PARTITION = 58; + END_TXN_ON_PARTITION_RESPONSE = 59; - END_TXN_ON_SUBSCRIPTION = 60; - END_TXN_ON_SUBSCRIPTION_RESPONSE = 61; - TC_CLIENT_CONNECT_REQUEST = 62; - TC_CLIENT_CONNECT_RESPONSE = 63; + END_TXN_ON_SUBSCRIPTION = 60; + END_TXN_ON_SUBSCRIPTION_RESPONSE = 61; + TC_CLIENT_CONNECT_REQUEST = 62; + TC_CLIENT_CONNECT_RESPONSE = 63; - } + WATCH_TOPIC_LIST = 64; + WATCH_TOPIC_LIST_SUCCESS = 65; + WATCH_TOPIC_UPDATE = 66; + WATCH_TOPIC_LIST_CLOSE = 67; + TOPIC_MIGRATED = 68; + } - required Type type = 1; - optional CommandConnect connect = 2; - optional CommandConnected connected = 3; + required Type type = 1; - optional CommandSubscribe subscribe = 4; - optional CommandProducer producer = 5; - optional CommandSend send = 6; - optional CommandSendReceipt send_receipt = 7; - optional CommandSendError send_error = 8; - optional CommandMessage message = 9; - optional CommandAck ack = 10; - optional CommandFlow flow = 11; - optional CommandUnsubscribe unsubscribe = 12; + optional CommandConnect connect = 2; + optional CommandConnected connected = 3; - optional CommandSuccess success = 13; - optional CommandError error = 14; + optional CommandSubscribe subscribe = 4; + optional CommandProducer producer = 5; + optional CommandSend send = 6; + optional CommandSendReceipt send_receipt = 7; + optional CommandSendError send_error = 8; + optional CommandMessage message = 9; + optional CommandAck ack = 10; + optional CommandFlow flow = 11; + optional CommandUnsubscribe unsubscribe = 12; - optional CommandCloseProducer close_producer = 15; - optional CommandCloseConsumer close_consumer = 16; + optional CommandSuccess success = 13; + optional CommandError error = 14; - optional CommandProducerSuccess producer_success = 17; - optional CommandPing ping = 18; - optional CommandPong pong = 19; - optional CommandRedeliverUnacknowledgedMessages redeliverUnacknowledgedMessages = 20; + optional CommandCloseProducer close_producer = 15; + optional CommandCloseConsumer close_consumer = 16; - optional CommandPartitionedTopicMetadata partitionMetadata = 21; - optional CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; + optional CommandProducerSuccess producer_success = 17; + optional CommandPing ping = 18; + optional CommandPong pong = 19; + optional CommandRedeliverUnacknowledgedMessages redeliverUnacknowledgedMessages = 20; - optional CommandLookupTopic lookupTopic = 23; - optional CommandLookupTopicResponse lookupTopicResponse = 24; + optional CommandPartitionedTopicMetadata partitionMetadata = 21; + optional CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; - optional CommandConsumerStats consumerStats = 25; - optional CommandConsumerStatsResponse consumerStatsResponse = 26; + optional CommandLookupTopic lookupTopic = 23; + optional CommandLookupTopicResponse lookupTopicResponse = 24; - optional CommandReachedEndOfTopic reachedEndOfTopic = 27; + optional CommandConsumerStats consumerStats = 25; + optional CommandConsumerStatsResponse consumerStatsResponse = 26; - optional CommandSeek seek = 28; + optional CommandReachedEndOfTopic reachedEndOfTopic = 27; - optional CommandGetLastMessageId getLastMessageId = 29; - optional CommandGetLastMessageIdResponse getLastMessageIdResponse = 30; + optional CommandSeek seek = 28; - optional CommandActiveConsumerChange active_consumer_change = 31; + optional CommandGetLastMessageId getLastMessageId = 29; + optional CommandGetLastMessageIdResponse getLastMessageIdResponse = 30; - optional CommandGetTopicsOfNamespace getTopicsOfNamespace = 32; - optional CommandGetTopicsOfNamespaceResponse getTopicsOfNamespaceResponse = 33; + optional CommandActiveConsumerChange active_consumer_change = 31; - optional CommandGetSchema getSchema = 34; - optional CommandGetSchemaResponse getSchemaResponse = 35; + optional CommandGetTopicsOfNamespace getTopicsOfNamespace = 32; + optional CommandGetTopicsOfNamespaceResponse getTopicsOfNamespaceResponse = 33; - optional CommandAuthChallenge authChallenge = 36; - optional CommandAuthResponse authResponse = 37; + optional CommandGetSchema getSchema = 34; + optional CommandGetSchemaResponse getSchemaResponse = 35; - optional CommandAckResponse ackResponse = 38; + optional CommandAuthChallenge authChallenge = 36; + optional CommandAuthResponse authResponse = 37; - optional CommandGetOrCreateSchema getOrCreateSchema = 39; - optional CommandGetOrCreateSchemaResponse getOrCreateSchemaResponse = 40; + optional CommandAckResponse ackResponse = 38; - // transaction related - optional CommandNewTxn newTxn = 50; - optional CommandNewTxnResponse newTxnResponse = 51; - optional CommandAddPartitionToTxn addPartitionToTxn = 52; - optional CommandAddPartitionToTxnResponse addPartitionToTxnResponse = 53; - optional CommandAddSubscriptionToTxn addSubscriptionToTxn = 54; - optional CommandAddSubscriptionToTxnResponse addSubscriptionToTxnResponse = 55; - optional CommandEndTxn endTxn = 56; - optional CommandEndTxnResponse endTxnResponse = 57; - optional CommandEndTxnOnPartition endTxnOnPartition = 58; - optional CommandEndTxnOnPartitionResponse endTxnOnPartitionResponse = 59; - optional CommandEndTxnOnSubscription endTxnOnSubscription = 60; - optional CommandEndTxnOnSubscriptionResponse endTxnOnSubscriptionResponse = 61; - optional CommandTcClientConnectRequest tcClientConnectRequest = 62; - optional CommandTcClientConnectResponse tcClientConnectResponse = 63; + optional CommandGetOrCreateSchema getOrCreateSchema = 39; + optional CommandGetOrCreateSchemaResponse getOrCreateSchemaResponse = 40; + + // transaction related + optional CommandNewTxn newTxn = 50; + optional CommandNewTxnResponse newTxnResponse = 51; + optional CommandAddPartitionToTxn addPartitionToTxn= 52; + optional CommandAddPartitionToTxnResponse addPartitionToTxnResponse = 53; + optional CommandAddSubscriptionToTxn addSubscriptionToTxn = 54; + optional CommandAddSubscriptionToTxnResponse addSubscriptionToTxnResponse = 55; + optional CommandEndTxn endTxn = 56; + optional CommandEndTxnResponse endTxnResponse = 57; + optional CommandEndTxnOnPartition endTxnOnPartition = 58; + optional CommandEndTxnOnPartitionResponse endTxnOnPartitionResponse = 59; + optional CommandEndTxnOnSubscription endTxnOnSubscription = 60; + optional CommandEndTxnOnSubscriptionResponse endTxnOnSubscriptionResponse = 61; + optional CommandTcClientConnectRequest tcClientConnectRequest = 62; + optional CommandTcClientConnectResponse tcClientConnectResponse = 63; + + optional CommandWatchTopicList watchTopicList = 64; + optional CommandWatchTopicListSuccess watchTopicListSuccess = 65; + optional CommandWatchTopicUpdate watchTopicUpdate = 66; + optional CommandWatchTopicListClose watchTopicListClose = 67; + + optional CommandTopicMigrated topicMigrated = 68; } diff --git a/src/connection.rs b/src/connection.rs index d5390063..5d9d40a3 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -1633,6 +1633,7 @@ pub(crate) mod messages { close_producer: Some(proto::CommandCloseProducer { producer_id, request_id, + ..Default::default() }), ..Default::default() }, @@ -1753,6 +1754,7 @@ pub(crate) mod messages { close_consumer: Some(proto::CommandCloseConsumer { consumer_id, request_id, + ..Default::default() }), ..Default::default() }, @@ -1790,6 +1792,7 @@ pub(crate) mod messages { unsubscribe: Some(proto::CommandUnsubscribe { consumer_id, request_id, + ..Default::default() }), ..Default::default() }, diff --git a/src/message.rs b/src/message.rs index a302ec28..3d15c881 100644 --- a/src/message.rs +++ b/src/message.rs @@ -178,6 +178,7 @@ impl Message { Some(CommandCloseConsumer { consumer_id, request_id, + .. }), .. } => Some(RequestKey::CloseConsumer { From f1055c6caf475eae33657a20fdb009daa0c2a913 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 19:10:01 -0700 Subject: [PATCH 02/24] error: add SchemaRegistry variant for external schema errors --- src/error.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/error.rs b/src/error.rs index 3d513b77..7728a0a9 100644 --- a/src/error.rs +++ b/src/error.rs @@ -16,6 +16,7 @@ pub enum Error { Producer(ProducerError), ServiceDiscovery(ServiceDiscoveryError), Authentication(AuthenticationError), + SchemaRegistry(String), Custom(String), Executor, } @@ -57,6 +58,7 @@ impl fmt::Display for Error { Error::Producer(e) => write!(f, "producer error: {e}"), Error::ServiceDiscovery(e) => write!(f, "service discovery error: {e}"), Error::Authentication(e) => write!(f, "authentication error: {e}"), + Error::SchemaRegistry(e) => write!(f, "schema registry error: {e}"), Error::Custom(e) => write!(f, "error: {e}"), Error::Executor => write!(f, "could not spawn task"), } @@ -72,6 +74,7 @@ impl std::error::Error for Error { Error::Producer(e) => e.source(), Error::ServiceDiscovery(e) => e.source(), Error::Authentication(e) => e.source(), + Error::SchemaRegistry(_) => None, Error::Custom(_) => None, Error::Executor => None, } From 766ff83914950b01da46c4ab3c6c0f0eb9e47585 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 19:11:35 -0700 Subject: [PATCH 03/24] feat: add schema_id_util with magic-byte framing (PIP-420) --- src/lib.rs | 1 + src/schema/mod.rs | 1 + src/schema/schema_id_util.rs | 141 +++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 src/schema/mod.rs create mode 100644 src/schema/schema_id_util.rs diff --git a/src/lib.rs b/src/lib.rs index 3931ec89..64ffc152 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -224,6 +224,7 @@ pub mod producer; pub mod reader; mod retry_op; pub mod routing_policy; +pub mod schema; mod service_discovery; mod test_utils; diff --git a/src/schema/mod.rs b/src/schema/mod.rs new file mode 100644 index 00000000..8ed56d2c --- /dev/null +++ b/src/schema/mod.rs @@ -0,0 +1 @@ +pub mod schema_id_util; diff --git a/src/schema/schema_id_util.rs b/src/schema/schema_id_util.rs new file mode 100644 index 00000000..38df3279 --- /dev/null +++ b/src/schema/schema_id_util.rs @@ -0,0 +1,141 @@ +/// Magic byte for single-schema framing (PIP-420). +pub const MAGIC_BYTE_VALUE: u8 = 0xFF; +/// Magic byte for key-value schema framing (PIP-420). +pub const MAGIC_BYTE_KEY_VALUE: u8 = 0xFE; + +/// Parsed schema ID information from magic-byte framed data. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SchemaIdInfo { + Single(Vec), + KeyValue { key_id: Vec, value_id: Vec }, +} + +pub fn add_magic_header(schema_id: &[u8]) -> Vec { + let mut result = Vec::with_capacity(1 + schema_id.len()); + result.push(MAGIC_BYTE_VALUE); + result.extend_from_slice(schema_id); + result +} + +pub fn strip_magic_header(data: &[u8]) -> Option { + if data.is_empty() { + return None; + } + match data[0] { + MAGIC_BYTE_VALUE => Some(SchemaIdInfo::Single(data[1..].to_vec())), + MAGIC_BYTE_KEY_VALUE => { + if data.len() < 5 { + return None; + } + let key_len = + u32::from_be_bytes([data[1], data[2], data[3], data[4]]) as usize; + if data.len() < 5 + key_len { + return None; + } + let key_id = data[5..5 + key_len].to_vec(); + let value_id = data[5 + key_len..].to_vec(); + Some(SchemaIdInfo::KeyValue { key_id, value_id }) + } + _ => None, + } +} + +pub fn generate_kv_schema_id( + key_schema_id: Option<&[u8]>, + value_schema_id: Option<&[u8]>, +) -> Option> { + if key_schema_id.is_none() && value_schema_id.is_none() { + return None; + } + let key_bytes = key_schema_id.unwrap_or(&[]); + let val_bytes = value_schema_id.unwrap_or(&[]); + let key_len = key_bytes.len() as u32; + + let mut result = Vec::with_capacity(1 + 4 + key_bytes.len() + val_bytes.len()); + result.push(MAGIC_BYTE_KEY_VALUE); + result.extend_from_slice(&key_len.to_be_bytes()); + result.extend_from_slice(key_bytes); + result.extend_from_slice(val_bytes); + Some(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add_magic_header() { + let id = vec![0x00, 0x00, 0x00, 0x01]; // schema ID = 1 + let framed = add_magic_header(&id); + assert_eq!(framed[0], MAGIC_BYTE_VALUE); + assert_eq!(&framed[1..], &id); + } + + #[test] + fn test_add_magic_header_empty() { + let framed = add_magic_header(&[]); + assert_eq!(framed, vec![MAGIC_BYTE_VALUE]); + } + + #[test] + fn test_strip_magic_header_single() { + let id = vec![0x00, 0x00, 0x00, 0x05]; + let framed = add_magic_header(&id); + let info = strip_magic_header(&framed).unwrap(); + assert_eq!(info, SchemaIdInfo::Single(id)); + } + + #[test] + fn test_strip_magic_header_empty_input() { + assert_eq!(strip_magic_header(&[]), None); + } + + #[test] + fn test_strip_magic_header_no_magic() { + assert_eq!(strip_magic_header(&[0x00, 0x01, 0x02]), None); + } + + #[test] + fn test_generate_kv_schema_id_both() { + let key_id = vec![0x00, 0x00, 0x00, 0x01]; + let val_id = vec![0x00, 0x00, 0x00, 0x02]; + let framed = generate_kv_schema_id(Some(&key_id), Some(&val_id)).unwrap(); + // [0xFE] [4-byte key_len BE] [key_id] [value_id] + assert_eq!(framed[0], MAGIC_BYTE_KEY_VALUE); + let key_len = u32::from_be_bytes([framed[1], framed[2], framed[3], framed[4]]); + assert_eq!(key_len, 4); + assert_eq!(&framed[5..9], &key_id); + assert_eq!(&framed[9..], &val_id); + } + + #[test] + fn test_generate_kv_schema_id_key_only() { + let key_id = vec![0x00, 0x00, 0x00, 0x03]; + let framed = generate_kv_schema_id(Some(&key_id), None).unwrap(); + assert_eq!(framed[0], MAGIC_BYTE_KEY_VALUE); + let key_len = u32::from_be_bytes([framed[1], framed[2], framed[3], framed[4]]); + assert_eq!(key_len, 4); + assert_eq!(&framed[5..9], &key_id); + assert!(framed[9..].is_empty()); + } + + #[test] + fn test_generate_kv_schema_id_neither() { + assert_eq!(generate_kv_schema_id(None, None), None); + } + + #[test] + fn test_roundtrip_kv() { + let key_id = vec![0x01, 0x02]; + let val_id = vec![0x03, 0x04, 0x05]; + let framed = generate_kv_schema_id(Some(&key_id), Some(&val_id)).unwrap(); + let info = strip_magic_header(&framed).unwrap(); + assert_eq!( + info, + SchemaIdInfo::KeyValue { + key_id: key_id, + value_id: val_id, + } + ); + } +} From 3e36b010962f4bfd1c555db9f451b54189dbd144 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 19:12:48 -0700 Subject: [PATCH 04/24] feat: add PulsarSchema trait and EncodeData struct --- src/schema/mod.rs | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/schema/mod.rs b/src/schema/mod.rs index 8ed56d2c..19faf659 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -1 +1,53 @@ pub mod schema_id_util; + +use async_trait::async_trait; + +use crate::{ + message::proto, + Error, Payload, +}; + +/// Data returned from encoding a message with a PulsarSchema. +#[derive(Debug, Clone)] +pub struct EncodeData { + /// The serialized message payload. + pub payload: Vec, + /// Schema identifier from an external registry. + /// Present only for external schemas. Written to MessageMetadata.schema_id + /// with magic-byte framing. + pub schema_id: Option>, +} + +/// Unified schema interface for Pulsar messages. +/// +/// Both built-in Pulsar schemas and external schemas (e.g., Confluent Schema +/// Registry) implement this trait. Named `PulsarSchema` to avoid collision +/// with the protobuf `proto::Schema` type. +#[async_trait] +pub trait PulsarSchema: Send + Sync + 'static { + /// Schema metadata sent to the broker during producer/consumer creation. + fn schema_info(&self) -> proto::Schema; + + /// Encode a message for the given topic. + /// + /// Built-in schemas return `EncodeData` with `schema_id = None`. + /// External schemas return `EncodeData` with a pre-framed `schema_id` + /// (magic byte included) ready to write to `MessageMetadata.schema_id`. + async fn encode(&self, topic: &str, message: T) -> Result; + + /// Decode a message. + /// + /// For built-in schemas, `schema_id` is `None`. + /// For external schemas, `schema_id` identifies the schema in the registry. + async fn decode( + &self, + topic: &str, + payload: &Payload, + schema_id: Option<&[u8]>, + ) -> Result; + + /// Release resources (connections, caches) when producer/consumer closes. + async fn close(&self) -> Result<(), Error> { + Ok(()) + } +} From f8a8664ed23b74b1c4c3a9d8708fc7cc52183b19 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 19:14:45 -0700 Subject: [PATCH 05/24] feat: add DefaultPulsarSchema bridging old traits to PulsarSchema --- src/schema/default.rs | 145 ++++++++++++++++++++++++++++++++++++++++++ src/schema/mod.rs | 1 + 2 files changed, 146 insertions(+) create mode 100644 src/schema/default.rs diff --git a/src/schema/default.rs b/src/schema/default.rs new file mode 100644 index 00000000..6bcba6d0 --- /dev/null +++ b/src/schema/default.rs @@ -0,0 +1,145 @@ +use std::marker::PhantomData; + +use async_trait::async_trait; + +use crate::{ + client::{DeserializeMessage, SerializeMessage}, + message::proto, + schema::{EncodeData, PulsarSchema}, + Error, Payload, +}; + +/// Wraps existing `SerializeMessage`/`DeserializeMessage` implementations +/// into the `PulsarSchema` trait for backward compatibility. +pub struct DefaultPulsarSchema { + schema_info: proto::Schema, + _phantom: PhantomData, +} + +impl DefaultPulsarSchema { + pub fn new(schema_info: proto::Schema) -> Self { + Self { + schema_info, + _phantom: PhantomData, + } + } +} + +#[async_trait] +impl PulsarSchema for DefaultPulsarSchema +where + T: SerializeMessage + DeserializeMessage> + Send + Sync + 'static, + E: std::error::Error + Send + Sync + 'static, +{ + fn schema_info(&self) -> proto::Schema { + self.schema_info.clone() + } + + async fn encode(&self, _topic: &str, message: T) -> Result { + let msg = T::serialize_message(message)?; + Ok(EncodeData { + payload: msg.payload, + schema_id: None, + }) + } + + async fn decode( + &self, + _topic: &str, + payload: &Payload, + _schema_id: Option<&[u8]>, + ) -> Result { + let result: Result = T::deserialize_message(payload); + result.map_err(|e| Error::Custom(e.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::producer; + + // Test type that implements both SerializeMessage and DeserializeMessage + #[derive(Debug, Clone, PartialEq)] + struct TestMsg { + data: String, + } + + impl SerializeMessage for TestMsg { + fn serialize_message(input: Self) -> Result { + Ok(producer::Message { + payload: input.data.into_bytes(), + ..Default::default() + }) + } + } + + impl DeserializeMessage for TestMsg { + type Output = Result; + + fn deserialize_message(payload: &Payload) -> Self::Output { + let data = String::from_utf8(payload.data.clone()) + .map_err(|e| Error::Custom(e.to_string()))?; + Ok(TestMsg { data }) + } + } + + #[tokio::test] + async fn test_default_schema_encode() { + let schema = DefaultPulsarSchema::::new(proto::Schema { + r#type: proto::schema::Type::String as i32, + ..Default::default() + }); + + let msg = TestMsg { + data: "hello".to_string(), + }; + let encoded = schema.encode("test-topic", msg).await.unwrap(); + assert_eq!(encoded.payload, b"hello"); + assert!(encoded.schema_id.is_none()); + } + + #[tokio::test] + async fn test_default_schema_decode() { + let schema = DefaultPulsarSchema::::new(proto::Schema::default()); + + let payload = Payload { + metadata: proto::MessageMetadata::default(), + data: b"world".to_vec(), + }; + let decoded = schema.decode("test-topic", &payload, None).await.unwrap(); + assert_eq!(decoded.data, "world"); + } + + #[tokio::test] + async fn test_default_schema_roundtrip() { + let schema = DefaultPulsarSchema::::new(proto::Schema::default()); + + let original = TestMsg { + data: "roundtrip".to_string(), + }; + let encoded = schema + .encode("test-topic", original.clone()) + .await + .unwrap(); + + let payload = Payload { + metadata: proto::MessageMetadata::default(), + data: encoded.payload, + }; + let decoded = schema.decode("test-topic", &payload, None).await.unwrap(); + assert_eq!(decoded, original); + } + + #[tokio::test] + async fn test_default_schema_info() { + let info = proto::Schema { + r#type: proto::schema::Type::String as i32, + name: "test".to_string(), + ..Default::default() + }; + let schema = DefaultPulsarSchema::::new(info.clone()); + assert_eq!(schema.schema_info().r#type, info.r#type); + assert_eq!(schema.schema_info().name, info.name); + } +} diff --git a/src/schema/mod.rs b/src/schema/mod.rs index 19faf659..08f7e17c 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -1,3 +1,4 @@ +pub mod default; pub mod schema_id_util; use async_trait::async_trait; From ae3bccc1f15fdf43d7733bbc09e16a5100839b5a Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 19:15:05 -0700 Subject: [PATCH 06/24] feat: add KeyValueSchema composing two inner PulsarSchema instances --- src/schema/key_value.rs | 218 ++++++++++++++++++++++++++++++++++++++++ src/schema/mod.rs | 1 + 2 files changed, 219 insertions(+) create mode 100644 src/schema/key_value.rs diff --git a/src/schema/key_value.rs b/src/schema/key_value.rs new file mode 100644 index 00000000..17680366 --- /dev/null +++ b/src/schema/key_value.rs @@ -0,0 +1,218 @@ +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::{ + message::proto, + schema::{schema_id_util, EncodeData, PulsarSchema}, + Error, Payload, +}; + +use super::schema_id_util::SchemaIdInfo; + +/// Composes two inner schemas for key-value message types. +pub struct KeyValueSchema { + key_schema: Arc>, + value_schema: Arc>, +} + +impl KeyValueSchema { + pub fn new( + key_schema: Arc>, + value_schema: Arc>, + ) -> Self { + Self { + key_schema, + value_schema, + } + } +} + +/// Combine key and value payloads into a single byte vector. +/// Format: [4-byte key_len BE] [key_bytes] [value_bytes] +fn combine_kv_payload(key: &[u8], value: &[u8]) -> Vec { + let key_len = key.len() as u32; + let mut result = Vec::with_capacity(4 + key.len() + value.len()); + result.extend_from_slice(&key_len.to_be_bytes()); + result.extend_from_slice(key); + result.extend_from_slice(value); + result +} + +/// Split a combined key-value payload into key and value Payloads. +fn split_kv_payload(payload: &Payload) -> Result<(Payload, Payload), Error> { + if payload.data.len() < 4 { + return Err(Error::Custom( + "KeyValue payload too short to contain key length".to_string(), + )); + } + let key_len = + u32::from_be_bytes([payload.data[0], payload.data[1], payload.data[2], payload.data[3]]) + as usize; + if payload.data.len() < 4 + key_len { + return Err(Error::Custom( + "KeyValue payload too short for declared key length".to_string(), + )); + } + let key_data = payload.data[4..4 + key_len].to_vec(); + let val_data = payload.data[4 + key_len..].to_vec(); + Ok(( + Payload { + metadata: payload.metadata.clone(), + data: key_data, + }, + Payload { + metadata: payload.metadata.clone(), + data: val_data, + }, + )) +} + +#[async_trait] +impl PulsarSchema<(K, V)> for KeyValueSchema +where + K: Send + 'static, + V: Send + 'static, +{ + fn schema_info(&self) -> proto::Schema { + proto::Schema { + r#type: proto::schema::Type::KeyValue as i32, + ..Default::default() + } + } + + async fn encode(&self, topic: &str, message: (K, V)) -> Result { + let (key, value) = message; + let key_data = self.key_schema.encode(topic, key).await?; + let value_data = self.value_schema.encode(topic, value).await?; + let schema_id = schema_id_util::generate_kv_schema_id( + key_data.schema_id.as_deref(), + value_data.schema_id.as_deref(), + ); + Ok(EncodeData { + payload: combine_kv_payload(&key_data.payload, &value_data.payload), + schema_id, + }) + } + + async fn decode( + &self, + topic: &str, + payload: &Payload, + schema_id: Option<&[u8]>, + ) -> Result<(K, V), Error> { + let (key_id, value_id) = match schema_id { + Some(data) => match schema_id_util::strip_magic_header(data) { + Some(SchemaIdInfo::KeyValue { key_id, value_id }) => { + (Some(key_id), Some(value_id)) + } + _ => (None, None), + }, + None => (None, None), + }; + let (key_payload, value_payload) = split_kv_payload(payload)?; + let key = self + .key_schema + .decode(topic, &key_payload, key_id.as_deref()) + .await?; + let value = self + .value_schema + .decode(topic, &value_payload, value_id.as_deref()) + .await?; + Ok((key, value)) + } + + async fn close(&self) -> Result<(), Error> { + self.key_schema.close().await?; + self.value_schema.close().await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::schema_id_util; + + /// Simple mock schema for testing. + struct MockSchema { + schema_id: Option>, + } + + #[async_trait] + impl PulsarSchema for MockSchema { + fn schema_info(&self) -> proto::Schema { + proto::Schema::default() + } + + async fn encode(&self, _topic: &str, message: String) -> Result { + Ok(EncodeData { + payload: message.into_bytes(), + schema_id: self.schema_id.clone(), + }) + } + + async fn decode( + &self, + _topic: &str, + payload: &Payload, + _schema_id: Option<&[u8]>, + ) -> Result { + String::from_utf8(payload.data.clone()).map_err(|e| Error::Custom(e.to_string())) + } + } + + #[tokio::test] + async fn test_kv_encode_decode_roundtrip() { + let kv_schema = KeyValueSchema::new( + Arc::new(MockSchema { schema_id: None }), + Arc::new(MockSchema { schema_id: None }), + ); + + let encoded = kv_schema + .encode("topic", ("key".to_string(), "value".to_string())) + .await + .unwrap(); + + assert!(encoded.schema_id.is_none()); + + let payload = Payload { + metadata: proto::MessageMetadata::default(), + data: encoded.payload, + }; + let (k, v) = kv_schema.decode("topic", &payload, None).await.unwrap(); + assert_eq!(k, "key"); + assert_eq!(v, "value"); + } + + #[tokio::test] + async fn test_kv_encode_with_schema_ids() { + let kv_schema = KeyValueSchema::new( + Arc::new(MockSchema { + schema_id: Some(schema_id_util::add_magic_header(&[0x01])), + }), + Arc::new(MockSchema { + schema_id: Some(schema_id_util::add_magic_header(&[0x02])), + }), + ); + + let encoded = kv_schema + .encode("topic", ("key".to_string(), "value".to_string())) + .await + .unwrap(); + + let framed = encoded.schema_id.unwrap(); + assert_eq!(framed[0], schema_id_util::MAGIC_BYTE_KEY_VALUE); + } + + #[tokio::test] + async fn test_kv_schema_info() { + let kv_schema = KeyValueSchema::new( + Arc::new(MockSchema { schema_id: None }) as Arc>, + Arc::new(MockSchema { schema_id: None }) as Arc>, + ); + + let info = kv_schema.schema_info(); + assert_eq!(info.r#type, proto::schema::Type::KeyValue as i32); + } +} diff --git a/src/schema/mod.rs b/src/schema/mod.rs index 08f7e17c..90f51759 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -1,4 +1,5 @@ pub mod default; +pub mod key_value; pub mod schema_id_util; use async_trait::async_trait; From 5ba8a8e587078009c9bab61a6cae192cf2ab882b Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 19:18:02 -0700 Subject: [PATCH 07/24] feat: re-export PulsarSchema, EncodeData, and schema sub-types Co-Authored-By: Claude Opus 4.6 --- src/lib.rs | 1 + src/schema/mod.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 64ffc152..e15ac1d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -210,6 +210,7 @@ pub use message::{ Payload, }; pub use producer::{MultiTopicProducer, Producer, ProducerOptions}; +pub use schema::{EncodeData, PulsarSchema}; pub mod authentication; mod client; diff --git a/src/schema/mod.rs b/src/schema/mod.rs index 90f51759..9610591c 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -53,3 +53,8 @@ pub trait PulsarSchema: Send + Sync + 'static { Ok(()) } } + +// Re-exports +pub use default::DefaultPulsarSchema; +pub use key_value::KeyValueSchema; +pub use schema_id_util::SchemaIdInfo; From cb1a37d0be0c789476812e1ad2b92b2b1c383cca Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 20:46:30 -0700 Subject: [PATCH 08/24] =?UTF-8?q?refactor:=20Message=20=E2=80=94=20repl?= =?UTF-8?q?ace=20=5Fphantom=20with=20decoded=20field,=20add=20value()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- src/consumer/data.rs | 3 +++ src/consumer/message.rs | 43 ++++++++++++++++++++++++++++------------- src/consumer/topic.rs | 2 +- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 38200c00..4dd1a28f 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -10,6 +10,9 @@ use crate::{ pub type MessageIdDataReceiver = mpsc::Receiver>; +pub type DecodedMessageReceiver = + mpsc::Receiver), Error>>; + pub enum EngineEvent { Message(Option), EngineMessage(Option>), diff --git a/src/consumer/message.rs b/src/consumer/message.rs index e76e810d..dae33b20 100644 --- a/src/consumer/message.rs +++ b/src/consumer/message.rs @@ -1,47 +1,64 @@ -use std::marker::PhantomData; - use crate::{ consumer::data::MessageData, message::proto::{MessageIdData, MessageMetadata}, DeserializeMessage, Payload, }; -/// a message received by a consumer +/// A message received by a consumer. /// -/// it is generic over the type it can be deserialized to -#[derive(Debug)] +/// Generic over the type it can be deserialized to. pub struct Message { - /// origin topic of the message + /// Origin topic of the message. pub topic: String, - /// contains the message's data and other metadata + /// Contains the message's data and other metadata. pub payload: Payload, - /// contains the message's id and batch size data + /// Contains the message's id and batch size data. pub message_id: MessageData, - pub(super) _phantom: PhantomData, + /// Pre-decoded value from PulsarSchema. None when using DeserializeMessage path. + pub(super) decoded: Option, +} + +// Manual Debug impl — avoids requiring T: Debug +impl std::fmt::Debug for Message { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Message") + .field("topic", &self.topic) + .field("payload", &self.payload) + .field("message_id", &self.message_id) + .field("has_decoded", &self.decoded.is_some()) + .finish() + } } impl Message { - /// Pulsar metadata for the message + /// Pulsar metadata for the message. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn metadata(&self) -> &MessageMetadata { &self.payload.metadata } - /// Get Pulsar message id for the message + /// Get Pulsar message id for the message. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn message_id(&self) -> &MessageIdData { &self.message_id.id } - /// Get message key (partition key) + /// Get message key (partition key). #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn key(&self) -> Option { self.payload.metadata.partition_key.clone() } + + /// Returns the pre-decoded value if this message was decoded via PulsarSchema. + /// Returns None when using the DeserializeMessage path. + pub fn value(&self) -> Option<&T> { + self.decoded.as_ref() + } } impl Message { - /// directly deserialize a message + /// Directly deserialize a message using the DeserializeMessage trait. + /// This continues to work unchanged for backward compatibility. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn deserialize(&self) -> T::Output { T::deserialize_message(&self.payload) diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 1c2ca023..c3a1dd66 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -303,7 +303,7 @@ impl TopicConsumer { batch_size: payload.metadata.num_messages_in_batch, }, payload, - _phantom: PhantomData, + decoded: None, } } From 284d4fa6fb1b603490c67862c7c345a8f8431b32 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 20:58:26 -0700 Subject: [PATCH 09/24] refactor: TopicConsumer uses unified DecodedMessageReceiver channel Co-Authored-By: Claude Opus 4.6 --- src/consumer/builder.rs | 4 ++-- src/consumer/mod.rs | 12 +++++------ src/consumer/multi.rs | 8 +++---- src/consumer/topic.rs | 46 ++++++++++++++++++++++++++++++++--------- src/reader.rs | 10 ++++----- 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 5d8342ed..7a94dd85 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -268,7 +268,7 @@ impl ConsumerBuilder { /// creates a [Consumer] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub async fn build(self) -> Result, Error> { + pub async fn build(self) -> Result, Error> { // would this clone() consume too much memory? let (config, joined_topics) = self.clone().validate().await?; @@ -318,7 +318,7 @@ impl ConsumerBuilder { /// creates a [Reader] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub async fn into_reader(self) -> Result, Error> { + pub async fn into_reader(self) -> Result, Error> { // would this clone() consume too much memory? let (mut config, mut joined_topics) = self.clone().validate().await?; diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 3111c8df..5e3849da 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -40,7 +40,7 @@ use crate::{ DeserializeMessage, Pulsar, }; -enum InnerConsumer { +enum InnerConsumer { Single(TopicConsumer), Multi(MultiTopicConsumer), } @@ -79,11 +79,11 @@ enum InnerConsumer { /// # Ok(()) /// # } /// ``` -pub struct Consumer { +pub struct Consumer { inner: InnerConsumer, } -impl Consumer { +impl Consumer { /// creates a [ConsumerBuilder] from a client instance #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn builder(pulsar: &Pulsar) -> ConsumerBuilder { @@ -411,7 +411,7 @@ impl Consumer { } //TODO: why does T need to be 'static? -impl Stream for Consumer { +impl Stream for Consumer { type Item = Result, Error>; #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] @@ -495,7 +495,7 @@ mod tests { dur: Duration, ) -> Result, String> where - T: DeserializeMessage + 'static, + T: DeserializeMessage + Send + 'static, { match tokio::time::timeout(dur, c.next()).await { Err(_) => Err(format!("timed out waiting for next() after {:?}", dur)), @@ -1529,7 +1529,7 @@ mod tests { } /// Build an Exclusive consumer that starts from Earliest. - async fn earliest_consumer( + async fn earliest_consumer( client: &Pulsar, topic: &str, name: &str, diff --git a/src/consumer/multi.rs b/src/consumer/multi.rs index 42340427..de3c59ef 100644 --- a/src/consumer/multi.rs +++ b/src/consumer/multi.rs @@ -22,7 +22,7 @@ use crate::{ /// A consumer that can subscribe on multiple topics, from a regex matching /// topic names -pub struct MultiTopicConsumer { +pub struct MultiTopicConsumer { pub(super) namespace: String, pub(super) topic_regex: Option, pub(super) pulsar: Pulsar, @@ -40,7 +40,7 @@ pub struct MultiTopicConsumer { pub(super) disc_last_message_received: Option>, } -impl MultiTopicConsumer { +impl MultiTopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn topics(&self) -> Vec { self.topics.iter().map(|s| s.to_string()).collect() @@ -327,7 +327,7 @@ impl MultiTopicConsumer { } } -impl Debug for MultiTopicConsumer { +impl Debug for MultiTopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( @@ -338,7 +338,7 @@ impl Debug for MultiTopicConsumer } } -impl Stream for MultiTopicConsumer { +impl Stream for MultiTopicConsumer { type Item = Result, Error>; #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index c3a1dd66..913176bb 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -19,7 +19,7 @@ use crate::{ connection::Connection, consumer::{ config::ConsumerConfig, - data::{DeadLetterPolicy, EngineMessage, MessageData, MessageIdDataReceiver}, + data::{DeadLetterPolicy, DecodedMessageReceiver, EngineMessage, MessageData}, engine::ConsumerEngine, message::Message, }, @@ -31,11 +31,11 @@ use crate::{ }; // this is entirely public for use in reader.rs -pub struct TopicConsumer { +pub struct TopicConsumer { pub(crate) consumer_id: u64, pub(crate) config: ConsumerConfig, topic: String, - messages: Pin>, + messages: Pin>>, engine_tx: mpsc::UnboundedSender>, data_type: PhantomData T::Output>, pub(crate) dead_letter_policy: Option, @@ -43,7 +43,7 @@ pub struct TopicConsumer { pub(super) messages_received: u64, } -impl TopicConsumer { +impl TopicConsumer { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub(super) async fn new( client: Pulsar, @@ -132,11 +132,32 @@ impl TopicConsumer { return Err(Error::Executor); } + // Create the decoded channel (unified type for both schema and no-schema paths) + let (decoded_tx, decoded_rx) = + mpsc::channel::), Error>>( + receiver_queue_size as usize, + ); + + // Pass-through adapter: converts (id, payload) -> (id, payload, None) + let adapter_task = client.executor.spawn(Box::pin(async move { + let mut raw_rx = rx; + let mut decoded_tx = decoded_tx; + while let Some(result) = raw_rx.next().await { + let mapped = result.map(|(id, payload)| (id, payload, None)); + if decoded_tx.send(mapped).await.is_err() { + break; + } + } + })); + if adapter_task.is_err() { + return Err(Error::Executor); + } + Ok(TopicConsumer { consumer_id, config, topic, - messages: Box::pin(rx), + messages: Box::pin(decoded_rx), engine_tx, data_type: PhantomData, dead_letter_policy, @@ -295,7 +316,12 @@ impl TopicConsumer { } #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - fn create_message(&self, message_id: MessageIdData, payload: Payload) -> Message { + fn create_message( + &self, + message_id: MessageIdData, + payload: Payload, + decoded: Option, + ) -> Message { Message { topic: self.topic.clone(), message_id: MessageData { @@ -303,7 +329,7 @@ impl TopicConsumer { batch_size: payload.metadata.num_messages_in_batch, }, payload, - decoded: None, + decoded, } } @@ -318,7 +344,7 @@ impl TopicConsumer { } } -impl Stream for TopicConsumer { +impl Stream for TopicConsumer { type Item = Result, Error>; #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] @@ -326,10 +352,10 @@ impl Stream for TopicConsumer { match self.messages.as_mut().poll_next(cx) { Poll::Pending => Poll::Pending, Poll::Ready(None) => Poll::Ready(None), - Poll::Ready(Some(Ok((id, payload)))) => { + Poll::Ready(Some(Ok((id, payload, decoded)))) => { self.last_message_received = Some(Utc::now()); self.messages_received += 1; - Poll::Ready(Some(Ok(self.create_message(id, payload)))) + Poll::Ready(Some(Ok(self.create_message(id, payload, decoded)))) } Poll::Ready(Some(Err(e))) => { error!("we are using in the single-consumer and we got an error, {e}"); diff --git a/src/reader.rs b/src/reader.rs index dba1e4c2..465272c8 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -17,14 +17,14 @@ use crate::{ }; /// A client that acknowledges messages systematically -pub struct Reader { +pub struct Reader { pub(crate) consumer: TopicConsumer, pub(crate) state: Option>, } -impl Unpin for Reader {} +impl Unpin for Reader {} -pub enum State { +pub enum State { PollingConsumer, PollingAck( Message, @@ -32,7 +32,7 @@ pub enum State { ), } -impl Stream for Reader { +impl Stream for Reader { type Item = Result, Error>; #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] @@ -84,7 +84,7 @@ impl Stream for Reader { } } -impl Reader { +impl Reader { // this is totally useless as calling ConsumerBuilder::new(&pulsar_client) // does just the same /* From 2e9ef94e30f86fef4dad9bed6459c1888182c0af Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 21:02:14 -0700 Subject: [PATCH 10/24] feat: add with_schema() to ConsumerBuilder --- src/consumer/builder.rs | 68 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 7a94dd85..4c28ff40 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -1,5 +1,7 @@ use std::{ + any::Any, collections::{BTreeMap, VecDeque}, + sync::Arc, time::Duration, }; @@ -20,7 +22,6 @@ use crate::{ /// Builder structure for consumers /// /// This is the main way to create a [Consumer] or a [Reader] -#[derive(Clone)] pub struct ConsumerBuilder { pulsar: Pulsar, topics: Option>, @@ -35,6 +36,31 @@ pub struct ConsumerBuilder { consumer_options: Option, namespace: Option, topic_refresh: Option, + schema_info: Option, + schema_object: Option>, +} + +impl Clone for ConsumerBuilder { + fn clone(&self) -> Self { + ConsumerBuilder { + pulsar: self.pulsar.clone(), + topics: self.topics.clone(), + topic_regex: self.topic_regex.clone(), + subscription: self.subscription.clone(), + subscription_type: self.subscription_type, + consumer_id: self.consumer_id, + consumer_name: self.consumer_name.clone(), + batch_size: self.batch_size, + unacked_message_resend_delay: self.unacked_message_resend_delay, + consumer_options: self.consumer_options.clone(), + dead_letter_policy: self.dead_letter_policy.clone(), + namespace: self.namespace.clone(), + topic_refresh: self.topic_refresh, + schema_info: self.schema_info.clone(), + // schema_object is NOT cloned — only the original builder carries the schema. + schema_object: None, + } + } } impl ConsumerBuilder { @@ -56,6 +82,8 @@ impl ConsumerBuilder { consumer_options: None, namespace: None, topic_refresh: None, + schema_info: None, + schema_object: None, } } @@ -158,6 +186,18 @@ impl ConsumerBuilder { self } + /// Attach an external schema to this consumer. + /// The schema is type-erased internally; it is downcast back to + /// `PulsarSchema` when `build()` is called. + pub fn with_schema( + mut self, + schema: Arc>, + ) -> Self { + self.schema_info = Some(schema.schema_info()); + self.schema_object = Some(Box::new(schema)); + self + } + /// sets the dead letter policy #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn with_dead_letter_policy(mut self, dead_letter_policy: DeadLetterPolicy) -> Self { @@ -192,6 +232,8 @@ impl ConsumerBuilder { dead_letter_policy, namespace: _, topic_refresh: _, + schema_info: _, + schema_object: _, } = self; if consumer_name.is_none() { @@ -272,6 +314,19 @@ impl ConsumerBuilder { // would this clone() consume too much memory? let (config, joined_topics) = self.clone().validate().await?; + // Apply schema_info to options if set (takes precedence over .with_options()) + let mut config = config; + if let Some(ref info) = self.schema_info { + config.options.schema = Some(info.clone()); + } + + // Downcast schema_object: Box wrapping Arc> + let _schema: Option>> = + self.schema_object.as_ref().and_then(|obj| { + obj.downcast_ref::>>() + .cloned() + }); + let consumers = try_join_all(joined_topics.into_iter().map(|(topic, addr)| { TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone()) })) @@ -322,6 +377,17 @@ impl ConsumerBuilder { // would this clone() consume too much memory? let (mut config, mut joined_topics) = self.clone().validate().await?; + if let Some(ref info) = self.schema_info { + config.options.schema = Some(info.clone()); + } + + // Downcast schema (same pattern as build()) + let _schema: Option>> = + self.schema_object.as_ref().and_then(|obj| { + obj.downcast_ref::>>() + .cloned() + }); + // Internally, the reader interface is implemented as a consumer using an exclusive, // non-durable subscription config.options.durable = Some(false); From b98492065690e7e29b7841ce29ab4c2322f6a50f Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 21:06:28 -0700 Subject: [PATCH 11/24] feat: wire PulsarSchema decode task into TopicConsumer Co-Authored-By: Claude Sonnet 4.6 --- src/consumer/builder.rs | 9 ++++--- src/consumer/mod.rs | 4 +-- src/consumer/multi.rs | 2 +- src/consumer/topic.rs | 57 +++++++++++++++++++++++++++++++++-------- 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 4c28ff40..89dec18f 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -321,14 +321,15 @@ impl ConsumerBuilder { } // Downcast schema_object: Box wrapping Arc> - let _schema: Option>> = + let schema: Option>> = self.schema_object.as_ref().and_then(|obj| { obj.downcast_ref::>>() .cloned() }); let consumers = try_join_all(joined_topics.into_iter().map(|(topic, addr)| { - TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone()) + let schema = schema.clone(); + TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone(), schema) })) .await?; @@ -382,7 +383,7 @@ impl ConsumerBuilder { } // Downcast schema (same pattern as build()) - let _schema: Option>> = + let schema: Option>> = self.schema_object.as_ref().and_then(|obj| { obj.downcast_ref::>>() .cloned() @@ -404,7 +405,7 @@ impl ConsumerBuilder { } let (topic, addr) = joined_topics.pop().unwrap(); - let consumer = TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone()).await?; + let consumer = TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone(), schema).await?; Ok(Reader { consumer, diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 5e3849da..12681452 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -195,7 +195,7 @@ impl Consumer { let topic = c.topic().to_string(); let addr = client.lookup_topic(&topic).await?; let config = c.config().clone(); - InnerConsumer::Single(TopicConsumer::new(client, topic, addr, config).await?) + InnerConsumer::Single(TopicConsumer::new(client, topic, addr, config, None).await?) } InnerConsumer::Multi(c) => { c.seek(consumer_ids, message_id, timestamp).await?; @@ -210,7 +210,7 @@ impl Consumer { let topic_addr_pair = c.topics.iter().cloned().zip(addrs.iter().cloned()); let consumers = try_join_all(topic_addr_pair.map(|(topic, addr)| { - TopicConsumer::new(client.clone(), topic, addr, config.clone()) + TopicConsumer::new(client.clone(), topic, addr, config.clone(), None) })) .await?; diff --git a/src/consumer/multi.rs b/src/consumer/multi.rs index de3c59ef..1becdce6 100644 --- a/src/consumer/multi.rs +++ b/src/consumer/multi.rs @@ -190,7 +190,7 @@ impl MultiTopicConsumer TopicConsumer>>, ) -> Result, Error> { static CONSUMER_ID_GENERATOR: AtomicU64 = AtomicU64::new(0); @@ -138,19 +139,53 @@ impl TopicConsumer (id, payload, None) - let adapter_task = client.executor.spawn(Box::pin(async move { - let mut raw_rx = rx; - let mut decoded_tx = decoded_tx; - while let Some(result) = raw_rx.next().await { - let mapped = result.map(|(id, payload)| (id, payload, None)); - if decoded_tx.send(mapped).await.is_err() { - break; + if let Some(schema) = schema { + // Schema decode task: async decode via PulsarSchema + let decode_topic = topic.clone(); + let decode_task = client.executor.spawn(Box::pin(async move { + let mut raw_rx = rx; + let mut decoded_tx = decoded_tx; + while let Some(result) = raw_rx.next().await { + let mapped = match result { + Ok((id, payload)) => { + let schema_id_bytes = payload.metadata.schema_id.clone(); + match schema + .decode( + &decode_topic, + &payload, + schema_id_bytes.as_deref(), + ) + .await + { + Ok(decoded) => Ok((id, payload, Some(decoded))), + Err(e) => Err(e), + } + } + Err(e) => Err(e), + }; + if decoded_tx.send(mapped).await.is_err() { + break; + } } + })); + if decode_task.is_err() { + return Err(Error::Executor); + } + } else { + // No schema: pass-through adapter (id, payload) -> (id, payload, None) + let adapter_task = client.executor.spawn(Box::pin(async move { + let mut raw_rx = rx; + let mut decoded_tx = decoded_tx; + while let Some(result) = raw_rx.next().await { + let mapped = result.map(|(id, payload)| (id, payload, None)); + if decoded_tx.send(mapped).await.is_err() { + break; + } + } + })); + if adapter_task.is_err() { + return Err(Error::Executor); } - })); - if adapter_task.is_err() { - return Err(Error::Executor); } Ok(TopicConsumer { From db9a54dee670c71a52292aa281bdeabc8b462058 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 21:32:40 -0700 Subject: [PATCH 12/24] feat: add PulsarSchema support to Producer/ProducerBuilder with schema_id propagation - Add with_schema() to ProducerBuilder (type-erased via Box) - Add schema_object field to Producer for schema-aware encoding - send_schema_non_blocking() uses PulsarSchema::encode() before SerializeMessage - Add schema_id field to Message, ProducerMessage, and BatchItem - Propagate schema_id through batching pipeline to MessageMetadata - Wire schema_id into connection.rs MessageMetadata construction Co-Authored-By: Claude Opus 4.6 --- src/connection.rs | 1 + src/producer.rs | 129 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/src/connection.rs b/src/connection.rs index 5d9d40a3..00bdaa4d 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -1555,6 +1555,7 @@ pub(crate) mod messages { encryption_param: message.encryption_param, schema_version: message.schema_version, deliver_at_time: message.deliver_at_time, + schema_id: message.schema_id, ..Default::default() }, data: message.payload, diff --git a/src/producer.rs b/src/producer.rs index 7294ac99..537c26f0 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -1,5 +1,6 @@ //! Message publication use std::{ + any::Any, collections::{btree_map::Entry, BTreeMap, HashMap, VecDeque}, io::Write, pin::Pin, @@ -86,6 +87,9 @@ pub struct Message { /// UTC Unix timestamp in milliseconds, time at which the message should be /// delivered to consumers pub deliver_at_time: ::std::option::Option, + /// Schema ID from external schema registry (PIP-420). + /// Written to MessageMetadata.schema_id when present. + pub schema_id: Option>, } /// internal message type carrying options that must be defined @@ -118,6 +122,7 @@ pub(crate) struct ProducerMessage { /// UTC Unix timestamp in milliseconds, time at which the message should be /// delivered to consumers pub deliver_at_time: ::std::option::Option, + pub schema_id: Option>, } impl From for ProducerMessage { @@ -132,6 +137,7 @@ impl From for ProducerMessage { event_time: m.event_time, schema_version: m.schema_version, deliver_at_time: m.deliver_at_time, + schema_id: m.schema_id, ..Default::default() } } @@ -320,6 +326,10 @@ impl MultiTopicProducer { /// a producer for a single topic pub struct Producer { inner: ProducerInner, + /// Type-erased schema object (`Box` wrapping `Arc>`). + /// Cannot call schema.close() here due to type erasure — cleanup happens on Drop. + /// External schemas needing async cleanup should manage their own lifecycle. + schema_object: Option>, } impl Producer { @@ -434,6 +444,61 @@ impl Producer { } } + /// Sends a message using the attached PulsarSchema (PIP-420). + /// + /// When a schema has been registered via [`ProducerBuilder::with_schema`], + /// this method calls `PulsarSchema::encode` to obtain the payload and the + /// registry `schema_id` before sending. If the attached schema's type + /// parameter does not match `T`, this falls back to `SerializeMessage`. + /// + /// Use this method (instead of [`send_non_blocking`]) when you want + /// schema-aware encoding for messages produced with an external schema + /// registry. + /// + /// The `T: 'static` bound is required for Rust's `Any`-based runtime + /// type-checking inside the type-erased schema dispatch. + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + pub async fn send_schema_non_blocking( + &mut self, + message: T, + ) -> Result { + // Try PulsarSchema if available and the stored schema matches T. + if let Some(ref schema_obj) = self.schema_object { + if let Some(schema) = schema_obj + .downcast_ref::>>() + { + let topic = self.topic().to_string(); + let encode_data = schema.encode(&topic, message).await?; + + let serialized_message = Message { + payload: encode_data.payload, + schema_id: encode_data.schema_id, + ..Default::default() + }; + + return match &mut self.inner { + ProducerInner::Single(p) => p.send(serialized_message).await, + ProducerInner::Partitioned(p) => { + p.choose_partition(&serialized_message) + .send(serialized_message) + .await + } + }; + } + } + + // Fallback: use SerializeMessage (no schema attached or type mismatch). + let serialized_message = T::serialize_message(message)?; + match &mut self.inner { + ProducerInner::Single(p) => p.send(serialized_message).await, + ProducerInner::Partitioned(p) => { + p.choose_partition(&serialized_message) + .send(serialized_message) + .await + } + } + } + /// Sends a message /// /// this function is similar to send_non_blocking then waits the returned `SendFuture` @@ -763,7 +828,7 @@ impl TopicProducer { }, payload: message.payload, }; - let item = BatchItem::SingleMessage(tx, batched); + let item = BatchItem::SingleMessage(tx, batched, message.schema_id); msg_sender.send(item).await.map_err(|e| { Error::Producer(ProducerError::Custom(format!( "failed to send message to batch_process_loop: {e}" @@ -996,12 +1061,27 @@ impl std::ops::Drop for TopicProducer { /// Helper structure to prepare a producer /// /// generated from [Pulsar::producer] -#[derive(Clone)] pub struct ProducerBuilder { pulsar: Pulsar, topic: Option, name: Option, producer_options: Option, + schema_info: Option, + schema_object: Option>, +} + +impl Clone for ProducerBuilder { + fn clone(&self) -> Self { + ProducerBuilder { + pulsar: self.pulsar.clone(), + topic: self.topic.clone(), + name: self.name.clone(), + producer_options: self.producer_options.clone(), + schema_info: self.schema_info.clone(), + // schema_object is NOT cloned — only the original builder carries the schema. + schema_object: None, + } + } } impl ProducerBuilder { @@ -1013,6 +1093,8 @@ impl ProducerBuilder { topic: None, name: None, producer_options: None, + schema_info: None, + schema_object: None, } } @@ -1037,6 +1119,18 @@ impl ProducerBuilder { self } + /// Attach an external schema to this producer. + /// The schema is type-erased internally; it is downcast back to + /// `PulsarSchema` when `send_non_blocking()` is called. + pub fn with_schema( + mut self, + schema: Arc>, + ) -> Self { + self.schema_info = Some(schema.schema_info()); + self.schema_object = Some(Box::new(schema)); + self + } + /// creates a new producer #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn build(self) -> Result, Error> { @@ -1045,9 +1139,14 @@ impl ProducerBuilder { topic, name, producer_options, + schema_info, + schema_object, } = self; let topic = topic.ok_or_else(|| Error::Custom("topic not set".to_string()))?; - let options = producer_options.unwrap_or_default(); + let mut options = producer_options.unwrap_or_default(); + if let Some(info) = schema_info { + options.schema = Some(info); + } let mut producers: Vec> = try_join_all( pulsar @@ -1085,7 +1184,10 @@ impl ProducerBuilder { }), }; - Ok(Producer { inner: producer }) + Ok(Producer { + inner: producer, + schema_object, + }) } /// creates a new [MultiTopicProducer] @@ -1111,7 +1213,7 @@ struct BatchStorage { impl BatchStorage { #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn push_back(&mut self, item: BatchItem) { - if let BatchItem::SingleMessage(_, batched_msg) = &item { + if let BatchItem::SingleMessage(_, batched_msg, _) = &item { self.size += batched_msg.metadata.payload_size as usize; } self.storage.push_back(item); @@ -1143,6 +1245,7 @@ enum BatchItem { SingleMessage( oneshot::Sender>, BatchedMessage, + Option>, // schema_id for PIP-420 ), Flush(oneshot::Sender<()>), } @@ -1196,7 +1299,7 @@ async fn batch_process_loop( warn!("producer {}'s batch_process_loop exits when there are {} messages not flushed", producer_id, count); for item in batch_storage.get_messages() { - if let BatchItem::SingleMessage(tx, _) = item { + if let BatchItem::SingleMessage(tx, _, _) = item { let _ = tx.send(Err(Error::Producer(ProducerError::Closed))); } } @@ -1256,8 +1359,18 @@ async fn message_send_loop( } }; let counter = batch_items.len(); + + // Extract schema_id from first message (PIP-420: all messages in batch share same schema_id) + let batch_schema_id = batch_items.iter().find_map(|item| { + if let BatchItem::SingleMessage(_, _, schema_id) = item { + schema_id.clone() + } else { + None + } + }); + for item in batch_items { - if let BatchItem::SingleMessage(tx, batched_msg) = item { + if let BatchItem::SingleMessage(tx, batched_msg, _) = item { receipts.push(tx); batched_msg.serialize(&mut payload); } else { @@ -1278,6 +1391,7 @@ async fn message_send_loop( payload, num_messages_in_batch: Some(counter as i32), schema_version: schema_version.clone(), + schema_id: batch_schema_id, ..Default::default() }; @@ -1523,6 +1637,7 @@ mod tests { event_time: Some(42), schema_version: Some(vec![9, 9]), deliver_at_time: Some(123456789), + schema_id: None, }; let pm: ProducerMessage = m.clone().into(); From 61de0c3ab29bd406b71e74605ce8fd12ce8eb5ba Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 21:47:05 -0700 Subject: [PATCH 13/24] feat: add pulsar-schema-confluent as workspace member Move the Confluent Schema Registry integration crate into the pulsar-rs repo as a Cargo workspace member. This keeps the reference implementation co-located with the core library for easier development and CI. Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 3 + pulsar-schema-confluent/Cargo.toml | 18 ++ pulsar-schema-confluent/src/lib.rs | 305 +++++++++++++++++++++++++++++ 3 files changed, 326 insertions(+) create mode 100644 pulsar-schema-confluent/Cargo.toml create mode 100644 pulsar-schema-confluent/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index e470dc20..4865f3b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,6 @@ +[workspace] +members = ["pulsar-schema-confluent"] + [package] name = "pulsar" version = "6.4.1" diff --git a/pulsar-schema-confluent/Cargo.toml b/pulsar-schema-confluent/Cargo.toml new file mode 100644 index 00000000..c1b4cbd8 --- /dev/null +++ b/pulsar-schema-confluent/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pulsar-schema-confluent" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +description = "Confluent Schema Registry integration for pulsar-rs" +repository = "https://github.com/streamnative/pulsar-rs" + +[dependencies] +pulsar = { path = "..", default-features = false, features = ["tokio-runtime"] } +async-trait = "0.1" +schema_registry_converter = { version = "4.8", features = ["json"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +log = "0.4" + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/pulsar-schema-confluent/src/lib.rs b/pulsar-schema-confluent/src/lib.rs new file mode 100644 index 00000000..484d7b97 --- /dev/null +++ b/pulsar-schema-confluent/src/lib.rs @@ -0,0 +1,305 @@ +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Serialize}; + +use pulsar::{ + message::proto, + schema::{EncodeData, PulsarSchema}, + Error, Payload, +}; + +use schema_registry_converter::async_impl::schema_registry::{SrSettings, SrSettingsBuilder}; + +/// Authentication options for the Confluent Schema Registry. +pub enum ConfluentAuth { + Basic { username: String, password: String }, + Bearer { token: String }, +} + +/// Controls how the subject name is derived for a schema in the registry. +pub enum SubjectNameStrategy { + /// `-key` or `-value` + TopicName, + /// The fully-qualified Rust type name + RecordName, + /// `-` + TopicRecordName, +} + +/// Configuration for connecting to a Confluent Schema Registry. +pub struct ConfluentConfig { + pub registry_url: String, + pub auth: Option, + pub subject_name_strategy: SubjectNameStrategy, + pub timeout: Option, +} + +/// A [`PulsarSchema`] implementation backed by the Confluent Schema Registry. +/// +/// Encodes and decodes messages as JSON. Schema registration is a placeholder +/// for an initial release; `schema_id` will be `None` until a live registry is +/// wired up in a future iteration. +pub struct ConfluentJsonSchema { + /// Holds the connection settings for a future live-registry integration. + #[allow(dead_code)] + sr_settings: SrSettings, + subject_strategy: SubjectNameStrategy, + is_key: bool, + _phantom: PhantomData, +} + +impl ConfluentJsonSchema { + pub fn new(config: ConfluentConfig, is_key: bool) -> Self { + let sr_settings = if config.auth.is_some() || config.timeout.is_some() { + let mut builder: SrSettingsBuilder = + SrSettings::new_builder(config.registry_url); + if let Some(auth) = config.auth { + match auth { + ConfluentAuth::Basic { username, password } => { + builder.set_basic_authorization(&username, Some(&password)); + } + ConfluentAuth::Bearer { token } => { + builder.set_token_authorization(&token); + } + } + } + if let Some(timeout) = config.timeout { + builder.set_timeout(timeout); + } + builder.build().expect("Failed to build SrSettings") + } else { + SrSettings::new(config.registry_url) + }; + + Self { + sr_settings, + subject_strategy: config.subject_name_strategy, + is_key, + _phantom: PhantomData, + } + } + + fn subject_name(&self, topic: &str) -> String { + let suffix = if self.is_key { "key" } else { "value" }; + match &self.subject_strategy { + SubjectNameStrategy::TopicName => format!("{topic}-{suffix}"), + SubjectNameStrategy::RecordName => std::any::type_name::().to_string(), + SubjectNameStrategy::TopicRecordName => { + let type_name = std::any::type_name::(); + format!("{topic}-{type_name}") + } + } + } +} + +#[async_trait] +impl PulsarSchema for ConfluentJsonSchema +where + T: Serialize + DeserializeOwned + Send + Sync + 'static, +{ + fn schema_info(&self) -> proto::Schema { + proto::Schema { + r#type: 22, // External = 22 + ..Default::default() + } + } + + async fn encode(&self, topic: &str, message: T) -> Result { + let json_bytes = serde_json::to_vec(&message) + .map_err(|e| Error::SchemaRegistry(e.to_string()))?; + + // Schema registration and ID retrieval is a placeholder for now. + // A future iteration will wire the schema_registry_converter encode API + // to register schemas and extract the Confluent schema ID, then re-frame + // it with Pulsar's 0xFF magic byte via schema_id_util::add_magic_header(). + let _subject = self.subject_name(topic); + log::warn!( + "Confluent encode: schema registration not yet wired — schema_id will be None" + ); + + Ok(EncodeData { + payload: json_bytes, + schema_id: None, + }) + } + + async fn decode( + &self, + _topic: &str, + payload: &Payload, + schema_id: Option<&[u8]>, + ) -> Result { + // Deliberate simplification for initial release: + // JSON is forward-compatible, so direct deserialization works without + // fetching the writer's schema from the registry. + if let Some(id) = schema_id { + log::debug!( + "Confluent decode: received schema_id ({} bytes), using direct JSON deser", + id.len() + ); + } + serde_json::from_slice(&payload.data) + .map_err(|e| Error::SchemaRegistry(e.to_string())) + } +} + +/// Convenience factory for building Confluent-backed schemas. +pub struct ConfluentSchemaFactory; + +impl ConfluentSchemaFactory { + pub fn json( + config: ConfluentConfig, + ) -> ConfluentJsonSchema { + ConfluentJsonSchema::new(config, false) + } + + pub fn kv( + key_schema: ConfluentJsonSchema, + value_schema: ConfluentJsonSchema, + ) -> pulsar::schema::KeyValueSchema + where + K: Serialize + DeserializeOwned + Send + Sync + 'static, + V: Serialize + DeserializeOwned + Send + Sync + 'static, + { + pulsar::schema::KeyValueSchema::new( + Arc::new(key_schema) as Arc>, + Arc::new(value_schema) as Arc>, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] + struct TestEvent { + name: String, + value: i32, + } + + #[test] + fn test_subject_name_topic_strategy() { + let schema = ConfluentJsonSchema::::new( + ConfluentConfig { + registry_url: "http://localhost:8081".to_string(), + auth: None, + subject_name_strategy: SubjectNameStrategy::TopicName, + timeout: None, + }, + false, + ); + assert_eq!(schema.subject_name("my-topic"), "my-topic-value"); + } + + #[test] + fn test_subject_name_topic_strategy_key() { + let schema = ConfluentJsonSchema::::new( + ConfluentConfig { + registry_url: "http://localhost:8081".to_string(), + auth: None, + subject_name_strategy: SubjectNameStrategy::TopicName, + timeout: None, + }, + true, + ); + assert_eq!(schema.subject_name("my-topic"), "my-topic-key"); + } + + #[test] + fn test_schema_info_returns_external() { + let schema = ConfluentJsonSchema::::new( + ConfluentConfig { + registry_url: "http://localhost:8081".to_string(), + auth: None, + subject_name_strategy: SubjectNameStrategy::TopicName, + timeout: None, + }, + false, + ); + let info = schema.schema_info(); + assert_eq!(info.r#type, 22); + } + + #[test] + fn test_factory_json() { + let _schema = ConfluentSchemaFactory::json::(ConfluentConfig { + registry_url: "http://localhost:8081".to_string(), + auth: None, + subject_name_strategy: SubjectNameStrategy::TopicName, + timeout: None, + }); + } + + #[tokio::test] + async fn test_encode_decode_round_trip() { + let schema = ConfluentJsonSchema::::new( + ConfluentConfig { + registry_url: "http://localhost:8081".to_string(), + auth: None, + subject_name_strategy: SubjectNameStrategy::TopicName, + timeout: None, + }, + false, + ); + + let original = TestEvent { + name: "round-trip".to_string(), + value: 99, + }; + + let encode_data = schema + .encode("test-topic", original.clone()) + .await + .expect("encode failed"); + + let payload = Payload { + data: encode_data.payload, + metadata: Default::default(), + }; + + let decoded: TestEvent = schema + .decode("test-topic", &payload, encode_data.schema_id.as_deref()) + .await + .expect("decode failed"); + + assert_eq!( + decoded, original, + "round-trip encode/decode must preserve data" + ); + } + + #[ignore] // Requires a live Confluent Schema Registry + #[tokio::test] + async fn test_encode_returns_schema_id() { + let schema = ConfluentJsonSchema::::new( + ConfluentConfig { + registry_url: std::env::var("SCHEMA_REGISTRY_URL") + .unwrap_or_else(|_| "http://localhost:8081".to_string()), + auth: None, + subject_name_strategy: SubjectNameStrategy::TopicName, + timeout: None, + }, + false, + ); + let event = TestEvent { + name: "test".to_string(), + value: 42, + }; + let encode_data = schema + .encode("test-topic", event) + .await + .expect("encode failed"); + + assert!( + encode_data.schema_id.is_some(), + "encode() must return a non-None schema_id for external schema registry to work" + ); + let id = encode_data.schema_id.unwrap(); + assert!(id.len() >= 2, "schema_id too short"); + assert_eq!(id[0], 0xFF, "schema_id must start with Pulsar magic byte 0xFF"); + } +} From 79c289ecaaac09c60e04d45bf4e03ea9537a7a85 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 22:14:49 -0700 Subject: [PATCH 14/24] fix: address PR review critical issues (C1-C5) C1: Propagate schema through Consumer/MultiTopicConsumer for reconnect and seek. Store Arc> in both structs and pass to all TopicConsumer::new() calls. C2: Return explicit error on schema downcast type mismatch in ConsumerBuilder::build(), into_reader(), and Producer::send_schema_non_blocking() instead of silently falling through. C3: ConfluentJsonSchema::new() now returns Result instead of panicking on SrSettings build failure. C4: Schema decode errors now forward (id, payload, None) with a warning log instead of discarding the MessageIdData, allowing consumers to ack/nack failed messages. C5: send_schema_non_blocking() accepts optional Message metadata to preserve properties, partition_key, and event_time when using external schemas. Co-Authored-By: Claude Opus 4.6 --- pulsar-schema-confluent/src/lib.rs | 30 ++++++++++++++-------- src/consumer/builder.rs | 41 +++++++++++++++++++++++------- src/consumer/mod.rs | 16 ++++++++++-- src/consumer/multi.rs | 11 +++++++- src/consumer/topic.rs | 9 ++++++- src/producer.rs | 25 +++++++++++++----- 6 files changed, 102 insertions(+), 30 deletions(-) diff --git a/pulsar-schema-confluent/src/lib.rs b/pulsar-schema-confluent/src/lib.rs index 484d7b97..3094b420 100644 --- a/pulsar-schema-confluent/src/lib.rs +++ b/pulsar-schema-confluent/src/lib.rs @@ -52,7 +52,7 @@ pub struct ConfluentJsonSchema { } impl ConfluentJsonSchema { - pub fn new(config: ConfluentConfig, is_key: bool) -> Self { + pub fn new(config: ConfluentConfig, is_key: bool) -> Result { let sr_settings = if config.auth.is_some() || config.timeout.is_some() { let mut builder: SrSettingsBuilder = SrSettings::new_builder(config.registry_url); @@ -69,17 +69,19 @@ impl ConfluentJsonSchema { if let Some(timeout) = config.timeout { builder.set_timeout(timeout); } - builder.build().expect("Failed to build SrSettings") + builder.build().map_err(|e| { + Error::SchemaRegistry(format!("Failed to build SrSettings: {}", e)) + })? } else { SrSettings::new(config.registry_url) }; - Self { + Ok(Self { sr_settings, subject_strategy: config.subject_name_strategy, is_key, _phantom: PhantomData, - } + }) } fn subject_name(&self, topic: &str) -> String { @@ -152,7 +154,7 @@ pub struct ConfluentSchemaFactory; impl ConfluentSchemaFactory { pub fn json( config: ConfluentConfig, - ) -> ConfluentJsonSchema { + ) -> Result, Error> { ConfluentJsonSchema::new(config, false) } @@ -191,7 +193,8 @@ mod tests { timeout: None, }, false, - ); + ) + .expect("schema creation failed"); assert_eq!(schema.subject_name("my-topic"), "my-topic-value"); } @@ -205,7 +208,8 @@ mod tests { timeout: None, }, true, - ); + ) + .expect("schema creation failed"); assert_eq!(schema.subject_name("my-topic"), "my-topic-key"); } @@ -219,7 +223,8 @@ mod tests { timeout: None, }, false, - ); + ) + .expect("schema creation failed"); let info = schema.schema_info(); assert_eq!(info.r#type, 22); } @@ -231,7 +236,8 @@ mod tests { auth: None, subject_name_strategy: SubjectNameStrategy::TopicName, timeout: None, - }); + }) + .expect("factory json failed"); } #[tokio::test] @@ -244,7 +250,8 @@ mod tests { timeout: None, }, false, - ); + ) + .expect("schema creation failed"); let original = TestEvent { name: "round-trip".to_string(), @@ -284,7 +291,8 @@ mod tests { timeout: None, }, false, - ); + ) + .expect("schema creation failed"); let event = TestEvent { name: "test".to_string(), value: 42, diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 89dec18f..dd7110f7 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -322,10 +322,21 @@ impl ConsumerBuilder { // Downcast schema_object: Box wrapping Arc> let schema: Option>> = - self.schema_object.as_ref().and_then(|obj| { - obj.downcast_ref::>>() - .cloned() - }); + if let Some(ref obj) = self.schema_object { + Some( + obj.downcast_ref::>>() + .cloned() + .ok_or_else(|| { + Error::Custom(format!( + "Schema type mismatch: with_schema() was called with a different \ + type parameter than build::<{}>()", + std::any::type_name::() + )) + })?, + ) + } else { + None + }; let consumers = try_join_all(joined_topics.into_iter().map(|(topic, addr)| { let schema = schema.clone(); @@ -359,6 +370,7 @@ impl ConsumerBuilder { new_consumers: None, refresh, config, + schema: schema.clone(), disc_last_message_received: None, disc_messages_received: 0, }; @@ -369,7 +381,7 @@ impl ConsumerBuilder { } InnerConsumer::Multi(consumer) }; - Ok(Consumer { inner: consumer }) + Ok(Consumer { inner: consumer, schema }) } /// creates a [Reader] from this builder @@ -384,10 +396,21 @@ impl ConsumerBuilder { // Downcast schema (same pattern as build()) let schema: Option>> = - self.schema_object.as_ref().and_then(|obj| { - obj.downcast_ref::>>() - .cloned() - }); + if let Some(ref obj) = self.schema_object { + Some( + obj.downcast_ref::>>() + .cloned() + .ok_or_else(|| { + Error::Custom(format!( + "Schema type mismatch: with_schema() was called with a different \ + type parameter than into_reader::<{}>()", + std::any::type_name::() + )) + })?, + ) + } else { + None + }; // Internally, the reader interface is implemented as a consumer using an exclusive, // non-durable subscription diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 12681452..4fb20d25 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -81,6 +81,8 @@ enum InnerConsumer { /// ``` pub struct Consumer { inner: InnerConsumer, + /// Retained schema for re-creating TopicConsumers on seek/reconnect. + schema: Option>>, } impl Consumer { @@ -195,7 +197,9 @@ impl Consumer { let topic = c.topic().to_string(); let addr = client.lookup_topic(&topic).await?; let config = c.config().clone(); - InnerConsumer::Single(TopicConsumer::new(client, topic, addr, config, None).await?) + InnerConsumer::Single( + TopicConsumer::new(client, topic, addr, config, self.schema.clone()).await?, + ) } InnerConsumer::Multi(c) => { c.seek(consumer_ids, message_id, timestamp).await?; @@ -207,10 +211,17 @@ impl Consumer { try_join_all(topics.into_iter().map(|topic| client.lookup_topic(topic))) .await?; + let schema = self.schema.clone(); let topic_addr_pair = c.topics.iter().cloned().zip(addrs.iter().cloned()); let consumers = try_join_all(topic_addr_pair.map(|(topic, addr)| { - TopicConsumer::new(client.clone(), topic, addr, config.clone(), None) + TopicConsumer::new( + client.clone(), + topic, + addr, + config.clone(), + schema.clone(), + ) })) .await?; @@ -235,6 +246,7 @@ impl Consumer { new_consumers: None, refresh, config, + schema: self.schema.clone(), disc_last_message_received: None, disc_messages_received: 0, }) diff --git a/src/consumer/multi.rs b/src/consumer/multi.rs index 1becdce6..5a97b642 100644 --- a/src/consumer/multi.rs +++ b/src/consumer/multi.rs @@ -35,6 +35,8 @@ pub struct MultiTopicConsumer { >, pub(super) refresh: Pin + Send + Sync>>, pub(super) config: ConsumerConfig, + /// Retained schema for re-creating TopicConsumers on topic refresh. + pub(super) schema: Option>>, // Stats on disconnected consumers to keep metrics correct pub(super) disc_messages_received: u64, pub(super) disc_last_message_received: Option>, @@ -148,6 +150,7 @@ impl MultiTopicConsumer MultiTopicConsumer TopicConsumer Ok((id, payload, Some(decoded))), - Err(e) => Err(e), + Err(e) => { + log::warn!( + "schema decode failed for message {:?} on topic {}: {}. \ + Forwarding with decoded=None so it can be acked/nacked.", + id, decode_topic, e + ); + Ok((id, payload, None)) + } } } Err(e) => Err(e), diff --git a/src/producer.rs b/src/producer.rs index 537c26f0..9f240794 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -449,7 +449,7 @@ impl Producer { /// When a schema has been registered via [`ProducerBuilder::with_schema`], /// this method calls `PulsarSchema::encode` to obtain the payload and the /// registry `schema_id` before sending. If the attached schema's type - /// parameter does not match `T`, this falls back to `SerializeMessage`. + /// parameter does not match `T`, an error is returned. /// /// Use this method (instead of [`send_non_blocking`]) when you want /// schema-aware encoding for messages produced with an external schema @@ -461,6 +461,7 @@ impl Producer { pub async fn send_schema_non_blocking( &mut self, message: T, + message_metadata: Option, ) -> Result { // Try PulsarSchema if available and the stored schema matches T. if let Some(ref schema_obj) = self.schema_object { @@ -470,10 +471,16 @@ impl Producer { let topic = self.topic().to_string(); let encode_data = schema.encode(&topic, message).await?; - let serialized_message = Message { - payload: encode_data.payload, - schema_id: encode_data.schema_id, - ..Default::default() + let serialized_message = if let Some(mut msg) = message_metadata { + msg.payload = encode_data.payload; + msg.schema_id = encode_data.schema_id; + msg + } else { + Message { + payload: encode_data.payload, + schema_id: encode_data.schema_id, + ..Default::default() + } }; return match &mut self.inner { @@ -485,9 +492,15 @@ impl Producer { } }; } + // Schema was attached but downcast failed — type mismatch error. + return Err(Error::Custom(format!( + "Schema type mismatch: ProducerBuilder::with_schema() was called with a different \ + type parameter than send_schema_non_blocking::<{}>()", + std::any::type_name::() + ))); } - // Fallback: use SerializeMessage (no schema attached or type mismatch). + // No schema attached: use SerializeMessage. let serialized_message = T::serialize_message(message)?; match &mut self.inner { ProducerInner::Single(p) => p.send(serialized_message).await, From 79d3b3aa8c543421f0770dc70d4311dea7593b64 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 22:28:22 -0700 Subject: [PATCH 15/24] fix: address PR review important issues (I1-I5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I1: Add debug_assert! guards in ConsumerBuilder build()/into_reader() to catch accidental use of cloned builders (schema_object lost). I2: Replace pass-through adapter task with MessageReceiver enum that maps raw messages inline in poll_next — no extra task or channel hop when no schema is attached. I3: Validate batch schema_id consistency — reject entire batch when messages have conflicting schema_ids (PIP-420 requirement). I4: Mark producer::Message as #[non_exhaustive] so adding schema_id field is not a semver-breaking change. Add Message::new() constructor. I5: Change strip_magic_header() return type to Result> so corrupt data returns Err instead of being indistinguishable from absent framing. Add corruption test cases. Co-Authored-By: Claude Opus 4.6 --- examples/batching.rs | 5 +-- examples/producer.rs | 5 +-- examples/round_trip.rs | 5 +-- src/consumer/builder.rs | 16 ++++++++-- src/consumer/data.rs | 40 ++++++++++++++++++++++-- src/consumer/topic.rs | 42 ++++++++++--------------- src/lib.rs | 5 +-- src/producer.rs | 60 +++++++++++++++++++++++++++++++----- src/schema/key_value.rs | 2 +- src/schema/schema_id_util.rs | 49 ++++++++++++++++++++++------- 10 files changed, 162 insertions(+), 67 deletions(-) diff --git a/examples/batching.rs b/examples/batching.rs index 3aa5d9ff..22e89409 100644 --- a/examples/batching.rs +++ b/examples/batching.rs @@ -18,10 +18,7 @@ struct TestData { impl SerializeMessage for TestData { fn serialize_message(input: Self) -> Result { let payload = serde_json::to_vec(&input).map_err(|e| PulsarError::Custom(e.to_string()))?; - Ok(producer::Message { - payload, - ..Default::default() - }) + Ok(producer::Message::new(payload)) } } diff --git a/examples/producer.rs b/examples/producer.rs index b4ec41dd..30036977 100644 --- a/examples/producer.rs +++ b/examples/producer.rs @@ -16,10 +16,7 @@ struct TestData { impl SerializeMessage for TestData { fn serialize_message(input: Self) -> Result { let payload = serde_json::to_vec(&input).map_err(|e| PulsarError::Custom(e.to_string()))?; - Ok(producer::Message { - payload, - ..Default::default() - }) + Ok(producer::Message::new(payload)) } } diff --git a/examples/round_trip.rs b/examples/round_trip.rs index f8c17aaa..9867653f 100644 --- a/examples/round_trip.rs +++ b/examples/round_trip.rs @@ -16,10 +16,7 @@ struct TestData { impl SerializeMessage for TestData { fn serialize_message(input: Self) -> Result { let payload = serde_json::to_vec(&input).map_err(|e| PulsarError::Custom(e.to_string()))?; - Ok(producer::Message { - payload, - ..Default::default() - }) + Ok(producer::Message::new(payload)) } } diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index dd7110f7..f7ec84e1 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -57,7 +57,9 @@ impl Clone for ConsumerBuilder { namespace: self.namespace.clone(), topic_refresh: self.topic_refresh, schema_info: self.schema_info.clone(), - // schema_object is NOT cloned — only the original builder carries the schema. + // schema_object cannot be cloned (Box). The original builder + // retains it; clones are only valid for validate(). build() and + // into_reader() assert the original's schema_object is intact. schema_object: None, } } @@ -311,7 +313,12 @@ impl ConsumerBuilder { /// creates a [Consumer] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn build(self) -> Result, Error> { - // would this clone() consume too much memory? + // clone() is only used for validate(); schema_object is NOT cloneable. + // Guard: if schema_info is set, schema_object must still be present on `self`. + debug_assert!( + self.schema_info.is_none() || self.schema_object.is_some(), + "BUG: schema_info present but schema_object lost (was build() called on a clone?)" + ); let (config, joined_topics) = self.clone().validate().await?; // Apply schema_info to options if set (takes precedence over .with_options()) @@ -387,7 +394,10 @@ impl ConsumerBuilder { /// creates a [Reader] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn into_reader(self) -> Result, Error> { - // would this clone() consume too much memory? + debug_assert!( + self.schema_info.is_none() || self.schema_object.is_some(), + "BUG: schema_info present but schema_object lost (was into_reader() called on a clone?)" + ); let (mut config, mut joined_topics) = self.clone().validate().await?; if let Some(ref info) = self.schema_info { diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 4dd1a28f..52613b6c 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -1,6 +1,13 @@ -use std::sync::Arc; +use std::{ + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; -use futures::channel::{mpsc, oneshot}; +use futures::{ + channel::{mpsc, oneshot}, + Stream, +}; use crate::{ connection::Connection, @@ -13,6 +20,35 @@ pub type MessageIdDataReceiver = mpsc::Receiver = mpsc::Receiver), Error>>; +/// Unified receiver that avoids spawning a pass-through task when no schema is +/// attached. When polling, `Raw` maps `(id, payload)` → `(id, payload, None)` +/// inline without an extra async task or channel hop. +pub enum MessageReceiver { + /// No schema: poll the raw engine receiver directly. + Raw(MessageIdDataReceiver), + /// Schema attached: poll the decoded receiver (async decode task fills it). + Decoded(DecodedMessageReceiver), +} + +impl Stream for MessageReceiver { + type Item = Result<(MessageIdData, Payload, Option), Error>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // SAFETY: we only access the inner receivers which are Unpin (mpsc::Receiver). + let this = self.get_mut(); + match this { + MessageReceiver::Raw(rx) => match Pin::new(rx).poll_next(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(None) => Poll::Ready(None), + Poll::Ready(Some(result)) => { + Poll::Ready(Some(result.map(|(id, payload)| (id, payload, None)))) + } + }, + MessageReceiver::Decoded(rx) => Pin::new(rx).poll_next(cx), + } + } +} + pub enum EngineEvent { Message(Option), EngineMessage(Option>), diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 305c8cf6..88625ad3 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -19,7 +19,7 @@ use crate::{ connection::Connection, consumer::{ config::ConsumerConfig, - data::{DeadLetterPolicy, DecodedMessageReceiver, EngineMessage, MessageData}, + data::{DeadLetterPolicy, EngineMessage, MessageData, MessageReceiver}, engine::ConsumerEngine, message::Message, }, @@ -35,7 +35,7 @@ pub struct TopicConsumer { pub(crate) consumer_id: u64, pub(crate) config: ConsumerConfig, topic: String, - messages: Pin>>, + messages: Pin>>, engine_tx: mpsc::UnboundedSender>, data_type: PhantomData T::Output>, pub(crate) dead_letter_policy: Option, @@ -133,14 +133,15 @@ impl TopicConsumer), Error>>( - receiver_queue_size as usize, - ); - - if let Some(schema) = schema { - // Schema decode task: async decode via PulsarSchema + // Build the message receiver. + // When a schema is attached, spawn an async decode task that writes to a + // decoded channel. Otherwise, wrap the raw receiver directly — no extra + // task or channel hop needed (I2 fix). + let messages: MessageReceiver = if let Some(schema) = schema { + let (decoded_tx, decoded_rx) = + mpsc::channel::), Error>>( + receiver_queue_size as usize, + ); let decode_topic = topic.clone(); let decode_task = client.executor.spawn(Box::pin(async move { let mut raw_rx = rx; @@ -178,28 +179,17 @@ impl TopicConsumer (id, payload, None) - let adapter_task = client.executor.spawn(Box::pin(async move { - let mut raw_rx = rx; - let mut decoded_tx = decoded_tx; - while let Some(result) = raw_rx.next().await { - let mapped = result.map(|(id, payload)| (id, payload, None)); - if decoded_tx.send(mapped).await.is_err() { - break; - } - } - })); - if adapter_task.is_err() { - return Err(Error::Executor); - } - } + // No schema: use raw receiver directly, mapped inline in poll_next. + MessageReceiver::Raw(rx) + }; Ok(TopicConsumer { consumer_id, config, topic, - messages: Box::pin(decoded_rx), + messages: Box::pin(messages), engine_tx, data_type: PhantomData, dead_letter_policy, diff --git a/src/lib.rs b/src/lib.rs index e15ac1d9..30b9e975 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,10 +44,7 @@ //! fn serialize_message(input: Self) -> Result { //! let payload = //! serde_json::to_vec(&input).map_err(|e| PulsarError::Custom(e.to_string()))?; -//! Ok(producer::Message { -//! payload, -//! ..Default::default() -//! }) +//! Ok(producer::Message::new(payload)) //! } //! } //! diff --git a/src/producer.rs b/src/producer.rs index 9f240794..86ea5052 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -68,6 +68,7 @@ impl Future for SendFuture { /// this is actually a subset of the fields of a message, because batching, /// compression and encryption should be handled by the producer #[derive(Debug, Clone, Default)] +#[non_exhaustive] pub struct Message { /// serialized data pub payload: Vec, @@ -92,6 +93,17 @@ pub struct Message { pub schema_id: Option>, } +impl Message { + /// Create a `Message` carrying the given payload with all other fields + /// set to their defaults. + pub fn new(payload: Vec) -> Self { + Self { + payload, + ..Default::default() + } + } +} + /// internal message type carrying options that must be defined /// by the producer #[derive(Debug, Clone, Default)] @@ -1091,7 +1103,8 @@ impl Clone for ProducerBuilder { name: self.name.clone(), producer_options: self.producer_options.clone(), schema_info: self.schema_info.clone(), - // schema_object is NOT cloned — only the original builder carries the schema. + // schema_object cannot be cloned (Box). The original builder + // retains it; clones are only valid for non-schema paths. schema_object: None, } } @@ -1373,14 +1386,45 @@ async fn message_send_loop( }; let counter = batch_items.len(); - // Extract schema_id from first message (PIP-420: all messages in batch share same schema_id) - let batch_schema_id = batch_items.iter().find_map(|item| { - if let BatchItem::SingleMessage(_, _, schema_id) = item { - schema_id.clone() - } else { - None + // Extract and validate schema_id across the batch (PIP-420). + // All messages must share the same schema_id; mixed batches are rejected. + let batch_schema_id = { + let mut first_id: Option>> = None; + let mut conflict = false; + for item in batch_items.iter() { + if let BatchItem::SingleMessage(_, _, schema_id) = item { + match &first_id { + None => first_id = Some(schema_id.clone()), + Some(prev) if prev != schema_id => { + conflict = true; + break; + } + _ => {} + } + } } - }); + if conflict { + error!( + "producer {}: batch contains messages with different schema_ids — \ + rejecting entire batch", + producer_id + ); + // Fail all messages in the batch + for item in batch_items { + if let BatchItem::SingleMessage(tx, _, _) = item { + let _ = tx.send(Err(Error::Custom( + "Batch rejected: messages have conflicting schema_ids" + .to_string(), + ))); + } + } + if let Some(flush_tx) = flush_tx { + let _ = flush_tx.send(()); + } + continue; + } + first_id.flatten() + }; for item in batch_items { if let BatchItem::SingleMessage(tx, batched_msg, _) = item { diff --git a/src/schema/key_value.rs b/src/schema/key_value.rs index 17680366..52e5d6e4 100644 --- a/src/schema/key_value.rs +++ b/src/schema/key_value.rs @@ -102,7 +102,7 @@ where schema_id: Option<&[u8]>, ) -> Result<(K, V), Error> { let (key_id, value_id) = match schema_id { - Some(data) => match schema_id_util::strip_magic_header(data) { + Some(data) => match schema_id_util::strip_magic_header(data)? { Some(SchemaIdInfo::KeyValue { key_id, value_id }) => { (Some(key_id), Some(value_id)) } diff --git a/src/schema/schema_id_util.rs b/src/schema/schema_id_util.rs index 38df3279..e79f0911 100644 --- a/src/schema/schema_id_util.rs +++ b/src/schema/schema_id_util.rs @@ -17,26 +17,39 @@ pub fn add_magic_header(schema_id: &[u8]) -> Vec { result } -pub fn strip_magic_header(data: &[u8]) -> Option { +/// Parse a magic-byte-framed schema ID from raw bytes. +/// +/// Returns: +/// - `Ok(None)` — no magic byte present (data has no schema framing) +/// - `Ok(Some(info))` — valid framing parsed successfully +/// - `Err(...)` — magic byte present but data is truncated / corrupt +pub fn strip_magic_header(data: &[u8]) -> Result, crate::Error> { if data.is_empty() { - return None; + return Ok(None); } match data[0] { - MAGIC_BYTE_VALUE => Some(SchemaIdInfo::Single(data[1..].to_vec())), + MAGIC_BYTE_VALUE => Ok(Some(SchemaIdInfo::Single(data[1..].to_vec()))), MAGIC_BYTE_KEY_VALUE => { if data.len() < 5 { - return None; + return Err(crate::Error::Custom(format!( + "corrupt KV schema_id: expected at least 5 bytes, got {}", + data.len() + ))); } let key_len = u32::from_be_bytes([data[1], data[2], data[3], data[4]]) as usize; if data.len() < 5 + key_len { - return None; + return Err(crate::Error::Custom(format!( + "corrupt KV schema_id: key_len={} but only {} bytes remain", + key_len, + data.len() - 5 + ))); } let key_id = data[5..5 + key_len].to_vec(); let value_id = data[5 + key_len..].to_vec(); - Some(SchemaIdInfo::KeyValue { key_id, value_id }) + Ok(Some(SchemaIdInfo::KeyValue { key_id, value_id })) } - _ => None, + _ => Ok(None), } } @@ -81,18 +94,32 @@ mod tests { fn test_strip_magic_header_single() { let id = vec![0x00, 0x00, 0x00, 0x05]; let framed = add_magic_header(&id); - let info = strip_magic_header(&framed).unwrap(); + let info = strip_magic_header(&framed).unwrap().unwrap(); assert_eq!(info, SchemaIdInfo::Single(id)); } #[test] fn test_strip_magic_header_empty_input() { - assert_eq!(strip_magic_header(&[]), None); + assert!(strip_magic_header(&[]).unwrap().is_none()); } #[test] fn test_strip_magic_header_no_magic() { - assert_eq!(strip_magic_header(&[0x00, 0x01, 0x02]), None); + assert!(strip_magic_header(&[0x00, 0x01, 0x02]).unwrap().is_none()); + } + + #[test] + fn test_strip_magic_header_corrupt_kv() { + // KV magic byte but truncated data — should return Err, not None + let data = vec![MAGIC_BYTE_KEY_VALUE, 0x00]; + assert!(strip_magic_header(&data).is_err()); + } + + #[test] + fn test_strip_magic_header_corrupt_kv_key_len() { + // KV magic byte, valid key_len=10 but only 2 bytes of data after header + let data = vec![MAGIC_BYTE_KEY_VALUE, 0x00, 0x00, 0x00, 0x0A, 0x01, 0x02]; + assert!(strip_magic_header(&data).is_err()); } #[test] @@ -129,7 +156,7 @@ mod tests { let key_id = vec![0x01, 0x02]; let val_id = vec![0x03, 0x04, 0x05]; let framed = generate_kv_schema_id(Some(&key_id), Some(&val_id)).unwrap(); - let info = strip_magic_header(&framed).unwrap(); + let info = strip_magic_header(&framed).unwrap().unwrap(); assert_eq!( info, SchemaIdInfo::KeyValue { From d9efa438558cda0af1f96ddb9f8ec10ff1fea24f Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 22:47:22 -0700 Subject: [PATCH 16/24] fix: address PR review round 2 (6 findings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Replace debug_assert! with runtime Result error in ConsumerBuilder and ProducerBuilder — debug_assert! is stripped in release builds. 2. Add decode_error field to consumer Message so callers can distinguish "no schema attached" from "decode failed" without relying on logs alone. Expose via message.decode_error() accessor. 3. Clear schema_info in Clone impls (both builders) to prevent clones from negotiating External schema they cannot decode/encode. 4. Log warning when KV decode receives Single schema_id framing — indicates protocol/configuration mismatch. 5. Use std::sync::Once guard for confluent encode placeholder warning to prevent per-message log flooding in high-throughput producers. 6. Fix with_schema() doc to reference send_schema_non_blocking() instead of send_non_blocking(). Co-Authored-By: Claude Opus 4.6 --- pulsar-schema-confluent/src/lib.rs | 12 +++++++++--- src/consumer/builder.rs | 30 ++++++++++++++++++------------ src/consumer/data.rs | 7 ++++--- src/consumer/message.rs | 19 +++++++++++++++++-- src/consumer/topic.rs | 15 +++++++++------ src/producer.rs | 16 ++++++++++++---- src/schema/key_value.rs | 10 +++++++++- 7 files changed, 78 insertions(+), 31 deletions(-) diff --git a/pulsar-schema-confluent/src/lib.rs b/pulsar-schema-confluent/src/lib.rs index 3094b420..4d4c4fbc 100644 --- a/pulsar-schema-confluent/src/lib.rs +++ b/pulsar-schema-confluent/src/lib.rs @@ -118,9 +118,15 @@ where // to register schemas and extract the Confluent schema ID, then re-frame // it with Pulsar's 0xFF magic byte via schema_id_util::add_magic_header(). let _subject = self.subject_name(topic); - log::warn!( - "Confluent encode: schema registration not yet wired — schema_id will be None" - ); + { + use std::sync::Once; + static WARN_ONCE: Once = Once::new(); + WARN_ONCE.call_once(|| { + log::warn!( + "Confluent encode: schema registration not yet wired — schema_id will be None" + ); + }); + } Ok(EncodeData { payload: json_bytes, diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index f7ec84e1..5225f02b 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -56,10 +56,10 @@ impl Clone for ConsumerBuilder { dead_letter_policy: self.dead_letter_policy.clone(), namespace: self.namespace.clone(), topic_refresh: self.topic_refresh, - schema_info: self.schema_info.clone(), - // schema_object cannot be cloned (Box). The original builder - // retains it; clones are only valid for validate(). build() and - // into_reader() assert the original's schema_object is intact. + // schema_object cannot be cloned (Box). Clear schema_info + // too so that a clone never negotiates an External schema it cannot + // decode. Clones are only used for validate(). + schema_info: None, schema_object: None, } } @@ -315,10 +315,13 @@ impl ConsumerBuilder { pub async fn build(self) -> Result, Error> { // clone() is only used for validate(); schema_object is NOT cloneable. // Guard: if schema_info is set, schema_object must still be present on `self`. - debug_assert!( - self.schema_info.is_none() || self.schema_object.is_some(), - "BUG: schema_info present but schema_object lost (was build() called on a clone?)" - ); + if self.schema_info.is_some() && self.schema_object.is_none() { + return Err(Error::Custom( + "schema_object lost — was build() called on a cloned ConsumerBuilder? \ + Only the original builder retains the schema." + .to_string(), + )); + } let (config, joined_topics) = self.clone().validate().await?; // Apply schema_info to options if set (takes precedence over .with_options()) @@ -394,10 +397,13 @@ impl ConsumerBuilder { /// creates a [Reader] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn into_reader(self) -> Result, Error> { - debug_assert!( - self.schema_info.is_none() || self.schema_object.is_some(), - "BUG: schema_info present but schema_object lost (was into_reader() called on a clone?)" - ); + if self.schema_info.is_some() && self.schema_object.is_none() { + return Err(Error::Custom( + "schema_object lost — was into_reader() called on a cloned ConsumerBuilder? \ + Only the original builder retains the schema." + .to_string(), + )); + } let (mut config, mut joined_topics) = self.clone().validate().await?; if let Some(ref info) = self.schema_info { diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 52613b6c..448fcb74 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -17,8 +17,9 @@ use crate::{ pub type MessageIdDataReceiver = mpsc::Receiver>; +/// (message_id, payload, decoded_value, decode_error_description) pub type DecodedMessageReceiver = - mpsc::Receiver), Error>>; + mpsc::Receiver, Option), Error>>; /// Unified receiver that avoids spawning a pass-through task when no schema is /// attached. When polling, `Raw` maps `(id, payload)` → `(id, payload, None)` @@ -31,7 +32,7 @@ pub enum MessageReceiver { } impl Stream for MessageReceiver { - type Item = Result<(MessageIdData, Payload, Option), Error>; + type Item = Result<(MessageIdData, Payload, Option, Option), Error>; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // SAFETY: we only access the inner receivers which are Unpin (mpsc::Receiver). @@ -41,7 +42,7 @@ impl Stream for MessageReceiver { Poll::Pending => Poll::Pending, Poll::Ready(None) => Poll::Ready(None), Poll::Ready(Some(result)) => { - Poll::Ready(Some(result.map(|(id, payload)| (id, payload, None)))) + Poll::Ready(Some(result.map(|(id, payload)| (id, payload, None, None)))) } }, MessageReceiver::Decoded(rx) => Pin::new(rx).poll_next(cx), diff --git a/src/consumer/message.rs b/src/consumer/message.rs index dae33b20..687d70a6 100644 --- a/src/consumer/message.rs +++ b/src/consumer/message.rs @@ -14,8 +14,12 @@ pub struct Message { pub payload: Payload, /// Contains the message's id and batch size data. pub message_id: MessageData, - /// Pre-decoded value from PulsarSchema. None when using DeserializeMessage path. + /// Pre-decoded value from PulsarSchema. None when using DeserializeMessage path + /// or when schema decode failed (check [`decode_error`](Self::decode_error)). pub(super) decoded: Option, + /// If schema decode failed, contains the error description. + /// `None` when no schema is attached or when decoding succeeded. + pub(super) decode_error: Option, } // Manual Debug impl — avoids requiring T: Debug @@ -26,6 +30,7 @@ impl std::fmt::Debug for Message { .field("payload", &self.payload) .field("message_id", &self.message_id) .field("has_decoded", &self.decoded.is_some()) + .field("decode_error", &self.decode_error) .finish() } } @@ -50,10 +55,20 @@ impl Message { } /// Returns the pre-decoded value if this message was decoded via PulsarSchema. - /// Returns None when using the DeserializeMessage path. + /// + /// Returns `None` when: + /// - Using the `DeserializeMessage` path (no schema attached), or + /// - Schema decode failed — check [`decode_error()`](Self::decode_error) for details. pub fn value(&self) -> Option<&T> { self.decoded.as_ref() } + + /// If schema decode failed for this message, returns the error description. + /// + /// Returns `None` when no schema is attached or when decoding succeeded. + pub fn decode_error(&self) -> Option<&str> { + self.decode_error.as_deref() + } } impl Message { diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 88625ad3..759d534c 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -139,7 +139,7 @@ impl TopicConsumer = if let Some(schema) = schema { let (decoded_tx, decoded_rx) = - mpsc::channel::), Error>>( + mpsc::channel::, Option), Error>>( receiver_queue_size as usize, ); let decode_topic = topic.clone(); @@ -158,14 +158,15 @@ impl TopicConsumer Ok((id, payload, Some(decoded))), + Ok(decoded) => Ok((id, payload, Some(decoded), None)), Err(e) => { + let err_msg = format!("{}", e); log::warn!( "schema decode failed for message {:?} on topic {}: {}. \ Forwarding with decoded=None so it can be acked/nacked.", - id, decode_topic, e + id, decode_topic, err_msg ); - Ok((id, payload, None)) + Ok((id, payload, None, Some(err_msg))) } } } @@ -353,6 +354,7 @@ impl TopicConsumer, + decode_error: Option, ) -> Message { Message { topic: self.topic.clone(), @@ -362,6 +364,7 @@ impl TopicConsumer Stream for TopicCons match self.messages.as_mut().poll_next(cx) { Poll::Pending => Poll::Pending, Poll::Ready(None) => Poll::Ready(None), - Poll::Ready(Some(Ok((id, payload, decoded)))) => { + Poll::Ready(Some(Ok((id, payload, decoded, decode_error)))) => { self.last_message_received = Some(Utc::now()); self.messages_received += 1; - Poll::Ready(Some(Ok(self.create_message(id, payload, decoded)))) + Poll::Ready(Some(Ok(self.create_message(id, payload, decoded, decode_error)))) } Poll::Ready(Some(Err(e))) => { error!("we are using in the single-consumer and we got an error, {e}"); diff --git a/src/producer.rs b/src/producer.rs index 86ea5052..c6af76f8 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -1102,9 +1102,10 @@ impl Clone for ProducerBuilder { topic: self.topic.clone(), name: self.name.clone(), producer_options: self.producer_options.clone(), - schema_info: self.schema_info.clone(), - // schema_object cannot be cloned (Box). The original builder - // retains it; clones are only valid for non-schema paths. + // schema_object cannot be cloned (Box). Clear schema_info + // too so that a clone never negotiates an External schema it cannot + // encode. Clones are only used for non-schema paths. + schema_info: None, schema_object: None, } } @@ -1147,7 +1148,7 @@ impl ProducerBuilder { /// Attach an external schema to this producer. /// The schema is type-erased internally; it is downcast back to - /// `PulsarSchema` when `send_non_blocking()` is called. + /// `PulsarSchema` when [`send_schema_non_blocking()`](Producer::send_schema_non_blocking) is called. pub fn with_schema( mut self, schema: Arc>, @@ -1160,6 +1161,13 @@ impl ProducerBuilder { /// creates a new producer #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub async fn build(self) -> Result, Error> { + if self.schema_info.is_some() && self.schema_object.is_none() { + return Err(Error::Custom( + "schema_object lost — was build() called on a cloned ProducerBuilder? \ + Only the original builder retains the schema." + .to_string(), + )); + } let ProducerBuilder { pulsar, topic, diff --git a/src/schema/key_value.rs b/src/schema/key_value.rs index 52e5d6e4..9235830d 100644 --- a/src/schema/key_value.rs +++ b/src/schema/key_value.rs @@ -106,7 +106,15 @@ where Some(SchemaIdInfo::KeyValue { key_id, value_id }) => { (Some(key_id), Some(value_id)) } - _ => (None, None), + Some(SchemaIdInfo::Single(_)) => { + log::warn!( + "KV decode received Single schema_id framing on topic {} — \ + possible protocol/configuration mismatch; falling back to no schema IDs", + topic + ); + (None, None) + } + None => (None, None), }, None => (None, None), }; From 26ac9de6e142e9fefa7c8273268d68482887cc36 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 23:15:35 -0700 Subject: [PATCH 17/24] fix: carry schema through build_multi_topic and fix clippy warnings - build_multi_topic() now propagates schema_info and schema_object from ProducerBuilder into MultiTopicProducer, so multi-topic and partitioned-topic producers created via with_schema() correctly negotiate schemas with the broker and encode messages with schema_id. - Changed build_multi_topic() return type to Result for consistency with build() and to guard against inconsistent schema state. - Added send_schema_non_blocking() to MultiTopicProducer for schema- aware message sending across topics. - Extracted get_or_create_producer() helper to reduce duplication. - Added 7 unit tests covering schema propagation in multi-topic and partitioned-topic producer flows. - Fixed clippy warnings: redundant returns in oauth2.rs, redundant field names in schema_id_util.rs, manual_is_multiple_of in examples. Co-Authored-By: Claude Opus 4.6 --- examples/batching.rs | 4 +- examples/round_trip.rs | 4 +- src/authentication/oauth2.rs | 4 +- src/client.rs | 8 +- src/producer.rs | 361 +++++++++++++++++++++++++++++++++-- src/schema/schema_id_util.rs | 2 +- 6 files changed, 364 insertions(+), 19 deletions(-) diff --git a/examples/batching.rs b/examples/batching.rs index 22e89409..354607fd 100644 --- a/examples/batching.rs +++ b/examples/batching.rs @@ -70,7 +70,7 @@ async fn main() -> Result<(), pulsar::Error> { v.push(receipt_rx); println!("sent"); counter += 1; - if counter % 4 == 0 { + if counter.is_multiple_of(4) { //producer.send_batch().await.unwrap(); println!("sent {counter} messages"); break; @@ -98,7 +98,7 @@ async fn main() -> Result<(), pulsar::Error> { } println!("got message: {data:?}"); counter += 1; - if counter % 4 == 0 { + if counter.is_multiple_of(4) { println!("sent {counter} messages"); break; } diff --git a/examples/round_trip.rs b/examples/round_trip.rs index 9867653f..fb0abf91 100644 --- a/examples/round_trip.rs +++ b/examples/round_trip.rs @@ -60,7 +60,7 @@ async fn main() -> Result<(), pulsar::Error> { .await .unwrap(); counter += 1; - if counter % 1000 == 0 { + if counter.is_multiple_of(1000) { println!("sent {counter} messages"); } } @@ -86,7 +86,7 @@ async fn main() -> Result<(), pulsar::Error> { panic!("Unexpected payload: {}", &data.data); } counter += 1; - if counter % 1000 == 0 { + if counter.is_multiple_of(1000) { println!("received {counter} messages"); } } diff --git a/src/authentication/oauth2.rs b/src/authentication/oauth2.rs index 44386277..6f29c196 100644 --- a/src/authentication/oauth2.rs +++ b/src/authentication/oauth2.rs @@ -244,9 +244,9 @@ impl OAuth2Authentication { .await?; if let Some(token_endpoint) = metadata.token_endpoint() { self.token_url = Some(token_endpoint.clone()); - return Ok(token_endpoint.clone()); + Ok(token_endpoint.clone()) } else { - return Err(Box::from("token url not exists")); + Err(Box::from("token url not exists")) } } } diff --git a/src/client.rs b/src/client.rs index 2bf36272..b5c6cd25 100644 --- a/src/client.rs +++ b/src/client.rs @@ -598,7 +598,13 @@ async fn run_producer( client: Pulsar, mut messages: mpsc::UnboundedReceiver, ) { - let mut producer = client.producer().build_multi_topic(); + let mut producer = match client.producer().build_multi_topic() { + Ok(p) => p, + Err(e) => { + error!("failed to build multi-topic producer: {e}"); + return; + } + }; while let Some(SendMessage { topic, message: payload, diff --git a/src/producer.rs b/src/producer.rs index c6af76f8..22a7081d 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -202,7 +202,7 @@ impl ProducerOptions { /// # let topic = "topic"; /// # let message = "data".to_owned(); /// let pulsar: Pulsar<_> = Pulsar::builder(addr, TokioExecutor).build().await?; -/// let mut producer = pulsar.producer().with_name("name").build_multi_topic(); +/// let mut producer = pulsar.producer().with_name("name").build_multi_topic()?; /// let send_1 = producer.send_non_blocking(topic, &message).await?; /// let send_2 = producer.send_non_blocking(topic, &message).await?; /// send_1.await?; @@ -215,6 +215,11 @@ pub struct MultiTopicProducer { producers: BTreeMap>, options: ProducerOptions, name: Option, + /// Type-erased schema object, moved from `ProducerBuilder::with_schema`. + /// Used by [`send_schema_non_blocking`](Self::send_schema_non_blocking) to + /// encode messages at the multi-topic level before delegating to per-topic + /// producers. + schema_object: Option>, } impl MultiTopicProducer { @@ -263,7 +268,71 @@ impl MultiTopicProducer { ) -> Result { let message = T::serialize_message(message)?; let topic = topic.into(); - let producer = match self.producers.entry(topic) { + let producer = self.get_or_create_producer(&topic).await?; + producer.send_non_blocking(message).await + } + + /// Sends a message on a topic using the attached PulsarSchema (PIP-420). + /// + /// When a schema has been registered via + /// [`ProducerBuilder::with_schema`], this method calls + /// `PulsarSchema::encode` to obtain the payload and the registry + /// `schema_id`, then delegates the pre-encoded [`Message`] to the + /// per-topic producer. + /// + /// If no schema is attached, falls back to [`SerializeMessage`]. + #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] + pub async fn send_schema_non_blocking< + T: SerializeMessage + Sized + Send + 'static, + S: Into, + >( + &mut self, + topic: S, + message: T, + message_metadata: Option, + ) -> Result { + let topic = topic.into(); + + // If a PulsarSchema is attached, encode via it. + if let Some(ref schema_obj) = self.schema_object { + if let Some(schema) = schema_obj + .downcast_ref::>>() + { + let encode_data = schema.encode(&topic, message).await?; + + let serialized_message = if let Some(mut msg) = message_metadata { + msg.payload = encode_data.payload; + msg.schema_id = encode_data.schema_id; + msg + } else { + Message { + payload: encode_data.payload, + schema_id: encode_data.schema_id, + ..Default::default() + } + }; + + let producer = self.get_or_create_producer(&topic).await?; + return producer.send_non_blocking(serialized_message).await; + } + // Downcast failed — type mismatch. + return Err(Error::Custom(format!( + "Schema type mismatch: ProducerBuilder::with_schema() was called with a different \ + type parameter than send_schema_non_blocking::<{}>()", + std::any::type_name::() + ))); + } + + // No schema attached: fall back to SerializeMessage. + let serialized_message = T::serialize_message(message)?; + let producer = self.get_or_create_producer(&topic).await?; + producer.send_non_blocking(serialized_message).await + } + + /// Get or lazily create a per-topic producer. + async fn get_or_create_producer(&mut self, topic: &str) -> Result<&mut Producer, Error> { + // Use the entry API. We need the topic as owned String for the key. + match self.producers.entry(topic.to_string()) { Entry::Vacant(entry) => { let mut builder = self .client @@ -274,12 +343,10 @@ impl MultiTopicProducer { builder = builder.with_name(name); } let producer = builder.build().await?; - entry.insert(producer) + Ok(entry.insert(producer)) } - Entry::Occupied(entry) => entry.into_mut(), - }; - - producer.send_non_blocking(message).await + Entry::Occupied(entry) => Ok(entry.into_mut()), + } } /// sends a list of messages on a topic @@ -1225,14 +1292,33 @@ impl ProducerBuilder { } /// creates a new [MultiTopicProducer] + /// + /// If [`with_schema`](Self::with_schema) was called, the schema is carried + /// into the multi-topic producer so that: + /// 1. `options.schema` is set — lazily-created per-topic producers negotiate + /// the schema with the broker. + /// 2. [`MultiTopicProducer::send_schema_non_blocking`] can encode via the + /// attached `PulsarSchema`. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub fn build_multi_topic(self) -> MultiTopicProducer { - MultiTopicProducer { + pub fn build_multi_topic(self) -> Result, Error> { + if self.schema_info.is_some() && self.schema_object.is_none() { + return Err(Error::Custom( + "schema_object lost — was build_multi_topic() called on a cloned \ + ProducerBuilder? Only the original builder retains the schema." + .to_string(), + )); + } + let mut options = self.producer_options.unwrap_or_default(); + if let Some(info) = self.schema_info { + options.schema = Some(info); + } + Ok(MultiTopicProducer { client: self.pulsar, producers: Default::default(), - options: self.producer_options.unwrap_or_default(), + options, name: self.name, - } + schema_object: self.schema_object, + }) } } @@ -1923,6 +2009,259 @@ mod tests { } } + // ---- Schema / multi-topic tests ---- + + /// Minimal PulsarSchema for testing — encodes to payload bytes and + /// returns a fixed schema_id. + struct TestSchema { + schema_id: Option>, + } + + #[async_trait::async_trait] + impl crate::schema::PulsarSchema for TestSchema { + fn schema_info(&self) -> proto::Schema { + proto::Schema { + name: "test-schema".to_string(), + r#type: proto::schema::Type::String as i32, + ..Default::default() + } + } + + async fn encode( + &self, + _topic: &str, + message: String, + ) -> Result { + Ok(crate::schema::EncodeData { + payload: message.into_bytes(), + schema_id: self.schema_id.clone(), + }) + } + + async fn decode( + &self, + _topic: &str, + payload: &crate::Payload, + _schema_id: Option<&[u8]>, + ) -> Result { + String::from_utf8(payload.data.clone()).map_err(|e| Error::Custom(e.to_string())) + } + } + + #[test] + fn build_multi_topic_carries_schema_into_options() { + // with_schema sets schema_info; build_multi_topic should merge it + // into options.schema so per-topic producers negotiate it with the + // broker. + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let pulsar: Pulsar<_> = Pulsar::builder("pulsar://127.0.0.1:6650", TokioExecutor) + .build() + .await + .unwrap(); + + let schema: Arc> = + Arc::new(TestSchema { schema_id: Some(vec![1, 2, 3]) }); + + let producer = pulsar + .producer() + .with_schema(schema) + .build_multi_topic() + .unwrap(); + + // The schema should have been merged into options. + assert!( + producer.options().schema.is_some(), + "schema_info should be merged into options.schema" + ); + assert_eq!( + producer.options().schema.as_ref().unwrap().name, + "test-schema" + ); + }); + } + + #[test] + fn build_multi_topic_carries_schema_object() { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let pulsar: Pulsar<_> = Pulsar::builder("pulsar://127.0.0.1:6650", TokioExecutor) + .build() + .await + .unwrap(); + + let schema: Arc> = + Arc::new(TestSchema { schema_id: None }); + + let producer = pulsar + .producer() + .with_schema(schema) + .build_multi_topic() + .unwrap(); + + assert!( + producer.schema_object.is_some(), + "schema_object should be carried into MultiTopicProducer" + ); + }); + } + + #[test] + fn build_multi_topic_without_schema_has_none() { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let pulsar: Pulsar<_> = Pulsar::builder("pulsar://127.0.0.1:6650", TokioExecutor) + .build() + .await + .unwrap(); + + let producer = pulsar.producer().build_multi_topic().unwrap(); + + assert!(producer.options().schema.is_none()); + assert!(producer.schema_object.is_none()); + }); + } + + #[test] + fn cloned_builder_loses_schema_on_build_multi_topic() { + // Clone clears both schema_info and schema_object. The clone builds + // successfully but the resulting MultiTopicProducer has no schema, + // which is the intended safety behavior. + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let pulsar: Pulsar<_> = Pulsar::builder("pulsar://127.0.0.1:6650", TokioExecutor) + .build() + .await + .unwrap(); + + let schema: Arc> = + Arc::new(TestSchema { schema_id: None }); + + let builder = pulsar.producer().with_schema(schema); + let cloned = builder.clone(); + + // Original should carry schema: + let original = builder.build_multi_topic().unwrap(); + assert!( + original.options().schema.is_some(), + "original should have schema" + ); + assert!( + original.schema_object.is_some(), + "original should have schema_object" + ); + + // Clone should succeed but without schema: + let cloned_producer = cloned.build_multi_topic().unwrap(); + assert!( + cloned_producer.options().schema.is_none(), + "cloned builder should not have schema" + ); + assert!( + cloned_producer.schema_object.is_none(), + "cloned builder should not have schema_object" + ); + }); + } + + #[tokio::test] + async fn multi_topic_producer_send_schema_non_blocking() { + let _result = log::set_logger(&TEST_LOGGER); + log::set_max_level(LevelFilter::Debug); + let pulsar: Pulsar<_> = Pulsar::builder("pulsar://127.0.0.1:6650", TokioExecutor) + .build() + .await + .unwrap(); + + let topic = format!("multi_topic_schema_{}", rand::random::()); + + let schema: Arc> = + Arc::new(TestSchema { schema_id: Some(vec![0, 1, 2, 3]) }); + + let mut producer = pulsar + .producer() + .with_schema(schema) + .build_multi_topic() + .unwrap(); + + // send_schema_non_blocking should encode via PulsarSchema + let _receipt = producer + .send_schema_non_blocking(&topic, "hello-schema".to_string(), None) + .await + .unwrap() + .await + .unwrap(); + } + + #[tokio::test] + async fn partitioned_topic_producer_with_schema() { + let _result = log::set_logger(&TEST_LOGGER); + log::set_max_level(LevelFilter::Debug); + let pulsar: Pulsar<_> = Pulsar::builder("pulsar://127.0.0.1:6650", TokioExecutor) + .build() + .await + .unwrap(); + + let topic = format!("partitioned_schema_{}", rand::random::()); + let partition_count = 3; + test_utils::create_partitioned_topic("public", "default", &topic, partition_count).await; + + let schema: Arc> = + Arc::new(TestSchema { schema_id: Some(vec![10, 20]) }); + + // Build a single-topic producer (partitioned) with schema + let mut producer = pulsar + .producer() + .with_topic(&topic) + .with_schema(schema) + .build() + .await + .unwrap(); + + // Send via schema path + for i in 0..5 { + let _receipt = producer + .send_schema_non_blocking(format!("msg-{i}"), None) + .await + .unwrap() + .await + .unwrap(); + } + } + + #[tokio::test] + async fn multi_topic_partitioned_producer_with_schema() { + let _result = log::set_logger(&TEST_LOGGER); + log::set_max_level(LevelFilter::Debug); + let pulsar: Pulsar<_> = Pulsar::builder("pulsar://127.0.0.1:6650", TokioExecutor) + .build() + .await + .unwrap(); + + let topic = format!("multi_partitioned_schema_{}", rand::random::()); + let partition_count = 3; + test_utils::create_partitioned_topic("public", "default", &topic, partition_count).await; + + let schema: Arc> = + Arc::new(TestSchema { schema_id: Some(vec![7, 8, 9]) }); + + let mut producer = pulsar + .producer() + .with_schema(schema) + .build_multi_topic() + .unwrap(); + + // send_schema_non_blocking to a partitioned topic + for i in 0..6 { + let _receipt = producer + .send_schema_non_blocking(&topic, format!("multi-msg-{i}"), None) + .await + .unwrap() + .await + .unwrap(); + } + } + struct TestCustomRoutingPolicy {} impl CustomRoutingPolicy for TestCustomRoutingPolicy { diff --git a/src/schema/schema_id_util.rs b/src/schema/schema_id_util.rs index e79f0911..1f474274 100644 --- a/src/schema/schema_id_util.rs +++ b/src/schema/schema_id_util.rs @@ -160,7 +160,7 @@ mod tests { assert_eq!( info, SchemaIdInfo::KeyValue { - key_id: key_id, + key_id, value_id: val_id, } ); From c22e159e56050665c918ca7d8476050bd4dd137f Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 23:24:45 -0700 Subject: [PATCH 18/24] fix: address review round 3 (C2, I1-I3) C2: Fix KV encode double-framing of magic bytes. Inner schemas return schema IDs with a 0xFF magic prefix. KeyValueSchema::encode() now strips that prefix via strip_single_magic_prefix() before wrapping in the 0xFE KV frame, preventing double-framing on the wire. Added a round-trip test with a RecordingSchema mock that verifies inner decode() receives raw IDs without the 0xFF prefix. I1: KV decode now returns an error when it encounters Single (0xFF) framing on a KeyValue message, instead of silently falling back to (None, None). A Single schema_id on a KV message indicates a real configuration problem that should surface immediately. I2: Confluent placeholder encode() now logs a warning on every call instead of only once. The once-guarded warning was insufficient because after the first message, subsequent encodes silently returned schema_id: None with no indication. I3: Fixed MessageReceiver doc comment: Raw maps to 4-tuple (id, payload, None, None), not 3-tuple (id, payload, None). Co-Authored-By: Claude Opus 4.6 --- pulsar-schema-confluent/src/lib.rs | 17 ++-- src/consumer/data.rs | 5 +- src/schema/key_value.rs | 151 +++++++++++++++++++++++++++-- src/schema/schema_id_util.rs | 54 +++++++++++ 4 files changed, 207 insertions(+), 20 deletions(-) diff --git a/pulsar-schema-confluent/src/lib.rs b/pulsar-schema-confluent/src/lib.rs index 4d4c4fbc..27f4cd80 100644 --- a/pulsar-schema-confluent/src/lib.rs +++ b/pulsar-schema-confluent/src/lib.rs @@ -113,20 +113,17 @@ where let json_bytes = serde_json::to_vec(&message) .map_err(|e| Error::SchemaRegistry(e.to_string()))?; - // Schema registration and ID retrieval is a placeholder for now. + // Schema registration and ID retrieval is not yet implemented. // A future iteration will wire the schema_registry_converter encode API // to register schemas and extract the Confluent schema ID, then re-frame // it with Pulsar's 0xFF magic byte via schema_id_util::add_magic_header(). let _subject = self.subject_name(topic); - { - use std::sync::Once; - static WARN_ONCE: Once = Once::new(); - WARN_ONCE.call_once(|| { - log::warn!( - "Confluent encode: schema registration not yet wired — schema_id will be None" - ); - }); - } + log::warn!( + "ConfluentJsonSchema::encode(): schema registration not yet implemented \ + for subject '{}' on topic '{topic}' — returning schema_id: None. \ + Messages will be sent without external schema framing.", + _subject, + ); Ok(EncodeData { payload: json_bytes, diff --git a/src/consumer/data.rs b/src/consumer/data.rs index 448fcb74..9da1b577 100644 --- a/src/consumer/data.rs +++ b/src/consumer/data.rs @@ -22,8 +22,9 @@ pub type DecodedMessageReceiver = mpsc::Receiver, Option), Error>>; /// Unified receiver that avoids spawning a pass-through task when no schema is -/// attached. When polling, `Raw` maps `(id, payload)` → `(id, payload, None)` -/// inline without an extra async task or channel hop. +/// attached. When polling, `Raw` maps `(id, payload)` → +/// `(id, payload, None, None)` inline without an extra async task or channel +/// hop. pub enum MessageReceiver { /// No schema: poll the raw engine receiver directly. Raw(MessageIdDataReceiver), diff --git a/src/schema/key_value.rs b/src/schema/key_value.rs index 9235830d..55194b51 100644 --- a/src/schema/key_value.rs +++ b/src/schema/key_value.rs @@ -85,9 +85,21 @@ where let (key, value) = message; let key_data = self.key_schema.encode(topic, key).await?; let value_data = self.value_schema.encode(topic, value).await?; + + // Inner schemas return schema IDs framed with a 0xFF magic prefix. + // Strip that prefix before KV-framing to avoid double-framing — the KV + // envelope (0xFE + length-delimited) is the only framing on the wire. + // On decode, strip_magic_header extracts raw inner IDs which are passed + // directly to inner schemas' decode(), completing the round-trip. let schema_id = schema_id_util::generate_kv_schema_id( - key_data.schema_id.as_deref(), - value_data.schema_id.as_deref(), + key_data + .schema_id + .as_deref() + .map(schema_id_util::strip_single_magic_prefix), + value_data + .schema_id + .as_deref() + .map(schema_id_util::strip_single_magic_prefix), ); Ok(EncodeData { payload: combine_kv_payload(&key_data.payload, &value_data.payload), @@ -107,12 +119,11 @@ where (Some(key_id), Some(value_id)) } Some(SchemaIdInfo::Single(_)) => { - log::warn!( - "KV decode received Single schema_id framing on topic {} — \ - possible protocol/configuration mismatch; falling back to no schema IDs", - topic - ); - (None, None) + return Err(Error::Custom(format!( + "KV decode received Single (0xFF) schema_id framing on topic \ + {topic} — expected KeyValue (0xFE) framing. This indicates a \ + protocol or producer configuration mismatch." + ))); } None => (None, None), }, @@ -211,6 +222,103 @@ mod tests { let framed = encoded.schema_id.unwrap(); assert_eq!(framed[0], schema_id_util::MAGIC_BYTE_KEY_VALUE); + + // Verify inner IDs are stored WITHOUT the 0xFF magic prefix (no double-framing). + // The KV frame should contain raw inner IDs: [0xFE, key_len(4), 0x01, 0x02] + let info = schema_id_util::strip_magic_header(&framed).unwrap().unwrap(); + match info { + schema_id_util::SchemaIdInfo::KeyValue { key_id, value_id } => { + // Raw inner IDs — no 0xFF prefix + assert_eq!(key_id, vec![0x01], "key_id should be raw, without 0xFF prefix"); + assert_eq!(value_id, vec![0x02], "value_id should be raw, without 0xFF prefix"); + } + other => panic!("expected KeyValue, got {:?}", other), + } + } + + #[tokio::test] + async fn test_kv_encode_decode_roundtrip_with_schema_ids() { + // Verifies that inner schema IDs survive the encode→decode round-trip + // without double-framing. The recording mock captures the schema_id + // received by decode() so we can assert it matches the raw (unframed) ID. + use std::sync::Mutex; + + struct RecordingSchema { + encode_id: Option>, + decoded_ids: Arc>>>>, + } + + #[async_trait] + impl PulsarSchema for RecordingSchema { + fn schema_info(&self) -> proto::Schema { + proto::Schema::default() + } + async fn encode(&self, _topic: &str, msg: String) -> Result { + Ok(EncodeData { + payload: msg.into_bytes(), + schema_id: self.encode_id.clone(), + }) + } + async fn decode( + &self, + _topic: &str, + payload: &Payload, + schema_id: Option<&[u8]>, + ) -> Result { + self.decoded_ids + .lock() + .unwrap() + .push(schema_id.map(|s| s.to_vec())); + String::from_utf8(payload.data.clone()).map_err(|e| Error::Custom(e.to_string())) + } + } + + let key_decoded = Arc::new(Mutex::new(Vec::new())); + let val_decoded = Arc::new(Mutex::new(Vec::new())); + + let kv_schema = KeyValueSchema::new( + Arc::new(RecordingSchema { + encode_id: Some(schema_id_util::add_magic_header(&[0x00, 0x01])), + decoded_ids: key_decoded.clone(), + }), + Arc::new(RecordingSchema { + encode_id: Some(schema_id_util::add_magic_header(&[0x00, 0x02])), + decoded_ids: val_decoded.clone(), + }), + ); + + let encoded = kv_schema + .encode("t", ("k".to_string(), "v".to_string())) + .await + .unwrap(); + + let payload = Payload { + metadata: proto::MessageMetadata::default(), + data: encoded.payload, + }; + let (k, v) = kv_schema + .decode("t", &payload, encoded.schema_id.as_deref()) + .await + .unwrap(); + assert_eq!(k, "k"); + assert_eq!(v, "v"); + + // Inner decode() should receive raw IDs (without 0xFF prefix) + let key_ids = key_decoded.lock().unwrap(); + assert_eq!(key_ids.len(), 1); + assert_eq!( + key_ids[0].as_deref(), + Some([0x00, 0x01].as_slice()), + "inner key decode should receive raw ID without 0xFF prefix" + ); + + let val_ids = val_decoded.lock().unwrap(); + assert_eq!(val_ids.len(), 1); + assert_eq!( + val_ids[0].as_deref(), + Some([0x00, 0x02].as_slice()), + "inner value decode should receive raw ID without 0xFF prefix" + ); } #[tokio::test] @@ -223,4 +331,31 @@ mod tests { let info = kv_schema.schema_info(); assert_eq!(info.r#type, proto::schema::Type::KeyValue as i32); } + + #[tokio::test] + async fn test_kv_decode_rejects_single_framed_schema_id() { + let kv_schema = KeyValueSchema::new( + Arc::new(MockSchema { schema_id: None }), + Arc::new(MockSchema { schema_id: None }), + ); + + // Build a valid KV payload + let payload = Payload { + metadata: proto::MessageMetadata::default(), + data: combine_kv_payload(b"key", b"value"), + }; + + // Pass a Single-framed schema_id (0xFF prefix) — should be an error, + // not silently degraded to (None, None). + let single_framed = schema_id_util::add_magic_header(&[0x00, 0x01]); + let result = kv_schema + .decode("topic", &payload, Some(&single_framed)) + .await; + assert!(result.is_err(), "KV decode with Single framing should error"); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("Single (0xFF)"), + "error should mention Single framing, got: {err_msg}" + ); + } } diff --git a/src/schema/schema_id_util.rs b/src/schema/schema_id_util.rs index 1f474274..0e16880e 100644 --- a/src/schema/schema_id_util.rs +++ b/src/schema/schema_id_util.rs @@ -53,6 +53,31 @@ pub fn strip_magic_header(data: &[u8]) -> Result, crate::Er } } +/// Strip the single-value magic prefix (`0xFF`) from an inner schema ID. +/// +/// Inner `PulsarSchema::encode()` returns schema IDs framed with the `0xFF` +/// magic byte (via `add_magic_header`). When composing a key-value schema ID, +/// we need the **raw** inner IDs — the KV frame (`0xFE`) provides its own +/// length-delimited envelope, so the inner `0xFF` prefix must be removed to +/// avoid double-framing. +/// +/// Returns the raw bytes after stripping. If the prefix is absent the input is +/// returned unchanged (defensive — inner schemas *should* always frame). +pub fn strip_single_magic_prefix(data: &[u8]) -> &[u8] { + if data.first() == Some(&MAGIC_BYTE_VALUE) { + &data[1..] + } else { + data + } +} + +/// Build a KV-framed schema ID from **raw** (unframed) inner schema IDs. +/// +/// Callers must strip the inner `0xFF` magic prefix before calling this +/// function — use [`strip_single_magic_prefix`] on values returned by inner +/// `PulsarSchema::encode()`. +/// +/// Wire format: `[0xFE] [4-byte key_len BE] [raw_key_id] [raw_value_id]` pub fn generate_kv_schema_id( key_schema_id: Option<&[u8]>, value_schema_id: Option<&[u8]>, @@ -165,4 +190,33 @@ mod tests { } ); } + + #[test] + fn test_strip_single_magic_prefix_with_prefix() { + let framed = add_magic_header(&[0x00, 0x01]); + assert_eq!(strip_single_magic_prefix(&framed), &[0x00, 0x01]); + } + + #[test] + fn test_strip_single_magic_prefix_without_prefix() { + let raw = vec![0x00, 0x01]; + assert_eq!(strip_single_magic_prefix(&raw), &[0x00, 0x01]); + } + + #[test] + fn test_strip_single_magic_prefix_empty() { + assert_eq!(strip_single_magic_prefix(&[]), &[] as &[u8]); + } + + #[test] + fn test_strip_single_magic_prefix_only_magic_byte() { + assert_eq!(strip_single_magic_prefix(&[MAGIC_BYTE_VALUE]), &[] as &[u8]); + } + + #[test] + fn test_strip_single_magic_prefix_kv_magic_untouched() { + // KV magic byte (0xFE) should NOT be stripped + let data = vec![MAGIC_BYTE_KEY_VALUE, 0x01, 0x02]; + assert_eq!(strip_single_magic_prefix(&data), data.as_slice()); + } } From f3296cad84bfc658cfe4631902f20536191afc0f Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 12 Mar 2026 23:31:37 -0700 Subject: [PATCH 19/24] fix: restore backward-compatible public API - Remove #[non_exhaustive] from producer::Message. This was a source-breaking change: downstream SerializeMessage impls that construct `Message { payload, ..Default::default() }` would stop compiling. The `Message::new()` constructor remains available as a convenience but is no longer required. - Revert build_multi_topic() to return MultiTopicProducer directly instead of Result. The only error path was a defensive guard for an inconsistent state that cannot occur through normal API usage (Clone clears both schema fields together). Replaced with debug_assert! for development-time safety without breaking every existing caller. Co-Authored-By: Claude Opus 4.6 --- src/client.rs | 8 +------- src/producer.rs | 47 ++++++++++++++++++++++------------------------- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/src/client.rs b/src/client.rs index b5c6cd25..2bf36272 100644 --- a/src/client.rs +++ b/src/client.rs @@ -598,13 +598,7 @@ async fn run_producer( client: Pulsar, mut messages: mpsc::UnboundedReceiver, ) { - let mut producer = match client.producer().build_multi_topic() { - Ok(p) => p, - Err(e) => { - error!("failed to build multi-topic producer: {e}"); - return; - } - }; + let mut producer = client.producer().build_multi_topic(); while let Some(SendMessage { topic, message: payload, diff --git a/src/producer.rs b/src/producer.rs index 22a7081d..504eb631 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -68,7 +68,6 @@ impl Future for SendFuture { /// this is actually a subset of the fields of a message, because batching, /// compression and encryption should be handled by the producer #[derive(Debug, Clone, Default)] -#[non_exhaustive] pub struct Message { /// serialized data pub payload: Vec, @@ -202,7 +201,7 @@ impl ProducerOptions { /// # let topic = "topic"; /// # let message = "data".to_owned(); /// let pulsar: Pulsar<_> = Pulsar::builder(addr, TokioExecutor).build().await?; -/// let mut producer = pulsar.producer().with_name("name").build_multi_topic()?; +/// let mut producer = pulsar.producer().with_name("name").build_multi_topic(); /// let send_1 = producer.send_non_blocking(topic, &message).await?; /// let send_2 = producer.send_non_blocking(topic, &message).await?; /// send_1.await?; @@ -1300,25 +1299,27 @@ impl ProducerBuilder { /// 2. [`MultiTopicProducer::send_schema_non_blocking`] can encode via the /// attached `PulsarSchema`. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub fn build_multi_topic(self) -> Result, Error> { - if self.schema_info.is_some() && self.schema_object.is_none() { - return Err(Error::Custom( - "schema_object lost — was build_multi_topic() called on a cloned \ - ProducerBuilder? Only the original builder retains the schema." - .to_string(), - )); - } + pub fn build_multi_topic(self) -> MultiTopicProducer { + // Clone clears both schema_info and schema_object together, so this + // state (schema_info present but schema_object missing) should never + // occur through normal API usage. + debug_assert!( + !(self.schema_info.is_some() && self.schema_object.is_none()), + "schema_object lost — was build_multi_topic() called on a builder \ + in an inconsistent state? Only the original (non-cloned) builder \ + retains the schema." + ); let mut options = self.producer_options.unwrap_or_default(); if let Some(info) = self.schema_info { options.schema = Some(info); } - Ok(MultiTopicProducer { + MultiTopicProducer { client: self.pulsar, producers: Default::default(), options, name: self.name, schema_object: self.schema_object, - }) + } } } @@ -2066,8 +2067,7 @@ mod tests { let producer = pulsar .producer() .with_schema(schema) - .build_multi_topic() - .unwrap(); + .build_multi_topic(); // The schema should have been merged into options. assert!( @@ -2096,8 +2096,7 @@ mod tests { let producer = pulsar .producer() .with_schema(schema) - .build_multi_topic() - .unwrap(); + .build_multi_topic(); assert!( producer.schema_object.is_some(), @@ -2115,7 +2114,7 @@ mod tests { .await .unwrap(); - let producer = pulsar.producer().build_multi_topic().unwrap(); + let producer = pulsar.producer().build_multi_topic(); assert!(producer.options().schema.is_none()); assert!(producer.schema_object.is_none()); @@ -2141,7 +2140,7 @@ mod tests { let cloned = builder.clone(); // Original should carry schema: - let original = builder.build_multi_topic().unwrap(); + let original = builder.build_multi_topic(); assert!( original.options().schema.is_some(), "original should have schema" @@ -2152,7 +2151,7 @@ mod tests { ); // Clone should succeed but without schema: - let cloned_producer = cloned.build_multi_topic().unwrap(); + let cloned_producer = cloned.build_multi_topic(); assert!( cloned_producer.options().schema.is_none(), "cloned builder should not have schema" @@ -2181,12 +2180,11 @@ mod tests { let mut producer = pulsar .producer() .with_schema(schema) - .build_multi_topic() - .unwrap(); + .build_multi_topic(); // send_schema_non_blocking should encode via PulsarSchema let _receipt = producer - .send_schema_non_blocking(&topic, "hello-schema".to_string(), None) + .send_schema_non_blocking::(&topic, "hello-schema".to_string(), None) .await .unwrap() .await @@ -2248,13 +2246,12 @@ mod tests { let mut producer = pulsar .producer() .with_schema(schema) - .build_multi_topic() - .unwrap(); + .build_multi_topic(); // send_schema_non_blocking to a partitioned topic for i in 0..6 { let _receipt = producer - .send_schema_non_blocking(&topic, format!("multi-msg-{i}"), None) + .send_schema_non_blocking::(&topic, format!("multi-msg-{i}"), None) .await .unwrap() .await From 22aa649697dcc5a05a962ce3ec82a57184484aa2 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 13 Mar 2026 00:15:56 -0700 Subject: [PATCH 20/24] fix: address review round 4 (I1-I3, Message backward compat) I1: Replace per-call log::warn! in ConfluentJsonSchema::encode() with AtomicBool-gated warn-once + log::debug! for subsequent calls. Prevents log flooding at high throughput. I2: Replace debug_assert! in build_multi_topic() with log::error! + schema drop. Same invariant that build() catches via Result::Err, but without breaking the backward-compatible return type. Prevents silent corruption in release builds where debug_assert is stripped. I3: Update MultiTopicProducer::send_schema_non_blocking doc to document the type-mismatch error case, matching the single-topic variant. Backward compat: Remove schema_id from public producer::Message. Move it to internal ProducerMessage only. Both send_schema_non_blocking variants now construct ProducerMessage directly and call send_raw(), keeping schema_id off the public API surface. Add Producer::send_raw() helper for ProducerMessage dispatch with partition routing. Co-Authored-By: Claude Opus 4.6 --- pulsar-schema-confluent/src/lib.rs | 24 ++++++--- src/producer.rs | 81 +++++++++++++++++++----------- 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/pulsar-schema-confluent/src/lib.rs b/pulsar-schema-confluent/src/lib.rs index 27f4cd80..00772b55 100644 --- a/pulsar-schema-confluent/src/lib.rs +++ b/pulsar-schema-confluent/src/lib.rs @@ -1,4 +1,5 @@ use std::marker::PhantomData; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -48,6 +49,8 @@ pub struct ConfluentJsonSchema { sr_settings: SrSettings, subject_strategy: SubjectNameStrategy, is_key: bool, + /// Ensures the "not yet implemented" warning is emitted at most once. + warned: AtomicBool, _phantom: PhantomData, } @@ -80,6 +83,7 @@ impl ConfluentJsonSchema { sr_settings, subject_strategy: config.subject_name_strategy, is_key, + warned: AtomicBool::new(false), _phantom: PhantomData, }) } @@ -117,13 +121,19 @@ where // A future iteration will wire the schema_registry_converter encode API // to register schemas and extract the Confluent schema ID, then re-frame // it with Pulsar's 0xFF magic byte via schema_id_util::add_magic_header(). - let _subject = self.subject_name(topic); - log::warn!( - "ConfluentJsonSchema::encode(): schema registration not yet implemented \ - for subject '{}' on topic '{topic}' — returning schema_id: None. \ - Messages will be sent without external schema framing.", - _subject, - ); + let subject = self.subject_name(topic); + if !self.warned.swap(true, Ordering::Relaxed) { + log::warn!( + "ConfluentJsonSchema::encode(): schema registration not yet implemented \ + for subject '{subject}' on topic '{topic}' — returning schema_id: None. \ + Messages will be sent without external schema framing.", + ); + } else { + log::debug!( + "ConfluentJsonSchema::encode(): schema_id: None for subject '{subject}' \ + on topic '{topic}' (not-yet-implemented placeholder)", + ); + } Ok(EncodeData { payload: json_bytes, diff --git a/src/producer.rs b/src/producer.rs index 504eb631..1acd3f05 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -87,9 +87,6 @@ pub struct Message { /// UTC Unix timestamp in milliseconds, time at which the message should be /// delivered to consumers pub deliver_at_time: ::std::option::Option, - /// Schema ID from external schema registry (PIP-420). - /// Written to MessageMetadata.schema_id when present. - pub schema_id: Option>, } impl Message { @@ -148,7 +145,6 @@ impl From for ProducerMessage { event_time: m.event_time, schema_version: m.schema_version, deliver_at_time: m.deliver_at_time, - schema_id: m.schema_id, ..Default::default() } } @@ -276,8 +272,9 @@ impl MultiTopicProducer { /// When a schema has been registered via /// [`ProducerBuilder::with_schema`], this method calls /// `PulsarSchema::encode` to obtain the payload and the registry - /// `schema_id`, then delegates the pre-encoded [`Message`] to the - /// per-topic producer. + /// `schema_id`, then delegates the pre-encoded message to the + /// per-topic producer. If the attached schema's type parameter does + /// not match `T`, an error is returned. /// /// If no schema is attached, falls back to [`SerializeMessage`]. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] @@ -299,20 +296,23 @@ impl MultiTopicProducer { { let encode_data = schema.encode(&topic, message).await?; - let serialized_message = if let Some(mut msg) = message_metadata { + // Build a Message for metadata, then convert to + // ProducerMessage so we can attach schema_id without + // exposing it on the public Message type. + let msg = if let Some(mut msg) = message_metadata { msg.payload = encode_data.payload; - msg.schema_id = encode_data.schema_id; msg } else { Message { payload: encode_data.payload, - schema_id: encode_data.schema_id, ..Default::default() } }; + let mut producer_msg: ProducerMessage = msg.into(); + producer_msg.schema_id = encode_data.schema_id; let producer = self.get_or_create_producer(&topic).await?; - return producer.send_non_blocking(serialized_message).await; + return producer.send_raw(producer_msg).await; } // Downcast failed — type mismatch. return Err(Error::Custom(format!( @@ -549,24 +549,30 @@ impl Producer { let topic = self.topic().to_string(); let encode_data = schema.encode(&topic, message).await?; - let serialized_message = if let Some(mut msg) = message_metadata { + // Build a Message for metadata / partition routing, then + // convert to ProducerMessage so we can attach schema_id + // without exposing it on the public Message type. + let msg = if let Some(mut msg) = message_metadata { msg.payload = encode_data.payload; - msg.schema_id = encode_data.schema_id; msg } else { Message { payload: encode_data.payload, - schema_id: encode_data.schema_id, ..Default::default() } }; return match &mut self.inner { - ProducerInner::Single(p) => p.send(serialized_message).await, + ProducerInner::Single(p) => { + let mut pm: ProducerMessage = msg.into(); + pm.schema_id = encode_data.schema_id; + p.send_raw(pm).await + } ProducerInner::Partitioned(p) => { - p.choose_partition(&serialized_message) - .send(serialized_message) - .await + let tp = p.choose_partition(&msg); + let mut pm: ProducerMessage = msg.into(); + pm.schema_id = encode_data.schema_id; + tp.send_raw(pm).await } }; } @@ -590,6 +596,26 @@ impl Producer { } } + /// Send a pre-built [`ProducerMessage`] directly (internal use). + /// + /// This by-passes `Message` → `ProducerMessage` conversion so callers + /// can set fields that are not part of the public [`Message`] type + /// (e.g. `schema_id`). + pub(crate) async fn send_raw(&mut self, message: ProducerMessage) -> Result { + match &mut self.inner { + ProducerInner::Single(p) => p.send_raw(message).await, + ProducerInner::Partitioned(p) => { + // Build a lightweight Message for partition routing only. + let routing_hint = Message { + partition_key: message.partition_key.clone(), + ordering_key: message.ordering_key.clone(), + ..Default::default() + }; + p.choose_partition(&routing_hint).send_raw(message).await + } + } + } + /// Sends a message /// /// this function is similar to send_non_blocking then waits the returned `SendFuture` @@ -1300,17 +1326,17 @@ impl ProducerBuilder { /// attached `PulsarSchema`. #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn build_multi_topic(self) -> MultiTopicProducer { - // Clone clears both schema_info and schema_object together, so this - // state (schema_info present but schema_object missing) should never - // occur through normal API usage. - debug_assert!( - !(self.schema_info.is_some() && self.schema_object.is_none()), - "schema_object lost — was build_multi_topic() called on a builder \ - in an inconsistent state? Only the original (non-cloned) builder \ - retains the schema." - ); let mut options = self.producer_options.unwrap_or_default(); - if let Some(info) = self.schema_info { + if self.schema_info.is_some() && self.schema_object.is_none() { + // Same invariant that build() checks via Result::Err. We cannot + // return Result here (backward-compat), so log an error and drop + // the schema config to prevent silent corruption at send time. + log::error!( + "build_multi_topic(): schema_info present but schema_object missing — \ + was this called on a cloned builder? Only the original builder retains \ + the schema. The schema will be dropped to avoid silent corruption." + ); + } else if let Some(info) = self.schema_info { options.schema = Some(info); } MultiTopicProducer { @@ -1789,7 +1815,6 @@ mod tests { event_time: Some(42), schema_version: Some(vec![9, 9]), deliver_at_time: Some(123456789), - schema_id: None, }; let pm: ProducerMessage = m.clone().into(); From 6ab55ee6640bf3ad514483423ab199f420af7325 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 13 Mar 2026 08:53:25 -0700 Subject: [PATCH 21/24] style: run rustfmt (stable 1.94.0) to fix CI format check Co-Authored-By: Claude Opus 4.6 --- pulsar-schema-confluent/src/lib.rs | 21 +++++++------ src/consumer/builder.rs | 16 +++++++--- src/consumer/mod.rs | 8 +---- src/consumer/topic.rs | 49 +++++++++++++++-------------- src/producer.rs | 50 +++++++++++++----------------- src/schema/default.rs | 5 +-- src/schema/key_value.rs | 34 ++++++++++++++------ src/schema/mod.rs | 5 +-- src/schema/schema_id_util.rs | 3 +- 9 files changed, 97 insertions(+), 94 deletions(-) diff --git a/pulsar-schema-confluent/src/lib.rs b/pulsar-schema-confluent/src/lib.rs index 00772b55..7d2066d0 100644 --- a/pulsar-schema-confluent/src/lib.rs +++ b/pulsar-schema-confluent/src/lib.rs @@ -57,8 +57,7 @@ pub struct ConfluentJsonSchema { impl ConfluentJsonSchema { pub fn new(config: ConfluentConfig, is_key: bool) -> Result { let sr_settings = if config.auth.is_some() || config.timeout.is_some() { - let mut builder: SrSettingsBuilder = - SrSettings::new_builder(config.registry_url); + let mut builder: SrSettingsBuilder = SrSettings::new_builder(config.registry_url); if let Some(auth) = config.auth { match auth { ConfluentAuth::Basic { username, password } => { @@ -72,9 +71,9 @@ impl ConfluentJsonSchema { if let Some(timeout) = config.timeout { builder.set_timeout(timeout); } - builder.build().map_err(|e| { - Error::SchemaRegistry(format!("Failed to build SrSettings: {}", e)) - })? + builder + .build() + .map_err(|e| Error::SchemaRegistry(format!("Failed to build SrSettings: {}", e)))? } else { SrSettings::new(config.registry_url) }; @@ -114,8 +113,8 @@ where } async fn encode(&self, topic: &str, message: T) -> Result { - let json_bytes = serde_json::to_vec(&message) - .map_err(|e| Error::SchemaRegistry(e.to_string()))?; + let json_bytes = + serde_json::to_vec(&message).map_err(|e| Error::SchemaRegistry(e.to_string()))?; // Schema registration and ID retrieval is not yet implemented. // A future iteration will wire the schema_registry_converter encode API @@ -156,8 +155,7 @@ where id.len() ); } - serde_json::from_slice(&payload.data) - .map_err(|e| Error::SchemaRegistry(e.to_string())) + serde_json::from_slice(&payload.data).map_err(|e| Error::SchemaRegistry(e.to_string())) } } @@ -321,6 +319,9 @@ mod tests { ); let id = encode_data.schema_id.unwrap(); assert!(id.len() >= 2, "schema_id too short"); - assert_eq!(id[0], 0xFF, "schema_id must start with Pulsar magic byte 0xFF"); + assert_eq!( + id[0], 0xFF, + "schema_id must start with Pulsar magic byte 0xFF" + ); } } diff --git a/src/consumer/builder.rs b/src/consumer/builder.rs index 5225f02b..9717042a 100644 --- a/src/consumer/builder.rs +++ b/src/consumer/builder.rs @@ -312,7 +312,9 @@ impl ConsumerBuilder { /// creates a [Consumer] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub async fn build(self) -> Result, Error> { + pub async fn build( + self, + ) -> Result, Error> { // clone() is only used for validate(); schema_object is NOT cloneable. // Guard: if schema_info is set, schema_object must still be present on `self`. if self.schema_info.is_some() && self.schema_object.is_none() { @@ -391,12 +393,17 @@ impl ConsumerBuilder { } InnerConsumer::Multi(consumer) }; - Ok(Consumer { inner: consumer, schema }) + Ok(Consumer { + inner: consumer, + schema, + }) } /// creates a [Reader] from this builder #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] - pub async fn into_reader(self) -> Result, Error> { + pub async fn into_reader( + self, + ) -> Result, Error> { if self.schema_info.is_some() && self.schema_object.is_none() { return Err(Error::Custom( "schema_object lost — was into_reader() called on a cloned ConsumerBuilder? \ @@ -444,7 +451,8 @@ impl ConsumerBuilder { } let (topic, addr) = joined_topics.pop().unwrap(); - let consumer = TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone(), schema).await?; + let consumer = + TopicConsumer::new(self.pulsar.clone(), topic, addr, config.clone(), schema).await?; Ok(Reader { consumer, diff --git a/src/consumer/mod.rs b/src/consumer/mod.rs index 4fb20d25..613121de 100644 --- a/src/consumer/mod.rs +++ b/src/consumer/mod.rs @@ -215,13 +215,7 @@ impl Consumer { let topic_addr_pair = c.topics.iter().cloned().zip(addrs.iter().cloned()); let consumers = try_join_all(topic_addr_pair.map(|(topic, addr)| { - TopicConsumer::new( - client.clone(), - topic, - addr, - config.clone(), - schema.clone(), - ) + TopicConsumer::new(client.clone(), topic, addr, config.clone(), schema.clone()) })) .await?; diff --git a/src/consumer/topic.rs b/src/consumer/topic.rs index 759d534c..c48bccd1 100644 --- a/src/consumer/topic.rs +++ b/src/consumer/topic.rs @@ -138,40 +138,36 @@ impl TopicConsumer = if let Some(schema) = schema { - let (decoded_tx, decoded_rx) = - mpsc::channel::, Option), Error>>( - receiver_queue_size as usize, - ); + let (decoded_tx, decoded_rx) = mpsc::channel::< + Result<(MessageIdData, Payload, Option, Option), Error>, + >(receiver_queue_size as usize); let decode_topic = topic.clone(); let decode_task = client.executor.spawn(Box::pin(async move { let mut raw_rx = rx; let mut decoded_tx = decoded_tx; while let Some(result) = raw_rx.next().await { - let mapped = match result { - Ok((id, payload)) => { - let schema_id_bytes = payload.metadata.schema_id.clone(); - match schema - .decode( - &decode_topic, - &payload, - schema_id_bytes.as_deref(), - ) - .await - { - Ok(decoded) => Ok((id, payload, Some(decoded), None)), - Err(e) => { - let err_msg = format!("{}", e); - log::warn!( + let mapped = + match result { + Ok((id, payload)) => { + let schema_id_bytes = payload.metadata.schema_id.clone(); + match schema + .decode(&decode_topic, &payload, schema_id_bytes.as_deref()) + .await + { + Ok(decoded) => Ok((id, payload, Some(decoded), None)), + Err(e) => { + let err_msg = format!("{}", e); + log::warn!( "schema decode failed for message {:?} on topic {}: {}. \ Forwarding with decoded=None so it can be acked/nacked.", id, decode_topic, err_msg ); - Ok((id, payload, None, Some(err_msg))) + Ok((id, payload, None, Some(err_msg))) + } } } - } - Err(e) => Err(e), - }; + Err(e) => Err(e), + }; if decoded_tx.send(mapped).await.is_err() { break; } @@ -390,7 +386,12 @@ impl Stream for TopicCons Poll::Ready(Some(Ok((id, payload, decoded, decode_error)))) => { self.last_message_received = Some(Utc::now()); self.messages_received += 1; - Poll::Ready(Some(Ok(self.create_message(id, payload, decoded, decode_error)))) + Poll::Ready(Some(Ok(self.create_message( + id, + payload, + decoded, + decode_error, + )))) } Poll::Ready(Some(Err(e))) => { error!("we are using in the single-consumer and we got an error, {e}"); diff --git a/src/producer.rs b/src/producer.rs index 1acd3f05..de426390 100644 --- a/src/producer.rs +++ b/src/producer.rs @@ -291,8 +291,8 @@ impl MultiTopicProducer { // If a PulsarSchema is attached, encode via it. if let Some(ref schema_obj) = self.schema_object { - if let Some(schema) = schema_obj - .downcast_ref::>>() + if let Some(schema) = + schema_obj.downcast_ref::>>() { let encode_data = schema.encode(&topic, message).await?; @@ -543,8 +543,8 @@ impl Producer { ) -> Result { // Try PulsarSchema if available and the stored schema matches T. if let Some(ref schema_obj) = self.schema_object { - if let Some(schema) = schema_obj - .downcast_ref::>>() + if let Some(schema) = + schema_obj.downcast_ref::>>() { let topic = self.topic().to_string(); let encode_data = schema.encode(&topic, message).await?; @@ -1392,7 +1392,7 @@ enum BatchItem { SingleMessage( oneshot::Sender>, BatchedMessage, - Option>, // schema_id for PIP-420 + Option>, // schema_id for PIP-420 ), Flush(oneshot::Sender<()>), } @@ -2086,13 +2086,11 @@ mod tests { .await .unwrap(); - let schema: Arc> = - Arc::new(TestSchema { schema_id: Some(vec![1, 2, 3]) }); + let schema: Arc> = Arc::new(TestSchema { + schema_id: Some(vec![1, 2, 3]), + }); - let producer = pulsar - .producer() - .with_schema(schema) - .build_multi_topic(); + let producer = pulsar.producer().with_schema(schema).build_multi_topic(); // The schema should have been merged into options. assert!( @@ -2118,10 +2116,7 @@ mod tests { let schema: Arc> = Arc::new(TestSchema { schema_id: None }); - let producer = pulsar - .producer() - .with_schema(schema) - .build_multi_topic(); + let producer = pulsar.producer().with_schema(schema).build_multi_topic(); assert!( producer.schema_object.is_some(), @@ -2199,13 +2194,11 @@ mod tests { let topic = format!("multi_topic_schema_{}", rand::random::()); - let schema: Arc> = - Arc::new(TestSchema { schema_id: Some(vec![0, 1, 2, 3]) }); + let schema: Arc> = Arc::new(TestSchema { + schema_id: Some(vec![0, 1, 2, 3]), + }); - let mut producer = pulsar - .producer() - .with_schema(schema) - .build_multi_topic(); + let mut producer = pulsar.producer().with_schema(schema).build_multi_topic(); // send_schema_non_blocking should encode via PulsarSchema let _receipt = producer @@ -2229,8 +2222,9 @@ mod tests { let partition_count = 3; test_utils::create_partitioned_topic("public", "default", &topic, partition_count).await; - let schema: Arc> = - Arc::new(TestSchema { schema_id: Some(vec![10, 20]) }); + let schema: Arc> = Arc::new(TestSchema { + schema_id: Some(vec![10, 20]), + }); // Build a single-topic producer (partitioned) with schema let mut producer = pulsar @@ -2265,13 +2259,11 @@ mod tests { let partition_count = 3; test_utils::create_partitioned_topic("public", "default", &topic, partition_count).await; - let schema: Arc> = - Arc::new(TestSchema { schema_id: Some(vec![7, 8, 9]) }); + let schema: Arc> = Arc::new(TestSchema { + schema_id: Some(vec![7, 8, 9]), + }); - let mut producer = pulsar - .producer() - .with_schema(schema) - .build_multi_topic(); + let mut producer = pulsar.producer().with_schema(schema).build_multi_topic(); // send_schema_non_blocking to a partitioned topic for i in 0..6 { diff --git a/src/schema/default.rs b/src/schema/default.rs index 6bcba6d0..366f4b08 100644 --- a/src/schema/default.rs +++ b/src/schema/default.rs @@ -118,10 +118,7 @@ mod tests { let original = TestMsg { data: "roundtrip".to_string(), }; - let encoded = schema - .encode("test-topic", original.clone()) - .await - .unwrap(); + let encoded = schema.encode("test-topic", original.clone()).await.unwrap(); let payload = Payload { metadata: proto::MessageMetadata::default(), diff --git a/src/schema/key_value.rs b/src/schema/key_value.rs index 55194b51..5632e2ae 100644 --- a/src/schema/key_value.rs +++ b/src/schema/key_value.rs @@ -46,9 +46,12 @@ fn split_kv_payload(payload: &Payload) -> Result<(Payload, Payload), Error> { "KeyValue payload too short to contain key length".to_string(), )); } - let key_len = - u32::from_be_bytes([payload.data[0], payload.data[1], payload.data[2], payload.data[3]]) - as usize; + let key_len = u32::from_be_bytes([ + payload.data[0], + payload.data[1], + payload.data[2], + payload.data[3], + ]) as usize; if payload.data.len() < 4 + key_len { return Err(Error::Custom( "KeyValue payload too short for declared key length".to_string(), @@ -115,9 +118,7 @@ where ) -> Result<(K, V), Error> { let (key_id, value_id) = match schema_id { Some(data) => match schema_id_util::strip_magic_header(data)? { - Some(SchemaIdInfo::KeyValue { key_id, value_id }) => { - (Some(key_id), Some(value_id)) - } + Some(SchemaIdInfo::KeyValue { key_id, value_id }) => (Some(key_id), Some(value_id)), Some(SchemaIdInfo::Single(_)) => { return Err(Error::Custom(format!( "KV decode received Single (0xFF) schema_id framing on topic \ @@ -225,12 +226,22 @@ mod tests { // Verify inner IDs are stored WITHOUT the 0xFF magic prefix (no double-framing). // The KV frame should contain raw inner IDs: [0xFE, key_len(4), 0x01, 0x02] - let info = schema_id_util::strip_magic_header(&framed).unwrap().unwrap(); + let info = schema_id_util::strip_magic_header(&framed) + .unwrap() + .unwrap(); match info { schema_id_util::SchemaIdInfo::KeyValue { key_id, value_id } => { // Raw inner IDs — no 0xFF prefix - assert_eq!(key_id, vec![0x01], "key_id should be raw, without 0xFF prefix"); - assert_eq!(value_id, vec![0x02], "value_id should be raw, without 0xFF prefix"); + assert_eq!( + key_id, + vec![0x01], + "key_id should be raw, without 0xFF prefix" + ); + assert_eq!( + value_id, + vec![0x02], + "value_id should be raw, without 0xFF prefix" + ); } other => panic!("expected KeyValue, got {:?}", other), } @@ -351,7 +362,10 @@ mod tests { let result = kv_schema .decode("topic", &payload, Some(&single_framed)) .await; - assert!(result.is_err(), "KV decode with Single framing should error"); + assert!( + result.is_err(), + "KV decode with Single framing should error" + ); let err_msg = result.unwrap_err().to_string(); assert!( err_msg.contains("Single (0xFF)"), diff --git a/src/schema/mod.rs b/src/schema/mod.rs index 9610591c..efbfcc8c 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -4,10 +4,7 @@ pub mod schema_id_util; use async_trait::async_trait; -use crate::{ - message::proto, - Error, Payload, -}; +use crate::{message::proto, Error, Payload}; /// Data returned from encoding a message with a PulsarSchema. #[derive(Debug, Clone)] diff --git a/src/schema/schema_id_util.rs b/src/schema/schema_id_util.rs index 0e16880e..3559b1d2 100644 --- a/src/schema/schema_id_util.rs +++ b/src/schema/schema_id_util.rs @@ -36,8 +36,7 @@ pub fn strip_magic_header(data: &[u8]) -> Result, crate::Er data.len() ))); } - let key_len = - u32::from_be_bytes([data[1], data[2], data[3], data[4]]) as usize; + let key_len = u32::from_be_bytes([data[1], data[2], data[3], data[4]]) as usize; if data.len() < 5 + key_len { return Err(crate::Error::Custom(format!( "corrupt KV schema_id: key_len={} but only {} bytes remain", From ae49c8e9fffe069c174a2a62f4d6da0e1c6d9f4b Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 13 Mar 2026 12:16:50 -0700 Subject: [PATCH 22/24] fix: minimize PulsarApi.proto diff to only PIP-420 fields Restore master's proto and add only the two fields needed: - Schema.Type.External = 22 - MessageMetadata.schema_id = 32 Removes unrelated upstream formatting and feature additions that were polluting the PR diff. Co-Authored-By: Claude Opus 4.6 --- PulsarApi.proto | 1506 ++++++++++++++++++++++------------------------- 1 file changed, 709 insertions(+), 797 deletions(-) diff --git a/PulsarApi.proto b/PulsarApi.proto index ed49f7bc..2b0d7d8c 100644 --- a/PulsarApi.proto +++ b/PulsarApi.proto @@ -23,690 +23,645 @@ option java_package = "org.apache.pulsar.common.api.proto"; option optimize_for = LITE_RUNTIME; message Schema { - enum Type { - None = 0; - String = 1; - Json = 2; - Protobuf = 3; - Avro = 4; - Bool = 5; - Int8 = 6; - Int16 = 7; - Int32 = 8; - Int64 = 9; - Float = 10; - Double = 11; - Date = 12; - Time = 13; - Timestamp = 14; - KeyValue = 15; - Instant = 16; - LocalDate = 17; - LocalTime = 18; - LocalDateTime = 19; - ProtobufNative = 20; - AutoConsume = 21; - External = 22; - } - - required string name = 1; - required bytes schema_data = 3; - required Type type = 4; - repeated KeyValue properties = 5; + enum Type { + None = 0; + String = 1; + Json = 2; + Protobuf = 3; + Avro = 4; + Bool = 5; + Int8 = 6; + Int16 = 7; + Int32 = 8; + Int64 = 9; + Float = 10; + Double = 11; + Date = 12; + Time = 13; + Timestamp = 14; + KeyValue = 15; + Instant = 16; + LocalDate = 17; + LocalTime = 18; + LocalDateTime = 19; + ProtobufNative = 20; + External = 22; + } + + required string name = 1; + required bytes schema_data = 3; + required Type type = 4; + repeated KeyValue properties = 5; } message MessageIdData { - required uint64 ledgerId = 1; - required uint64 entryId = 2; - optional int32 partition = 3 [default = -1]; - optional int32 batch_index = 4 [default = -1]; - repeated int64 ack_set = 5; - optional int32 batch_size = 6; + required uint64 ledgerId = 1; + required uint64 entryId = 2; + optional int32 partition = 3 [default = -1]; + optional int32 batch_index = 4 [default = -1]; + repeated int64 ack_set = 5; + optional int32 batch_size = 6; - // For the chunk message id, we need to specify the first chunk message id. - optional MessageIdData first_chunk_message_id = 7; + // For the chunk message id, we need to specify the first chunk message id. + optional MessageIdData first_chunk_message_id = 7; } message KeyValue { - required string key = 1; - required string value = 2; + required string key = 1; + required string value = 2; } message KeyLongValue { - required string key = 1; - required uint64 value = 2; + required string key = 1; + required uint64 value = 2; } message IntRange { - required int32 start = 1; - required int32 end = 2; + required int32 start = 1; + required int32 end = 2; } message EncryptionKeys { - required string key = 1; - required bytes value = 2; - repeated KeyValue metadata = 3; + required string key = 1; + required bytes value = 2; + repeated KeyValue metadata = 3; } enum CompressionType { - NONE = 0; - LZ4 = 1; - ZLIB = 2; - ZSTD = 3; - SNAPPY = 4; + NONE = 0; + LZ4 = 1; + ZLIB = 2; + ZSTD = 3; + SNAPPY = 4; } enum ProducerAccessMode { - Shared = 0; // By default multiple producers can publish on a topic - Exclusive = 1; // Require exclusive access for producer. Fail immediately if there's already a producer connected. - WaitForExclusive = 2; // Producer creation is pending until it can acquire exclusive access - ExclusiveWithFencing = 3; // Require exclusive access for producer. Fence out old producer. + Shared = 0; // By default multiple producers can publish on a topic + Exclusive = 1; // Require exclusive access for producer. Fail immediately if there's already a producer connected. + WaitForExclusive = 2; // Producer creation is pending until it can acquire exclusive access + ExclusiveWithFencing = 3; // Require exclusive access for producer. Fence out old producer. } message MessageMetadata { - required string producer_name = 1; - required uint64 sequence_id = 2; - required uint64 publish_time = 3; - repeated KeyValue properties = 4; - - // Property set on replicated message, - // includes the source cluster name - optional string replicated_from = 5; - //key to decide partition for the msg - optional string partition_key = 6; - // Override namespace's replication - repeated string replicate_to = 7; - optional CompressionType compression = 8 [default = NONE]; - optional uint32 uncompressed_size = 9 [default = 0]; - // Removed below checksum field from Metadata as - // it should be part of send-command which keeps checksum of header + payload - //optional sfixed64 checksum = 10; - // differentiate single and batch message metadata - optional int32 num_messages_in_batch = 11 [default = 1]; - - // the timestamp that this event occurs. it is typically set by applications. - // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. - optional uint64 event_time = 12 [default = 0]; - // Contains encryption key name, encrypted key and metadata to describe the key - repeated EncryptionKeys encryption_keys = 13; - // Algorithm used to encrypt data key - optional string encryption_algo = 14; - // Additional parameters required by encryption - optional bytes encryption_param = 15; - optional bytes schema_version = 16; - - optional bool partition_key_b64_encoded = 17 [ default = false ]; - // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. - optional bytes ordering_key = 18; - - // Mark the message to be delivered at or after the specified timestamp - optional int64 deliver_at_time = 19; - - // Identify whether a message is a "marker" message used for - // internal metadata instead of application published data. - // Markers will generally not be propagated back to clients - optional int32 marker_type = 20; - - // transaction related message info - optional uint64 txnid_least_bits = 22; - optional uint64 txnid_most_bits = 23; - - /// Add highest sequence id to support batch message with external sequence id - optional uint64 highest_sequence_id = 24 [default = 0]; - - // Indicate if the message payload value is set - optional bool null_value = 25 [default = false]; - optional string uuid = 26; - optional int32 num_chunks_from_msg = 27; - optional int32 total_chunk_msg_size = 28; - optional int32 chunk_id = 29; - - // Indicate if the message partition key is set - optional bool null_partition_key = 30 [default = false]; - - // Indicates the indexes of messages retained in the batch after compaction. When a batch is compacted, - // some messages may be removed (compacted out). For example, if the original batch contains: - // `k0 => v0, k1 => v1, k2 => v2, k1 => null`, the compacted batch will retain only `k0 => v0` and `k2 => v2`. - // In this case, this field will be set to `[0, 2]`, and the payload buffer will only include the retained messages. - // - // Note: Batches compacted by older versions of the compaction service do not include this field. For such batches, - // the `compacted_out` field in `SingleMessageMetadata` must be checked to identify and filter out compacted - // messages (e.g., `k1 => v1` and `k1 => null` in the example above). - repeated int32 compacted_batch_indexes = 31; - optional bytes schema_id = 32; + required string producer_name = 1; + required uint64 sequence_id = 2; + required uint64 publish_time = 3; + repeated KeyValue properties = 4; + + // Property set on replicated message, + // includes the source cluster name + optional string replicated_from = 5; + //key to decide partition for the msg + optional string partition_key = 6; + // Override namespace's replication + repeated string replicate_to = 7; + optional CompressionType compression = 8 [default = NONE]; + optional uint32 uncompressed_size = 9 [default = 0]; + // Removed below checksum field from Metadata as + // it should be part of send-command which keeps checksum of header + payload + //optional sfixed64 checksum = 10; + // differentiate single and batch message metadata + optional int32 num_messages_in_batch = 11 [default = 1]; + + // the timestamp that this event occurs. it is typically set by applications. + // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. + optional uint64 event_time = 12 [default = 0]; + // Contains encryption key name, encrypted key and metadata to describe the key + repeated EncryptionKeys encryption_keys = 13; + // Algorithm used to encrypt data key + optional string encryption_algo = 14; + // Additional parameters required by encryption + optional bytes encryption_param = 15; + optional bytes schema_version = 16; + + optional bool partition_key_b64_encoded = 17 [default = false]; + // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. + optional bytes ordering_key = 18; + + // Mark the message to be delivered at or after the specified timestamp + optional int64 deliver_at_time = 19; + + // Identify whether a message is a "marker" message used for + // internal metadata instead of application published data. + // Markers will generally not be propagated back to clients + optional int32 marker_type = 20; + + // transaction related message info + optional uint64 txnid_least_bits = 22; + optional uint64 txnid_most_bits = 23; + + /// Add highest sequence id to support batch message with external sequence id + optional uint64 highest_sequence_id = 24 [default = 0]; + + // Indicate if the message payload value is set + optional bool null_value = 25 [default = false]; + optional string uuid = 26; + optional int32 num_chunks_from_msg = 27; + optional int32 total_chunk_msg_size = 28; + optional int32 chunk_id = 29; + + // Indicate if the message partition key is set + optional bool null_partition_key = 30 [default = false]; + optional bytes schema_id = 32; } message SingleMessageMetadata { - repeated KeyValue properties = 1; - optional string partition_key = 2; - required int32 payload_size = 3; - optional bool compacted_out = 4 [default = false]; - - // the timestamp that this event occurs. it is typically set by applications. - // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. - optional uint64 event_time = 5 [default = 0]; - optional bool partition_key_b64_encoded = 6 [ default = false ]; - // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. - optional bytes ordering_key = 7; - // Allows consumer retrieve the sequence id that the producer set. - optional uint64 sequence_id = 8; - // Indicate if the message payload value is set - optional bool null_value = 9 [ default = false ]; - // Indicate if the message partition key is set - optional bool null_partition_key = 10 [ default = false]; + repeated KeyValue properties = 1; + optional string partition_key = 2; + required int32 payload_size = 3; + optional bool compacted_out = 4 [default = false]; + + // the timestamp that this event occurs. it is typically set by applications. + // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. + optional uint64 event_time = 5 [default = 0]; + optional bool partition_key_b64_encoded = 6 [default = false]; + // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. + optional bytes ordering_key = 7; + // Allows consumer retrieve the sequence id that the producer set. + optional uint64 sequence_id = 8; + // Indicate if the message payload value is set + optional bool null_value = 9 [default = false]; + // Indicate if the message partition key is set + optional bool null_partition_key = 10 [default = false]; } // metadata added for entry from broker message BrokerEntryMetadata { - optional uint64 broker_timestamp = 1; - optional uint64 index = 2; + optional uint64 broker_timestamp = 1; + optional uint64 index = 2; } enum ServerError { - UnknownError = 0; - MetadataError = 1; // Error with ZK/metadata - PersistenceError = 2; // Error writing reading from BK - AuthenticationError = 3; // Non valid authentication - AuthorizationError = 4; // Not authorized to use resource - - ConsumerBusy = 5; // Unable to subscribe/unsubscribe because - // other consumers are connected - ServiceNotReady = 6; // Any error that requires client retry operation with a fresh lookup - ProducerBlockedQuotaExceededError = 7; // Unable to create producer because backlog quota exceeded - ProducerBlockedQuotaExceededException = 8; // Exception while creating producer because quota exceeded - ChecksumError = 9; // Error while verifying message checksum - UnsupportedVersionError = 10; // Error when an older client/version doesn't support a required feature - TopicNotFound = 11; // Topic not found - SubscriptionNotFound = 12; // Subscription not found - ConsumerNotFound = 13; // Consumer not found - TooManyRequests = 14; // Error with too many simultaneously request - TopicTerminatedError = 15; // The topic has been terminated - - ProducerBusy = 16; // Producer with same name is already connected - InvalidTopicName = 17; // The topic name is not valid - - IncompatibleSchema = 18; // Specified schema was incompatible with topic schema - ConsumerAssignError = 19; // Dispatcher assign consumer error - - TransactionCoordinatorNotFound = 20; // Transaction coordinator not found error - InvalidTxnStatus = 21; // Invalid txn status error - NotAllowedError = 22; // Not allowed error - - TransactionConflict = 23; // Ack with transaction conflict - TransactionNotFound = 24; // Transaction not found - - ProducerFenced = 25; // When a producer asks and fail to get exclusive producer access, - // or loses the eclusive status after a reconnection, the broker will - // use this error to indicate that this producer is now permanently - // fenced. Applications are now supposed to close it and create a - // new producer + UnknownError = 0; + MetadataError = 1; // Error with ZK/metadata + PersistenceError = 2; // Error writing reading from BK + AuthenticationError = 3; // Non valid authentication + AuthorizationError = 4; // Not authorized to use resource + + ConsumerBusy = 5; // Unable to subscribe/unsubscribe because + // other consumers are connected + ServiceNotReady = 6; // Any error that requires client retry operation with a fresh lookup + ProducerBlockedQuotaExceededError = 7; // Unable to create producer because backlog quota exceeded + ProducerBlockedQuotaExceededException = 8; // Exception while creating producer because quota exceeded + ChecksumError = 9; // Error while verifying message checksum + UnsupportedVersionError = 10; // Error when an older client/version doesn't support a required feature + TopicNotFound = 11; // Topic not found + SubscriptionNotFound = 12; // Subscription not found + ConsumerNotFound = 13; // Consumer not found + TooManyRequests = 14; // Error with too many simultaneously request + TopicTerminatedError = 15; // The topic has been terminated + + ProducerBusy = 16; // Producer with same name is already connected + InvalidTopicName = 17; // The topic name is not valid + + IncompatibleSchema = 18; // Specified schema was incompatible with topic schema + ConsumerAssignError = 19; // Dispatcher assign consumer error + + TransactionCoordinatorNotFound = 20; // Transaction coordinator not found error + InvalidTxnStatus = 21; // Invalid txn status error + NotAllowedError = 22; // Not allowed error + + TransactionConflict = 23; // Ack with transaction conflict + TransactionNotFound = 24; // Transaction not found + + ProducerFenced = 25; // When a producer asks and fail to get exclusive producer access, + // or loses the exclusive status after a reconnection, the broker will + // use this error to indicate that this producer is now permanently + // fenced. Applications are now supposed to close it and create a + // new producer } enum AuthMethod { - AuthMethodNone = 0; - AuthMethodYcaV1 = 1; - AuthMethodAthens = 2; + AuthMethodNone = 0; + AuthMethodYcaV1 = 1; + AuthMethodAthens = 2; } // Each protocol version identify new features that are // incrementally added to the protocol enum ProtocolVersion { - v0 = 0; // Initial versioning - v1 = 1; // Added application keep-alive - v2 = 2; // Added RedeliverUnacknowledgedMessages Command - v3 = 3; // Added compression with LZ4 and ZLib - v4 = 4; // Added batch message support - v5 = 5; // Added disconnect client w/o closing connection - v6 = 6; // Added checksum computation for metadata + payload - v7 = 7; // Added CommandLookupTopic - Binary Lookup - v8 = 8; // Added CommandConsumerStats - Client fetches broker side consumer stats - v9 = 9; // Added end of topic notification - v10 = 10;// Added proxy to broker - v11 = 11;// C++ consumers before this version are not correctly handling the checksum field - v12 = 12;// Added get topic's last messageId from broker - // Added CommandActiveConsumerChange - // Added CommandGetTopicsOfNamespace - v13 = 13; // Schema-registry : added avro schema format for json - v14 = 14; // Add CommandAuthChallenge and CommandAuthResponse for mutual auth - // Added Key_Shared subscription - v15 = 15; // Add CommandGetOrCreateSchema and CommandGetOrCreateSchemaResponse - v16 = 16; // Add support for broker entry metadata - v17 = 17; // Added support ack receipt - v18 = 18; // Add client support for broker entry metadata - v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse - v20 = 20; // Add client support for topic migration redirection CommandTopicMigrated - v21 = 21; // Carry the AUTO_CONSUME schema to the Broker after this version + v0 = 0; // Initial versioning + v1 = 1; // Added application keep-alive + v2 = 2; // Added RedeliverUnacknowledgedMessages Command + v3 = 3; // Added compression with LZ4 and ZLib + v4 = 4; // Added batch message support + v5 = 5; // Added disconnect client w/o closing connection + v6 = 6; // Added checksum computation for metadata + payload + v7 = 7; // Added CommandLookupTopic - Binary Lookup + v8 = 8; // Added CommandConsumerStats - Client fetches broker side consumer stats + v9 = 9; // Added end of topic notification + v10 = 10;// Added proxy to broker + v11 = 11;// C++ consumers before this version are not correctly handling the checksum field + v12 = 12;// Added get topic's last messageId from broker + // Added CommandActiveConsumerChange + // Added CommandGetTopicsOfNamespace + v13 = 13; // Schema-registry : added avro schema format for json + v14 = 14; // Add CommandAuthChallenge and CommandAuthResponse for mutual auth + // Added Key_Shared subscription + v15 = 15; // Add CommandGetOrCreateSchema and CommandGetOrCreateSchemaResponse + v16 = 16; // Add support for broker entry metadata + v17 = 17; // Added support ack receipt + v18 = 18; // Add client support for broker entry metadata + v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse } message CommandConnect { - required string client_version = 1; // The version of the client. Proxy should forward client's client_version. - optional AuthMethod auth_method = 2; // Deprecated. Use "auth_method_name" instead. - optional string auth_method_name = 5; - optional bytes auth_data = 3; - optional int32 protocol_version = 4 [default = 0]; + required string client_version = 1; + optional AuthMethod auth_method = 2; // Deprecated. Use "auth_method_name" instead. + optional string auth_method_name = 5; + optional bytes auth_data = 3; + optional int32 protocol_version = 4 [default = 0]; - // Client can ask to be proxyied to a specific broker - // This is only honored by a Pulsar proxy - optional string proxy_to_broker_url = 6; + // Client can ask to be proxyied to a specific broker + // This is only honored by a Pulsar proxy + optional string proxy_to_broker_url = 6; - // Original principal that was verified by - // a Pulsar proxy. In this case the auth info above - // will be the auth of the proxy itself - optional string original_principal = 7; + // Original principal that was verified by + // a Pulsar proxy. In this case the auth info above + // will be the auth of the proxy itself + optional string original_principal = 7; - // Original auth role and auth Method that was passed - // to the proxy. In this case the auth info above - // will be the auth of the proxy itself - optional string original_auth_data = 8; - optional string original_auth_method = 9; + // Original auth role and auth Method that was passed + // to the proxy. In this case the auth info above + // will be the auth of the proxy itself + optional string original_auth_data = 8; + optional string original_auth_method = 9; - // Feature flags - optional FeatureFlags feature_flags = 10; - - optional string proxy_version = 11; // Version of the proxy. Should only be forwarded by a proxy. + // Feature flags + optional FeatureFlags feature_flags = 10; } -// Please also add a new enum for the class "PulsarClientException.FailedFeatureCheck" when adding a new feature flag. message FeatureFlags { optional bool supports_auth_refresh = 1 [default = false]; optional bool supports_broker_entry_metadata = 2 [default = false]; optional bool supports_partial_producer = 3 [default = false]; - optional bool supports_topic_watchers = 4 [default = false]; - optional bool supports_get_partitioned_metadata_without_auto_creation = 5 [default = false]; - optional bool supports_repl_dedup_by_lid_and_eid = 6 [default = false]; - optional bool supports_topic_watcher_reconcile = 7 [default = false]; } message CommandConnected { - required string server_version = 1; - optional int32 protocol_version = 2 [default = 0]; - optional int32 max_message_size = 3; - optional FeatureFlags feature_flags = 4; + required string server_version = 1; + optional int32 protocol_version = 2 [default = 0]; + optional int32 max_message_size = 3; } message CommandAuthResponse { - optional string client_version = 1; // The version of the client. Proxy should forward client's client_version. - optional AuthData response = 2; - optional int32 protocol_version = 3 [default = 0]; + optional string client_version = 1; + optional AuthData response = 2; + optional int32 protocol_version = 3 [default = 0]; } message CommandAuthChallenge { - optional string server_version = 1; - optional AuthData challenge = 2; - optional int32 protocol_version = 3 [default = 0]; + optional string server_version = 1; + optional AuthData challenge = 2; + optional int32 protocol_version = 3 [default = 0]; } // To support mutual authentication type, such as Sasl, reuse this command to mutual auth. message AuthData { - optional string auth_method_name = 1; - optional bytes auth_data = 2; + optional string auth_method_name = 1; + optional bytes auth_data = 2; } enum KeySharedMode { - AUTO_SPLIT = 0; - STICKY = 1; + AUTO_SPLIT = 0; + STICKY = 1; } message KeySharedMeta { - required KeySharedMode keySharedMode = 1; - repeated IntRange hashRanges = 3; - optional bool allowOutOfOrderDelivery = 4 [default = false]; + required KeySharedMode keySharedMode = 1; + repeated IntRange hashRanges = 3; + optional bool allowOutOfOrderDelivery = 4 [default = false]; } message CommandSubscribe { - enum SubType { - Exclusive = 0; - Shared = 1; - Failover = 2; - Key_Shared = 3; - } - required string topic = 1; - required string subscription = 2; - required SubType subType = 3; - - required uint64 consumer_id = 4; - required uint64 request_id = 5; - optional string consumer_name = 6; - optional int32 priority_level = 7; - - // Signal wether the subscription should be backed by a - // durable cursor or not - optional bool durable = 8 [default = true]; - - // If specified, the subscription will position the cursor - // markd-delete position on the particular message id and - // will send messages from that point - optional MessageIdData start_message_id = 9; - - /// Add optional metadata key=value to this consumer - repeated KeyValue metadata = 10; - - optional bool read_compacted = 11; - - optional Schema schema = 12; - enum InitialPosition { - Latest = 0; - Earliest = 1; - } - // Signal whether the subscription will initialize on latest - // or not -- earliest - optional InitialPosition initialPosition = 13 [default = Latest]; - - // Mark the subscription as "replicated". Pulsar will make sure - // to periodically sync the state of replicated subscriptions - // across different clusters (when using geo-replication). - optional bool replicate_subscription_state = 14; - - // If true, the subscribe operation will cause a topic to be - // created if it does not exist already (and if topic auto-creation - // is allowed by broker. - // If false, the subscribe operation will fail if the topic - // does not exist. - optional bool force_topic_creation = 15 [default = true]; - - // If specified, the subscription will reset cursor's position back - // to specified seconds and will send messages from that point - optional uint64 start_message_rollback_duration_sec = 16 [default = 0]; - - optional KeySharedMeta keySharedMeta = 17; - - repeated KeyValue subscription_properties = 18; - - // The consumer epoch, when exclusive and failover consumer redeliver unack message will increase the epoch - optional uint64 consumer_epoch = 19; + enum SubType { + Exclusive = 0; + Shared = 1; + Failover = 2; + Key_Shared = 3; + } + required string topic = 1; + required string subscription = 2; + required SubType subType = 3; + + required uint64 consumer_id = 4; + required uint64 request_id = 5; + optional string consumer_name = 6; + optional int32 priority_level = 7; + + // Signal wether the subscription should be backed by a + // durable cursor or not + optional bool durable = 8 [default = true]; + + // If specified, the subscription will position the cursor + // markd-delete position on the particular message id and + // will send messages from that point + optional MessageIdData start_message_id = 9; + + /// Add optional metadata key=value to this consumer + repeated KeyValue metadata = 10; + + optional bool read_compacted = 11; + + optional Schema schema = 12; + enum InitialPosition { + Latest = 0; + Earliest = 1; + } + // Signal whether the subscription will initialize on latest + // or not -- earliest + optional InitialPosition initialPosition = 13 [default = Latest]; + + // Mark the subscription as "replicated". Pulsar will make sure + // to periodically sync the state of replicated subscriptions + // across different clusters (when using geo-replication). + optional bool replicate_subscription_state = 14; + + // If true, the subscribe operation will cause a topic to be + // created if it does not exist already (and if topic auto-creation + // is allowed by broker. + // If false, the subscribe operation will fail if the topic + // does not exist. + optional bool force_topic_creation = 15 [default = true]; + + // If specified, the subscription will reset cursor's position back + // to specified seconds and will send messages from that point + optional uint64 start_message_rollback_duration_sec = 16 [default = 0]; + + optional KeySharedMeta keySharedMeta = 17; + + repeated KeyValue subscription_properties = 18; + + // The consumer epoch, when exclusive and failover consumer redeliver unack message will increase the epoch + optional uint64 consumer_epoch = 19; } message CommandPartitionedTopicMetadata { - required string topic = 1; - required uint64 request_id = 2; - // TODO - Remove original_principal, original_auth_data, original_auth_method - // Original principal that was verified by - // a Pulsar proxy. - optional string original_principal = 3; + required string topic = 1; + required uint64 request_id = 2; + // TODO - Remove original_principal, original_auth_data, original_auth_method + // Original principal that was verified by + // a Pulsar proxy. + optional string original_principal = 3; - // Original auth role and auth Method that was passed - // to the proxy. - optional string original_auth_data = 4; - optional string original_auth_method = 5; - optional bool metadata_auto_creation_enabled = 6 [default = true]; + // Original auth role and auth Method that was passed + // to the proxy. + optional string original_auth_data = 4; + optional string original_auth_method = 5; } message CommandPartitionedTopicMetadataResponse { - enum LookupType { - Success = 0; - Failed = 1; - } - optional uint32 partitions = 1; // Optional in case of error - required uint64 request_id = 2; - optional LookupType response = 3; - optional ServerError error = 4; - optional string message = 5; + enum LookupType { + Success = 0; + Failed = 1; + } + optional uint32 partitions = 1; // Optional in case of error + required uint64 request_id = 2; + optional LookupType response = 3; + optional ServerError error = 4; + optional string message = 5; } message CommandLookupTopic { - required string topic = 1; - required uint64 request_id = 2; - optional bool authoritative = 3 [default = false]; + required string topic = 1; + required uint64 request_id = 2; + optional bool authoritative = 3 [default = false]; - // TODO - Remove original_principal, original_auth_data, original_auth_method - // Original principal that was verified by - // a Pulsar proxy. - optional string original_principal = 4; + // TODO - Remove original_principal, original_auth_data, original_auth_method + // Original principal that was verified by + // a Pulsar proxy. + optional string original_principal = 4; - // Original auth role and auth Method that was passed - // to the proxy. - optional string original_auth_data = 5; - optional string original_auth_method = 6; - // - optional string advertised_listener_name = 7; - // The properties used for topic lookup - repeated KeyValue properties = 8; + // Original auth role and auth Method that was passed + // to the proxy. + optional string original_auth_data = 5; + optional string original_auth_method = 6; + // + optional string advertised_listener_name = 7; } message CommandLookupTopicResponse { - enum LookupType { - Redirect = 0; - Connect = 1; - Failed = 2; - } - - optional string brokerServiceUrl = 1; // Optional in case of error - optional string brokerServiceUrlTls = 2; - optional LookupType response = 3; - required uint64 request_id = 4; - optional bool authoritative = 5 [default = false]; - optional ServerError error = 6; - optional string message = 7; - - // If it's true, indicates to the client that it must - // always connect through the service url after the - // lookup has been completed. - optional bool proxy_through_service_url = 8 [default = false]; + enum LookupType { + Redirect = 0; + Connect = 1; + Failed = 2; + } + + optional string brokerServiceUrl = 1; // Optional in case of error + optional string brokerServiceUrlTls = 2; + optional LookupType response = 3; + required uint64 request_id = 4; + optional bool authoritative = 5 [default = false]; + optional ServerError error = 6; + optional string message = 7; + + // If it's true, indicates to the client that it must + // always connect through the service url after the + // lookup has been completed. + optional bool proxy_through_service_url = 8 [default = false]; } /// Create a new Producer on a topic, assigning the given producer_id, /// all messages sent with this producer_id will be persisted on the topic message CommandProducer { - required string topic = 1; - required uint64 producer_id = 2; - required uint64 request_id = 3; + required string topic = 1; + required uint64 producer_id = 2; + required uint64 request_id = 3; - /// If a producer name is specified, the name will be used, - /// otherwise the broker will generate a unique name - optional string producer_name = 4; + /// If a producer name is specified, the name will be used, + /// otherwise the broker will generate a unique name + optional string producer_name = 4; - optional bool encrypted = 5 [default = false]; + optional bool encrypted = 5 [default = false]; - /// Add optional metadata key=value to this producer - repeated KeyValue metadata = 6; + /// Add optional metadata key=value to this producer + repeated KeyValue metadata = 6; - optional Schema schema = 7; + optional Schema schema = 7; - // If producer reconnect to broker, the epoch of this producer will +1 - optional uint64 epoch = 8 [default = 0]; + // If producer reconnect to broker, the epoch of this producer will +1 + optional uint64 epoch = 8 [default = 0]; - // Indicate the name of the producer is generated or user provided - // Use default true here is in order to be forward compatible with the client - optional bool user_provided_producer_name = 9 [default = true]; + // Indicate the name of the producer is generated or user provided + // Use default true here is in order to be forward compatible with the client + optional bool user_provided_producer_name = 9 [default = true]; - // Require that this producers will be the only producer allowed on the topic - optional ProducerAccessMode producer_access_mode = 10 [default = Shared]; + // Require that this producers will be the only producer allowed on the topic + optional ProducerAccessMode producer_access_mode = 10 [default = Shared]; - // Topic epoch is used to fence off producers that reconnects after a new - // exclusive producer has already taken over. This id is assigned by the - // broker on the CommandProducerSuccess. The first time, the client will - // leave it empty and then it will always carry the same epoch number on - // the subsequent reconnections. - optional uint64 topic_epoch = 11; + // Topic epoch is used to fence off producers that reconnects after a new + // exclusive producer has already taken over. This id is assigned by the + // broker on the CommandProducerSuccess. The first time, the client will + // leave it empty and then it will always carry the same epoch number on + // the subsequent reconnections. + optional uint64 topic_epoch = 11; - optional bool txn_enabled = 12 [default = false]; + optional bool txn_enabled = 12 [default = false]; - // Name of the initial subscription of the topic. - // If this field is not set, the initial subscription will not be created. - // If this field is set but the broker's `allowAutoSubscriptionCreation` - // is disabled, the producer will fail to be created. - optional string initial_subscription_name = 13; + // Name of the initial subscription of the topic. + // If this field is not set, the initial subscription will not be created. + // If this field is set but the broker's `allowAutoSubscriptionCreation` + // is disabled, the producer will fail to be created. + optional string initial_subscription_name = 13; } message CommandSend { - required uint64 producer_id = 1; - required uint64 sequence_id = 2; - optional int32 num_messages = 3 [default = 1]; - optional uint64 txnid_least_bits = 4 [default = 0]; - optional uint64 txnid_most_bits = 5 [default = 0]; - - /// Add highest sequence id to support batch message with external sequence id - optional uint64 highest_sequence_id = 6 [default = 0]; - optional bool is_chunk =7 [default = false]; + required uint64 producer_id = 1; + required uint64 sequence_id = 2; + optional int32 num_messages = 3 [default = 1]; + optional uint64 txnid_least_bits = 4 [default = 0]; + optional uint64 txnid_most_bits = 5 [default = 0]; - // Specify if the message being published is a Pulsar marker or not - optional bool marker = 8 [default = false]; + /// Add highest sequence id to support batch message with external sequence id + optional uint64 highest_sequence_id = 6 [default = 0]; + optional bool is_chunk = 7 [default = false]; - // Message id of this message, currently is used in replicator for shadow topic. - optional MessageIdData message_id = 9; + // Specify if the message being published is a Pulsar marker or not + optional bool marker = 8 [default = false]; } message CommandSendReceipt { - required uint64 producer_id = 1; - required uint64 sequence_id = 2; - optional MessageIdData message_id = 3; - optional uint64 highest_sequence_id = 4 [default = 0]; + required uint64 producer_id = 1; + required uint64 sequence_id = 2; + optional MessageIdData message_id = 3; + optional uint64 highest_sequence_id = 4 [default = 0]; } message CommandSendError { - required uint64 producer_id = 1; - required uint64 sequence_id = 2; - required ServerError error = 3; - required string message = 4; + required uint64 producer_id = 1; + required uint64 sequence_id = 2; + required ServerError error = 3; + required string message = 4; } message CommandMessage { - required uint64 consumer_id = 1; - required MessageIdData message_id = 2; - optional uint32 redelivery_count = 3 [default = 0]; - repeated int64 ack_set = 4; - optional uint64 consumer_epoch = 5; + required uint64 consumer_id = 1; + required MessageIdData message_id = 2; + optional uint32 redelivery_count = 3 [default = 0]; + repeated int64 ack_set = 4; + optional uint64 consumer_epoch = 5; } message CommandAck { - enum AckType { - Individual = 0; - Cumulative = 1; - } + enum AckType { + Individual = 0; + Cumulative = 1; + } - required uint64 consumer_id = 1; - required AckType ack_type = 2; + required uint64 consumer_id = 1; + required AckType ack_type = 2; - // In case of individual acks, the client can pass a list of message ids - repeated MessageIdData message_id = 3; + // In case of individual acks, the client can pass a list of message ids + repeated MessageIdData message_id = 3; - // Acks can contain a flag to indicate the consumer - // received an invalid message that got discarded - // before being passed on to the application. - enum ValidationError { - UncompressedSizeCorruption = 0; - DecompressionError = 1; - ChecksumMismatch = 2; - BatchDeSerializeError = 3; - DecryptionError = 4; - } + // Acks can contain a flag to indicate the consumer + // received an invalid message that got discarded + // before being passed on to the application. + enum ValidationError { + UncompressedSizeCorruption = 0; + DecompressionError = 1; + ChecksumMismatch = 2; + BatchDeSerializeError = 3; + DecryptionError = 4; + } - optional ValidationError validation_error = 4; - repeated KeyLongValue properties = 5; + optional ValidationError validation_error = 4; + repeated KeyLongValue properties = 5; - optional uint64 txnid_least_bits = 6 [default = 0]; - optional uint64 txnid_most_bits = 7 [default = 0]; - optional uint64 request_id = 8; + optional uint64 txnid_least_bits = 6 [default = 0]; + optional uint64 txnid_most_bits = 7 [default = 0]; + optional uint64 request_id = 8; } message CommandAckResponse { - required uint64 consumer_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; - optional uint64 request_id = 6; + required uint64 consumer_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; + optional uint64 request_id = 6; } // changes on active consumer message CommandActiveConsumerChange { - required uint64 consumer_id = 1; - optional bool is_active = 2 [default = false]; + required uint64 consumer_id = 1; + optional bool is_active = 2 [default = false]; } message CommandFlow { - required uint64 consumer_id = 1; + required uint64 consumer_id = 1; - // Max number of messages to prefetch, in addition - // of any number previously specified - required uint32 messagePermits = 2; + // Max number of messages to prefetch, in addition + // of any number previously specified + required uint32 messagePermits = 2; } message CommandUnsubscribe { - required uint64 consumer_id = 1; - required uint64 request_id = 2; - optional bool force = 3 [default = false]; + required uint64 consumer_id = 1; + required uint64 request_id = 2; } // Reset an existing consumer to a particular message id message CommandSeek { - required uint64 consumer_id = 1; - required uint64 request_id = 2; + required uint64 consumer_id = 1; + required uint64 request_id = 2; - optional MessageIdData message_id = 3; - optional uint64 message_publish_time = 4; + optional MessageIdData message_id = 3; + optional uint64 message_publish_time = 4; } // Message sent by broker to client when a topic // has been forcefully terminated and there are no more // messages left to consume message CommandReachedEndOfTopic { - required uint64 consumer_id = 1; -} - -message CommandTopicMigrated { - enum ResourceType { - Producer = 0; - Consumer = 1; - } - required uint64 resource_id = 1; - required ResourceType resource_type = 2; - optional string brokerServiceUrl = 3; - optional string brokerServiceUrlTls = 4; - + required uint64 consumer_id = 1; } - message CommandCloseProducer { - required uint64 producer_id = 1; - required uint64 request_id = 2; - optional string assignedBrokerServiceUrl = 3; - optional string assignedBrokerServiceUrlTls = 4; + required uint64 producer_id = 1; + required uint64 request_id = 2; } message CommandCloseConsumer { - required uint64 consumer_id = 1; - required uint64 request_id = 2; - optional string assignedBrokerServiceUrl = 3; - optional string assignedBrokerServiceUrlTls = 4; + required uint64 consumer_id = 1; + required uint64 request_id = 2; } message CommandRedeliverUnacknowledgedMessages { - required uint64 consumer_id = 1; - repeated MessageIdData message_ids = 2; - optional uint64 consumer_epoch = 3; + required uint64 consumer_id = 1; + repeated MessageIdData message_ids = 2; + optional uint64 consumer_epoch = 3; } message CommandSuccess { - required uint64 request_id = 1; - optional Schema schema = 2; + required uint64 request_id = 1; + optional Schema schema = 2; } /// Response from CommandProducer message CommandProducerSuccess { - required uint64 request_id = 1; - required string producer_name = 2; + required uint64 request_id = 1; + required string producer_name = 2; - // The last sequence id that was stored by this producer in the previous session - // This will only be meaningful if deduplication has been enabled. - optional int64 last_sequence_id = 3 [default = -1]; - optional bytes schema_version = 4; + // The last sequence id that was stored by this producer in the previous session + // This will only be meaningful if deduplication has been enabled. + optional int64 last_sequence_id = 3 [default = -1]; + optional bytes schema_version = 4; - // The topic epoch assigned by the broker. This field will only be set if we - // were requiring exclusive access when creating the producer. - optional uint64 topic_epoch = 5; + // The topic epoch assigned by the broker. This field will only be set if we + // were requiring exclusive access when creating the producer. + optional uint64 topic_epoch = 5; - // If producer is not "ready", the client will avoid to timeout the request - // for creating the producer. Instead it will wait indefinitely until it gets - // a subsequent `CommandProducerSuccess` with `producer_ready==true`. - optional bool producer_ready = 6 [default = true]; + // If producer is not "ready", the client will avoid to timeout the request + // for creating the producer. Instead it will wait indefinitely until it gets + // a subsequent `CommandProducerSuccess` with `producer_ready==true`. + optional bool producer_ready = 6 [default = true]; } message CommandError { - required uint64 request_id = 1; - required ServerError error = 2; - required string message = 3; + required uint64 request_id = 1; + required ServerError error = 2; + required string message = 3; } // Commands to probe the state of connection. @@ -718,439 +673,396 @@ message CommandPong { } message CommandConsumerStats { - required uint64 request_id = 1; - // required string topic_name = 2; - // required string subscription_name = 3; - required uint64 consumer_id = 4; + required uint64 request_id = 1; + // required string topic_name = 2; + // required string subscription_name = 3; + required uint64 consumer_id = 4; } message CommandConsumerStatsResponse { - required uint64 request_id = 1; - optional ServerError error_code = 2; - optional string error_message = 3; + required uint64 request_id = 1; + optional ServerError error_code = 2; + optional string error_message = 3; - /// Total rate of messages delivered to the consumer. msg/s - optional double msgRateOut = 4; + /// Total rate of messages delivered to the consumer. msg/s + optional double msgRateOut = 4; - /// Total throughput delivered to the consumer. bytes/s - optional double msgThroughputOut = 5; + /// Total throughput delivered to the consumer. bytes/s + optional double msgThroughputOut = 5; - /// Total rate of messages redelivered by this consumer. msg/s - optional double msgRateRedeliver = 6; + /// Total rate of messages redelivered by this consumer. msg/s + optional double msgRateRedeliver = 6; - /// Name of the consumer - optional string consumerName = 7; + /// Name of the consumer + optional string consumerName = 7; - /// Number of available message permits for the consumer - optional uint64 availablePermits = 8; + /// Number of available message permits for the consumer + optional uint64 availablePermits = 8; - /// Number of unacknowledged messages for the consumer - optional uint64 unackedMessages = 9; + /// Number of unacknowledged messages for the consumer + optional uint64 unackedMessages = 9; - /// Flag to verify if consumer is blocked due to reaching threshold of unacked messages - optional bool blockedConsumerOnUnackedMsgs = 10; + /// Flag to verify if consumer is blocked due to reaching threshold of unacked messages + optional bool blockedConsumerOnUnackedMsgs = 10; - /// Address of this consumer - optional string address = 11; + /// Address of this consumer + optional string address = 11; - /// Timestamp of connection - optional string connectedSince = 12; + /// Timestamp of connection + optional string connectedSince = 12; - /// Whether this subscription is Exclusive or Shared or Failover - optional string type = 13; + /// Whether this subscription is Exclusive or Shared or Failover + optional string type = 13; - /// Total rate of messages expired on this subscription. msg/s - optional double msgRateExpired = 14; + /// Total rate of messages expired on this subscription. msg/s + optional double msgRateExpired = 14; - /// Number of messages in the subscription backlog - optional uint64 msgBacklog = 15; + /// Number of messages in the subscription backlog + optional uint64 msgBacklog = 15; - /// Total rate of messages ack. msg/s - optional double messageAckRate = 16; + /// Total rate of messages ack. msg/s + optional double messageAckRate = 16; } message CommandGetLastMessageId { - required uint64 consumer_id = 1; - required uint64 request_id = 2; + required uint64 consumer_id = 1; + required uint64 request_id = 2; } message CommandGetLastMessageIdResponse { - required MessageIdData last_message_id = 1; - required uint64 request_id = 2; - optional MessageIdData consumer_mark_delete_position = 3; + required MessageIdData last_message_id = 1; + required uint64 request_id = 2; + optional MessageIdData consumer_mark_delete_position = 3; } message CommandGetTopicsOfNamespace { - enum Mode { - PERSISTENT = 0; - NON_PERSISTENT = 1; - ALL = 2; - } - required uint64 request_id = 1; - required string namespace = 2; - optional Mode mode = 3 [default = PERSISTENT]; - optional string topics_pattern = 4; - optional string topics_hash = 5; - // Context properties from the client - repeated KeyValue properties = 6; + enum Mode { + PERSISTENT = 0; + NON_PERSISTENT = 1; + ALL = 2; + } + required uint64 request_id = 1; + required string namespace = 2; + optional Mode mode = 3 [default = PERSISTENT]; + optional string topics_pattern = 4; + optional string topics_hash = 5; } message CommandGetTopicsOfNamespaceResponse { - required uint64 request_id = 1; - repeated string topics = 2; - // true iff the topic list was filtered by the pattern supplied by the client - optional bool filtered = 3 [default = false]; - // hash computed from the names of matching topics - optional string topics_hash = 4; - // if false, topics is empty and the list of matching topics has not changed - optional bool changed = 5 [default = true]; -} - -message CommandWatchTopicList { - required uint64 request_id = 1; - required uint64 watcher_id = 2; - required string namespace = 3; - required string topics_pattern = 4; - // Only present when the client reconnects: - optional string topics_hash = 5; -} - -message CommandWatchTopicListSuccess { - required uint64 request_id = 1; - required uint64 watcher_id = 2; - repeated string topic = 3; - required string topics_hash = 4; -} - -message CommandWatchTopicUpdate { - required uint64 watcher_id = 1; - repeated string new_topics = 2; - repeated string deleted_topics = 3; - required string topics_hash = 4; -} - -message CommandWatchTopicListClose { - required uint64 request_id = 1; - required uint64 watcher_id = 2; + required uint64 request_id = 1; + repeated string topics = 2; + // true iff the topic list was filtered by the pattern supplied by the client + optional bool filtered = 3 [default = false]; + // hash computed from the names of matching topics + optional string topics_hash = 4; + // if false, topics is empty and the list of matching topics has not changed + optional bool changed = 5 [default = true]; } message CommandGetSchema { - required uint64 request_id = 1; - required string topic = 2; + required uint64 request_id = 1; + required string topic = 2; - optional bytes schema_version = 3; + optional bytes schema_version = 3; } message CommandGetSchemaResponse { - required uint64 request_id = 1; - optional ServerError error_code = 2; - optional string error_message = 3; + required uint64 request_id = 1; + optional ServerError error_code = 2; + optional string error_message = 3; - optional Schema schema = 4; - optional bytes schema_version = 5; + optional Schema schema = 4; + optional bytes schema_version = 5; } message CommandGetOrCreateSchema { - required uint64 request_id = 1; - required string topic = 2; - required Schema schema = 3; + required uint64 request_id = 1; + required string topic = 2; + required Schema schema = 3; } message CommandGetOrCreateSchemaResponse { - required uint64 request_id = 1; - optional ServerError error_code = 2; - optional string error_message = 3; + required uint64 request_id = 1; + optional ServerError error_code = 2; + optional string error_message = 3; - optional bytes schema_version = 4; + optional bytes schema_version = 4; } /// --- transaction related --- enum TxnAction { - COMMIT = 0; - ABORT = 1; + COMMIT = 0; + ABORT = 1; } message CommandTcClientConnectRequest { - required uint64 request_id = 1; - required uint64 tc_id = 2 [default = 0]; + required uint64 request_id = 1; + required uint64 tc_id = 2 [default = 0]; } message CommandTcClientConnectResponse { - required uint64 request_id = 1; - optional ServerError error = 2; - optional string message = 3; + required uint64 request_id = 1; + optional ServerError error = 2; + optional string message = 3; } message CommandNewTxn { - required uint64 request_id = 1; - optional uint64 txn_ttl_seconds = 2 [default = 0]; - optional uint64 tc_id = 3 [default = 0]; + required uint64 request_id = 1; + optional uint64 txn_ttl_seconds = 2 [default = 0]; + optional uint64 tc_id = 3 [default = 0]; } message CommandNewTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandAddPartitionToTxn { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - repeated string partitions = 4; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + repeated string partitions = 4; } message CommandAddPartitionToTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message Subscription { - required string topic = 1; - required string subscription = 2; + required string topic = 1; + required string subscription = 2; } message CommandAddSubscriptionToTxn { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - repeated Subscription subscription = 4; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + repeated Subscription subscription = 4; } message CommandAddSubscriptionToTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandEndTxn { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional TxnAction txn_action = 4; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional TxnAction txn_action = 4; } message CommandEndTxnResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandEndTxnOnPartition { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional string topic = 4; - optional TxnAction txn_action = 5; - optional uint64 txnid_least_bits_of_low_watermark = 6; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional string topic = 4; + optional TxnAction txn_action = 5; + optional uint64 txnid_least_bits_of_low_watermark = 6; } message CommandEndTxnOnPartitionResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message CommandEndTxnOnSubscription { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional Subscription subscription= 4; - optional TxnAction txn_action = 5; - optional uint64 txnid_least_bits_of_low_watermark = 6; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional Subscription subscription = 4; + optional TxnAction txn_action = 5; + optional uint64 txnid_least_bits_of_low_watermark = 6; } message CommandEndTxnOnSubscriptionResponse { - required uint64 request_id = 1; - optional uint64 txnid_least_bits = 2 [default = 0]; - optional uint64 txnid_most_bits = 3 [default = 0]; - optional ServerError error = 4; - optional string message = 5; + required uint64 request_id = 1; + optional uint64 txnid_least_bits = 2 [default = 0]; + optional uint64 txnid_most_bits = 3 [default = 0]; + optional ServerError error = 4; + optional string message = 5; } message BaseCommand { - enum Type { - CONNECT = 2; - CONNECTED = 3; - SUBSCRIBE = 4; - - PRODUCER = 5; + enum Type { + CONNECT = 2; + CONNECTED = 3; + SUBSCRIBE = 4; - SEND = 6; - SEND_RECEIPT= 7; - SEND_ERROR = 8; + PRODUCER = 5; - MESSAGE = 9; - ACK = 10; - FLOW = 11; + SEND = 6; + SEND_RECEIPT = 7; + SEND_ERROR = 8; - UNSUBSCRIBE = 12; + MESSAGE = 9; + ACK = 10; + FLOW = 11; - SUCCESS = 13; - ERROR = 14; + UNSUBSCRIBE = 12; - CLOSE_PRODUCER = 15; - CLOSE_CONSUMER = 16; + SUCCESS = 13; + ERROR = 14; - PRODUCER_SUCCESS = 17; + CLOSE_PRODUCER = 15; + CLOSE_CONSUMER = 16; - PING = 18; - PONG = 19; + PRODUCER_SUCCESS = 17; - REDELIVER_UNACKNOWLEDGED_MESSAGES = 20; + PING = 18; + PONG = 19; - PARTITIONED_METADATA = 21; - PARTITIONED_METADATA_RESPONSE = 22; + REDELIVER_UNACKNOWLEDGED_MESSAGES = 20; - LOOKUP = 23; - LOOKUP_RESPONSE = 24; + PARTITIONED_METADATA = 21; + PARTITIONED_METADATA_RESPONSE = 22; - CONSUMER_STATS = 25; - CONSUMER_STATS_RESPONSE = 26; + LOOKUP = 23; + LOOKUP_RESPONSE = 24; - REACHED_END_OF_TOPIC = 27; + CONSUMER_STATS = 25; + CONSUMER_STATS_RESPONSE = 26; - SEEK = 28; + REACHED_END_OF_TOPIC = 27; - GET_LAST_MESSAGE_ID = 29; - GET_LAST_MESSAGE_ID_RESPONSE = 30; + SEEK = 28; - ACTIVE_CONSUMER_CHANGE = 31; + GET_LAST_MESSAGE_ID = 29; + GET_LAST_MESSAGE_ID_RESPONSE = 30; + ACTIVE_CONSUMER_CHANGE = 31; - GET_TOPICS_OF_NAMESPACE = 32; - GET_TOPICS_OF_NAMESPACE_RESPONSE = 33; - GET_SCHEMA = 34; - GET_SCHEMA_RESPONSE = 35; + GET_TOPICS_OF_NAMESPACE = 32; + GET_TOPICS_OF_NAMESPACE_RESPONSE = 33; - AUTH_CHALLENGE = 36; - AUTH_RESPONSE = 37; + GET_SCHEMA = 34; + GET_SCHEMA_RESPONSE = 35; - ACK_RESPONSE = 38; + AUTH_CHALLENGE = 36; + AUTH_RESPONSE = 37; - GET_OR_CREATE_SCHEMA = 39; - GET_OR_CREATE_SCHEMA_RESPONSE = 40; + ACK_RESPONSE = 38; - // transaction related - NEW_TXN = 50; - NEW_TXN_RESPONSE = 51; + GET_OR_CREATE_SCHEMA = 39; + GET_OR_CREATE_SCHEMA_RESPONSE = 40; - ADD_PARTITION_TO_TXN = 52; - ADD_PARTITION_TO_TXN_RESPONSE = 53; + // transaction related + NEW_TXN = 50; + NEW_TXN_RESPONSE = 51; - ADD_SUBSCRIPTION_TO_TXN = 54; - ADD_SUBSCRIPTION_TO_TXN_RESPONSE = 55; + ADD_PARTITION_TO_TXN = 52; + ADD_PARTITION_TO_TXN_RESPONSE = 53; - END_TXN = 56; - END_TXN_RESPONSE = 57; + ADD_SUBSCRIPTION_TO_TXN = 54; + ADD_SUBSCRIPTION_TO_TXN_RESPONSE = 55; - END_TXN_ON_PARTITION = 58; - END_TXN_ON_PARTITION_RESPONSE = 59; + END_TXN = 56; + END_TXN_RESPONSE = 57; - END_TXN_ON_SUBSCRIPTION = 60; - END_TXN_ON_SUBSCRIPTION_RESPONSE = 61; - TC_CLIENT_CONNECT_REQUEST = 62; - TC_CLIENT_CONNECT_RESPONSE = 63; + END_TXN_ON_PARTITION = 58; + END_TXN_ON_PARTITION_RESPONSE = 59; - WATCH_TOPIC_LIST = 64; - WATCH_TOPIC_LIST_SUCCESS = 65; - WATCH_TOPIC_UPDATE = 66; - WATCH_TOPIC_LIST_CLOSE = 67; + END_TXN_ON_SUBSCRIPTION = 60; + END_TXN_ON_SUBSCRIPTION_RESPONSE = 61; + TC_CLIENT_CONNECT_REQUEST = 62; + TC_CLIENT_CONNECT_RESPONSE = 63; - TOPIC_MIGRATED = 68; - } + } - required Type type = 1; + required Type type = 1; - optional CommandConnect connect = 2; - optional CommandConnected connected = 3; + optional CommandConnect connect = 2; + optional CommandConnected connected = 3; - optional CommandSubscribe subscribe = 4; - optional CommandProducer producer = 5; - optional CommandSend send = 6; - optional CommandSendReceipt send_receipt = 7; - optional CommandSendError send_error = 8; - optional CommandMessage message = 9; - optional CommandAck ack = 10; - optional CommandFlow flow = 11; - optional CommandUnsubscribe unsubscribe = 12; + optional CommandSubscribe subscribe = 4; + optional CommandProducer producer = 5; + optional CommandSend send = 6; + optional CommandSendReceipt send_receipt = 7; + optional CommandSendError send_error = 8; + optional CommandMessage message = 9; + optional CommandAck ack = 10; + optional CommandFlow flow = 11; + optional CommandUnsubscribe unsubscribe = 12; - optional CommandSuccess success = 13; - optional CommandError error = 14; + optional CommandSuccess success = 13; + optional CommandError error = 14; - optional CommandCloseProducer close_producer = 15; - optional CommandCloseConsumer close_consumer = 16; + optional CommandCloseProducer close_producer = 15; + optional CommandCloseConsumer close_consumer = 16; - optional CommandProducerSuccess producer_success = 17; - optional CommandPing ping = 18; - optional CommandPong pong = 19; - optional CommandRedeliverUnacknowledgedMessages redeliverUnacknowledgedMessages = 20; + optional CommandProducerSuccess producer_success = 17; + optional CommandPing ping = 18; + optional CommandPong pong = 19; + optional CommandRedeliverUnacknowledgedMessages redeliverUnacknowledgedMessages = 20; - optional CommandPartitionedTopicMetadata partitionMetadata = 21; - optional CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; + optional CommandPartitionedTopicMetadata partitionMetadata = 21; + optional CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; - optional CommandLookupTopic lookupTopic = 23; - optional CommandLookupTopicResponse lookupTopicResponse = 24; + optional CommandLookupTopic lookupTopic = 23; + optional CommandLookupTopicResponse lookupTopicResponse = 24; - optional CommandConsumerStats consumerStats = 25; - optional CommandConsumerStatsResponse consumerStatsResponse = 26; + optional CommandConsumerStats consumerStats = 25; + optional CommandConsumerStatsResponse consumerStatsResponse = 26; - optional CommandReachedEndOfTopic reachedEndOfTopic = 27; + optional CommandReachedEndOfTopic reachedEndOfTopic = 27; - optional CommandSeek seek = 28; + optional CommandSeek seek = 28; - optional CommandGetLastMessageId getLastMessageId = 29; - optional CommandGetLastMessageIdResponse getLastMessageIdResponse = 30; + optional CommandGetLastMessageId getLastMessageId = 29; + optional CommandGetLastMessageIdResponse getLastMessageIdResponse = 30; - optional CommandActiveConsumerChange active_consumer_change = 31; + optional CommandActiveConsumerChange active_consumer_change = 31; - optional CommandGetTopicsOfNamespace getTopicsOfNamespace = 32; - optional CommandGetTopicsOfNamespaceResponse getTopicsOfNamespaceResponse = 33; + optional CommandGetTopicsOfNamespace getTopicsOfNamespace = 32; + optional CommandGetTopicsOfNamespaceResponse getTopicsOfNamespaceResponse = 33; - optional CommandGetSchema getSchema = 34; - optional CommandGetSchemaResponse getSchemaResponse = 35; + optional CommandGetSchema getSchema = 34; + optional CommandGetSchemaResponse getSchemaResponse = 35; - optional CommandAuthChallenge authChallenge = 36; - optional CommandAuthResponse authResponse = 37; + optional CommandAuthChallenge authChallenge = 36; + optional CommandAuthResponse authResponse = 37; - optional CommandAckResponse ackResponse = 38; + optional CommandAckResponse ackResponse = 38; - optional CommandGetOrCreateSchema getOrCreateSchema = 39; - optional CommandGetOrCreateSchemaResponse getOrCreateSchemaResponse = 40; + optional CommandGetOrCreateSchema getOrCreateSchema = 39; + optional CommandGetOrCreateSchemaResponse getOrCreateSchemaResponse = 40; - // transaction related - optional CommandNewTxn newTxn = 50; - optional CommandNewTxnResponse newTxnResponse = 51; - optional CommandAddPartitionToTxn addPartitionToTxn= 52; - optional CommandAddPartitionToTxnResponse addPartitionToTxnResponse = 53; - optional CommandAddSubscriptionToTxn addSubscriptionToTxn = 54; - optional CommandAddSubscriptionToTxnResponse addSubscriptionToTxnResponse = 55; - optional CommandEndTxn endTxn = 56; - optional CommandEndTxnResponse endTxnResponse = 57; - optional CommandEndTxnOnPartition endTxnOnPartition = 58; - optional CommandEndTxnOnPartitionResponse endTxnOnPartitionResponse = 59; - optional CommandEndTxnOnSubscription endTxnOnSubscription = 60; - optional CommandEndTxnOnSubscriptionResponse endTxnOnSubscriptionResponse = 61; - optional CommandTcClientConnectRequest tcClientConnectRequest = 62; - optional CommandTcClientConnectResponse tcClientConnectResponse = 63; - - optional CommandWatchTopicList watchTopicList = 64; - optional CommandWatchTopicListSuccess watchTopicListSuccess = 65; - optional CommandWatchTopicUpdate watchTopicUpdate = 66; - optional CommandWatchTopicListClose watchTopicListClose = 67; - - optional CommandTopicMigrated topicMigrated = 68; + // transaction related + optional CommandNewTxn newTxn = 50; + optional CommandNewTxnResponse newTxnResponse = 51; + optional CommandAddPartitionToTxn addPartitionToTxn = 52; + optional CommandAddPartitionToTxnResponse addPartitionToTxnResponse = 53; + optional CommandAddSubscriptionToTxn addSubscriptionToTxn = 54; + optional CommandAddSubscriptionToTxnResponse addSubscriptionToTxnResponse = 55; + optional CommandEndTxn endTxn = 56; + optional CommandEndTxnResponse endTxnResponse = 57; + optional CommandEndTxnOnPartition endTxnOnPartition = 58; + optional CommandEndTxnOnPartitionResponse endTxnOnPartitionResponse = 59; + optional CommandEndTxnOnSubscription endTxnOnSubscription = 60; + optional CommandEndTxnOnSubscriptionResponse endTxnOnSubscriptionResponse = 61; + optional CommandTcClientConnectRequest tcClientConnectRequest = 62; + optional CommandTcClientConnectResponse tcClientConnectResponse = 63; } From ee64fb236e99c94020612dcc50e3c98c41affef5 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Fri, 13 Mar 2026 12:19:44 -0700 Subject: [PATCH 23/24] fix: remove needless ..Default::default() after proto revert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After minimizing PulsarApi.proto, CommandCloseProducer, CommandCloseConsumer, and CommandUnsubscribe have only the fields already specified — the struct update syntax is redundant. Rust 1.94.0 clippy flags this as an error. Co-Authored-By: Claude Opus 4.6 --- src/connection.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/connection.rs b/src/connection.rs index 00bdaa4d..2b34b8fe 100644 --- a/src/connection.rs +++ b/src/connection.rs @@ -1634,7 +1634,6 @@ pub(crate) mod messages { close_producer: Some(proto::CommandCloseProducer { producer_id, request_id, - ..Default::default() }), ..Default::default() }, @@ -1755,7 +1754,6 @@ pub(crate) mod messages { close_consumer: Some(proto::CommandCloseConsumer { consumer_id, request_id, - ..Default::default() }), ..Default::default() }, @@ -1793,7 +1791,6 @@ pub(crate) mod messages { unsubscribe: Some(proto::CommandUnsubscribe { consumer_id, request_id, - ..Default::default() }), ..Default::default() }, From 81435638a7790fa16c42dd3a93c7c26f9eb9e07d Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Mon, 16 Mar 2026 12:27:07 -0700 Subject: [PATCH 24/24] fix: revert unrelated changes (clippy lints, doc cosmetics) - Revert is_multiple_of() back to % operator in examples - Restore original doc comment style in consumer/message.rs - Remove unnecessary .. on CommandCloseConsumer destructure Co-Authored-By: Claude Opus 4.6 --- examples/batching.rs | 4 ++-- examples/round_trip.rs | 4 ++-- src/consumer/message.rs | 19 +++++++++---------- src/message.rs | 1 - 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/examples/batching.rs b/examples/batching.rs index 354607fd..22e89409 100644 --- a/examples/batching.rs +++ b/examples/batching.rs @@ -70,7 +70,7 @@ async fn main() -> Result<(), pulsar::Error> { v.push(receipt_rx); println!("sent"); counter += 1; - if counter.is_multiple_of(4) { + if counter % 4 == 0 { //producer.send_batch().await.unwrap(); println!("sent {counter} messages"); break; @@ -98,7 +98,7 @@ async fn main() -> Result<(), pulsar::Error> { } println!("got message: {data:?}"); counter += 1; - if counter.is_multiple_of(4) { + if counter % 4 == 0 { println!("sent {counter} messages"); break; } diff --git a/examples/round_trip.rs b/examples/round_trip.rs index fb0abf91..9867653f 100644 --- a/examples/round_trip.rs +++ b/examples/round_trip.rs @@ -60,7 +60,7 @@ async fn main() -> Result<(), pulsar::Error> { .await .unwrap(); counter += 1; - if counter.is_multiple_of(1000) { + if counter % 1000 == 0 { println!("sent {counter} messages"); } } @@ -86,7 +86,7 @@ async fn main() -> Result<(), pulsar::Error> { panic!("Unexpected payload: {}", &data.data); } counter += 1; - if counter.is_multiple_of(1000) { + if counter % 1000 == 0 { println!("received {counter} messages"); } } diff --git a/src/consumer/message.rs b/src/consumer/message.rs index 687d70a6..5dbeb3d1 100644 --- a/src/consumer/message.rs +++ b/src/consumer/message.rs @@ -4,15 +4,15 @@ use crate::{ DeserializeMessage, Payload, }; -/// A message received by a consumer. +/// a message received by a consumer /// -/// Generic over the type it can be deserialized to. +/// it is generic over the type it can be deserialized to pub struct Message { - /// Origin topic of the message. + /// origin topic of the message pub topic: String, - /// Contains the message's data and other metadata. + /// contains the message's data and other metadata pub payload: Payload, - /// Contains the message's id and batch size data. + /// contains the message's id and batch size data pub message_id: MessageData, /// Pre-decoded value from PulsarSchema. None when using DeserializeMessage path /// or when schema decode failed (check [`decode_error`](Self::decode_error)). @@ -36,19 +36,19 @@ impl std::fmt::Debug for Message { } impl Message { - /// Pulsar metadata for the message. + /// Pulsar metadata for the message #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn metadata(&self) -> &MessageMetadata { &self.payload.metadata } - /// Get Pulsar message id for the message. + /// Get Pulsar message id for the message #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn message_id(&self) -> &MessageIdData { &self.message_id.id } - /// Get message key (partition key). + /// Get message key (partition key) #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn key(&self) -> Option { self.payload.metadata.partition_key.clone() @@ -72,8 +72,7 @@ impl Message { } impl Message { - /// Directly deserialize a message using the DeserializeMessage trait. - /// This continues to work unchanged for backward compatibility. + /// directly deserialize a message #[cfg_attr(feature = "telemetry", tracing::instrument(skip_all))] pub fn deserialize(&self) -> T::Output { T::deserialize_message(&self.payload) diff --git a/src/message.rs b/src/message.rs index 3d15c881..a302ec28 100644 --- a/src/message.rs +++ b/src/message.rs @@ -178,7 +178,6 @@ impl Message { Some(CommandCloseConsumer { consumer_id, request_id, - .. }), .. } => Some(RequestKey::CloseConsumer {