|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | +package org.apache.pulsar.broker.service.schema; |
| 20 | + |
| 21 | +import static org.assertj.core.api.Assertions.assertThat; |
| 22 | +import java.util.HashSet; |
| 23 | +import java.util.List; |
| 24 | +import java.util.Map; |
| 25 | +import org.apache.avro.util.ClassSecurityValidator; |
| 26 | +import org.apache.avro.util.ClassSecurityValidator.ClassSecurityPredicate; |
| 27 | +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; |
| 28 | +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; |
| 29 | +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; |
| 30 | +import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; |
| 31 | +import org.apache.pulsar.broker.transaction.buffer.metadata.AbortTxnMetadata; |
| 32 | +import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; |
| 33 | +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; |
| 34 | +import org.apache.pulsar.client.api.Schema; |
| 35 | +import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; |
| 36 | +import org.apache.pulsar.common.events.ActionType; |
| 37 | +import org.apache.pulsar.common.events.EventType; |
| 38 | +import org.apache.pulsar.common.events.PulsarEvent; |
| 39 | +import org.apache.pulsar.common.events.TopicPoliciesEvent; |
| 40 | +import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; |
| 41 | +import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; |
| 42 | +import org.apache.pulsar.common.policies.data.PersistencePolicies; |
| 43 | +import org.apache.pulsar.common.policies.data.PublishRate; |
| 44 | +import org.apache.pulsar.common.policies.data.RetentionPolicies; |
| 45 | +import org.apache.pulsar.common.policies.data.SubscribeRate; |
| 46 | +import org.apache.pulsar.common.policies.data.TopicPolicies; |
| 47 | +import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; |
| 48 | +import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; |
| 49 | +import org.apache.pulsar.metadata.api.MetadataEvent; |
| 50 | +import org.apache.pulsar.metadata.api.NotificationType; |
| 51 | +import org.apache.pulsar.metadata.api.extended.CreateOption; |
| 52 | +import org.testng.annotations.AfterMethod; |
| 53 | +import org.testng.annotations.BeforeMethod; |
| 54 | +import org.testng.annotations.Test; |
| 55 | + |
| 56 | +/** |
| 57 | + * Verifies that the broker can serialize every type it writes to its system topics with nothing declared |
| 58 | + * in advance — building the schema from the class is what trusts it, exactly as for an application POJO. |
| 59 | + * |
| 60 | + * <p>Each test pins the global validator to Avro's hardcoded {@code DEFAULT_TRUSTED_CLASSES}, so that |
| 61 | + * anything not covered by building the schema fails here rather than being carried by trust some other |
| 62 | + * test happened to declare first. |
| 63 | + */ |
| 64 | +@Test(groups = "broker") |
| 65 | +public class PulsarInternalAvroTypesTrustTest { |
| 66 | + |
| 67 | + private ClassSecurityPredicate previousValidator; |
| 68 | + |
| 69 | + @BeforeMethod |
| 70 | + public void useProductionBaselineValidator() { |
| 71 | + previousValidator = ClassSecurityValidator.getGlobal(); |
| 72 | + // DEFAULT_TRUSTED_CLASSES excludes the system-properties predicate, unlike DEFAULT. |
| 73 | + ClassSecurityValidator.setGlobal(ClassSecurityValidator.DEFAULT_TRUSTED_CLASSES); |
| 74 | + } |
| 75 | + |
| 76 | + @AfterMethod(alwaysRun = true) |
| 77 | + public void restoreValidator() { |
| 78 | + ClassSecurityValidator.setGlobal(previousValidator); |
| 79 | + } |
| 80 | + |
| 81 | + @Test |
| 82 | + public void testMetadataEventRoundTrips() { |
| 83 | + // Published by PulsarMetadataEventSynchronizer. Exercises the HashSet "java-class" property and |
| 84 | + // the CreateOption / NotificationType enums. |
| 85 | + MetadataEvent event = new MetadataEvent("/path", "value".getBytes(), new HashSet<>(List.of( |
| 86 | + CreateOption.Ephemeral)), 1L, 2L, "cluster", NotificationType.Created); |
| 87 | + |
| 88 | + Schema<MetadataEvent> schema = Schema.AVRO(MetadataEvent.class); |
| 89 | + |
| 90 | + assertThat(schema.decode(schema.encode(event))).isEqualTo(event); |
| 91 | + } |
| 92 | + |
| 93 | + @Test |
| 94 | + public void testPulsarEventRoundTrips() { |
| 95 | + // Published to the __change_events system topic, and reaches TopicPolicies through |
| 96 | + // TopicPoliciesEvent. |
| 97 | + PulsarEvent event = PulsarEvent.builder() |
| 98 | + .eventType(EventType.TOPIC_POLICY) |
| 99 | + .actionType(ActionType.UPDATE) |
| 100 | + .replicateTo(new HashSet<>(List.of("cluster-a"))) |
| 101 | + .topicPoliciesEvent(TopicPoliciesEvent.builder() |
| 102 | + .domain("persistent") |
| 103 | + .tenant("public") |
| 104 | + .namespace("default") |
| 105 | + .topic("t1") |
| 106 | + // Populate the nested types too: Avro only resolves the class behind a schema |
| 107 | + // node when it actually writes it, so leaving these null would skip the enum |
| 108 | + // and the policy impl types entirely and hide a missing allow-list entry. |
| 109 | + .policies(TopicPolicies.builder() |
| 110 | + .subscriptionTypesEnabled(List.of(SubType.Shared, SubType.Key_Shared)) |
| 111 | + .replicationClusters(List.of("cluster-a")) |
| 112 | + .backLogQuotaMap(Map.of("destination_storage", |
| 113 | + (BacklogQuotaImpl) BacklogQuotaImpl.builder().limitSize(1024L).build())) |
| 114 | + .dispatchRate(DispatchRateImpl.builder().dispatchThrottlingRateInMsg(10).build()) |
| 115 | + .retentionPolicies(new RetentionPolicies(1, 2)) |
| 116 | + .persistence(new PersistencePolicies(1, 1, 1, 1.0)) |
| 117 | + .inactiveTopicPolicies(new InactiveTopicPolicies( |
| 118 | + InactiveTopicDeleteMode.delete_when_no_subscriptions, 60, true)) |
| 119 | + .publishRate(new PublishRate(10, 1024)) |
| 120 | + .subscribeRate(new SubscribeRate(10, 30)) |
| 121 | + .build()) |
| 122 | + .build()) |
| 123 | + .build(); |
| 124 | + |
| 125 | + Schema<PulsarEvent> schema = Schema.AVRO(PulsarEvent.class); |
| 126 | + |
| 127 | + assertThat(schema.decode(schema.encode(event))).isEqualTo(event); |
| 128 | + } |
| 129 | + |
| 130 | + @Test |
| 131 | + public void testTransactionBufferSnapshotRoundTrips() { |
| 132 | + // The original snapshot format lives in ...buffer.metadata, not ...buffer.metadata.v2, so it is |
| 133 | + // only covered if the parent package is trusted. |
| 134 | + TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); |
| 135 | + snapshot.setTopicName("persistent://public/default/t1"); |
| 136 | + snapshot.setMaxReadPositionLedgerId(1L); |
| 137 | + snapshot.setMaxReadPositionEntryId(2L); |
| 138 | + AbortTxnMetadata aborted = new AbortTxnMetadata(); |
| 139 | + aborted.setTxnIdMostBits(3L); |
| 140 | + aborted.setTxnIdLeastBits(4L); |
| 141 | + snapshot.setAborts(List.of(aborted)); |
| 142 | + |
| 143 | + Schema<TransactionBufferSnapshot> schema = Schema.AVRO(TransactionBufferSnapshot.class); |
| 144 | + |
| 145 | + // These snapshot classes do not define equals(), so compare field by field. |
| 146 | + assertThat(schema.decode(schema.encode(snapshot))) |
| 147 | + .usingRecursiveComparison().isEqualTo(snapshot); |
| 148 | + } |
| 149 | + |
| 150 | + @Test |
| 151 | + public void testTransactionBufferSnapshotIndexesRoundTrip() { |
| 152 | + TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); |
| 153 | + indexes.setTopicName("persistent://public/default/t1"); |
| 154 | + indexes.setIndexList(List.of()); |
| 155 | + |
| 156 | + Schema<TransactionBufferSnapshotIndexes> schema = Schema.AVRO(TransactionBufferSnapshotIndexes.class); |
| 157 | + |
| 158 | + assertThat(schema.decode(schema.encode(indexes))) |
| 159 | + .usingRecursiveComparison().isEqualTo(indexes); |
| 160 | + } |
| 161 | + |
| 162 | + @Test |
| 163 | + public void testJsonSchemaOverBrokerInternalTypesRoundTrips() { |
| 164 | + // Schema.JSON also derives an Avro schema from the POJO, but reads and writes it with Jackson |
| 165 | + // rather than ReflectDatumReader/Writer, so it does not resolve classes reflectively and does not |
| 166 | + // need its types in the allow-list. These are the broker's own JSON-schema types, used by the |
| 167 | + // extensible load balancer; pin the behaviour so a future change to the JSON read/write path |
| 168 | + // cannot start requiring trust without a test noticing. |
| 169 | + Schema<ServiceUnitStateData> unitStateSchema = Schema.JSON(ServiceUnitStateData.class); |
| 170 | + ServiceUnitStateData unitState = new ServiceUnitStateData( |
| 171 | + ServiceUnitState.Owned, "dst-broker", "src-broker", 1L); |
| 172 | + assertThat(unitStateSchema.decode(unitStateSchema.encode(unitState))) |
| 173 | + .usingRecursiveComparison().isEqualTo(unitState); |
| 174 | + |
| 175 | + Schema<BrokerLoadData> brokerLoadSchema = Schema.JSON(BrokerLoadData.class); |
| 176 | + assertThat(brokerLoadSchema.decode(brokerLoadSchema.encode(new BrokerLoadData()))).isNotNull(); |
| 177 | + |
| 178 | + Schema<TopBundlesLoadData> topBundlesSchema = Schema.JSON(TopBundlesLoadData.class); |
| 179 | + assertThat(topBundlesSchema.decode(topBundlesSchema.encode(new TopBundlesLoadData()))).isNotNull(); |
| 180 | + } |
| 181 | + |
| 182 | + @Test |
| 183 | + public void testTrustDoesNotLeakToUnrelatedApplicationClasses() { |
| 184 | + // Serializing the broker's own types must not widen the set for anything else. Build a schema |
| 185 | + // first so Pulsar's predicate is actually installed on top of the pinned baseline - otherwise |
| 186 | + // the assertion below would be answered by the baseline alone and could never fail, whatever |
| 187 | + // auto-registration did. |
| 188 | + Schema.AVRO(MetadataEvent.class); |
| 189 | + assertThat(ClassSecurityValidator.getGlobal().isTrusted(MetadataEvent.class)).isTrue(); |
| 190 | + |
| 191 | + assertThat(ClassSecurityValidator.getGlobal().isTrusted(UnusedPojo.class)).isFalse(); |
| 192 | + } |
| 193 | + |
| 194 | + /** Never passed to a schema, so nothing should have trusted it. */ |
| 195 | + public static class UnusedPojo { |
| 196 | + private String field = "value"; |
| 197 | + |
| 198 | + public String getField() { |
| 199 | + return field; |
| 200 | + } |
| 201 | + |
| 202 | + public void setField(String field) { |
| 203 | + this.field = field; |
| 204 | + } |
| 205 | + } |
| 206 | +} |
0 commit comments