Skip to content

Commit 0c1c390

Browse files
mneethirajmneethiraj
authored
ATLAS-5073: ensure propertyKey is created before using it (#401)
1 parent be7b208 commit 0c1c390

File tree

6 files changed

+133
-33
lines changed

6 files changed

+133
-33
lines changed

‎addons/kafka-bridge/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -342,12 +342,6 @@
342342
</resources>
343343
</configuration>
344344
</execution>
345-
</executions>
346-
</plugin>
347-
348-
<plugin>
349-
<artifactId>maven-resources-plugin</artifactId>
350-
<executions>
351345
<execution>
352346
<id>copy-solr-resources</id>
353347
<goals>

‎repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@
9090
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
9191
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT;
9292
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
93+
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
94+
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
9395
import static org.apache.atlas.repository.Constants.BACKING_INDEX;
9496
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
9597
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
@@ -121,6 +123,11 @@
121123
import static org.apache.atlas.repository.Constants.PROPERTY_KEY_AUDIT_REDUCTION_NAME;
122124
import static org.apache.atlas.repository.Constants.PROPERTY_KEY_INDEX_RECOVERY_NAME;
123125
import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
126+
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_CATEGORY_KEY;
127+
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_END1_KEY;
128+
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_END2_KEY;
129+
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_LABEL_KEY;
130+
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
124131
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
125132
import static org.apache.atlas.repository.Constants.RELATIONSHIP_TYPE_PROPERTY_KEY;
126133
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
@@ -131,8 +138,13 @@
131138
import static org.apache.atlas.repository.Constants.TASK_TYPE_PROPERTY_KEY;
132139
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
133140
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
141+
import static org.apache.atlas.repository.Constants.TYPEDESCRIPTION_PROPERTY_KEY;
134142
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
143+
import static org.apache.atlas.repository.Constants.TYPEOPTIONS_PROPERTY_KEY;
135144
import static org.apache.atlas.repository.Constants.TYPESERVICETYPE_PROPERTY_KEY;
145+
import static org.apache.atlas.repository.Constants.TYPEVERSION_PROPERTY_KEY;
146+
import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY;
147+
import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
136148
import static org.apache.atlas.repository.Constants.VERTEX_INDEX;
137149
import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY;
138150
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
@@ -631,9 +643,21 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep
631643
// create fulltext indexes
632644
createFullTextIndex(management, ENTITY_TEXT_PROPERTY_KEY, String.class, SINGLE);
633645

646+
createPropertyKey(management, TYPE_CATEGORY_PROPERTY_KEY, String.class, SINGLE);
647+
createPropertyKey(management, TYPEDESCRIPTION_PROPERTY_KEY, String.class, SINGLE);
648+
createPropertyKey(management, TYPEVERSION_PROPERTY_KEY, String.class, SINGLE);
649+
createPropertyKey(management, VERSION_PROPERTY_KEY, Long.class, SINGLE);
650+
createPropertyKey(management, TYPEOPTIONS_PROPERTY_KEY, String.class, SINGLE);
634651
createPropertyKey(management, IS_PROXY_KEY, Boolean.class, SINGLE);
635652
createPropertyKey(management, PROVENANCE_TYPE_KEY, Integer.class, SINGLE);
636653
createPropertyKey(management, HOME_ID_KEY, String.class, SINGLE);
654+
createPropertyKey(management, ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.class, SINGLE);
655+
createPropertyKey(management, ATTRIBUTE_KEY_PROPERTY_KEY, String.class, SINGLE);
656+
createPropertyKey(management, RELATIONSHIPTYPE_END1_KEY, String.class, SINGLE);
657+
createPropertyKey(management, RELATIONSHIPTYPE_END2_KEY, String.class, SINGLE);
658+
createPropertyKey(management, RELATIONSHIPTYPE_CATEGORY_KEY, String.class, SINGLE);
659+
createPropertyKey(management, RELATIONSHIPTYPE_LABEL_KEY, String.class, SINGLE);
660+
createPropertyKey(management, RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, String.class, SINGLE);
637661

638662
commit(management);
639663

‎repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasAbstractDefStoreV2.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
2828
import org.apache.atlas.model.typedef.AtlasStructDef;
2929
import org.apache.atlas.query.AtlasDSL;
30+
import org.apache.atlas.repository.graphdb.AtlasCardinality;
31+
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
3032
import org.apache.atlas.repository.graphdb.AtlasVertex;
3133
import org.apache.atlas.repository.store.graph.AtlasDefStore;
3234
import org.apache.atlas.type.AtlasType;
@@ -169,4 +171,10 @@ public void deleteByGuid(String guid, AtlasVertex preDeleteResult) throws AtlasB
169171
public boolean isInvalidTypeDefName(String typeName) {
170172
return INVALID_TYPEDEF_NAMES_LIST.contains(typeName);
171173
}
174+
175+
protected static void createPropertyKey(String propertyKey, Class<?> clz, AtlasCardinality cardinality, AtlasGraphManagement management) {
176+
if (!management.containsPropertyKey(propertyKey)) {
177+
management.makePropertyKey(propertyKey, clz, cardinality);
178+
}
179+
}
172180
}

‎repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEnumDefStoreV2.java‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@
2525
import org.apache.atlas.model.typedef.AtlasEnumDef;
2626
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
2727
import org.apache.atlas.repository.Constants;
28+
import org.apache.atlas.repository.IndexException;
29+
import org.apache.atlas.repository.graphdb.AtlasCardinality;
30+
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
2831
import org.apache.atlas.repository.graphdb.AtlasVertex;
2932
import org.apache.atlas.type.AtlasTypeRegistry;
3033
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
@@ -37,6 +40,8 @@
3740
import java.util.Iterator;
3841
import java.util.List;
3942

43+
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.encodePropertyKey;
44+
4045
/**
4146
* EnumDef store in v2 format.
4247
*/
@@ -239,14 +244,11 @@ private void toVertex(AtlasEnumDef enumDef, AtlasVertex vertex) throws AtlasBase
239244
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, enumDef.getName(), "values");
240245
}
241246

247+
createPropertyKeys(enumDef);
248+
242249
List<String> values = new ArrayList<>(enumDef.getElementDefs().size());
243250

244251
for (AtlasEnumElementDef element : enumDef.getElementDefs()) {
245-
// Validate the enum element
246-
if (StringUtils.isEmpty(element.getValue()) || null == element.getOrdinal()) {
247-
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, enumDef.getName(), "elementValue");
248-
}
249-
250252
String elemKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef, element.getValue());
251253

252254
AtlasGraphUtilsV2.setProperty(vertex, elemKey, element.getOrdinal());
@@ -259,10 +261,9 @@ private void toVertex(AtlasEnumDef enumDef, AtlasVertex vertex) throws AtlasBase
259261

260262
values.add(element.getValue());
261263
}
262-
AtlasGraphUtilsV2.setProperty(vertex, AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef), values);
263264

264-
String defaultValueKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef, "defaultValue");
265-
AtlasGraphUtilsV2.setProperty(vertex, defaultValueKey, enumDef.getDefaultValue());
265+
AtlasGraphUtilsV2.setProperty(vertex, AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef), values);
266+
AtlasGraphUtilsV2.setProperty(vertex, AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef, "defaultValue"), enumDef.getDefaultValue());
266267
}
267268

268269
private AtlasEnumDef toEnumDef(AtlasVertex vertex) {
@@ -281,7 +282,13 @@ private static AtlasEnumDef toEnumDef(AtlasVertex vertex, AtlasEnumDef enumDef,
281282
typeDefStore.vertexToTypeDef(vertex, ret);
282283

283284
List<AtlasEnumElementDef> elements = new ArrayList<>();
284-
List<String> elemValues = vertex.getProperty(AtlasGraphUtilsV2.getTypeDefPropertyKey(ret), List.class);
285+
Object names = vertex.getProperty(AtlasGraphUtilsV2.getTypeDefPropertyKey(ret), Object.class);
286+
List<String> elemValues = names instanceof List ? (List<String>) names : new ArrayList<>();
287+
288+
if (names == null) {
289+
LOG.warn("failed to load element names for enum {}", ret.getName());
290+
}
291+
285292
for (String elemValue : elemValues) {
286293
String elemKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(ret, elemValue);
287294
String descKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(elemKey, "description");
@@ -299,4 +306,34 @@ private static AtlasEnumDef toEnumDef(AtlasVertex vertex, AtlasEnumDef enumDef,
299306

300307
return ret;
301308
}
309+
310+
private void createPropertyKeys(AtlasEnumDef enumDef) throws AtlasBaseException {
311+
AtlasGraphManagement management = typeDefStore.atlasGraph.getManagementSystem();
312+
313+
// create property keys first
314+
for (AtlasEnumElementDef element : enumDef.getElementDefs()) {
315+
// Validate the enum element
316+
if (StringUtils.isEmpty(element.getValue()) || null == element.getOrdinal()) {
317+
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, enumDef.getName(), "elementValue");
318+
}
319+
320+
String elemKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef, element.getValue());
321+
322+
createPropertyKey(encodePropertyKey(elemKey), Integer.class, AtlasCardinality.SINGLE, management);
323+
}
324+
325+
String typeDefKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef);
326+
String defaultValueKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(enumDef, "defaultValue");
327+
328+
createPropertyKey(encodePropertyKey(typeDefKey), Object.class, AtlasCardinality.SINGLE, management);
329+
createPropertyKey(encodePropertyKey(defaultValueKey), String.class, AtlasCardinality.SINGLE, management);
330+
331+
try {
332+
management.commit();
333+
} catch (Exception e) {
334+
LOG.error("PropertyKey creation failed", e);
335+
336+
throw new AtlasBaseException(new IndexException("Index commit failed", e));
337+
}
338+
}
302339
}

‎repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java‎

Lines changed: 54 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
2929
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
3030
import org.apache.atlas.repository.Constants;
31+
import org.apache.atlas.repository.IndexException;
32+
import org.apache.atlas.repository.graphdb.AtlasCardinality;
33+
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
3134
import org.apache.atlas.repository.graphdb.AtlasVertex;
3235
import org.apache.atlas.type.AtlasRelationshipType;
3336
import org.apache.atlas.type.AtlasStructType;
@@ -64,28 +67,19 @@ public AtlasStructDefStoreV2(AtlasTypeDefGraphStoreV2 typeDefStore, AtlasTypeReg
6467
public static void updateVertexPreCreate(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex, AtlasTypeDefGraphStoreV2 typeDefStore) throws AtlasBaseException {
6568
List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size());
6669

67-
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
68-
// Validate the mandatory features of an attribute (compatibility with legacy type system)
69-
if (StringUtils.isEmpty(attributeDef.getName())) {
70-
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "name");
71-
}
72-
73-
if (StringUtils.isEmpty(attributeDef.getTypeName())) {
74-
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "typeName");
75-
}
70+
createPropertyKeys(structDef, typeDefStore);
7671

77-
String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, attributeDef.getName());
78-
String encodedPropertyKey = AtlasGraphUtilsV2.encodePropertyKey(propertyKey);
72+
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
73+
String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, attributeDef.getName());
7974

80-
vertex.setProperty(encodedPropertyKey, toJsonFromAttribute(structType.getAttribute(attributeDef.getName())));
75+
vertex.setProperty(AtlasGraphUtilsV2.encodePropertyKey(propertyKey), toJsonFromAttribute(structType.getAttribute(attributeDef.getName())));
8176

8277
attrNames.add(attributeDef.getName());
8378
}
8479

85-
String typeNamePropertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef);
86-
String encodedtypeNamePropertyKey = AtlasGraphUtilsV2.encodePropertyKey(typeNamePropertyKey);
80+
String typeNamePropertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef);
8781

88-
vertex.setProperty(encodedtypeNamePropertyKey, attrNames);
82+
vertex.setProperty(AtlasGraphUtilsV2.encodePropertyKey(typeNamePropertyKey), attrNames);
8983
}
9084

9185
public static void updateVertexPreUpdate(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex, AtlasTypeDefGraphStoreV2 typeDefStore) throws AtlasBaseException {
@@ -99,12 +93,15 @@ public static void updateVertexPreUpdate(AtlasStructDef structDef, AtlasStructTy
9993

10094
String structDefPropertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef);
10195
String encodedStructDefPropertyKey = encodePropertyKey(structDefPropertyKey);
102-
List<String> currAttrNames = vertex.getProperty(encodedStructDefPropertyKey, List.class);
96+
Object names = vertex.getProperty(encodedStructDefPropertyKey, Object.class);
97+
List<String> currAttrNames = names instanceof List ? (List<String>) names : new ArrayList<>();
10398

10499
// delete attributes that are not present in updated structDef
105100
if (CollectionUtils.isNotEmpty(currAttrNames)) {
106101
List<String> removedAttributes = null;
107102

103+
createPropertyKeys(structDef, typeDefStore);
104+
108105
for (String currAttrName : currAttrNames) {
109106
if (!attrNames.contains(currAttrName)) {
110107
if (RequestContext.get().isInTypePatching()) {
@@ -136,6 +133,10 @@ public static void updateVertexPreUpdate(AtlasStructDef structDef, AtlasStructTy
136133
vertex.removeProperty(propertyKey);
137134
}
138135
}
136+
} else {
137+
if (names == null) {
138+
LOG.warn("failed to load attribute names for type {}", structDef.getName());
139+
}
139140
}
140141

141142
typeDefStore.updateTypeVertex(structDef, vertex);
@@ -207,7 +208,8 @@ public static AtlasStructDef toStructDef(AtlasVertex vertex, AtlasStructDef stru
207208
List<AtlasAttributeDef> attributeDefs = new ArrayList<>();
208209
String typePropertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(ret);
209210
String encodedTypePropertyKey = AtlasGraphUtilsV2.encodePropertyKey(typePropertyKey);
210-
List<String> attrNames = vertex.getProperty(encodedTypePropertyKey, List.class);
211+
Object names = vertex.getProperty(encodedTypePropertyKey, Object.class);
212+
List<String> attrNames = names instanceof List ? (List<String>) names : new ArrayList<>();
211213

212214
if (CollectionUtils.isNotEmpty(attrNames)) {
213215
for (String attrName : attrNames) {
@@ -223,6 +225,10 @@ public static AtlasStructDef toStructDef(AtlasVertex vertex, AtlasStructDef stru
223225

224226
attributeDefs.add(toAttributeDefFromJson(structDef, AtlasType.fromJson(attrJson, Map.class), typeDefStore));
225227
}
228+
} else {
229+
if (names == null) {
230+
LOG.warn("failed to load attribute names for type {}", structDef);
231+
}
226232
}
227233

228234
ret.setAttributeDefs(attributeDefs);
@@ -638,4 +644,35 @@ private static void addReferencesForAttribute(AtlasVertex vertex, AtlasAttribute
638644
}
639645
}
640646
}
647+
648+
private static void createPropertyKeys(AtlasStructDef structDef, AtlasTypeDefGraphStoreV2 typeDefStore) throws AtlasBaseException {
649+
AtlasGraphManagement management = typeDefStore.atlasGraph.getManagementSystem();
650+
651+
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
652+
// Validate the mandatory features of an attribute (compatibility with legacy type system)
653+
if (StringUtils.isEmpty(attributeDef.getName())) {
654+
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "name");
655+
}
656+
657+
if (StringUtils.isEmpty(attributeDef.getTypeName())) {
658+
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "typeName");
659+
}
660+
661+
String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, attributeDef.getName());
662+
663+
createPropertyKey(AtlasGraphUtilsV2.encodePropertyKey(propertyKey), String.class, AtlasCardinality.SINGLE, management);
664+
}
665+
666+
String typeNamePropertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef);
667+
668+
createPropertyKey(AtlasGraphUtilsV2.encodePropertyKey(typeNamePropertyKey), Object.class, AtlasCardinality.SINGLE, management);
669+
670+
try {
671+
management.commit();
672+
} catch (Exception e) {
673+
LOG.error("PropertyKey creation failed", e);
674+
675+
throw new AtlasBaseException(new IndexException("Index commit failed", e));
676+
}
677+
}
641678
}

0 commit comments

Comments
 (0)