Skip to content

Commit 0ae3bf9

Browse files
lonvialonvia
authored
Merge pull request #890 from lonvia/cleanup-propoerties
Use DatabaseProperties class to distribute import configuration to backend code
2 parents a1aa580 + 6cb2e6b commit 0ae3bf9

37 files changed

+352
-401
lines changed

‎app/es_embedded/src/main/java/de/komoot/photon/Server.java‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -179,21 +179,14 @@ private void setupDirectories(URL directoryName) throws IOException, URISyntaxEx
179179

180180
}
181181

182-
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException {
182+
public void recreateIndex(DatabaseProperties dbProperties) throws IOException {
183183
deleteIndex();
184184

185185
loadIndexSettings().createIndex(esClient, PhotonIndex.NAME);
186186

187-
createAndPutIndexMapping(languages, supportStructuredQueries);
188-
189-
DatabaseProperties dbProperties = new DatabaseProperties()
190-
.setLanguages(languages)
191-
.setImportDate(importDate)
192-
.setSupportGeometries(supportGeometries);
187+
createAndPutIndexMapping(dbProperties.getLanguages(), dbProperties.getSupportStructuredQueries());
193188

194189
saveToDatabase(dbProperties);
195-
196-
return dbProperties;
197190
}
198191

199192
private void createAndPutIndexMapping(String[] languages, boolean supportStructuredQueries)
@@ -264,7 +257,6 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException
264257
public DatabaseProperties loadFromDatabase() {
265258
GetResponse response = esClient.prepareGet(PhotonIndex.NAME, PhotonIndex.TYPE, PROPERTY_DOCUMENT_ID).execute().actionGet();
266259

267-
// We are currently at the database version where versioning was introduced.
268260
if (!response.isExists()) {
269261
throw new UsageException("Cannot find database properties. Your database version is too old. Please reimport.");
270262
}
@@ -275,31 +267,27 @@ public DatabaseProperties loadFromDatabase() {
275267
throw new UsageException("Found database properties but no '" + BASE_FIELD +"' field. Database corrupt?");
276268
}
277269

278-
String version = properties.getOrDefault(FIELD_VERSION, "");
279-
if (!DATABASE_VERSION.equals(version)) {
280-
LOGGER.error("Database has incompatible version '{}'. Expected: {}",
281-
version, DATABASE_VERSION);
282-
throw new UsageException("Incompatible database.");
283-
}
270+
final var dbProps = new DatabaseProperties();
271+
272+
dbProps.setVersion(properties.get(FIELD_VERSION));
284273

285-
String langString = properties.get(FIELD_LANGUAGES);
274+
final String langString = properties.get(FIELD_LANGUAGES);
275+
dbProps.setLanguages(langString == null ? null : langString.split(","));
286276

287277
String importDateString = properties.get(FIELD_IMPORT_DATE);
278+
dbProps.setImportDate(importDateString == null ? null : Date.from(Instant.parse(importDateString)));
288279

289-
String supportGeometries = properties.get(FIELD_SUPPORT_GEOMETRIES);
280+
dbProps.setSupportGeometries(Boolean.parseBoolean(properties.get(FIELD_SUPPORT_GEOMETRIES)));
290281

291-
return new DatabaseProperties(langString == null ? null : langString.split(","),
292-
importDateString == null ? null : Date.from(Instant.parse(importDateString)),
293-
false,
294-
Boolean.parseBoolean(supportGeometries));
282+
return dbProps;
295283
}
296284

297-
public Importer createImporter(String[] languages, ConfigExtraTags extraTags) {
298-
return new de.komoot.photon.elasticsearch.Importer(esClient, languages, extraTags);
285+
public Importer createImporter(DatabaseProperties dbProperties) {
286+
return new de.komoot.photon.elasticsearch.Importer(esClient, dbProperties);
299287
}
300288

301-
public Updater createUpdater(String[] languages, ConfigExtraTags extraTags) {
302-
return new de.komoot.photon.elasticsearch.Updater(esClient, languages, extraTags);
289+
public Updater createUpdater(DatabaseProperties dbProperties) {
290+
return new de.komoot.photon.elasticsearch.Updater(esClient, dbProperties);
303291
}
304292

305293
public SearchHandler createSearchHandler(String[] languages, int queryTimeoutSec) {

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/Importer.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package de.komoot.photon.elasticsearch;
22

3+
import de.komoot.photon.DatabaseProperties;
34
import de.komoot.photon.PhotonDoc;
45
import de.komoot.photon.ConfigExtraTags;
56
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -19,14 +20,12 @@ public class Importer implements de.komoot.photon.Importer {
1920

2021
private final Client esClient;
2122
private BulkRequestBuilder bulkRequest;
22-
private final String[] languages;
23-
private final ConfigExtraTags extraTags;
23+
private final DatabaseProperties dbProperties;
2424

25-
public Importer(Client esClient, String[] languages, ConfigExtraTags extraTags) {
25+
public Importer(Client esClient, DatabaseProperties dbProperties) {
2626
this.esClient = esClient;
2727
this.bulkRequest = esClient.prepareBulk();
28-
this.languages = languages;
29-
this.extraTags = extraTags;
28+
this.dbProperties = dbProperties;
3029
}
3130

3231
@Override
@@ -40,7 +39,7 @@ public void add(Iterable<PhotonDoc> docs) {
4039
final String uid = PhotonDoc.makeUid(placeID, objectId++);
4140
try {
4241
bulkRequest.add(esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).
43-
setSource(PhotonDocConverter.convert(doc, languages, extraTags)).setId(uid));
42+
setSource(PhotonDocConverter.convert(doc, dbProperties)).setId(uid));
4443
} catch (IOException e) {
4544
LOGGER.error("Could not bulk add document {}", uid, e);
4645
return;

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/PhotonDocConverter.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package de.komoot.photon.elasticsearch;
22

33
import de.komoot.photon.Constants;
4+
import de.komoot.photon.DatabaseProperties;
45
import de.komoot.photon.PhotonDoc;
56
import de.komoot.photon.ConfigExtraTags;
67
import de.komoot.photon.nominatim.model.AddressType;
@@ -12,14 +13,13 @@
1213

1314
import java.io.IOException;
1415
import java.util.HashMap;
15-
import java.util.HashSet;
1616
import java.util.Map;
1717
import java.util.Set;
1818

1919
import static de.komoot.photon.Utils.buildClassificationString;
2020

2121
public class PhotonDocConverter {
22-
public static XContentBuilder convert(PhotonDoc doc, String[] languages, ConfigExtraTags extraTags) throws IOException {
22+
public static XContentBuilder convert(PhotonDoc doc, DatabaseProperties dbProperties) throws IOException {
2323
final AddressType atype = doc.getAddressType();
2424
XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
2525
.field(Constants.OSM_ID, doc.getOsmId())
@@ -60,14 +60,14 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages, ConfigE
6060
builder.field("postcode", doc.getPostcode());
6161
}
6262

63-
writeName(builder, doc, languages);
63+
writeName(builder, doc, dbProperties.getLanguages());
6464

6565
for (AddressType entry : doc.getAddressParts().keySet()) {
6666
Map<String, String> fNames = new HashMap<>();
6767

6868
doc.copyAddressName(fNames, "default", entry, "name");
6969

70-
for (String language : languages) {
70+
for (String language : dbProperties.getLanguages()) {
7171
doc.copyAddressName(fNames, language, entry, "name:" + language);
7272
}
7373

@@ -77,8 +77,8 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages, ConfigE
7777
String countryCode = doc.getCountryCode();
7878
if (countryCode != null)
7979
builder.field(Constants.COUNTRYCODE, countryCode);
80-
writeContext(builder, doc.getContextByLanguage(languages));
81-
writeExtraTags(builder, extraTags.filterExtraTags(doc.getExtratags()));
80+
writeContext(builder, doc.getContextByLanguage(dbProperties.getLanguages()));
81+
writeExtraTags(builder, dbProperties.configExtraTags().filterExtraTags(doc.getExtratags()));
8282
writeExtent(builder, doc.getBbox());
8383

8484
builder.endObject();

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/Updater.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package de.komoot.photon.elasticsearch;
22

3+
import de.komoot.photon.DatabaseProperties;
34
import de.komoot.photon.PhotonDoc;
45
import de.komoot.photon.ConfigExtraTags;
56
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -17,14 +18,12 @@ public class Updater implements de.komoot.photon.Updater {
1718

1819
private final Client esClient;
1920
private BulkRequestBuilder bulkRequest;
20-
private final String[] languages;
21-
private final ConfigExtraTags extraTags;
21+
private final DatabaseProperties dbProperties;
2222

23-
public Updater(Client esClient, String[] languages, ConfigExtraTags extraTags) {
23+
public Updater(Client esClient, DatabaseProperties dbProperties) {
2424
this.esClient = esClient;
2525
this.bulkRequest = esClient.prepareBulk();
26-
this.languages = languages;
27-
this.extraTags = extraTags;
26+
this.dbProperties = dbProperties;
2827
}
2928

3029
@Override
@@ -47,7 +46,7 @@ public void addOrUpdate(Iterable<PhotonDoc> docs) {
4746
try {
4847
bulkRequest.add(
4948
esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE)
50-
.setSource(PhotonDocConverter.convert(doc, languages, extraTags))
49+
.setSource(PhotonDocConverter.convert(doc, dbProperties))
5150
.setId(uid));
5251
} catch (IOException ex) {
5352
LOGGER.error("Parse error in document {}: {}", placeID, ex);

‎app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java‎

Lines changed: 9 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,22 @@
66
import org.locationtech.jts.geom.PrecisionModel;
77
import de.komoot.photon.searcher.PhotonResult;
88
import org.junit.jupiter.api.AfterEach;
9-
import org.junit.jupiter.api.io.TempDir;
109
import org.locationtech.jts.io.ParseException;
1110
import org.locationtech.jts.io.WKTReader;
1211

1312
import java.io.IOException;
1413
import java.nio.file.Path;
1514
import java.util.*;
16-
import java.util.stream.Collectors;
1715

1816
/**
1917
* Start an ES server with some test data that then can be queried in tests that extend this class
2018
*/
2119
public class ESBaseTester {
22-
@TempDir
23-
protected Path dataDirectory;
24-
2520
public static final String TEST_CLUSTER_NAME = "photon-test";
2621
protected static final GeometryFactory FACTORY = new GeometryFactory(new PrecisionModel(), 4326);
2722

2823
private ElasticTestServer server;
24+
final DatabaseProperties dbProperties = new DatabaseProperties();
2925

3026
protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) throws ParseException {
3127
Point location = FACTORY.createPoint(new Coordinate(lon, lat));
@@ -46,43 +42,19 @@ public void tearDown() throws IOException {
4642
shutdownES();
4743
}
4844

49-
public void setUpES() throws IOException {
50-
setUpES(dataDirectory, false,"en");
51-
}
52-
53-
public void setUpESWithGeometry() throws IOException {
54-
setUpES(dataDirectory, true,"en");
55-
}
56-
/**
57-
* Setup the ES server
58-
*
59-
* @throws IOException
60-
*/
61-
public void setUpES(Path testDirectory, boolean supportGeometries, String... languages) throws IOException {
45+
public void setUpES(Path testDirectory) throws IOException {
6246
server = new ElasticTestServer(testDirectory.toString());
6347
server.start(TEST_CLUSTER_NAME, new String[]{});
64-
server.recreateIndex(languages, new Date(), false, supportGeometries);
48+
server.recreateIndex(dbProperties);
6549
refresh();
6650
}
6751

6852
protected Importer makeImporter() {
69-
return server.createImporter(new String[]{"en"}, new ConfigExtraTags());
70-
}
71-
72-
protected Importer makeImporterWithExtra(String... extraTags) {
73-
return server.createImporter(new String[]{"en"}, new ConfigExtraTags(Arrays.stream(extraTags).collect(Collectors.toList())));
74-
}
75-
76-
protected Importer makeImporterWithLanguages(String... languages) {
77-
return server.createImporter(languages, new ConfigExtraTags());
53+
return server.createImporter(dbProperties);
7854
}
7955

8056
protected Updater makeUpdater() {
81-
return server.createUpdater(new String[]{"en"}, new ConfigExtraTags());
82-
}
83-
84-
protected Updater makeUpdaterWithExtra(String... extraTags) {
85-
return server.createUpdater(new String[]{"en"}, new ConfigExtraTags(Arrays.stream(extraTags).collect(Collectors.toList())));
57+
return server.createUpdater(dbProperties);
8658
}
8759

8860
protected ElasticTestServer getServer() {
@@ -97,6 +69,10 @@ protected void refresh() {
9769
server.refresh();
9870
}
9971

72+
protected DatabaseProperties getProperties() {
73+
return dbProperties;
74+
}
75+
10076
/**
10177
* Shutdown the ES node
10278
*/

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticResultTest.java‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
package de.komoot.photon.elasticsearch;
22

3+
import de.komoot.photon.*;
4+
import de.komoot.photon.Importer;
35
import org.locationtech.jts.geom.Coordinate;
46
import org.locationtech.jts.geom.Point;
5-
import de.komoot.photon.ESBaseTester;
6-
import de.komoot.photon.Importer;
7-
import de.komoot.photon.PhotonDoc;
8-
import de.komoot.photon.ConfigExtraTags;
97
import de.komoot.photon.query.PhotonRequest;
108
import de.komoot.photon.searcher.PhotonResult;
119
import de.komoot.photon.searcher.SearchHandler;
@@ -23,9 +21,6 @@
2321

2422
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
2523
class ElasticResultTest extends ESBaseTester {
26-
@TempDir
27-
private static Path instanceTestDirectory;
28-
2924
private Map<String, String> makeMap(String... kv) {
3025
Map<String, String> result = new HashMap<>();
3126
for (int i = 0; i < kv.length; i += 2) {
@@ -42,10 +37,12 @@ protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String
4237

4338

4439
@BeforeAll
45-
void setUp() throws Exception {
46-
setUpES(instanceTestDirectory, false, "en", "de", "fr", "it");
47-
Importer instance = getServer().createImporter(new String[]{"en", "de", "fr", "it"},
48-
new ConfigExtraTags(List.of("population", "capital")));
40+
void setUp(@TempDir Path dataDirectory) throws Exception {
41+
setUpES(dataDirectory);
42+
final var dbProperties = new DatabaseProperties();
43+
dbProperties.setLanguages(new String[]{"en", "de", "fr", "it"});
44+
dbProperties.setExtraTags(List.of("population", "capital"));
45+
Importer instance = getServer().createImporter(dbProperties);
4946

5047
instance.add(List.of(
5148
createDoc(45.2, -7.45, 123, 1123, "place", "city")

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ImporterTest.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66
import de.komoot.photon.searcher.PhotonResult;
77
import org.junit.jupiter.api.BeforeEach;
88
import org.junit.jupiter.api.Test;
9+
import org.junit.jupiter.api.io.TempDir;
910
import org.locationtech.jts.io.ParseException;
1011
import org.locationtech.jts.io.WKTReader;
1112

1213
import java.io.IOException;
14+
import java.nio.file.Path;
1315
import java.util.Collections;
1416
import java.util.HashMap;
1517
import java.util.Map;
@@ -20,13 +22,13 @@
2022
class ImporterTest extends ESBaseTester {
2123

2224
@BeforeEach
23-
public void setUp() throws IOException {
24-
setUpES();
25+
public void setUp(@TempDir Path dataDirectory) throws IOException {
26+
setUpES(dataDirectory);
2527
}
2628

2729
@Test
2830
void testAddSimpleDoc() throws ParseException {
29-
Importer instance = makeImporterWithExtra("");
31+
Importer instance = makeImporter();
3032

3133
instance.add(List.of(new PhotonDoc(1234, "N", 1000, "place", "city")
3234
.geometry(new WKTReader().read("MULTIPOLYGON (((6.111933 51.2659309, 6.1119417 51.2659247, 6.1119554 51.2659249, 6.1119868 51.2659432, 6.111964 51.2659591, 6.1119333 51.2659391, 6.111933 51.2659309)))"))
@@ -48,7 +50,7 @@ void testAddSimpleDoc() throws ParseException {
4850

4951
@Test
5052
void testAddHousenumberMultiDoc() {
51-
Importer instance = makeImporterWithExtra("");
53+
Importer instance = makeImporter();
5254

5355
instance.add(List.of(
5456
new PhotonDoc(4432, "N", 100, "building", "yes").houseNumber("34"),
@@ -79,7 +81,8 @@ void testAddHousenumberMultiDoc() {
7981

8082
@Test
8183
void testSelectedExtraTagsCanBeIncluded() {
82-
Importer instance = makeImporterWithExtra("maxspeed", "website");
84+
getProperties().setExtraTags(List.of("maxspeed", "website"));
85+
Importer instance = makeImporter();
8386

8487
Map<String, String> extratags = new HashMap<>();
8588
extratags.put("website", "foo");

0 commit comments

Comments
 (0)