Skip to content

Commit 8595801

Browse files
lonvialonvia
committed
use DBProperties for configuring importer and updater
1 parent 8ba9005 commit 8595801

File tree

12 files changed

+116
-97
lines changed

12 files changed

+116
-97
lines changed

‎app/es_embedded/src/main/java/de/komoot/photon/Server.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -179,21 +179,14 @@ private void setupDirectories(URL directoryName) throws IOException, URISyntaxEx
179179

180180
}
181181

182-
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException {
182+
public void recreateIndex(DatabaseProperties dbProperties) throws IOException {
183183
deleteIndex();
184184

185185
loadIndexSettings().createIndex(esClient, PhotonIndex.NAME);
186186

187-
createAndPutIndexMapping(languages, supportStructuredQueries);
188-
189-
DatabaseProperties dbProperties = new DatabaseProperties()
190-
.setLanguages(languages)
191-
.setImportDate(importDate)
192-
.setSupportGeometries(supportGeometries);
187+
createAndPutIndexMapping(dbProperties.getLanguages(), dbProperties.getSupportStructuredQueries());
193188

194189
saveToDatabase(dbProperties);
195-
196-
return dbProperties;
197190
}
198191

199192
private void createAndPutIndexMapping(String[] languages, boolean supportStructuredQueries)
@@ -289,12 +282,12 @@ public DatabaseProperties loadFromDatabase() {
289282
return dbProps;
290283
}
291284

292-
public Importer createImporter(String[] languages, ConfigExtraTags extraTags) {
293-
return new de.komoot.photon.elasticsearch.Importer(esClient, languages, extraTags);
285+
public Importer createImporter(DatabaseProperties dbProperties) {
286+
return new de.komoot.photon.elasticsearch.Importer(esClient, dbProperties);
294287
}
295288

296-
public Updater createUpdater(String[] languages, ConfigExtraTags extraTags) {
297-
return new de.komoot.photon.elasticsearch.Updater(esClient, languages, extraTags);
289+
public Updater createUpdater(DatabaseProperties dbProperties) {
290+
return new de.komoot.photon.elasticsearch.Updater(esClient, dbProperties);
298291
}
299292

300293
public SearchHandler createSearchHandler(String[] languages, int queryTimeoutSec) {

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/Importer.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package de.komoot.photon.elasticsearch;
22

3+
import de.komoot.photon.DatabaseProperties;
34
import de.komoot.photon.PhotonDoc;
45
import de.komoot.photon.ConfigExtraTags;
56
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -19,14 +20,12 @@ public class Importer implements de.komoot.photon.Importer {
1920

2021
private final Client esClient;
2122
private BulkRequestBuilder bulkRequest;
22-
private final String[] languages;
23-
private final ConfigExtraTags extraTags;
23+
private final DatabaseProperties dbProperties;
2424

25-
public Importer(Client esClient, String[] languages, ConfigExtraTags extraTags) {
25+
public Importer(Client esClient, DatabaseProperties dbProperties) {
2626
this.esClient = esClient;
2727
this.bulkRequest = esClient.prepareBulk();
28-
this.languages = languages;
29-
this.extraTags = extraTags;
28+
this.dbProperties = dbProperties;
3029
}
3130

3231
@Override
@@ -40,7 +39,7 @@ public void add(Iterable<PhotonDoc> docs) {
4039
final String uid = PhotonDoc.makeUid(placeID, objectId++);
4140
try {
4241
bulkRequest.add(esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).
43-
setSource(PhotonDocConverter.convert(doc, languages, extraTags)).setId(uid));
42+
setSource(PhotonDocConverter.convert(doc, dbProperties)).setId(uid));
4443
} catch (IOException e) {
4544
LOGGER.error("Could not bulk add document {}", uid, e);
4645
return;

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/PhotonDocConverter.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package de.komoot.photon.elasticsearch;
22

33
import de.komoot.photon.Constants;
4+
import de.komoot.photon.DatabaseProperties;
45
import de.komoot.photon.PhotonDoc;
56
import de.komoot.photon.ConfigExtraTags;
67
import de.komoot.photon.nominatim.model.AddressType;
@@ -12,14 +13,13 @@
1213

1314
import java.io.IOException;
1415
import java.util.HashMap;
15-
import java.util.HashSet;
1616
import java.util.Map;
1717
import java.util.Set;
1818

1919
import static de.komoot.photon.Utils.buildClassificationString;
2020

2121
public class PhotonDocConverter {
22-
public static XContentBuilder convert(PhotonDoc doc, String[] languages, ConfigExtraTags extraTags) throws IOException {
22+
public static XContentBuilder convert(PhotonDoc doc, DatabaseProperties dbProperties) throws IOException {
2323
final AddressType atype = doc.getAddressType();
2424
XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
2525
.field(Constants.OSM_ID, doc.getOsmId())
@@ -60,14 +60,14 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages, ConfigE
6060
builder.field("postcode", doc.getPostcode());
6161
}
6262

63-
writeName(builder, doc, languages);
63+
writeName(builder, doc, dbProperties.getLanguages());
6464

6565
for (AddressType entry : doc.getAddressParts().keySet()) {
6666
Map<String, String> fNames = new HashMap<>();
6767

6868
doc.copyAddressName(fNames, "default", entry, "name");
6969

70-
for (String language : languages) {
70+
for (String language : dbProperties.getLanguages()) {
7171
doc.copyAddressName(fNames, language, entry, "name:" + language);
7272
}
7373

@@ -77,8 +77,8 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages, ConfigE
7777
String countryCode = doc.getCountryCode();
7878
if (countryCode != null)
7979
builder.field(Constants.COUNTRYCODE, countryCode);
80-
writeContext(builder, doc.getContextByLanguage(languages));
81-
writeExtraTags(builder, extraTags.filterExtraTags(doc.getExtratags()));
80+
writeContext(builder, doc.getContextByLanguage(dbProperties.getLanguages()));
81+
writeExtraTags(builder, dbProperties.configExtraTags().filterExtraTags(doc.getExtratags()));
8282
writeExtent(builder, doc.getBbox());
8383

8484
builder.endObject();

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/Updater.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package de.komoot.photon.elasticsearch;
22

3+
import de.komoot.photon.DatabaseProperties;
34
import de.komoot.photon.PhotonDoc;
45
import de.komoot.photon.ConfigExtraTags;
56
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -17,14 +18,12 @@ public class Updater implements de.komoot.photon.Updater {
1718

1819
private final Client esClient;
1920
private BulkRequestBuilder bulkRequest;
20-
private final String[] languages;
21-
private final ConfigExtraTags extraTags;
21+
private final DatabaseProperties dbProperties;
2222

23-
public Updater(Client esClient, String[] languages, ConfigExtraTags extraTags) {
23+
public Updater(Client esClient, DatabaseProperties dbProperties) {
2424
this.esClient = esClient;
2525
this.bulkRequest = esClient.prepareBulk();
26-
this.languages = languages;
27-
this.extraTags = extraTags;
26+
this.dbProperties = dbProperties;
2827
}
2928

3029
@Override
@@ -47,7 +46,7 @@ public void addOrUpdate(Iterable<PhotonDoc> docs) {
4746
try {
4847
bulkRequest.add(
4948
esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE)
50-
.setSource(PhotonDocConverter.convert(doc, languages, extraTags))
49+
.setSource(PhotonDocConverter.convert(doc, dbProperties))
5150
.setId(uid));
5251
} catch (IOException ex) {
5352
LOGGER.error("Parse error in document {}: {}", placeID, ex);

‎app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,30 +59,47 @@ public void setUpESWithGeometry() throws IOException {
5959
* @throws IOException
6060
*/
6161
public void setUpES(Path testDirectory, boolean supportGeometries, String... languages) throws IOException {
62+
final var dbProperties = new DatabaseProperties();
63+
dbProperties.setLanguages(languages);
64+
dbProperties.setSupportGeometries(supportGeometries);
65+
dbProperties.setImportDate(new Date());
66+
6267
server = new ElasticTestServer(testDirectory.toString());
6368
server.start(TEST_CLUSTER_NAME, new String[]{});
64-
server.recreateIndex(languages, new Date(), false, supportGeometries);
69+
server.recreateIndex(dbProperties);
6570
refresh();
6671
}
6772

6873
protected Importer makeImporter() {
69-
return server.createImporter(new String[]{"en"}, new ConfigExtraTags());
74+
final var dbProperties = new DatabaseProperties();
75+
dbProperties.setLanguages(new String[]{"en"});
76+
return server.createImporter(dbProperties);
7077
}
7178

7279
protected Importer makeImporterWithExtra(String... extraTags) {
73-
return server.createImporter(new String[]{"en"}, new ConfigExtraTags(Arrays.stream(extraTags).collect(Collectors.toList())));
80+
final var dbProperties = new DatabaseProperties();
81+
dbProperties.setLanguages(new String[]{"en"});
82+
dbProperties.setExtraTags(Arrays.stream(extraTags).collect(Collectors.toList()));
83+
return server.createImporter(dbProperties);
7484
}
7585

7686
protected Importer makeImporterWithLanguages(String... languages) {
77-
return server.createImporter(languages, new ConfigExtraTags());
87+
final var dbProperties = new DatabaseProperties();
88+
dbProperties.setLanguages(languages);
89+
return server.createImporter(dbProperties);
7890
}
7991

8092
protected Updater makeUpdater() {
81-
return server.createUpdater(new String[]{"en"}, new ConfigExtraTags());
93+
final var dbProperties = new DatabaseProperties();
94+
dbProperties.setLanguages(new String[]{"en"});
95+
return server.createUpdater(dbProperties);
8296
}
8397

8498
protected Updater makeUpdaterWithExtra(String... extraTags) {
85-
return server.createUpdater(new String[]{"en"}, new ConfigExtraTags(Arrays.stream(extraTags).collect(Collectors.toList())));
99+
final var dbProperties = new DatabaseProperties();
100+
dbProperties.setLanguages(new String[]{"en"});
101+
dbProperties.setExtraTags(Arrays.stream(extraTags).collect(Collectors.toList()));
102+
return server.createUpdater(dbProperties);
86103
}
87104

88105
protected ElasticTestServer getServer() {

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticResultTest.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
package de.komoot.photon.elasticsearch;
22

3+
import de.komoot.photon.*;
4+
import de.komoot.photon.Importer;
35
import org.locationtech.jts.geom.Coordinate;
46
import org.locationtech.jts.geom.Point;
5-
import de.komoot.photon.ESBaseTester;
6-
import de.komoot.photon.Importer;
7-
import de.komoot.photon.PhotonDoc;
8-
import de.komoot.photon.ConfigExtraTags;
97
import de.komoot.photon.query.PhotonRequest;
108
import de.komoot.photon.searcher.PhotonResult;
119
import de.komoot.photon.searcher.SearchHandler;
@@ -44,8 +42,10 @@ protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String
4442
@BeforeAll
4543
void setUp() throws Exception {
4644
setUpES(instanceTestDirectory, false, "en", "de", "fr", "it");
47-
Importer instance = getServer().createImporter(new String[]{"en", "de", "fr", "it"},
48-
new ConfigExtraTags(List.of("population", "capital")));
45+
final var dbProperties = new DatabaseProperties();
46+
dbProperties.setLanguages(new String[]{"en", "de", "fr", "it"});
47+
dbProperties.setExtraTags(List.of("population", "capital"));
48+
Importer instance = getServer().createImporter(dbProperties);
4949

5050
instance.add(List.of(
5151
createDoc(45.2, -7.45, 123, 1123, "place", "city")

‎app/opensearch/src/main/java/de/komoot/photon/Server.java‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import java.io.File;
1919
import java.io.IOException;
20-
import java.util.Date;
2120

2221
public class Server {
2322
/**
@@ -120,24 +119,19 @@ public void shutdown() {
120119
}
121120
}
122121

123-
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException {
122+
public void recreateIndex(DatabaseProperties dbProperties) throws IOException {
124123
// delete any existing data
125124
if (client.indices().exists(e -> e.index(PhotonIndex.NAME)).value()) {
126125
client.indices().delete(d -> d.index(PhotonIndex.NAME));
127126
}
128127

129-
(new IndexSettingBuilder()).setShards(5).createIndex(client, PhotonIndex.NAME);
128+
new IndexSettingBuilder().setShards(5).createIndex(client, PhotonIndex.NAME);
130129

131-
(new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME);
130+
new IndexMapping(dbProperties.getSupportStructuredQueries())
131+
.addLanguages(dbProperties.getLanguages())
132+
.putMapping(client, PhotonIndex.NAME);
132133

133-
var dbProperties = new DatabaseProperties();
134-
dbProperties.setLanguages(languages);
135-
dbProperties.setImportDate(importDate);
136-
dbProperties.setSupportStructuredQueries(supportStructuredQueries);
137-
dbProperties.setSupportGeometries(supportGeometries);
138134
saveToDatabase(dbProperties);
139-
140-
return dbProperties;
141135
}
142136

143137
public void updateIndexSettings(String synonymFile) throws IOException {
@@ -180,13 +174,13 @@ public DatabaseProperties loadFromDatabase() throws IOException {
180174
return dbEntry.source();
181175
}
182176

183-
public Importer createImporter(String[] languages, ConfigExtraTags extraTags) {
184-
registerPhotonDocSerializer(languages, extraTags);
177+
public Importer createImporter(DatabaseProperties dbProperties) {
178+
registerPhotonDocSerializer(dbProperties);
185179
return new de.komoot.photon.opensearch.Importer(client);
186180
}
187181

188-
public Updater createUpdater(String[] languages, ConfigExtraTags extraTags) {
189-
registerPhotonDocSerializer(languages, extraTags);
182+
public Updater createUpdater(DatabaseProperties dbProperties) {
183+
registerPhotonDocSerializer(dbProperties);
190184
return new de.komoot.photon.opensearch.Updater(client);
191185
}
192186

@@ -202,10 +196,10 @@ public ReverseHandler createReverseHandler(int queryTimeoutSec) {
202196
return new OpenSearchReverseHandler(client, queryTimeoutSec);
203197
}
204198

205-
private void registerPhotonDocSerializer(String[] languages, ConfigExtraTags extraTags) {
199+
private void registerPhotonDocSerializer(DatabaseProperties dbProperties) {
206200
final var module = new SimpleModule("PhotonDocSerializer",
207201
new Version(1, 0, 0, null, null, null));
208-
module.addSerializer(PhotonDoc.class, new PhotonDocSerializer(languages, extraTags));
202+
module.addSerializer(PhotonDoc.class, new PhotonDocSerializer(dbProperties));
209203

210204
((JacksonJsonpMapper) client._transport().jsonpMapper()).objectMapper().registerModule(module);
211205
}

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/PhotonDocSerializer.java‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,21 @@
33
import com.fasterxml.jackson.core.JsonGenerator;
44
import com.fasterxml.jackson.databind.SerializerProvider;
55
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
6-
import de.komoot.photon.ConfigExtraTags;
7-
import de.komoot.photon.Constants;
8-
import de.komoot.photon.PhotonDoc;
9-
import de.komoot.photon.Utils;
6+
import de.komoot.photon.*;
107
import org.locationtech.jts.geom.Envelope;
118
import org.locationtech.jts.io.geojson.GeoJsonWriter;
129

1310
import java.io.IOException;
1411
import java.util.HashMap;
15-
import java.util.HashSet;
1612
import java.util.Map;
1713
import java.util.Set;
1814

1915
public class PhotonDocSerializer extends StdSerializer<PhotonDoc> {
20-
// Versioning of the json output format produced. This version appears
21-
// in JSON dumps and allows to track changes.
22-
public static final String FORMAT_VERSION = "1.0.0";
16+
private final DatabaseProperties dbProperties;
2317

24-
private final String[] languages;
25-
private final ConfigExtraTags extraTags;
26-
27-
public PhotonDocSerializer(String[] languages, ConfigExtraTags extraTags) {
18+
public PhotonDocSerializer(DatabaseProperties dbProperties) {
2819
super(PhotonDoc.class);
29-
this.languages = languages;
30-
this.extraTags = extraTags;
20+
this.dbProperties = dbProperties;
3121
}
3222

3323
@Override
@@ -71,14 +61,14 @@ public void serialize(PhotonDoc value, JsonGenerator gen, SerializerProvider pro
7161
gen.writeStringField("postcode", value.getPostcode());
7262
}
7363

74-
writeName(gen, value, languages);
64+
writeName(gen, value);
7565

7666
for (var entry : value.getAddressParts().keySet()) {
7767
Map<String, String> fNames = new HashMap<>();
7868

7969
value.copyAddressName(fNames, "default", entry, "name");
8070

81-
for (String language : languages) {
71+
for (String language : dbProperties.getLanguages()) {
8272
value.copyAddressName(fNames, language, entry, "name:" + language);
8373
}
8474

@@ -90,19 +80,19 @@ public void serialize(PhotonDoc value, JsonGenerator gen, SerializerProvider pro
9080
gen.writeStringField(Constants.COUNTRYCODE, countryCode);
9181
}
9282

93-
writeContext(gen, value.getContextByLanguage(languages));
94-
extraTags.writeFilteredExtraTags(gen, "extra", value.getExtratags());
83+
writeContext(gen, value.getContextByLanguage(dbProperties.getLanguages()));
84+
dbProperties.configExtraTags().writeFilteredExtraTags(gen, "extra", value.getExtratags());
9585
writeExtent(gen, value.getBbox());
9686

9787
gen.writeEndObject();
9888
}
9989

100-
private void writeName(JsonGenerator gen, PhotonDoc doc, String[] languages) throws IOException {
90+
private void writeName(JsonGenerator gen, PhotonDoc doc) throws IOException {
10191
Map<String, String> fNames = new HashMap<>();
10292

10393
doc.copyName(fNames, "default", "name");
10494

105-
for (String language : languages) {
95+
for (String language : dbProperties.getLanguages()) {
10696
doc.copyName(fNames, language, "name:" + language);
10797
}
10898

0 commit comments

Comments
 (0)