Skip to content

Commit 28755fc

Browse files
lonvialonvia
authored
Merge pull request #899 from lonvia/more-tests
Extend test coverage for Nominatim importer and json reader and dumper
2 parents 8f303c1 + fd5c47f commit 28755fc

File tree

18 files changed

+520
-479
lines changed

18 files changed

+520
-479
lines changed

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/PhotonDocSerializer.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import org.locationtech.jts.io.geojson.GeoJsonWriter;
99

1010
import java.io.IOException;
11-
import java.util.HashMap;
1211
import java.util.Map;
1312
import java.util.Set;
1413

‎src/main/java/de/komoot/photon/PhotonDoc.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ public PhotonDoc addAddresses(Map<String, String> address, String[] languages) {
253253
return this;
254254
}
255255

256+
List<String> langList = Arrays.asList(languages);
256257
Map<AddressType, Map<String, String>> overlay = new EnumMap<>(AddressType.class);
257258
for (var entry : address.entrySet()) {
258259
final String key = entry.getKey();
@@ -264,27 +265,27 @@ public PhotonDoc addAddresses(Map<String, String> address, String[] languages) {
264265
.stream()
265266
.filter(e -> key.startsWith(e.getValue()))
266267
.findFirst()
267-
.map(e -> {
268+
.ifPresent(e -> {
268269
var atype = e.getKey();
269270
if (atype == AddressType.OTHER) {
270271
final String[] parts = key.split(":", 0);
272+
final String intKey = parts[parts.length - 1];
271273
if (parts.length == 1) {
272274
context.addName("default", entry.getValue());
273-
} else if (Arrays.stream(languages).noneMatch(l -> l.equals(parts[parts.length - 1]))) {
274-
context.addName(parts[parts.length - 1], entry.getValue());
275+
} else if (langList.contains(intKey)) {
276+
context.addName(intKey, entry.getValue());
275277
}
276278
} else {
277279
int prefixLen = e.getValue().length();
278280
if (key.length() == prefixLen) {
279281
overlay.computeIfAbsent(atype, k -> new HashMap<>()).put("default", entry.getValue());
280282
} else if (key.charAt(prefixLen) == ':') {
281283
final String intKey = key.substring(prefixLen + 1);
282-
if (Arrays.stream(languages).noneMatch(l -> l.equals(intKey))) {
283-
overlay.computeIfAbsent(atype, k -> new HashMap<>()).put("default", entry.getValue());
284+
if (langList.contains(intKey)) {
285+
overlay.computeIfAbsent(atype, k -> new HashMap<>()).put(intKey, entry.getValue());
284286
}
285287
}
286288
}
287-
return true;
288289
});
289290
}
290291
}

‎src/main/java/de/komoot/photon/PhotonDocAddressSet.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ private void addPlaceAddress(PhotonDoc base, Map<String, String> address, String
4444

4545
final String place = address.get("place");
4646
if (place != null && !place.isBlank()) {
47-
placeAddress.put(AddressType.STREET, Map.of("name", place));
47+
placeAddress.put(AddressType.STREET, Map.of("default", place));
4848
}
4949
}
5050

@@ -75,7 +75,7 @@ private void addGenericAddress(PhotonDoc base, Map<String, String> address, Stri
7575
}
7676
}
7777

78-
placeAddress.put(AddressType.STREET, Map.of("name", place));
78+
placeAddress.put(AddressType.STREET, Map.of("default", place));
7979
} else {
8080
placeAddress = base.getAddressParts();
8181
}

‎src/main/java/de/komoot/photon/json/JsonDumper.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,13 @@ public void writeHeader(Map<String, NameMap> countryNames) throws IOException {
9191
if (!e.getKey().isBlank()) {
9292
writer.writeStartObject();
9393
writer.writeObjectField("country_code", e.getKey());
94-
writer.writeObjectField("name", e.getValue());
94+
writer.writeObjectFieldStart("name");
95+
for (var entry : e.getValue().entrySet()) {
96+
writer.writeStringField(
97+
convertNameKey(entry.getKey(), "name"),
98+
entry.getValue());
99+
}
100+
writer.writeEndObject();
95101
writer.writeEndObject();
96102
}
97103
}

‎src/main/java/de/komoot/photon/json/JsonReader.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import de.komoot.photon.UsageException;
1010
import de.komoot.photon.nominatim.ImportThread;
1111
import de.komoot.photon.nominatim.model.AddressRow;
12+
import de.komoot.photon.nominatim.model.NameMap;
1213
import org.apache.logging.log4j.LogManager;
1314
import org.apache.logging.log4j.Logger;
1415

@@ -22,7 +23,7 @@ public class JsonReader {
2223

2324
private final JsonParser parser;
2425
private NominatimDumpHeader header = null;
25-
private final Map<String, Map<String, String>> countryNames = new HashMap<>();
26+
private final Map<String, NameMap> countryNames = new HashMap<>();
2627
private final Map<Long, AddressRow> addressCache = new HashMap<>();
2728

2829
private boolean useFullGeometries = false;
@@ -153,7 +154,8 @@ public void readFile(ImportThread importThread) throws IOException {
153154
}
154155
while (parser.nextToken() != JsonToken.END_ARRAY) {
155156
var cinfo = parser.readValueAs(CountryInfo.class);
156-
countryNames.put(cinfo.getCountryCode().toUpperCase(), cinfo.getName());
157+
countryNames.put(cinfo.getCountryCode().toUpperCase(),
158+
NameMap.makeForPlace(cinfo.getName(), languages));
157159
}
158160
} else {
159161
LOGGER.warn("Unknown document type '{}'. Ignored.", docType);
@@ -208,7 +210,7 @@ private String readStartDocument() throws IOException {
208210
}
209211
}
210212

211-
LOGGER.error("Missing 'properties' field at {}", parser.currentLocation());
213+
LOGGER.error("Missing 'content' field at {}", parser.currentLocation());
212214
throw new UsageException("Invalid dump file.");
213215
}
214216

‎src/main/java/de/komoot/photon/nominatim/NominatimConnector.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,9 @@
55
import de.komoot.photon.nominatim.model.NameMap;
66
import org.apache.commons.dbcp2.BasicDataSource;
77
import org.springframework.jdbc.core.JdbcTemplate;
8-
import org.springframework.jdbc.core.RowMapper;
98
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
109
import org.springframework.transaction.support.TransactionTemplate;
1110

12-
import java.sql.ResultSet;
13-
import java.sql.SQLException;
1411
import java.util.Date;
1512
import java.util.HashMap;
1613
import java.util.List;

‎src/main/java/de/komoot/photon/nominatim/model/NameMap.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ NameMap setName(String field, Map<String, String> source, String... keys) {
3838
.map(source::get)
3939
.filter(Objects::nonNull)
4040
.findFirst()
41-
.map(k -> entries.add(new SimpleImmutableEntry<>(field, k)));
41+
.ifPresent(k -> entries.add(new SimpleImmutableEntry<>(field, k)));
4242
}
4343
return this;
4444
}

‎src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException {
3939
.rankAddress(rs.getInt("rank_address"))
4040
.postcode(rs.getString("postcode"));
4141

42+
final var admin_level = rs.getInt("admin_level");
43+
if (admin_level > 0 && admin_level < 15) {
44+
doc.adminLevel(admin_level);
45+
}
46+
4247
if (useGeometryColumn) {
4348
try {
4449
doc.geometry(dbutils.extractGeometry(rs, "geometry"));

‎src/test/java/de/komoot/photon/AssertUtil.java‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎src/test/java/de/komoot/photon/PhotonDocAddressSetTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ class PhotonDocAddressSetTest {
1919

2020
@BeforeEach
2121
public void setupPhotonDoc() {
22-
baseDoc.setAddressPartIfNew(AddressType.CITY, Map.of("name", "Hamburg"));
23-
baseDoc.setAddressPartIfNew(AddressType.STREET, Map.of("name", "Chaussee"));
22+
baseDoc.setAddressPartIfNew(AddressType.CITY, Map.of("default", "Hamburg"));
23+
baseDoc.setAddressPartIfNew(AddressType.STREET, Map.of("default", "Chaussee"));
2424
}
2525

2626
private void assertDocWithHousenumber(PhotonDoc doc, String housenumber) {
@@ -38,8 +38,8 @@ private void assertDocWithHousenumber(PhotonDoc doc, String housenumber) {
3838
private void assertDocWithHnrAndStreet(PhotonDoc doc, String housenumber, String street) {
3939
assertAll(
4040
() -> assertDocWithHousenumber(doc, housenumber),
41-
() -> assertEquals(Map.of("name", street), doc.getAddressParts().get(AddressType.STREET)),
42-
() -> assertEquals(Map.of("name", "Hamburg"), doc.getAddressParts().get(AddressType.CITY))
41+
() -> assertEquals(Map.of("default", street), doc.getAddressParts().get(AddressType.STREET)),
42+
() -> assertEquals(Map.of("default", "Hamburg"), doc.getAddressParts().get(AddressType.CITY))
4343
);
4444
}
4545

0 commit comments

Comments
 (0)