Merge pull request #903 from hircumg/feature/dedupe-arg

Added dedupe arg for requests

Author: lonvia

README.md

@@ -113,7 +113,7 @@ photon uses gradle for building. To build the package from source make sure you
 ./gradlew build
 ```
 
-This will build and test photon in the ElasticSearch and OpenSearch version. 
+This will build and test photon in the ElasticSearch and OpenSearch version.
 The final jar can be found in the `target` directory.
 
 ## Usage
@@ -431,6 +431,18 @@ http://localhost:2322/api?q=berlin&layer=city&layer=locality
 
 Example above will return both cities and localities.
 
+#### Dedupe results
+
+```
+http://localhost:2322/api?q=berlin&dedupe=1
+```
+
+Sometimes you have several objects in OSM identifying the same place or object in reality.
+The simplest case is a street being split into many different OSM ways due to different characteristics.
+Photon will attempt to detect such duplicates and only return one match.
+Setting `dedupe` parameter to `0` disables this deduplication mechanism and ensures that all results are returned.
+By default, Photon will attempt to deduplicate results which have the same `name`, `postcode`, and `osm_value` if exists.
+
 ### Results for Search and Reverse
 
 Photon returns a response in [GeocodeJson format](https://github.com/geocoders/geocodejson-spec/tree/master/draft)

src/main/java/de/komoot/photon/GenericSearchHandler.java

@@ -30,7 +30,9 @@ public void handle(@NotNull Context context) {
         var results = requestHandler.search(searchRequest);
 
         // Further filtering
-        results = new StreetDupesRemover(searchRequest.getLanguage()).execute(results);
+        if (searchRequest.getDedupe()){
+            results = new StreetDupesRemover(searchRequest.getLanguage()).execute(results);
+        }
 
         // Restrict to the requested limit.
         if (results.size() > searchRequest.getLimit()) {

src/main/java/de/komoot/photon/query/RequestBase.java

@@ -8,6 +8,7 @@ public class RequestBase {
     private String language = "default";
     private int limit = 15;
     private boolean debug = false;
+    private boolean dedupe = true;
     private boolean returnGeometry = false;
 
     private final List<TagFilter> osmTagFilters = new ArrayList<>(1);
@@ -25,6 +26,10 @@ public boolean getDebug() {
         return debug;
     }
 
+    public boolean getDedupe() {
+        return dedupe;
+    }
+
     public boolean getReturnGeometry() {
         return returnGeometry;
     }
@@ -55,6 +60,12 @@ public void setDebug(Boolean debug) {
         }
     }
 
+    public void setDedupe(Boolean dedupe) {
+        if (dedupe != null) {
+            this.dedupe = dedupe;
+        }
+    }
+
     public void setReturnGeometry(Boolean returnGeometry) {
         if (returnGeometry != null) {
             this.returnGeometry = returnGeometry;

src/main/java/de/komoot/photon/query/RequestFactoryBase.java

@@ -12,7 +12,7 @@
 
 public class RequestFactoryBase {
     protected static final Set<String> BASE_PARAMETERS = Set.of(
-            "lang", "limit", "debug", "geometry", "osm_tag", "layer");
+        "lang", "limit", "debug", "dedupe", "geometry", "osm_tag", "layer");
     private static final List<String> AVAILABLE_LAYERS = AddressType.getNames();
     private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(new PrecisionModel(), 4326);
 
@@ -37,6 +37,8 @@ protected void completeBaseRequest(RequestBase request, Context context) {
 
         request.setDebug(context.queryParamAsClass("debug", Boolean.class).getOrDefault(false));
 
+        request.setDedupe(context.queryParamAsClass("dedupe", Boolean.class).getOrDefault(true));
+
         request.addLayerFilters(context.queryParamsAsClass("layer", String.class)
                 .allowNullable()
                 .check(AVAILABLE_LAYERS::containsAll,

src/test/java/de/komoot/photon/api/ApiDedupeTest.java

@@ -0,0 +1,82 @@
+package de.komoot.photon.api;
+
+import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson;
+
+import de.komoot.photon.App;
+import de.komoot.photon.Importer;
+import de.komoot.photon.PhotonDoc;
+import java.nio.file.Path;
+import java.util.List;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/**
+ * Tests for dedupe works correctly
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class ApiDedupeTest extends ApiBaseTester {
+
+    @BeforeAll
+    void setUp(@TempDir Path dataDirectory) throws Exception {
+        setUpES(dataDirectory);
+        Importer instance = makeImporter();
+
+        instance.add(
+            List.of(
+                new PhotonDoc()
+                    .placeId(1000)
+                    .osmType("W")
+                    .osmId(1000)
+                    .tagKey("highway")
+                    .tagValue("residential")
+                    .postcode("1000")
+                    .rankAddress(26)
+                    .centroid(makePoint(15.94174, 45.80355))
+                    .names(makeDocNames("name", "Pfanove"))
+            )
+        );
+        instance.add(
+            List.of(
+                new PhotonDoc()
+                    .placeId(1001)
+                    .osmType("W")
+                    .osmId(1001)
+                    .tagKey("highway")
+                    .tagValue("residential")
+                    .postcode("1000")
+                    .rankAddress(26)
+                    .centroid(makePoint(15.94192, 45.802429))
+                    .names(makeDocNames("name", "Pfanove"))
+            )
+        );
+
+        instance.finish();
+        refresh();
+        startAPI();
+    }
+
+    @AfterAll
+    public void tearDown() {
+        App.shutdown();
+        shutdownES();
+    }
+
+    @ParameterizedTest
+    @ValueSource(
+        strings = {
+            "/api?q=Pfanove", // basic search
+            "/api?q=Pfanove&dedupe=1", // explicitly enabled dedupe
+        }
+    )
+    void testEnabledDedupe(String baseUrl) throws Exception {
+        assertThatJson(readURL(baseUrl)).isObject().node("features").isArray().hasSize(1);
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = { "/api?q=Pfanove&dedupe=0" })
+    void testDisabledDedupe(String baseUrl) throws Exception {
+        assertThatJson(readURL(baseUrl)).isObject().node("features").isArray().hasSize(2);
+    }
+}