add chunk-optimized unique method (#245)

* add chunk-optimized unique method

* use rafaqz's unique method

* test for use of iteration

* add test for predicate

* fix tabstops

Author: bjarthur

src/mapreduce.jl

@@ -66,3 +66,10 @@ function Base.count(f, v::AbstractDiskArray)
         count(f, v[chunk...])
     end
 end
+
+Base.unique(v::AbstractDiskArray) = unique(identity, v)
+function Base.unique(f, v::AbstractDiskArray)
+    reduce((unique(f, v[c...]) for c in eachchunk(v))) do acc, u
+        unique!(f, append!(acc, u))
+    end
+end

test/runtests.jl

@@ -1061,3 +1061,11 @@ end
     end
     @test count(a) + count(!, a) == length(a)
 end
+
+@testset "unique" begin
+    a = ChunkedDiskArray((1:100) .& 7, chunksize=(9,))
+    out = @capture_out @trace unique(a) DiskArrays
+    @test occursin("_iterate_disk", out) == false
+    @test length(unique(a)) == length(unique(identity, a)) == 8
+    @test unique(x->x>3, a) == [1,4]
+end