Make metadata getAll fetch limit configurable (#209)

Author: em3s

engine/src/main/kotlin/com/kakao/actionbase/v2/engine/Graph.kt

@@ -174,6 +174,8 @@ class Graph(
 
     val encoderPoolSize = config.encoderPoolSize
 
+    val metadataFetchLimit = config.metadataFetchLimit
+
     init {
         if (config.metastoreReloadInitialDelay != null && config.metastoreReloadInterval != null) {
             startMetastoreReload(config.metastoreReloadInitialDelay, config.metastoreReloadInterval, log)
@@ -834,10 +836,7 @@ class Graph(
         return onlineMetadataLabel.mutate(edges, EdgeOperation.INSERT, bulk = true).then()
     }
 
-    @Suppress("ForbiddenComment")
     private fun getOnlineMetadata(type: MetadataType): Mono<List<RowWithSchema>> {
-        // TODO: use configuration or pagination
-        val sufficientFetchSize = 1000
         val bound = Duration.ofMinutes(2)
         val lastTs = System.currentTimeMillis() - bound.toMillis()
 
@@ -846,7 +845,7 @@ class Graph(
                 name = Metadata.onlineMetadataLabelV2Entity.name,
                 srcSet = setOf(MetadataSyncEntity.Src(phase, type).toCompositeKey()),
                 indexName = Metadata.onlineMetadataLabelV2Entity.indices[0].name,
-                limit = sufficientFetchSize,
+                limit = metadataFetchLimit,
             )
 
         return singleStepQuery(scanFilter, emptySet())

engine/src/main/kotlin/com/kakao/actionbase/v2/engine/GraphConfig.kt

@@ -1,6 +1,7 @@
 package com.kakao.actionbase.v2.engine
 
 import com.kakao.actionbase.v2.engine.entity.DefaultStorageEntity
+import com.kakao.actionbase.v2.engine.service.ddl.DdlService
 import com.kakao.actionbase.v2.engine.warmup.WarmUpConfig
 
 import java.net.InetAddress
@@ -31,6 +32,7 @@ data class GraphConfig(
     // Aligned with nginx.conf proxy_read_timeout 300
     val mutationRequestTimeout: Long = 300_000,
     val hbase: Map<String, String> = emptyMap(),
+    val metadataFetchLimit: Int = DdlService.DEFAULT_METADATA_LIMIT,
 ) {
     companion object {
         val builder: Builder
@@ -58,6 +60,7 @@ data class GraphConfig(
         private var warmUp: WarmUpConfig = WarmUpConfig()
         private var artifactInfo: String? = null
         private var hbase: Map<String, String> = emptyMap()
+        private var metadataFetchLimit: Int = DdlService.DEFAULT_METADATA_LIMIT
 
         // Aligned with nginx.conf proxy_read_timeout 300
         private var mutationRequestTimeout: Long = 300_000
@@ -110,6 +113,12 @@ data class GraphConfig(
 
         fun withHBase(hbase: Map<String, String>) = apply { this.hbase = hbase }
 
+        fun withMetadataFetchLimit(limit: Int) =
+            apply {
+                require(limit > 0) { "ddlFetchLimit must be positive, got $limit" }
+                this.metadataFetchLimit = limit
+            }
+
         fun build(): GraphConfig =
             GraphConfig(
                 phase = phase,
@@ -133,6 +142,7 @@ data class GraphConfig(
                 artifactInfo = artifactInfo,
                 mutationRequestTimeout = mutationRequestTimeout,
                 hbase = hbase,
+                metadataFetchLimit = metadataFetchLimit,
             )
     }
 }

engine/src/main/kotlin/com/kakao/actionbase/v2/engine/service/ddl/DdlService.kt

@@ -25,7 +25,6 @@ abstract class DdlService<Entity : EdgeEntity, Create : DdlRequest, Update : Ddl
     protected val graph: Graph,
     private val label: Label,
     private val factory: EntityFactory<Entity>,
-    private val limit: Int = DEFAULT_METADATA_LIMIT,
 ) {
     fun createMetadataOnlyForTest(
         name: EntityName,
@@ -171,7 +170,7 @@ abstract class DdlService<Entity : EdgeEntity, Create : DdlRequest, Update : Ddl
             ScanFilter(
                 name = label.name,
                 srcSet = setOf(name.phaseServiceName),
-                limit = limit,
+                limit = graph.metadataFetchLimit,
             )
         return label
             .scan(scanFilter, emptySet(), EmptyEdgeIdEncoder.INSTANCE)

server/src/main/kotlin/com/kakao/actionbase/server/configuration/GraphConfiguration.kt

@@ -81,6 +81,7 @@ class GraphConfiguration {
                         .of(infoEndpoint)
                 withArtifactInfo(artifactInfo.toString())
                 withHBase(properties.hbase)
+                properties.metadataFetchLimit?.let { withMetadataFetchLimit(it) }
             }
         return builder.build()
     }

server/src/main/kotlin/com/kakao/actionbase/server/configuration/GraphProperties.kt

@@ -30,4 +30,5 @@ data class GraphProperties(
     val allowMirror: Boolean = false,
     val mutationRequestTimeout: Long?,
     val hbase: Map<String, String> = emptyMap(),
+    val metadataFetchLimit: Int?,
 )