feat!(config): support version lists in preprocessing pipeline config for better developer experience (#5083)

resolves #4703

### Config changes (non-breaking)

Support version as list in preprocessing part of config

### Description

Bumping prepro pipeline versions has been a bit of a pain due to having
to copy/paste/delete or comment/uncomment entries - in particular also
when nothing changed but the version (which is the most common case).

This PR makes bumping prepro versions easier by allowing version to be a
list, which acts as syntactic sugar for a copy for each entry in the
list with version substituted by each item's value.

To catch mistakes, we validate that each version appears at most once
and throw an error if that's not the case.

This change is fully backwards compatible. What's *breaking* is that we
can now take advantage of this new feature!

Example usage:
```yaml
preprocessing:
  - version: [1, 2, 3]  # Creates 3 separate deployments
    image: ghcr.io/loculus-project/preprocessing-nextclade
  - version: 4          # Creates 1 additional deployment
```

### Screenshot

<img width="1504" height="532" alt="image"
src="https://github.com/user-attachments/assets/6802daab-454c-4076-af90-33488dd72d68"
/>

### Manual testing

I tested that the validation for duplicates works.

Author: corneliusroemer

kubernetes/loculus/templates/_flattenPreprocessingVersions.tpl

@@ -0,0 +1,18 @@
+{{- define "loculus.flattenPreprocessingVersions" -}}
+{{- $preprocessing := . -}}
+{{- $flattened := list -}}
+{{- $seen := dict -}}
+{{- range $pc := $preprocessing -}}
+  {{- $versions := (kindIs "slice" $pc.version | ternary $pc.version (list $pc.version)) -}}
+  {{- range $v := $versions -}}
+    {{- if hasKey $seen (toString $v) -}}
+      {{- fail (printf "Duplicate preprocessing pipeline version %v found in organism configuration" $v) -}}
+    {{- end -}}
+    {{- $_ := set $seen (toString $v) true -}}
+    {{- $copy := deepCopy $pc -}}
+    {{- $_ := set $copy "version" $v -}}
+    {{- $flattened = append $flattened $copy -}}
+  {{- end -}}
+{{- end -}}
+{{- dict "items" $flattened | toJson -}}
+{{- end -}}

kubernetes/loculus/templates/loculus-preprocessing-config.yaml

@@ -3,7 +3,8 @@
 {{- $rawNucleotideSequences := (($organismConfig.schema | include "loculus.patchMetadataSchema" | fromYaml).nucleotideSequences) }}
 {{- $nucleotideSequences := ($rawNucleotideSequences | default "" ) }}
 {{- $nucleotideSequencesList := (eq $rawNucleotideSequences nil | ternary (list "main") $rawNucleotideSequences) }}
-{{- range $processingIndex, $processingConfig := $organismConfig.preprocessing }}
+{{- $flattened := include "loculus.flattenPreprocessingVersions" $organismConfig.preprocessing | fromJson }}
+{{- range $processingIndex, $processingConfig := $flattened.items }}
 {{- if $processingConfig.configFile }}
 {{- /* Use the enaDepositionConfig as the base config */}}
 {{- $enaDepositionConfig := dig "enaDeposition" "configFile" dict $organismConfig }}

kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml

@@ -7,7 +7,8 @@
 {{- $keycloakHost := $testconfig | ternary (printf "http://%s:8083" $.Values.localHost) "http://loculus-keycloak-service:8083" }}
 {{- if not .Values.disablePreprocessing }}
 {{- range $organism, $organismConfig := (include "loculus.enabledOrganisms" . | fromJson) }}
-{{- range $processingIndex, $processingConfig := $organismConfig.preprocessing }}
+{{- $flattened := include "loculus.flattenPreprocessingVersions" $organismConfig.preprocessing | fromJson }}
+{{- range $processingIndex, $processingConfig := $flattened.items }}
 {{- $thisDockerTag := $processingConfig.dockerTag | default $dockerTag }}
 {{- $replicas := $processingConfig.replicas | default 1 }}
 ---

kubernetes/loculus/values.schema.json

@@ -441,9 +441,21 @@
               "version": {
                 "groups": ["preprocessing"],
                 "docsIncludePrefix": false,
-                "type": "integer",
-                "minimum": 1,
-                "description": "Version of the preprocessing pipeline."
+                "oneOf": [
+                  {
+                    "type": "integer",
+                    "minimum": 1
+                  },
+                  {
+                    "type": "array",
+                    "items": {
+                      "type": "integer",
+                      "minimum": 1
+                    },
+                    "minItems": 1
+                  }
+                ],
+                "description": "Version of the preprocessing pipeline. Can be a single integer or an array of integers."
               },
               "image": {
                 "groups": ["preprocessing"],

kubernetes/loculus/values.yaml

@@ -1368,14 +1368,7 @@ defaultOrganisms:
     <<: *defaultOrganismConfig
     preprocessing:
       - <<: *preprocessing
-        version: 1
-        configFile:
-          <<: *preprocessingConfigFile      
-          genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L]
-          nextclade_dataset_name: nextstrain/ebola/sudan
-          nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/ebola/data_output
-      - <<: *preprocessing
-        version: 2
+        version: [1, 2]
         configFile:
           <<: *preprocessingConfigFile
           genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L]