make db persistent for testing migration

backend/docs/db/schema.sql

@@ -427,7 +427,6 @@ CREATE TABLE public.sequence_entries (
     released_at timestamp without time zone,
     is_revocation boolean DEFAULT false NOT NULL,
     original_data jsonb,
-    version_comment text,
     unprocessed_data jsonb
 );
 
@@ -469,11 +468,11 @@ CREATE VIEW public.sequence_entries_view AS
     se.released_at,
     se.is_revocation,
     se.unprocessed_data,
-    se.version_comment,
     sepd.started_processing_at,
     sepd.finished_processing_at,
     sepd.processed_data,
         CASE
+            WHEN se.is_revocation THEN jsonb_build_object('metadata', COALESCE((se.unprocessed_data -> 'metadata'::text), '{}'::jsonb), 'unalignedNucleotideSequences', '{}'::jsonb, 'alignedNucleotideSequences', '{}'::jsonb, 'nucleotideInsertions', '{}'::jsonb, 'alignedAminoAcidSequences', '{}'::jsonb, 'aminoAcidInsertions', '{}'::jsonb, 'files', 'null'::jsonb)
             WHEN (aem.external_metadata IS NULL) THEN sepd.processed_data
             ELSE (sepd.processed_data || jsonb_build_object('metadata', ((sepd.processed_data -> 'metadata'::text) || aem.external_metadata)))
         END AS joint_metadata,

backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt

@@ -165,14 +165,6 @@ open class ReleasedDataModel(
                     )
                 },
             ) +
-            conditionalMetadata(
-                rawProcessedData.isRevocation,
-                {
-                    mapOf(
-                        "versionComment" to TextNode(rawProcessedData.versionComment),
-                    )
-                },
-            ) +
             conditionalMetadata(
                 earliestReleaseDate != null,
                 {

backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt

@@ -52,6 +52,7 @@ import org.loculus.backend.api.FileIdAndNameAndReadUrl
 import org.loculus.backend.api.GeneticSequence
 import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Organism
+import org.loculus.backend.api.OriginalData
 import org.loculus.backend.api.OriginalDataWithFileUrls
 import org.loculus.backend.api.PreprocessingStatus.IN_PROCESSING
 import org.loculus.backend.api.PreprocessingStatus.PROCESSED
@@ -743,7 +744,6 @@ class SubmissionDatabaseService(
             SequenceEntriesView.accessionColumn,
             SequenceEntriesView.versionColumn,
             SequenceEntriesView.isRevocationColumn,
-            SequenceEntriesView.versionCommentColumn,
             SequenceEntriesView.jointDataColumn,
             SequenceEntriesView.submitterColumn,
             SequenceEntriesView.groupIdColumn,
@@ -786,7 +786,6 @@ class SubmissionDatabaseService(
                     DataUseTermsType.fromString(it[DataUseTermsTable.dataUseTermsTypeColumn]),
                     it[DataUseTermsTable.restrictedUntilColumn],
                 ),
-                versionComment = it[SequenceEntriesView.versionCommentColumn],
                 dataUseTermsChangeDate = it[DataUseTermsTable.changeDateColumn],
             )
         }
@@ -962,16 +961,14 @@ class SubmissionDatabaseService(
 
         SequenceEntriesTable.insert(
             SequenceEntriesTable.select(
-                SequenceEntriesTable.accessionColumn, SequenceEntriesTable.versionColumn.plus(1),
-                when (versionComment) {
-                    null -> Op.nullOp()
-                    else -> stringParam(versionComment)
-                },
+                SequenceEntriesTable.accessionColumn,
+                SequenceEntriesTable.versionColumn.plus(1),
                 SequenceEntriesTable.submissionIdColumn,
                 stringParam(authenticatedUser.username),
                 SequenceEntriesTable.groupIdColumn,
                 dateTimeParam(dateProvider.getCurrentDateTime()),
-                booleanParam(true), SequenceEntriesTable.organismColumn,
+                booleanParam(true),
+                SequenceEntriesTable.organismColumn,
             ).where {
                 (
                     SequenceEntriesTable.accessionColumn inList
@@ -982,7 +979,6 @@ class SubmissionDatabaseService(
             columns = listOf(
                 SequenceEntriesTable.accessionColumn,
                 SequenceEntriesTable.versionColumn,
-                SequenceEntriesTable.versionCommentColumn,
                 SequenceEntriesTable.submissionIdColumn,
                 SequenceEntriesTable.submitterColumn,
                 SequenceEntriesTable.groupIdColumn,
@@ -992,6 +988,28 @@ class SubmissionDatabaseService(
             ),
         )
 
+        if (versionComment != null) {
+            log.debug { "Adding version comment for revocation: $versionComment" }
+            val metadata = mapOf("versionComment" to versionComment)
+            val originalData = compressionService.compressSequencesInOriginalData(
+                OriginalData(
+                    metadata = metadata,
+                    unalignedNucleotideSequences = emptyMap(),
+                ),
+                organism,
+            )
+            SequenceEntriesTable.update(
+                where = {
+                    (SequenceEntriesTable.accessionColumn inList accessions) and
+                        SequenceEntriesTable.isMaxVersion and
+                        (SequenceEntriesTable.isRevocationColumn eq true)
+                },
+            ) {
+                it[originalDataColumn] = originalData
+                it[unprocessedDataColumn] = originalData
+            }
+        }
+
         auditLogger.log(
             authenticatedUser.username,
             "Revoked ${accessions.size} sequences: " +
@@ -1532,7 +1550,6 @@ data class RawProcessedData(
     override val accession: Accession,
     override val version: Version,
     val isRevocation: Boolean,
-    val versionComment: String?,
     val submitter: String,
     val groupId: Int,
     val groupName: String,

backend/src/main/kotlin/org/loculus/backend/service/submission/dbtables/SequenceEntriesTable.kt

@@ -24,7 +24,6 @@ object SequenceEntriesTable : Table(SEQUENCE_ENTRIES_TABLE_NAME) {
 
     val accessionColumn = varchar("accession", 255)
     val versionColumn = long("version")
-    val versionCommentColumn = varchar("version_comment", 255).nullable()
     val organismColumn = varchar("organism", 255)
     val submissionIdColumn = varchar("submission_id", 255)
     val submitterColumn = varchar("submitter", 255)

backend/src/main/kotlin/org/loculus/backend/service/submission/dbtables/SequenceEntriesView.kt

@@ -45,7 +45,6 @@ object SequenceEntriesView : Table(SEQUENCE_ENTRIES_VIEW_NAME) {
     val statusColumn = varchar("status", 255)
     val processingResultColumn = varchar("processing_result", 255).nullable()
     val isRevocationColumn = bool("is_revocation").default(false)
-    val versionCommentColumn = varchar("version_comment", 255).nullable()
     val errorsColumn = jacksonSerializableJsonb<List<PreprocessingAnnotation>>("errors").nullable()
     val warningsColumn = jacksonSerializableJsonb<List<PreprocessingAnnotation>>("warnings").nullable()
     val pipelineVersionColumn = long("pipeline_version").nullable()

backend/src/main/resources/db/migration/V1.27__move_version_comment_to_metadata.sql

@@ -0,0 +1,108 @@
+-- Moves revocation versionComment from the dedicated version_comment column
+-- into the unprocessed_data JSONB metadata, and updates the view to automatically
+-- construct joint_metadata for revocations from unprocessed_data.
+-- This unifies how versionComment is stored for both revisions and revocations.
+
+-- Step 1: For existing revocations with non-null version_comment and no unprocessed_data,
+-- create unprocessed_data with versionComment in metadata
+UPDATE sequence_entries
+SET unprocessed_data = jsonb_build_object(
+    'metadata', jsonb_build_object('versionComment', version_comment),
+    'unalignedNucleotideSequences', '{}'::jsonb
+)
+WHERE is_revocation = true AND version_comment IS NOT NULL AND unprocessed_data IS NULL;
+
+-- For revocations that already have unprocessed_data (unlikely but safe),
+-- merge versionComment into existing metadata
+UPDATE sequence_entries
+SET unprocessed_data = jsonb_set(
+    unprocessed_data,
+    '{metadata}',
+    COALESCE(unprocessed_data -> 'metadata', '{}'::jsonb) || jsonb_build_object('versionComment', version_comment)
+)
+WHERE is_revocation = true AND version_comment IS NOT NULL AND unprocessed_data IS NOT NULL;
+
+-- Step 1b: Ensure all revocations have unprocessed_data (even those without version_comment)
+UPDATE sequence_entries
+SET unprocessed_data = jsonb_build_object(
+    'metadata', '{}'::jsonb,
+    'unalignedNucleotideSequences', '{}'::jsonb
+)
+WHERE is_revocation = true AND unprocessed_data IS NULL;
+
+-- Step 2: Drop the view (must be done before dropping the column)
+DROP VIEW IF EXISTS sequence_entries_view;
+
+-- Step 3: Drop the version_comment column
+ALTER TABLE sequence_entries DROP COLUMN version_comment;
+
+-- Step 4: Recreate the view without version_comment.
+-- For revocations, joint_metadata is constructed from unprocessed_data
+-- so versionComment survives pipeline version changes.
+CREATE VIEW sequence_entries_view AS
+SELECT
+    se.accession,
+    se.version,
+    se.organism,
+    se.submission_id,
+    se.submitter,
+    se.approver,
+    se.group_id,
+    se.submitted_at,
+    se.released_at,
+    se.is_revocation,
+    se.unprocessed_data,
+    sepd.started_processing_at,
+    sepd.finished_processing_at,
+    sepd.processed_data,
+    CASE
+        WHEN se.is_revocation THEN
+            jsonb_build_object(
+                'metadata', COALESCE(se.unprocessed_data -> 'metadata', '{}'::jsonb),
+                'unalignedNucleotideSequences', '{}'::jsonb,
+                'alignedNucleotideSequences', '{}'::jsonb,
+                'nucleotideInsertions', '{}'::jsonb,
+                'alignedAminoAcidSequences', '{}'::jsonb,
+                'aminoAcidInsertions', '{}'::jsonb,
+                'files', 'null'::jsonb
+            )
+        WHEN aem.external_metadata IS NULL THEN sepd.processed_data
+        ELSE sepd.processed_data ||
+            jsonb_build_object('metadata', (sepd.processed_data -> 'metadata') || aem.external_metadata)
+    END AS joint_metadata,
+    CASE
+        WHEN se.is_revocation THEN cpp.version
+        ELSE sepd.pipeline_version
+    END AS pipeline_version,
+    sepd.errors,
+    sepd.warnings,
+    CASE
+        WHEN se.released_at IS NOT NULL THEN 'APPROVED_FOR_RELEASE'
+        WHEN se.is_revocation THEN 'PROCESSED'
+        WHEN sepd.processing_status = 'IN_PROCESSING' THEN 'IN_PROCESSING'
+        WHEN sepd.processing_status = 'PROCESSED' THEN 'PROCESSED'
+        ELSE 'RECEIVED'
+    END AS status,
+    CASE
+        WHEN sepd.processing_status = 'IN_PROCESSING' THEN NULL
+        WHEN sepd.errors IS NOT NULL AND jsonb_array_length(sepd.errors) > 0 THEN 'HAS_ERRORS'
+        WHEN sepd.warnings IS NOT NULL AND jsonb_array_length(sepd.warnings) > 0 THEN 'HAS_WARNINGS'
+        ELSE 'NO_ISSUES'
+    END AS processing_result
+FROM sequence_entries se
+LEFT JOIN current_processing_pipeline cpp
+    ON se.organism = cpp.organism
+LEFT JOIN sequence_entries_preprocessed_data sepd
+    ON se.accession = sepd.accession
+    AND se.version = sepd.version
+    AND sepd.pipeline_version = cpp.version
+LEFT JOIN (
+    SELECT
+        em.accession,
+        em.version,
+        jsonb_merge_agg(em.external_metadata) AS external_metadata
+    FROM external_metadata em
+    GROUP BY em.accession, em.version
+) aem
+    ON aem.accession = se.accession
+    AND aem.version = se.version;

backend/src/test/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinderTest.kt

@@ -66,7 +66,6 @@ fun row(
         files = null,
     ),
     isRevocation = false,
-    versionComment = null,
     submitter = "foo",
     submissionId = "foo",
     submittedAtTimestamp = releasedAt,

kubernetes/loculus/values.yaml

@@ -2574,7 +2574,7 @@ secrets:
 runDevelopmentKeycloakDatabase: true
 runDevelopmentMainDatabase: true
 runDevelopmentS3: true
-developmentDatabasePersistence: false
+developmentDatabasePersistence: true
 enforceHTTPS: true
 registrationTermsMessage: >
   You must agree to the <a href="http://main.loculus.org/terms">terms of use</a>.