Fix clear before finalize

Author: ypatia

libtiledbvcf/src/stats/allele_count.cc

@@ -336,19 +336,20 @@ void AlleleCount::flush(bool finalize) {
     fragment_sample_names_.insert(sample_names_.begin(), sample_names_.end());
     sample_names_.clear();
 
-    // Clear buffers
-    contig_buffer_.clear();
-    contig_offsets_.clear();
-    pos_buffer_.clear();
-    ref_buffer_.clear();
-    ref_offsets_.clear();
-    alt_buffer_.clear();
-    alt_offsets_.clear();
-    filter_buffer_.clear();
-    filter_offsets_.clear();
-    gt_buffer_.clear();
-    gt_offsets_.clear();
-    count_buffer_.clear();
+    // Shrink buffers to zero size but keep storage (resize(0)) so REST/client
+    // can still read from the same memory if submit response or finalize uses it.
+    contig_buffer_.resize(0);
+    contig_offsets_.resize(0);
+    pos_buffer_.resize(0);
+    ref_buffer_.resize(0);
+    ref_offsets_.resize(0);
+    alt_buffer_.resize(0);
+    alt_offsets_.resize(0);
+    filter_buffer_.resize(0);
+    filter_offsets_.resize(0);
+    gt_buffer_.resize(0);
+    gt_offsets_.resize(0);
+    count_buffer_.resize(0);
   }
 
   if (finalize) {

libtiledbvcf/src/stats/variant_stats.cc

@@ -441,20 +441,22 @@ void VariantStats::flush(bool finalize) {
     fragment_sample_names_.insert(sample_names_.begin(), sample_names_.end());
     sample_names_.clear();
 
-    // Clear buffers
-    contig_buffer_.clear();
-    contig_offsets_.clear();
-    pos_buffer_.clear();
-    sample_buffer_.clear();
-    sample_offsets_.clear();
-    allele_buffer_.clear();
-    allele_offsets_.clear();
-    ac_buffer_.clear();
-    an_buffer_.clear();
-    n_hom_buffer_.clear();
+    // Shrink buffers to zero size but keep storage (resize(0)) so REST async
+    // upload can still read from the same memory if submit() returned before
+    // the server finished copying; avoids data corruption from buffer reuse.
+    contig_buffer_.resize(0);
+    contig_offsets_.resize(0);
+    pos_buffer_.resize(0);
+    sample_buffer_.resize(0);
+    sample_offsets_.resize(0);
+    allele_buffer_.resize(0);
+    allele_offsets_.resize(0);
+    ac_buffer_.resize(0);
+    an_buffer_.resize(0);
+    n_hom_buffer_.resize(0);
     if (array_version_ >= 3) {
-      max_length_buffer_.clear();
-      end_buffer_.clear();
+      max_length_buffer_.resize(0);
+      end_buffer_.resize(0);
     }
   }
 

libtiledbvcf/src/utils/buffer.cc

@@ -96,7 +96,9 @@ void Buffer::append(const void* data, size_t bytes) {
 }
 
 void Buffer::clear() {
-  offsets_.clear();
+  // Use resize(0) so storage is preserved; REST async upload may still read
+  // from these buffers after submit() returns, avoiding S3 BadDigest.
+  offsets_.resize(0);
   data_size_ = 0;
   offset_nelts_ = 0;
   data_effective_size_ = 0;

libtiledbvcf/src/write/writer.cc

@@ -993,6 +993,9 @@ std::pair<uint64_t, uint64_t> Writer::ingest_samples_v4(
                 // Write and finalize anchors stored in the anchor worker.
                 anchors_ingested += write_anchors(anchor_worker);
 
+                // Unset buffer sizes on the query (same ptrs, size 0). The Query
+                // must retain the submit response for finalize; do not discard
+                // the query or its state until submit_and_finalize() completes.
                 worker->buffers().clear_query_buffers(
                     query_.get(), dataset_->metadata().version);
                 // Finalize fragment for this contig async
@@ -1104,8 +1107,8 @@ std::pair<uint64_t, uint64_t> Writer::ingest_samples_v4(
         }
       }
 
-      // When an ingestion worker is finished, clear its query buffers
-      // only if the query buffers are not empty.
+      // When an ingestion worker is finished, unset buffer sizes on the query
+      // (Query retains submit response for finalize). Only do so if set.
       if (finished && utils::query_buffers_set(query_.get())) {
         worker->buffers().clear_query_buffers(
             query_.get(), dataset_->metadata().version);
@@ -1389,6 +1392,12 @@ void Writer::dataset_version(int32_t* version) const {
   *version = dataset_->metadata().version;
 }
 
+// REST write protocol: Submit (WRITE) returns serialized query state in the
+// response. The client must retain that exact state and send it as the
+// finalize request body. If the client discards the submit response before
+// calling finalize, the server cannot deserialize the query correctly. The
+// TileDB Query object must hold the submit response until submit_and_finalize()
+// completes, which sends the submit response.
 void Writer::finalize_query(std::unique_ptr<tiledb::Query> query) {
   if (utils::query_buffers_set(query.get())) {
     LOG_FATAL("Cannot submit_and_finalize query with buffers set.");