feat: replace private by published_at

Author: ThibaudDauce

udata/core/dataset/api.py

@@ -254,7 +254,10 @@ def parse_filters(datasets, args):
         if args.get("private") is not None:
             if current_user.is_anonymous:
                 abort(401)
-            datasets = datasets.filter(private=args["private"])
+            if args["private"]:
+                datasets = datasets.filter(published_at=None)
+            else:
+                datasets = datasets.filter(published_at__ne=None)
         return datasets
 
 
@@ -306,9 +309,7 @@ class DatasetListAPI(API):
     def get(self):
         """List or search all datasets"""
         args = dataset_parser.parse()
-        datasets = Dataset.objects.visible_by_user(
-            current_user, mongoengine.Q(private__ne=True, archived=None, deleted=None)
-        )
+        datasets = Dataset.objects.visible_by_user(current_user)
         datasets = dataset_parser.parse_filters(datasets, args)
         sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
         return datasets.order_by(sort).paginate(args["page"], args["page_size"])
@@ -829,7 +830,7 @@ class DatasetSuggestAPI(API):
     def get(self):
         """Datasets suggest endpoint using mongoDB contains"""
         args = suggest_parser.parse_args()
-        datasets_query = Dataset.objects(archived=None, deleted=None, private=False)
+        datasets_query = Dataset.objects.visible()
         datasets = datasets_query.filter(
             Q(title__icontains=args["q"]) | Q(acronym__icontains=args["q"])
         )

udata/core/dataset/api_fields.py

@@ -274,6 +274,7 @@
         "last_modified",
         "deleted",
         "private",
+        "published_at",
         "tags",
         "badges",
         "resources",
@@ -349,7 +350,10 @@
         "archived": fields.ISODateTime(description="The archival date if archived"),
         "featured": fields.Boolean(description="Is the dataset featured"),
         "private": fields.Boolean(
-            description="Is the dataset private to the owner or the organization"
+            description="Is the dataset private (DEPRECATED: use published_at instead)"
+        ),
+        "published_at": fields.ISODateTime(
+            description="Publication date (null if unpublished/private)"
         ),
         "tags": fields.List(fields.String),
         "badges": fields.List(

udata/core/dataset/apiv2.py

@@ -50,6 +50,7 @@
         "last_modified",
         "deleted",
         "private",
+        "published_at",
         "tags",
         "badges",
         "resources",
@@ -125,7 +126,10 @@
         "archived": fields.ISODateTime(description="The archival date if archived"),
         "featured": fields.Boolean(description="Is the dataset featured"),
         "private": fields.Boolean(
-            description="Is the dataset private to the owner or the organization"
+            description="Is the dataset private (DEPRECATED: use published_at instead)"
+        ),
+        "published_at": fields.ISODateTime(
+            description="Publication date (null if unpublished/private)"
         ),
         "tags": fields.List(fields.String),
         "badges": fields.List(
@@ -317,9 +321,7 @@ class DatasetListAPI(API):
     def get(self):
         """List or search all datasets"""
         args = dataset_parser.parse()
-        datasets = Dataset.objects.exclude("resources").visible_by_user(
-            current_user, mongoengine.Q(private__ne=True, archived=None, deleted=None)
-        )
+        datasets = Dataset.objects.exclude("resources").visible_by_user(current_user)
         datasets = dataset_parser.parse_filters(datasets, args)
         sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
         return datasets.order_by(sort).paginate(args["page"], args["page_size"])

udata/core/dataset/csv.py

@@ -38,6 +38,7 @@ class DatasetCsvAdapter(csv.Adapter):
         "last_modified",
         ("tags", lambda o: ",".join(o.tags)),
         ("archived", lambda o: o.archived or False),
+        "published_at",
         ("resources_count", lambda o: len(o.resources)),
         ("main_resources_count", lambda o: len([r for r in o.resources if r.type == "main"])),
         ("resources_formats", lambda o: ",".join(set(r.format for r in o.resources if r.format))),
@@ -71,6 +72,7 @@ class ResourcesCsvAdapter(csv.NestedAdapter):
         ),
         dataset_field("license"),
         dataset_field("private"),
+        dataset_field("published_at"),
         dataset_field("archived", lambda r: r.archived or False),
     )
     nested_fields = (

udata/core/dataset/factories.py

@@ -1,4 +1,5 @@
 import json
+from datetime import datetime
 from os.path import join
 
 import factory
@@ -19,6 +20,7 @@ class Meta:
     title = factory.Faker("sentence")
     description = factory.Faker("text")
     frequency = UpdateFrequency.UNKNOWN
+    published_at = factory.LazyFunction(datetime.utcnow)
     resources = factory.LazyAttribute(lambda o: ResourceFactory.build_batch(o.nb_resources))
 
     class Params:
@@ -31,7 +33,7 @@ class Params:
 
 
 class HiddenDatasetFactory(DatasetFactory):
-    private = True
+    published_at = None
 
 
 class ChecksumFactory(ModelFactory):

udata/core/dataset/forms.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 from udata.core.access_type.constants import (
     AccessAudienceCondition,
     AccessAudienceType,
@@ -213,13 +215,68 @@ class DatasetForm(ModelForm):
         _("Private"),
         description=_("Restrict the dataset visibility to you or your organization only."),
     )
+    published_at = fields.DateTimeField(_("Publication date"))
 
     owner = fields.CurrentUserField()
     organization = fields.PublishAsField(_("Publish as"))
     extras = fields.ExtrasField()
     resources = fields.NestedModelList(ResourceForm)
     contact_points = fields.ContactPointListField(validators=[validate_contact_point])
 
+    # ==================================================================================
+    # Backward compatibility layer for `private` field
+    # ==================================================================================
+    # The `private` boolean field has been replaced by `published_at` (datetime) on the
+    # Dataset model. However, we need to maintain API backward compatibility.
+    #
+    # Challenges encountered:
+    # 1. MongoEngine rejects unknown fields in constructor - we can't just add a
+    #    `private` property with a setter on the model because ModelForm.save() passes
+    #    self.data directly to the model constructor, which fails with FieldDoesNotExist.
+    #
+    # 2. WTForms BooleanField.process_formdata() doesn't update self.data when the value
+    #    is empty/None, so we can't rely on self.private.data to detect explicit None.
+    #
+    # 3. self.data is a read-only property computed from field values, so we can't
+    #    modify it directly (e.g., self.data.pop("private") fails).
+    #
+    # Solution:
+    # - Override __init__ to populate private.data from the existing instance
+    # - Override populate_obj to skip the private field (model has no such field)
+    # - Override save to:
+    #   a) Check self.formdata (raw JSON dict) to detect if private was explicitly sent
+    #   b) Convert private -> published_at before creating/updating the instance
+    #   c) Exclude private from the data dict when creating new instances
+    # ==================================================================================
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.instance:
+            self.private.data = self.instance.published_at is None
+
+    def populate_obj(self, obj):
+        for name, field in self._fields.items():
+            if name != "private":
+                field.populate_obj(obj, name)
+
+    def save(self, commit=True, **kwargs):
+        if self.formdata and "private" in self.formdata:
+            private_value = self.formdata.get("private")
+            if private_value is True:
+                self.published_at.data = None
+            elif private_value is False or private_value is None:
+                self.published_at.data = datetime.utcnow()
+
+        if self.instance:
+            self.populate_obj(self.instance)
+        else:
+            data = {k: v for k, v in self.data.items() if k != "private"}
+            self.instance = self.model_class(**data)
+
+        if commit:
+            self.instance.save(**kwargs)
+        return self.instance
+
 
 class ResourcesListForm(ModelForm):
     model_class = Dataset

udata/core/dataset/models.py

@@ -340,10 +340,15 @@ def default(cls):
 
 class DatasetQuerySet(OwnedQuerySet):
     def visible(self):
-        return self(private__ne=True, deleted=None, archived=None)
+        return self(published_at__ne=None, deleted=None, archived=None)
 
     def hidden(self):
-        return self(db.Q(private=True) | db.Q(deleted__ne=None) | db.Q(archived__ne=None))
+        return self(db.Q(published_at=None) | db.Q(deleted__ne=None) | db.Q(archived__ne=None))
+
+    def visible_by_user(self, user):
+        return super().visible_by_user(
+            user, db.Q(published_at__ne=None, archived=None, deleted=None)
+        )
 
     def with_badge(self, kind):
         return self(badges__kind=kind)
@@ -553,7 +558,7 @@ class Dataset(
     tags = field(db.TagListField())
     resources = field(db.ListField(db.EmbeddedDocumentField(Resource)), auditable=False)
 
-    private = field(db.BooleanField(default=False))
+    published_at = field(db.DateTimeField())
 
     frequency = field(db.EnumField(UpdateFrequency))
     frequency_date = field(db.DateTimeField(verbose_name=_("Future date of update")))
@@ -600,6 +605,11 @@ class Dataset(
     def __str__(self):
         return self.title or ""
 
+    def to_dict(self, exclude=None):
+        data = super().to_dict(exclude=exclude)
+        data["private"] = self.private
+        return data
+
     __metrics_keys__ = [
         "discussions",
         "discussions_open",
@@ -743,7 +753,12 @@ def is_visible(self):
 
     @property
     def is_hidden(self):
-        return self.private or self.deleted or self.archived
+        return self.published_at is None or self.deleted or self.archived
+
+    @property
+    def private(self) -> bool:
+        """Computed property for backward compatibility."""
+        return self.published_at is None
 
     @property
     def full_title(self):

udata/core/dataset/tasks.py

@@ -91,10 +91,15 @@ def get_queryset(model_cls):
     if model_cls.__name__ == "Resource":
         model_cls = getattr(udata_models, "Dataset")
     params = {}
-    attrs = ("private", "deleted", "deleted_at")
-    for attr in attrs:
+    # Dataset uses published_at instead of private; other models still use private
+    if model_cls.__name__ == "Dataset":
+        params["published_at__ne"] = None
+    elif getattr(model_cls, "private", None):
+        params["private"] = False
+    # Filter out deleted/soft-deleted items
+    for attr in ("deleted", "deleted_at"):
         if getattr(model_cls, attr, None):
-            params[attr] = False
+            params[attr] = None
     # no_cache to avoid eating up too much RAM
     return model_cls.objects.filter(**params).no_cache()
 

udata/core/organization/api.py

@@ -580,7 +580,7 @@ def get(self, org):
         args = dataset_parser.parse()
         qs = Dataset.objects.owned_by(org)
         if not OrganizationPrivatePermission(org).can():
-            qs = qs(private__ne=True)
+            qs = qs(published_at__ne=None)
         return qs.order_by(args["sort"]).paginate(args["page"], args["page_size"])
 
 

udata/migrations/2025-12-10-add-published_at-field.py

@@ -0,0 +1,48 @@
+"""
+Migration: Add published_at field and populate from private field history.
+
+Phase 1 (fast): Bulk update all public datasets with created_at_internal
+Phase 2 (slow): Refine dates using activity history for more accuracy
+"""
+import logging
+
+import click
+
+from udata.core.dataset.activities import UserUpdatedDataset
+from udata.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Phase 1: Bulk update with created_at_internal...")
+
+    # Avoid downtime: set a default value immediately so the system stays functional
+    result = db.dataset.update_many(
+        {"private": False, "published_at": {"$exists": False}},
+        [{"$set": {"published_at": "$created_at_internal"}}],
+    )
+    log.info(f"Phase 1 done: {result.modified_count} datasets updated")
+
+    log.info("Phase 2: Refining dates from activity history...")
+
+    datasets = Dataset.objects(published_at__ne=None).only("id", "created_at_internal")
+    count = datasets.count()
+    updated = 0
+
+    with click.progressbar(
+        datasets.no_cache().timeout(False), length=count, label="Refining dates"
+    ) as progress:
+        for dataset in progress:
+            activity = (
+                UserUpdatedDataset.objects(related_to=dataset.id, changes="private")
+                .order_by("-created_at")
+                .only("created_at")
+                .first()
+            )
+
+            if activity and activity.created_at != dataset.created_at_internal:
+                Dataset.objects(id=dataset.id).update_one(set__published_at=activity.created_at)
+                updated += 1
+
+    log.info(f"Phase 2 done: {updated} datasets refined with activity date")