Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions dandiapi/api/tests/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ class EmbargoedAssetBlobFactory(AssetBlobFactory):
class DraftAssetFactory(factory.django.DjangoModelFactory):
class Meta:
model = Asset
skip_postgeneration_save = True

path = factory.Faker('file_path', absolute=False, extension='nwb')
blob = factory.SubFactory(AssetBlobFactory)
Expand All @@ -238,6 +239,14 @@ def metadata(self) -> dict:
metadata.pop(key, None)
return metadata

@factory.post_generation
def versions(self, create: bool, extracted: list[Version]) -> None: # noqa: FBT001
if not create:
return
if extracted is None:
extracted = []
self.versions.add(*extracted)


class PublishedAssetFactory(DraftAssetFactory):
published = True
Expand Down
31 changes: 11 additions & 20 deletions dandiapi/api/tests/test_asset.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,13 @@
from dandiapi.api.services.publish import publish_asset
from dandiapi.api.tasks.scheduled import validate_pending_asset_metadata
from dandiapi.api.tests.factories import (
DandisetFactory,
DraftVersionFactory,
PublishedVersionFactory,
UserFactory,
)
from dandiapi.zarr.models import ZarrArchiveStatus
from dandiapi.zarr.tasks import ingest_zarr_archive
from dandiapi.zarr.tests.factories import ZarrArchiveFactory
from dandiapi.zarr.tests.factories import EmbargoedZarrArchiveFactory, ZarrArchiveFactory

from .fuzzy import HTTP_URL_RE, TIMESTAMP_RE, URN_RE, UTC_ISO_TIMESTAMP_RE, UUID_RE

Expand Down Expand Up @@ -146,7 +145,7 @@ def test_publish_asset(draft_asset: Asset):


@pytest.mark.django_db
def test_asset_total_size(asset_factory, asset_blob_factory, zarr_archive_factory):
def test_asset_total_size(asset_factory, asset_blob_factory):
# This asset blob should only be counted once,
# despite belonging to multiple assets and multiple versions.
asset_blob = asset_blob_factory()
Expand All @@ -165,10 +164,7 @@ def test_asset_total_size(asset_factory, asset_blob_factory, zarr_archive_factor

assert Asset.total_size() == asset_blob.size

zarr_archive = zarr_archive_factory()
# give it some size
zarr_archive.size = 100
zarr_archive.save() # save adjusted .size into DB
zarr_archive = ZarrArchiveFactory.create(size=100)

# adding of an asset with zarr should be reflected
asset3 = asset_factory(zarr=zarr_archive, blob=None)
Expand Down Expand Up @@ -237,18 +233,16 @@ def test_asset_full_metadata_zarr(draft_asset_factory):


@pytest.mark.django_db
def test_asset_full_metadata_access(
draft_asset_factory, asset_blob_factory, zarr_archive_factory, embargoed_zarr_archive_factory
):
def test_asset_full_metadata_access(draft_asset_factory, asset_blob_factory):
raw_metadata = {
'foo': 'bar',
'schemaVersion': settings.DANDI_SCHEMA_VERSION,
}
embargoed_zarr_asset: Asset = draft_asset_factory(
metadata=raw_metadata, blob=None, zarr=embargoed_zarr_archive_factory()
metadata=raw_metadata, blob=None, zarr=EmbargoedZarrArchiveFactory.create()
)
open_zarr_asset: Asset = draft_asset_factory(
metadata=raw_metadata, blob=None, zarr=zarr_archive_factory()
metadata=raw_metadata, blob=None, zarr=ZarrArchiveFactory.create()
)

embargoed_blob_asset: Asset = draft_asset_factory(
Expand Down Expand Up @@ -327,11 +321,11 @@ def test_asset_rest_list_include_metadata(api_client, version, asset, asset_fact


@pytest.mark.django_db
def test_asset_rest_list_zarr_only(api_client, draft_asset_factory, zarr_archive_factory):
def test_asset_rest_list_zarr_only(api_client, draft_asset_factory):
draft_version = DraftVersionFactory.create()
# Create two blob assets and one zarr asset
zarr_asset = draft_asset_factory(
blob=None, zarr=zarr_archive_factory(dandiset=draft_version.dandiset)
blob=None, zarr=ZarrArchiveFactory.create(dandiset=draft_version.dandiset)
)
draft_version.assets.add(zarr_asset)
draft_version.assets.add(draft_asset_factory())
Expand Down Expand Up @@ -1047,13 +1041,11 @@ def test_asset_create_zarr_validated(api_client, zarr_file_factory):


@pytest.mark.django_db
def test_asset_create_zarr_wrong_dandiset(api_client, zarr_archive_factory):
def test_asset_create_zarr_wrong_dandiset(api_client):
user = UserFactory.create()
draft_version = DraftVersionFactory.create(dandiset__owners=[user])
api_client.force_authenticate(user=user)

zarr_dandiset = DandisetFactory.create()
zarr_archive = zarr_archive_factory(dandiset=zarr_dandiset)
zarr_archive = ZarrArchiveFactory.create()

path = 'test/create/asset.txt'
metadata = {
Expand Down Expand Up @@ -1630,7 +1622,6 @@ def test_asset_rest_delete_zarr(
@pytest.mark.django_db
def test_asset_rest_delete_zarr_modified(
api_client,
zarr_archive_factory,
zarr_file_factory,
):
"""Ensure that a zarr can be associated to an asset, modified, then the asset deleted."""
Expand All @@ -1639,7 +1630,7 @@ def test_asset_rest_delete_zarr_modified(
api_client.force_authenticate(user=user)

# Ensure zarr is ingested
zarr_archive = zarr_archive_factory(
zarr_archive = ZarrArchiveFactory.create(
status=ZarrArchiveStatus.UPLOADED, dandiset=draft_version.dandiset
)
zarr_file_factory(zarr_archive=zarr_archive, size=100)
Expand Down
8 changes: 4 additions & 4 deletions dandiapi/api/tests/test_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,10 +311,10 @@ def test_audit_zarr_create(api_client):


@pytest.mark.django_db
def test_audit_upload_zarr_chunks(api_client, zarr_archive_factory):
def test_audit_upload_zarr_chunks(api_client):
user = UserFactory.create()
draft_version = DraftVersionFactory.create(dandiset__owners=[user])
zarr_archive = zarr_archive_factory(dandiset=draft_version.dandiset)
zarr_archive = ZarrArchiveFactory.create(dandiset=draft_version.dandiset)

# Request some chunk uploads.
paths = ['a.txt', 'b.txt', 'c.txt']
Expand Down Expand Up @@ -352,10 +352,10 @@ def test_audit_finalize_zarr(api_client, zarr_file_factory):


@pytest.mark.django_db
def test_audit_delete_zarr_chunks(api_client, zarr_archive_factory, zarr_file_factory):
def test_audit_delete_zarr_chunks(api_client, zarr_file_factory):
user = UserFactory.create()
draft_version = DraftVersionFactory.create(dandiset__owners=[user])
zarr_archive = zarr_archive_factory(dandiset=draft_version.dandiset)
zarr_archive = ZarrArchiveFactory.create(dandiset=draft_version.dandiset)
zarr_files = [zarr_file_factory(zarr_archive=zarr_archive) for i in range(2)]
ingest_zarr_archive(zarr_archive.zarr_id)

Expand Down
4 changes: 2 additions & 2 deletions dandiapi/api/tests/test_dandiset.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
PublishedVersionFactory,
UserFactory,
)
from dandiapi.zarr.tests.factories import ZarrArchiveFactory

from .fuzzy import (
DANDISET_ID_RE,
Expand Down Expand Up @@ -938,13 +939,12 @@ def test_dandiset_rest_delete(api_client, embargo_status, success):
@pytest.mark.django_db
def test_dandiset_rest_delete_with_zarrs(
api_client,
zarr_archive_factory,
draft_asset_factory,
):
user = UserFactory.create()
api_client.force_authenticate(user=user)
draft_version = DraftVersionFactory.create(dandiset__owners=[user])
zarr = zarr_archive_factory(dandiset=draft_version.dandiset)
zarr = ZarrArchiveFactory.create(dandiset=draft_version.dandiset)
asset = draft_asset_factory(blob=None, zarr=zarr)

# Add paths
Expand Down
13 changes: 6 additions & 7 deletions dandiapi/api/tests/test_permission.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
import pytest
from rest_framework.permissions import SAFE_METHODS

from dandiapi.api.tests.factories import DandisetFactory, DraftVersionFactory, UserFactory
from dandiapi.api.tests.factories import DraftVersionFactory, UserFactory
from dandiapi.zarr.tests.factories import ZarrArchiveFactory


@pytest.mark.parametrize(
Expand Down Expand Up @@ -73,19 +74,17 @@
def test_approved_or_readonly(
api_client,
draft_asset_factory,
zarr_archive_factory,
method,
url_format,
owner_required,
):
user = UserFactory.create()
dandiset = DandisetFactory.create()
version = DraftVersionFactory.create(dandiset=dandiset)
zarr = zarr_archive_factory(dandiset=dandiset)
version = DraftVersionFactory.create()
zarr = ZarrArchiveFactory.create(dandiset=version.dandiset)
asset = draft_asset_factory()
version.assets.add(asset)

url = url_format.format(dandiset=dandiset, asset=asset, zarr=zarr)
url = url_format.format(dandiset=version.dandiset, asset=asset, zarr=zarr)
response = getattr(api_client, method)(url)

# Safe method, read only is okay
Expand All @@ -106,7 +105,7 @@ def test_approved_or_readonly(
# denied after reading the request body
if url == '/api/zarr/' and method == 'post':
response = getattr(api_client, method)(
url, data={'name': 'test', 'dandiset': dandiset.identifier}
url, data={'name': 'test', 'dandiset': version.dandiset.identifier}
)
else:
response = getattr(api_client, method)(url)
Expand Down
26 changes: 15 additions & 11 deletions dandiapi/api/tests/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from dandiapi.api.models import Asset, Version
from dandiapi.api.tests.factories import DraftVersionFactory, UserFactory
from dandiapi.zarr.models import ZarrArchiveStatus
from dandiapi.zarr.tests.factories import ZarrArchiveFactory

from .fuzzy import HTTP_URL_RE, URN_RE, UTC_ISO_TIMESTAMP_RE

Expand Down Expand Up @@ -290,17 +291,20 @@ def test_validate_version_metadata_no_assets():


@pytest.mark.django_db
def test_validate_version_metadata_empty_zarr_asset(zarr_archive_factory, draft_asset_factory):
def test_validate_version_metadata_empty_zarr_asset(draft_asset_factory):
draft_version = DraftVersionFactory.create()
asset = draft_asset_factory(
draft_asset_factory(
blob=None,
zarr=zarr_archive_factory(
status=ZarrArchiveStatus.COMPLETE, checksum=EMPTY_CHECKSUM, size=0, file_count=0
zarr=ZarrArchiveFactory.create(
dandiset=draft_version.dandiset,
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note this change.

Previously, the draft_asset.versions[].dandiset and the draft_asset.zarr.dandiset didn't match. I think this is a correction, but I'm not certain about the intent of the data model.

status=ZarrArchiveStatus.COMPLETE,
checksum=EMPTY_CHECKSUM,
size=0,
file_count=0,
),
status=Asset.Status.VALID,
versions=[draft_version],
)
assert asset.size == 0
draft_version.assets.add(asset)

# Since the zarr asset has zero size, a validation error should be produced
tasks.validate_version_metadata_task(draft_version.id)
Expand All @@ -316,11 +320,12 @@ def test_validate_version_metadata_empty_zarr_asset(zarr_archive_factory, draft_


@pytest.mark.django_db
def test_validate_version_metadata_only_zarr_assets(zarr_archive_factory, draft_asset_factory):
def test_validate_version_metadata_only_zarr_assets(draft_asset_factory):
draft_version = DraftVersionFactory.create()
asset = draft_asset_factory(
draft_asset_factory(
blob=None,
zarr=zarr_archive_factory(
zarr=ZarrArchiveFactory.create(
dandiset=draft_version.dandiset,
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same change here, making the dandiset match.

status=ZarrArchiveStatus.COMPLETE,
checksum=compute_zarr_checksum(
[ZarrArchiveFile(path=Path('foo/bar'), size=100, digest=hashlib.md5().hexdigest())]
Expand All @@ -329,9 +334,8 @@ def test_validate_version_metadata_only_zarr_assets(zarr_archive_factory, draft_
file_count=1,
),
status=Asset.Status.VALID,
versions=[draft_version],
)
assert asset.size > 0
draft_version.assets.add(asset)

tasks.validate_version_metadata_task(draft_version.id)
draft_version.refresh_from_db()
Expand Down
5 changes: 2 additions & 3 deletions dandiapi/api/tests/test_unembargo.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from dandiapi.api.tests.factories import DandisetFactory, DraftVersionFactory, UserFactory
from dandiapi.zarr.models import ZarrArchive, ZarrArchiveStatus
from dandiapi.zarr.tasks import ingest_zarr_archive
from dandiapi.zarr.tests.factories import ZarrArchiveFactory
from dandiapi.zarr.tests.factories import EmbargoedZarrArchiveFactory, ZarrArchiveFactory

if TYPE_CHECKING:
from zarr_checksum.generators import ZarrArchiveFile
Expand Down Expand Up @@ -227,7 +227,6 @@ def test_remove_dandiset_manifest_tags():
def test_unembargo_dandiset(
asset_factory,
embargoed_asset_blob_factory,
embargoed_zarr_archive_factory,
zarr_file_factory,
mailoutbox,
):
Expand All @@ -242,7 +241,7 @@ def test_unembargo_dandiset(
blob_asset = asset_factory(blob=embargoed_blob, status=Asset.Status.VALID)
draft_version.assets.add(blob_asset)

zarr_archive: ZarrArchive = embargoed_zarr_archive_factory(
zarr_archive: ZarrArchive = EmbargoedZarrArchiveFactory.create(
dandiset=dandiset, status=ZarrArchiveStatus.UPLOADED
)
zarr_files: list[ZarrArchiveFile] = [
Expand Down
7 changes: 3 additions & 4 deletions dandiapi/api/tests/test_version.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
PublishedVersionFactory,
UserFactory,
)
from dandiapi.zarr.tests.factories import ZarrArchiveFactory

if TYPE_CHECKING:
from rest_framework.test import APIClient
Expand Down Expand Up @@ -390,11 +391,10 @@ def test_version_size(
asset_factory,
asset_blob_factory,
embargoed_asset_blob_factory,
zarr_archive_factory,
):
version.assets.add(asset_factory(blob=asset_blob_factory(size=100)))
version.assets.add(asset_factory(blob=embargoed_asset_blob_factory(size=200)))
version.assets.add(asset_factory(blob=None, zarr=zarr_archive_factory(size=400)))
version.assets.add(asset_factory(blob=None, zarr=ZarrArchiveFactory.create(size=400)))
add_version_asset_paths(version=version)

assert version.size == 700
Expand Down Expand Up @@ -830,15 +830,14 @@ def test_version_rest_publish_unembargo_in_progress(api_client: APIClient):
def test_version_rest_publish_zarr(
api_client,
draft_asset_factory,
zarr_archive_factory,
zarr_file_factory,
):
user = UserFactory.create()
draft_version = DraftVersionFactory.create(dandiset__owners=[user])
api_client.force_authenticate(user=user)

# create and ingest zarr archive
zarr_archive = zarr_archive_factory(dandiset=draft_version.dandiset, status='Uploaded')
zarr_archive = ZarrArchiveFactory.create(dandiset=draft_version.dandiset, status='Uploaded')
zarr_file_factory(zarr_archive=zarr_archive)
ingest_zarr_archive(zarr_archive.zarr_id)
zarr_archive.refresh_from_db()
Expand Down
Loading
Loading