diff --git a/coldfront/plugins/fasrc/utils.py b/coldfront/plugins/fasrc/utils.py index bf3f86502..acd655621 100644 --- a/coldfront/plugins/fasrc/utils.py +++ b/coldfront/plugins/fasrc/utils.py @@ -45,26 +45,7 @@ def produce_query_statement(self, vol_type, volumes=None): 'path_def': "substring(e.Path, size('/n/') + size(split(e.Path, '/')[2]) + 1)", 'unique':'datetime(e.DotsLFSUpdateDate) as begin_date' }, - 'isilon': { - 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(parent_resource__name='Tier 1')), - 'relation': 'Owns', - 'match': "(e:IsilonPath) MATCH (d:ConfigValue {Name: 'IsilonPath.Invocation'})", - 'server': 'Isilon', - 'validation_query': "r.DotsUpdateDate = d.DotsUpdateDate\ - AND NOT (e.Path =~ '.*/rc_admin/.*')\ - AND (e.Path =~ '.*labs.*')\ - AND (datetime() - duration('P31D') <= datetime(r.DotsUpdateDate))\ - AND NOT (e.SizeGB = 0)", - 'r_updated': 'DotsUpdateDate', - 'storage_type': 'Isilon', - 'usedgb': 'UsedGB', - 'tb_allocation': 'e.SizeGB / 1024.0', - 'sizebytes': 'e.SizeBytes', - 'usedbytes': 'UsedBytes', - 'server_replace': "'01.rc.fas.harvard.edu', ''", - 'path_def': "replace(e.Path, '/ifs/', '')", - 'unique': 'datetime(e.DotsUpdateDate) as begin_date' - }, + 'volume': { 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(parent_resource__name='Tier 2')), 'relation': 'Owns', diff --git a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py b/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py deleted file mode 100644 index 4d0cb7e70..000000000 --- a/coldfront/plugins/isilon/management/commands/pull_isilon_quotas.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging - -from django.core.management.base import BaseCommand - -from coldfront.core.allocation.models import Allocation, AllocationAttributeType -from coldfront.core.resource.models import Resource -from coldfront.plugins.isilon.utils import ( - IsilonConnection, - get_isilon_url, - print_log_error, - update_coldfront_quota_and_usage, -) - -logger = logging.getLogger(__name__) - -class Command(BaseCommand): - """Pull Isilon quotas - """ - help = 'Pull Isilon quotas' - - def handle(self, *args, **kwargs): - """For all active isilon and powerscale allocations, update quota and usage - """ - quota_bytes_attributetype = AllocationAttributeType.objects.get( - name='Quota_In_Bytes') - quota_tbs_attributetype = AllocationAttributeType.objects.get( - name='Storage Quota (TiB)') - # create isilon connections to all isilon clusters in coldfront - isilon_resources = Resource.objects.filter(resourceattribute__value__in=('isilon', 'powerscale')) - for resource in isilon_resources: - report = {"complete": 0, "no entry": [], "empty quota": []} - resource_name = get_isilon_url(resource) - # try connecting to the cluster. If it fails, display an error and - # replace the resource with a dummy resource - try: - api_instance = IsilonConnection(resource_name) - except Exception as e: - message = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' - print_log_error(e, message) - # isilon_clusters[resource.name] = None - continue - - # get all active allocations for this resource - isilon_allocations = Allocation.objects.filter( - status__name='Active', - resources__name=resource.name, - ) - - # get all allocation quotas and usoges - try: - rc_labs = api_instance.quota_client.list_quota_quotas( - path='/ifs/rc_labs/', recurse_path_children=True, - ) - l3_labs = api_instance.quota_client.list_quota_quotas( - path='/ifs/rc_fasse_labs/', recurse_path_children=True, - ) - except Exception as e: - err = f'Could not connect to {resource_name} - will not update quotas for allocations on this resource' - print_log_error(e, err) - # isilon_clusters[resource.name] = None - continue - quotas = rc_labs.quotas + l3_labs.quotas - for allocation in isilon_allocations: - # get the api_response entry for this allocation. If it doesn't exist, skip - try: - api_entry = next(e for e in quotas if e.path == f'/ifs/{allocation.path}') - except StopIteration as e: - err = f'no isilon quota entry for allocation {allocation}' - print_log_error(e, err) - report['no entry'].append(f'{allocation.pk} {allocation.path} {allocation}') - continue - # update the quota and usage for this allocation - quota = api_entry.thresholds.hard - usage = api_entry.usage.fslogical - if quota is None: - err = f'no hard threshold set for allocation {allocation}' - print_log_error(None, err) - report['empty quota'].append(f'{allocation.pk} {allocation.path} {allocation}') - continue - quota_tb = quota / 1024 / 1024 / 1024 / 1024 - usage_tb = usage / 1024 / 1024 / 1024 / 1024 - update_coldfront_quota_and_usage( - allocation, quota_bytes_attributetype, [quota, usage] - ) - update_coldfront_quota_and_usage( - allocation, quota_tbs_attributetype, [quota_tb, usage_tb] - ) - print("SUCCESS:update for allocation", allocation, "complete") - report['complete'] += 1 - print(report) - logger.warning("isilon update report for %s: %s", resource_name, report) diff --git a/coldfront/plugins/isilon/management/commands/sync_isilon_allocations.py b/coldfront/plugins/isilon/management/commands/sync_isilon_allocations.py new file mode 100644 index 000000000..15667f4fe --- /dev/null +++ b/coldfront/plugins/isilon/management/commands/sync_isilon_allocations.py @@ -0,0 +1,250 @@ +import logging +from datetime import datetime + +from django.core.management.base import BaseCommand +from django.core.exceptions import ValidationError + +from coldfront.core.allocation.models import ( + Allocation, + AllocationAttribute, + AllocationAttributeType, + AllocationStatusChoice, + AllocationUser, + AllocationUserStatusChoice, +) +from coldfront.core.project.models import Project +from coldfront.core.resource.models import Resource +from coldfront.plugins.isilon.utils import ( + IsilonConnection, + get_isilon_url, + print_log_error, + update_coldfront_quota_and_usage, +) + +logger = logging.getLogger(__name__) + +_DEACTIVATE_STATUSES = ("Active", "Pending Deactivation") +_OPEN_REQUEST_STATUSES = ("New", "In Progress", "Pending Activation", "On Hold") +_QUOTA_PATHS = ("/ifs/rc_labs/", "/ifs/rc_fasse_labs/") + + +class Command(BaseCommand): + """Sync ColdFront allocation records with live isilon/powerscale quota data. + + For each isilon/powerscale resource: + - Creates new allocation records for quotas found on the cluster with no + matching ColdFront allocation. + - Updates quota and usage attributes on existing allocations. + - Sets allocations not found on the cluster (Active or Pending Deactivation) + to Inactive. + """ + + help = "Sync isilon/powerscale allocation records with live cluster quota data" + + def handle(self, *args, **kwargs): + quota_bytes_attrtype = AllocationAttributeType.objects.get(name="Quota_In_Bytes") + quota_tib_attrtype = AllocationAttributeType.objects.get(name="Storage Quota (TiB)") + subdir_attrtype = AllocationAttributeType.objects.get(name="Subdirectory") + payment_attrtype = AllocationAttributeType.objects.get(name="RequiresPayment") + + status_active = AllocationStatusChoice.objects.get(name="Active") + status_inactive = AllocationStatusChoice.objects.get(name="Inactive") + alloc_user_status_active = AllocationUserStatusChoice.objects.get(name="Active") + + isilon_resources = Resource.objects.filter( + resourceattribute__value__in=("isilon", "powerscale") + ) + + for resource in isilon_resources: + report = { + "created": 0, + "updated": 0, + "deactivated": 0, + "skipped_no_project": [], + "skipped_open_request": [], + "errors": [], + } + resource_url = get_isilon_url(resource) + + try: + api = IsilonConnection(resource_url) + except Exception as e: + message = f"Could not connect to {resource_url} — skipping" + print_log_error(e, message) + continue + + # Step A: fetch all live quotas from the cluster + live_quotas = {} + try: + for quota_path in _QUOTA_PATHS: + result = api.quota_client.list_quota_quotas( + path=quota_path, recurse_path_children=True + ) + for q in result.quotas: + live_quotas[q.path] = q + except Exception as e: + message = f"Could not fetch quotas from {resource_url} — skipping" + print_log_error(e, message) + continue + + # Step B: create allocations for live quotas with no ColdFront record + for quota_path, quota_obj in live_quotas.items(): + # path is like /ifs/rc_labs/labname — strip leading /ifs/ + relative_path = quota_path.removeprefix("/ifs/") + path_parts = relative_path.strip("/").split("/") + if len(path_parts) < 2: + continue + lab_name = path_parts[1] + + try: + project = Project.objects.get(title=lab_name) + except Project.DoesNotExist: + logger.warning( + "No project found for isilon path %s (lab=%s) — skipping", + quota_path, + lab_name, + ) + report["skipped_no_project"].append(quota_path) + continue + + # Check whether an allocation already exists for this path + existing = project.allocation_set.filter( + resources=resource, + allocationattribute__allocation_attribute_type=subdir_attrtype, + allocationattribute__value=relative_path, + ).exclude(status__name="Merged") + if existing.exists(): + continue # handled in Step C + + # Skip if there is an open allocation request for this resource + if project.allocation_set.filter( + status__name__in=_OPEN_REQUEST_STATUSES, resources=resource + ).exists(): + logger.info( + "Open allocation request exists for project %s on %s — skipping creation", + lab_name, + resource_url, + ) + report["skipped_open_request"].append(quota_path) + continue + + try: + quota_bytes = quota_obj.thresholds.hard or 0 + quota_tib = quota_bytes / 1024**4 + usage_bytes = quota_obj.usage.fslogical or 0 + usage_tib = usage_bytes / 1024**4 + + allocation = Allocation.objects.create( + project=project, + status=status_active, + start_date=datetime.now(), + is_changeable=resource.is_allocatable, + justification=f"Allocation Information for {lab_name}", + ) + allocation.resources.add(resource) + + AllocationAttribute.objects.create( + allocation=allocation, + allocation_attribute_type=subdir_attrtype, + value=relative_path, + ) + AllocationAttribute.objects.create( + allocation=allocation, + allocation_attribute_type=quota_bytes_attrtype, + value=quota_bytes, + ) + AllocationAttribute.objects.create( + allocation=allocation, + allocation_attribute_type=quota_tib_attrtype, + value=quota_tib, + ) + AllocationAttribute.objects.create( + allocation=allocation, + allocation_attribute_type=payment_attrtype, + value=resource.requires_payment, + ) + + try: + AllocationUser.objects.get_or_create( + allocation=allocation, + user=project.pi, + defaults={"status": alloc_user_status_active}, + ) + except ValidationError as e: + logger.warning( + "Could not add PI %s to allocation %s: %s", + project.pi.username, + allocation.pk, + e, + ) + + logger.info( + "Created allocation for %s on %s (path=%s)", + lab_name, + resource_url, + relative_path, + ) + report["created"] += 1 + + except Exception as e: + message = f"Error creating allocation for {quota_path} on {resource_url}" + print_log_error(e, message) + report["errors"].append(quota_path) + + # Step C: update quota and usage for existing allocations + existing_allocations = Allocation.objects.filter( + resources=resource, + status__name__in=_DEACTIVATE_STATUSES, + ) + for allocation in existing_allocations: + alloc_path = f"/ifs/{allocation.path}" if allocation.path else None + if not alloc_path or alloc_path not in live_quotas: + continue # not found — handled in Step D + + quota_obj = live_quotas[alloc_path] + quota_bytes = quota_obj.thresholds.hard + usage_bytes = quota_obj.usage.fslogical + if quota_bytes is None: + logger.warning( + "No hard threshold for allocation %s (path=%s) — skipping update", + allocation.pk, + alloc_path, + ) + continue + + quota_tib = quota_bytes / 1024**4 + usage_tib = usage_bytes / 1024**4 if usage_bytes else 0 + + try: + update_coldfront_quota_and_usage( + allocation, quota_bytes_attrtype, [quota_bytes, usage_bytes or 0] + ) + update_coldfront_quota_and_usage( + allocation, quota_tib_attrtype, [quota_tib, usage_tib] + ) + report["updated"] += 1 + except Exception as e: + message = f"Error updating allocation {allocation.pk} (path={alloc_path})" + print_log_error(e, message) + report["errors"].append(str(allocation.pk)) + + # Step D: deactivate allocations not found on the cluster + for allocation in existing_allocations: + alloc_path = f"/ifs/{allocation.path}" if allocation.path else None + if alloc_path and alloc_path in live_quotas: + continue # found — already handled above + + allocation.status = status_inactive + allocation.save() + logger.warning( + "Deactivated allocation %s (path=%s) — not found on %s", + allocation.pk, + allocation.path, + resource_url, + ) + report["deactivated"] += 1 + + self.stdout.write( + f"sync_isilon_allocations report for {resource_url}: {report}" + ) + logger.warning("sync_isilon_allocations report for %s: %s", resource_url, report) diff --git a/coldfront/plugins/isilon/tasks.py b/coldfront/plugins/isilon/tasks.py index 5d7fcf018..ad48cc2c7 100644 --- a/coldfront/plugins/isilon/tasks.py +++ b/coldfront/plugins/isilon/tasks.py @@ -1,6 +1,6 @@ from django.core import management -def pull_isilon_quotas(): - """Pull Isilon quotas - """ - management.call_command('pull_isilon_quotas') + +def sync_isilon_allocations(): + """Sync isilon/powerscale allocation records with live cluster quota data.""" + management.call_command("sync_isilon_allocations")