Merge pull request #3334 from bsipocz/ENH_irsa_no_metadata_tables

ENH: do not include image metadata tables in catalog list

Author: bsipocz

CHANGES.rst

@@ -57,6 +57,14 @@ imcce
 - Changing RuntimeError to NoResultsWarning when an empty result is
   returned. [#3307]
 
+ipac.irsa
+^^^^^^^^^
+
+- Fix ``list_catalogs`` to not include image metadata tables, only
+  catalogs. The ``include_metadata_tables`` keyword argument allows opting
+  in to return all TAP tables, including non-spatial and metadata ones,
+  too. [#3334]
+
 SIMBAD
 ^^^^^^
 

astroquery/ipac/irsa/core.py

@@ -304,7 +304,7 @@ def query_region(self, coordinates=None, *, catalog=None, spatial='Cone',
         return response.to_table()
 
     @deprecated_renamed_argument("cache", None, since="0.4.7")
-    def list_catalogs(self, *, full=False, filter=None, cache=False):
+    def list_catalogs(self, *, full=False, filter=None, include_metadata_tables=False, cache=False):
         """
         Return information of available IRSA catalogs.
 
@@ -316,8 +316,19 @@ def list_catalogs(self, *, full=False, filter=None, cache=False):
         filter : str or None
             If specified we only return catalogs when their catalog_name
             contains the filter string.
+        include_metadata_tables : bool
+            If True returns not just the catalogs but all table holdings including the image metadata tables.
+            These are not suitable for spatial queries with e.g. ``query_region``.
         """
-        tap_tables = self.query_tap("SELECT * FROM TAP_SCHEMA.tables").to_table()
+
+        if include_metadata_tables:
+            more_filtering = ""
+        else:
+            # Filter out non-spatial catalogs and metadata tables with
+            # irsa_pos=y and irsa_dbms=21
+            more_filtering = "WHERE irsa_dbms=21 AND irsa_pos='y'"
+
+        tap_tables = self.query_tap(f"SELECT * FROM TAP_SCHEMA.tables {more_filtering}").to_table()
 
         if filter:
             mask = [filter in name for name in tap_tables['table_name']]

astroquery/ipac/irsa/tests/test_irsa_remote.py

@@ -72,14 +72,26 @@ def test_list_columns(self):
     def test_list_catalogs(self):
         catalogs = Irsa.list_catalogs()
         # Number of available catalogs may change over time, test only for significant drop.
-        # (at the time of writing there are 933 tables in the list).
-        assert len(catalogs) > 900
+        # (at the time of writing there are 521 catalogs in the list).
+        assert len(catalogs) > 520
         assert isinstance(catalogs, dict)
 
+        catalogs_full = Irsa.list_catalogs(full=True)
+        assert isinstance(catalogs_full, Table)
+
     def test_list_catalogs_filter(self):
-        spitzer_catalogs = Irsa.list_catalogs(filter='spitzer')
+        allwise_catalogs = Irsa.list_catalogs(filter='allwise')
+
+        assert len(allwise_catalogs) == 4
+
+    def test_list_catalogs_metadata(self):
+        catalogs = Irsa.list_catalogs(filter='wise')
+        all_tables = Irsa.list_catalogs(filter='wise', include_metadata_tables=True)
+
+        assert len(catalogs) < len(all_tables)
 
-        assert len(spitzer_catalogs) == 142
+        assert 'wise.wise_allwise_p3am_cdd' not in catalogs
+        assert 'wise.wise_allwise_p3am_cdd' in all_tables
 
     @pytest.mark.parametrize('servicetype', (None, 'sia', 'ssa'))
     def test_list_collections(self, servicetype):

docs/ipac/irsa/irsa.rst

@@ -234,14 +234,15 @@ available about the columns in a `~astropy.table.Table`.
 .. doctest-remote-data::
 
     >>> from astroquery.ipac.irsa import Irsa
-    >>> Irsa.list_columns(catalog="allwise_p3as_psd")
-    {'designation': 'WISE source designation',
+    >>> Irsa.list_columns(catalog="allwise_p3as_psd")  # doctest: +IGNORE_OUTPUT
+    {...
+     'designation': 'WISE source designation',
      'ra': 'right ascension (J2000)',
      'dec': 'declination (J2000)',
      'sigra': 'uncertainty in RA',
      'sigdec': 'uncertainty in DEC',
      ...
-     'htm20': 'HTM20 spatial index key'}
+     }
 
 
 Async queries
@@ -410,8 +411,9 @@ Spitzer.
 
    >>> from astropy.table import unique
    >>> unique(arp220_spectra, keys='dataid_collection')['dataid_collection']
-   <MaskedColumn name='dataid_collection' dtype='object' description='IVOA Identifier of collection' length=4>
+   <MaskedColumn name='dataid_collection' dtype='object' description='IVOA Identifier of collection' length=5>
             goals
+   herschel_herus
      sofia_fifils
    spitzer_irsenh
       spitzer_sha
@@ -425,7 +427,7 @@ will return a `~astropy.table.Table`.
 
    >>> from astroquery.ipac.irsa import Irsa
    >>> Irsa.list_collections(servicetype='SSA')
-   <Table length=35>
+   <Table length=37>
           collection
             object
    ------------------------
@@ -447,6 +449,7 @@ will return a `~astropy.table.Table`.
                        irts
                     iso_sws
                  sofia_exes
+             sofia_exes_enh
                sofia_fifils
              sofia_flitecam
               sofia_forcast
@@ -464,6 +467,7 @@ will return a `~astropy.table.Table`.
                 spitzer_sha
               spitzer_sings
               spitzer_ssgss
+                       swas
 
 
 Other Configurations