Merge pull request #1261 from BlueBrain/3939/fetch-distributions-in-my-data

3939 // Fetch resources in my-data when they dont have distributions

Author: Dinika

src/shared/containers/DataTableContainer.tsx

@@ -193,6 +193,9 @@ const DataTableContainer: React.FC<DataTableProps> = ({
 
   const [searchboxValue, setSearchboxValue] = useState<string>('');
   const [searchboxFocused, setSearchboxFocused] = useState<boolean>(false);
+  const [fetchingRowsForDownlaod, setFetchingRowsForDownload] = useState<
+    boolean
+  >(false);
   const nexus = useNexusContext();
   const history = useHistory();
   const location = useLocation();
@@ -465,7 +468,9 @@ const DataTableContainer: React.FC<DataTableProps> = ({
           <Table
             bordered
             loading={
-              tableData.dataResult.isLoading || tableData.tableResult.isLoading
+              tableData.dataResult.isLoading ||
+              tableData.tableResult.isLoading ||
+              fetchingRowsForDownlaod
             }
             rowClassName={'data-table-row'}
             title={() => renderTitle(options)}
@@ -487,7 +492,19 @@ const DataTableContainer: React.FC<DataTableProps> = ({
             rowSelection={{
               selectedRowKeys,
               onSelect: tableData.onSelectSingleRow,
-              onSelectAll: tableData.onSelectAll,
+              onSelectAll: async (
+                selected: boolean,
+                selectedRows: StudioTableRow[],
+                changedRows: StudioTableRow[]
+              ) => {
+                setFetchingRowsForDownload(true);
+                await tableData.onSelectAll(
+                  selected,
+                  selectedRows,
+                  changedRows
+                );
+                setFetchingRowsForDownload(false);
+              },
             }}
             rowKey={r => getStudioRowKey(r)}
             data-testid="dashboard-table"

src/shared/hooks/useAccessDataForTable.tsx

@@ -5,19 +5,7 @@ import { notification } from 'antd';
 import { ColumnType } from 'antd/lib/table/interface';
 import * as bodybuilder from 'bodybuilder';
 import json2csv, { Parser } from 'json2csv';
-import {
-  get,
-  has,
-  isArray,
-  isNil,
-  isString,
-  pick,
-  sum,
-  sumBy,
-  toNumber,
-  uniq,
-  uniqBy,
-} from 'lodash';
+import { isArray, isNil, isString, pick, sumBy, toNumber } from 'lodash';
 import * as React from 'react';
 import { useQuery } from 'react-query';
 import {
@@ -48,6 +36,7 @@ import { isNumeric, parseJsonMaybe } from '../utils/index';
 import { addColumnsForES, rowRender } from '../utils/parseESResults';
 import { sparqlQueryExecutor } from '../utils/querySparqlView';
 import { CartContext } from './useDataCart';
+import PromisePool from '@supercharge/promise-pool';
 
 export const EXPORT_CSV_FILENAME = 'nexus-query-result.csv';
 export const CSV_MEDIATYPE = 'text/csv';
@@ -415,6 +404,52 @@ const getTotalContentSize = (rows: TDataSource[]) => {
   return size;
 };
 
+export const fetchResourceForDownload = async (
+  selfUrl: string,
+  nexus: ReturnType<typeof useNexusContext>
+): Promise<Resource> => {
+  let compactResource;
+  let expandedResource;
+
+  try {
+    compactResource = await nexus.httpGet({
+      path: selfUrl,
+      headers: { Accept: 'application/json' },
+    });
+
+    const receivedExpandedId =
+      isValidUrl(compactResource['@id']) &&
+      compactResource['@id']?.startsWith('http');
+
+    if (receivedExpandedId) {
+      return compactResource;
+    }
+
+    expandedResource = await nexus.httpGet({
+      path: `${selfUrl}?format=expanded`,
+      headers: { Accept: 'application/json' },
+    });
+
+    return {
+      ...compactResource,
+      ['@id']:
+        expandedResource?.[0]?.['@id'] ??
+        expandedResource['@id'] ??
+        compactResource['@id'],
+    };
+  } catch (err) {
+    notification.warning({
+      message: <>Could not fetch a resource with id for download {selfUrl}</>,
+      description: <em>{(err as any)?.reason ?? (err as any)?.['@type']}</em>,
+      key: selfUrl,
+    });
+    // @ts-ignore TODO: Remove when we supprot es2022 in ts.
+    throw new Error(`Could not select resource ${selfUrl} for download`, {
+      cause: err,
+    });
+  }
+};
+
 export const useAccessDataForTable = (
   orgLabel: string,
   projectLabel: string,
@@ -446,34 +481,31 @@ export const useAccessDataForTable = (
     let rowsForLS = dataPanelLS?.selectedRows || [];
 
     if (selected) {
-      rowKeysForLS = [
-        ...rowKeysForLS,
-        ...changedRows.map(row => getStudioRowKey(row)),
-      ];
-
-      const futureResources = changedRows.map(row =>
-        fetchResourceForDownload(getStudioRowKey(row))
-      );
-
-      Promise.allSettled(futureResources)
-        .then(result => {
-          result.forEach(res => {
-            if (res.status === 'fulfilled') {
-              const localStorageResource = toLocalStorageResources(
-                res.value,
-                'studios'
-              );
-              rowsForLS = [...rowsForLS, ...localStorageResource];
-            }
-          });
+      const { results, errors } = await PromisePool.withConcurrency(4)
+        .for(changedRows)
+        .handleError(async err => {
+          console.log(
+            '@@error in selecting multiple resources for download in studios',
+            err
+          );
           return;
         })
-        .then(() => {
-          saveSelectedRowsToLocalStorage(rowKeysForLS, rowsForLS);
-        })
-        .catch(err => {
-          console.log('@@error', err);
+        .process(async row => {
+          const fetchedRow = await fetchResourceForDownload(
+            getStudioRowKey(row),
+            nexus
+          );
+          const localStorageResources = toLocalStorageResources(
+            fetchedRow,
+            'studios'
+          );
+          rowKeysForLS.push(getStudioRowKey(row));
+
+          return localStorageResources;
         });
+
+      rowsForLS = [...rowsForLS, ...results.flat()];
+      saveSelectedRowsToLocalStorage(rowKeysForLS, rowsForLS);
     } else {
       const rowKeysToRemove = changedRows.map(row => getStudioRowKey(row));
 
@@ -500,7 +532,7 @@ export const useAccessDataForTable = (
     let localStorageRows = dataPanelLS?.selectedRows || [];
 
     if (selected) {
-      const deltaResource = await fetchResourceForDownload(recordKey);
+      const deltaResource = await fetchResourceForDownload(recordKey, nexus);
       const localStorageResource = toLocalStorageResources(
         deltaResource,
         'studios'
@@ -515,53 +547,6 @@ export const useAccessDataForTable = (
     saveSelectedRowsToLocalStorage(localStorageRowKeys, localStorageRows);
   };
 
-  const fetchResourceForDownload = async (
-    selfUrl: string
-  ): Promise<Resource> => {
-    let compactResource;
-    let expandedResource;
-
-    try {
-      compactResource = await nexus.httpGet({
-        path: selfUrl,
-        headers: { Accept: 'application/json' },
-      });
-
-      const receivedExpandedId =
-        isValidUrl(compactResource['@id']) &&
-        !isUrlCurieFormat(compactResource['@id']);
-
-      if (receivedExpandedId) {
-        return compactResource;
-      }
-
-      expandedResource = await nexus.httpGet({
-        path: `${selfUrl}?format=expanded`,
-        headers: { Accept: 'application/json' },
-      });
-
-      return {
-        ...compactResource,
-        ['@id']:
-          expandedResource?.[0]?.['@id'] ??
-          expandedResource['@id'] ??
-          compactResource['@id'],
-      };
-    } catch (err) {
-      notification.warning({
-        message: (
-          <div>Could not fetch a resource with id for download {selfUrl}</div>
-        ),
-        description: <em>{(err as any)?.reason ?? (err as any)?.['@type']}</em>,
-        key: selfUrl,
-      });
-      // @ts-ignore TODO: Remove when we supprot es2022 in ts.
-      throw new Error(`Could not select resource ${selfUrl} for download`, {
-        cause: err,
-      });
-    }
-  };
-
   const saveSelectedRowsToLocalStorage = (
     rowKeys: React.Key[],
     rows: TDataSource[]