CMR-7354: Allow requester to specify verbosity of bulk granule update task status (#1257)

* Recognize three new query parameters on the ingest/granule-bulk-update/status/:task-id endpoint: show_progress, show_request, and show_granules

Author: mitchstartzel

ingest-app/docs/api.md

@@ -1309,7 +1309,16 @@ curl -i \
 
 To get a detailed task status for a given granule bulk update task, user can send an HTTP GET request to `%CMR-ENDPOINT%/granule-bulk-update/status/<task-id>`
 
-This returns the status of the bulk update task including the overall task status (IN_PROGRESS or COMPLETE), an overall task status message, the original request JSON body, and the status of each granule updated. The granule status includes the granule-ur, the granule update status (PENDING, UPDATED, SKIPPED, FAILED), and a status message. FAILED indicates an error occurred either updating the granule or during granule validation. SKIPPED indicates the update didn't happen because the update operation does not apply to the granule. The error will be reported in the granule status message.
+This returns the status of the bulk update task including the overall task status (IN_PROGRESS or COMPLETE), the name of the task, and the time the task began processing. Additionally, there are optional query parameters which will increase the verbosity of this response:
+
+`show_progress=true`
+When specified, this parameter will return a progress message indicating the number of granules which have been processed out of the total number of granules in the request.
+
+`show_granules=true`
+This parameter will return the individual granule status for all granules in the request, which includes the granule-ur and the granule update status (PENDING, UPDATED, SKIPPED, or FAILED). FAILED indicates an error occurred either updating the granule or during granule validation. SKIPPED indicates the update didn't happen because the update operation does not apply to the granule. The error will be reported in the granule status message. Note that this parameter can cause the response to become much larger, scaling with the size of the original bulk update request.
+
+`show_request=true`
+This parameter will return the original json body used to initiate the bulk update. Note that this parameter, like before, can cause the response to become much larger, scaling with the size of the original bulk update request.
 
 The only supported response format for granule bulk update task status is application/json.
 
@@ -1319,7 +1328,7 @@ Example of granule bulk update task status:
 curl -i \
   -H "Echo-Token: XXXX" \
   -H "Cmr-Pretty:true" \
-  %CMR-ENDPOINT%/granule-bulk-update/status/3
+  %CMR-ENDPOINT%/granule-bulk-update/status/3?show_granules=true&show_request=true
 
 {
   "status" : 200,
@@ -1347,4 +1356,3 @@ By default the bulk granule update jobs are checked for completion every 5 minut
 ```
 curl -XPOST -i -H "Echo-Token: XXXX" %CMR-ENDPOINT%/granule-bulk-update/status
 ```
-

ingest-app/src/cmr/ingest/api/bulk.clj

@@ -188,10 +188,51 @@
         :request-json-body (:request-json-body task-status)
         :collection-statuses collection-statuses}))))
 
+(defn- generate-status-progress-message
+  "Generate progress message for an in-progress bulk granule update"
+  [granule-statuses]
+  (let [gran-count (count granule-statuses)
+        pending-count (count (filter #(= (:status %) "PENDING") granule-statuses))]
+    (if (zero? pending-count)
+     (format "Complete.")
+     (format "Of %d total granules, %d granules have been processed and %d are still pending."
+      gran-count (- gran-count pending-count) pending-count))))
+
+
+(defn- get-granule-task-status-response-generator
+  "Generates the response for a bulk granule update task status request. Depending on the parameters
+   show_request, show_progress, and show_granules, the response will be more or less verbose.
+   show_progress and show_granules require a full query of granules in the request, which can cause
+   this response generation to be much more expensive."
+  [request-context task-id gran-task params]
+  (let [{:keys [name created-at status status-message request-json-body]} gran-task
+        {:keys [show_request show_progress show_granules]} params
+        ;this call makes the status response much more expensive to retrieve
+        granule-statuses (when (or (= "true" show_progress) (= "true" show_granules))
+                           (data-gran-bulk-update/get-bulk-update-granule-statuses-for-task
+                            request-context task-id))
+        response-fields {:status 200
+                         :created-at created-at
+                         :name name
+                         :task-status status
+                         :status-message status-message}
+        extra-fields (as-> {} intermediate
+                                (if (= "true" show_progress)
+                                  (assoc intermediate :progress (generate-status-progress-message
+                                                                 granule-statuses))
+                                  intermediate)
+                                (if (= "true" show_request)
+                                  (assoc intermediate :request-json-body request-json-body)
+                                  intermediate)
+                                (if (= "true" show_granules)
+                                  (assoc intermediate :granule-statuses granule-statuses)
+                                  intermediate))]
+     (conj response-fields extra-fields)))
+
 (defn get-granule-task-status
-  "Get the status for the given task including granule statuses"
-  [task-id request]
-  (let [{:keys [headers request-context]} request
+  "Get the status for the given bulk granule update task"
+  [request task-id]
+  (let [{:keys [headers request-context params]} request
         _ (validate-granule-bulk-update-result-format headers)
         gran-task (data-gran-bulk-update/get-granule-task-by-id request-context task-id)
         provider-id (:provider-id gran-task)]
@@ -203,18 +244,9 @@
     (api-core/verify-provider-exists request-context provider-id)
     (acl/verify-ingest-management-permission request-context :read :provider-object provider-id)
 
-    (let [{:keys [name created-at status status-message request-json-body]} gran-task
-          granule-statuses (data-gran-bulk-update/get-bulk-update-granule-statuses-for-task
-                            request-context task-id)]
-      (api-core/generate-ingest-response
-       (assoc headers "accept" mt/json)
-       {:status 200
-        :created-at created-at
-        :name name
-        :task-status status
-        :status-message status-message
-        :request-json-body request-json-body
-        :granule-statuses granule-statuses}))))
+    (api-core/generate-ingest-response
+     (assoc headers "accept" mt/json)
+     (get-granule-task-status-response-generator request-context task-id gran-task params))))
 
 (defn update-completed-granule-task-statuses
   "On demand capability to update granule task statuses. Marks bulk granule

ingest-app/src/cmr/ingest/api/routes.clj

@@ -108,7 +108,7 @@
        (context "/:task-id" [task-id]
          (GET "/"
            request
-           (bulk/get-granule-task-status task-id request)))))
+           (bulk/get-granule-task-status request task-id)))))
     ;; Provider ingest routes
     (api-core/set-default-error-format
      :xml

ingest-app/test/cmr/ingest/api/bulk_test.clj

@@ -0,0 +1,25 @@
+(ns cmr.ingest.api.bulk-test
+  "tests functions in bulk update api"
+  (:require
+    [clojure.test :refer :all]
+    [cmr.common.util :as u :refer [are3]]
+    [cmr.ingest.api.bulk :as bulk]))
+
+(deftest generate-status-progress-message
+  (are3 [granule-statuses message]
+    (is (= message (#'bulk/generate-status-progress-message granule-statuses)))
+
+    "All updates complete"
+    [{:granule-ur "GranUR1" :status "UPDATED"}
+     {:granule-ur "GranUR2" :status "UPDATED"}]
+    "Complete."
+
+    "Some updates complete"
+    [{:granule-ur "GranUR1" :status "PENDING"}
+     {:granule-ur "GranUR2" :status "FAILED"}
+     {:granule-ur "GranUR3" :status "UPDATED"}]
+    "Of 3 total granules, 2 granules have been processed and 1 are still pending."
+
+    "No updates complete"
+    [{:granule-ur "GranUR1" :status "PENDING"}]
+    "Of 1 total granules, 0 granules have been processed and 1 are still pending."))

system-int-test/src/cmr/system_int_test/utils/ingest_util.clj

@@ -696,6 +696,7 @@
   "Implementation function to get the bulk update task status for the given task id"
   [concept-type provider-id task-id options]
   (let [accept-format (get options :accept-format)
+        query-params (get options :query-params)
         token (:token options)
         task-status-url (if (= :collection concept-type)
                           (url/ingest-collection-bulk-update-task-status-url provider-id task-id)
@@ -704,6 +705,7 @@
                 :url task-status-url
                 :connection-manager (s/conn-mgr)
                 :throw-exceptions false}
+        params (merge params (when query-params {:query-params query-params}))
         params (merge params (when accept-format {:accept accept-format}))
         params (merge params (when token {:headers {transmit-config/token-header token}}))
         default-result-format (if (= :collection concept-type) :xml :json)

system-int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_flow_test.clj

@@ -179,13 +179,15 @@
                                   :status-message (format "Granule UR [SC:non-existent] in task-id [%s] does not exist."
                                                           task3-id)}]}]
             (testing "default result format"
-              (let [response (ingest/granule-bulk-update-task-status task3-id)]
+              (let [status-req-options {:query-params {:show_granules "true" :show_request "true"}}
+                    response (ingest/granule-bulk-update-task-status task3-id status-req-options)]
                 (is (= task3-expected
                        (dissoc response :created-at)))))
 
             (testing "JSON result format"
-              (let [response (ingest/granule-bulk-update-task-status
-                              task3-id {:accept-format :json})]
+              (let [status-req-options {:query-params {:show_granules "true" :show_request "true"}
+                                        :accept-format :json}
+                    response (ingest/granule-bulk-update-task-status task3-id status-req-options)]
                 (is (= task3-expected
                        (dissoc response :created-at)))))