[develop] Eliminate Wave Postprocessing Warnings and Suppress Creation of Empty Output Directories

dev/jobs/JGEFS_WAVE_STAT

@@ -21,14 +21,7 @@ for grid in ${wavepostGRD}; do
     process_grdID "${grid}"
     prod_dir_grid="COMOUT_WAVE_GRID_${grid}"
     declare -rx "${prod_dir_grid}"="${ROTDIR}/${RUN}.${PDY}/${cyc}/ensstat/products/wave/gridded"
-    if [[ ! -d "${!prod_dir_grid}" ]]; then
-        mkdir -m 775 -p "${!prod_dir_grid}"
-    fi
 done
-declare -rx COMOUT_WAVE_STATION_ENS="${ROTDIR}/${RUN}.${PDY}/${cyc}/ensstat/products/wave/station"
-if [[ ! -d "${COMOUT_WAVE_STATION_ENS}" ]]; then
-    mkdir -m 775 -p "${COMOUT_WAVE_STATION_ENS}"
-fi
 
 # Execute the Script
 #"${SCRglobal}/exgefs_wave_stat.sh"

dev/jobs/JGEFS_WAVE_STAT_PNT

@@ -14,13 +14,6 @@ source "${USHglobal}/jjob_shell_setup.sh"
 # and exit on error in a pipeline (e.g. if a command in "cmd | cmd2" fails)
 source "${USHglobal}/set_strict.sh"
 
-# Set COM Paths
-
-declare -rx COMOUT_WAVE_STATION_ENS="${ROTDIR}/${RUN}.${PDY}/${cyc}/ensstat/products/wave/station"
-if [[ ! -d "${COMOUT_WAVE_STATION_ENS}" ]]; then
-    mkdir -m 775 -p "${COMOUT_WAVE_STATION_ENS}"
-fi
-
 # Execute the Script
 #"${SCRglobal}/exgefs_wave_stat_pnt.sh"
 export err=$?

dev/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE

@@ -33,7 +33,6 @@ declare -rx COMOUT_CONF="${ROTDIR}/${RUN}.${PDY}/${cyc}/conf"
 declare -rx COMOUT_ATMOS_RESTART="${ROTDIR}/${RUN}.${PDY}/${cyc}/model/atmos/restart"
 
 mkdir -m 755 -p "${COMOUT_CHEM_ANALYSIS}"
-mkdir -m 755 -p "${COMOUT_ATMOS_RESTART}"
 mkdir -m 755 -p "${COMOUT_CONF}"
 
 ###############################################################

dev/jobs/JGLOBAL_ANALYSIS_STATS

@@ -33,7 +33,6 @@ declare -rx COMIN_ATMOS_ANALYSIS="${ROTDIR}/${RUN}.${PDY}/${cyc}/analysis/atmos"
 declare -rx COMOUT_ATMOS_ANALYSIS="${ROTDIR}/${RUN}.${PDY}/${cyc}/analysis/atmos"
 declare -rx COMOUT_ATMOS_ANLMON="${ROTDIR}/${RUN}.${PDY}/${cyc}/products/atmos/anlmon"
 mkdir -m 755 -p "${COMOUT_ATMOS_ANALYSIS}"
-mkdir -m 755 -p "${COMOUT_ATMOS_ANLMON}"
 mkdir -m 755 -p "${COMOUT_CONF}"
 
 if [[ "${DO_AERO_ANL:-NO}" == "YES" ]]; then

dev/jobs/JGLOBAL_ARCHIVE_TARS

@@ -55,7 +55,6 @@ declare -rx COMIN_SNOW_ANLMON="${COMIN}/products/snow/anlmon"
 declare -rx COMIN_ICE_ANALYSIS="${COMIN}/analysis/ice"
 declare -rx COMIN_ICE_BMATRIX="${COMIN}/bmatrix/ice"
 declare -rx COMIN_ICE_HISTORY="${COMIN}/model/ice/history"
-declare -rx COMIN_ICE_INPUT="${COMIN}/model/ice/input"
 declare -rx COMIN_ICE_RESTART="${COMIN}/model/ice/restart"
 declare -rx COMIN_ICE_GRIB="${COMIN}/products/ice/grib2"
 declare -rx COMIN_ICE_NETCDF="${COMIN}/products/ice/netcdf"
@@ -79,7 +78,6 @@ declare -rx COMOUT_CONF="${COMOUT}/conf"
 declare -rx COMOUT_ATMOS_TRACK="${COMOUT}/products/atmos/cyclone/tracks"
 
 mkdir -p "${COMOUT_CONF}"
-mkdir -p "${COMOUT_ATMOS_TRACK}"
 
 for grid in "0p25" "0p50" "1p00"; do
     declare -rx "COMIN_ATMOS_GRIB_${grid}"="${COMIN}/products/atmos/grib2/${grid}"

dev/jobs/JGLOBAL_ATMOS_SFCANL

@@ -37,8 +37,6 @@ declare -rx COMOUT_ATMOS_RESTART="${ROTDIR}/${RUN}.${PDY}/${cyc}/model/atmos/res
 
 declare -rx COMIN_ATMOS_ENKF_ANALYSIS_STAT="${ROTDIR}/enkfgdas.${PDY}/${cyc}/ensstat/analysis/atmos"
 
-mkdir -p "${COMOUT_ATMOS_RESTART}"
-
 # Use CFP to stage and save files in parallel
 export USE_CFP=YES
 

dev/jobs/JGLOBAL_FORECAST

@@ -95,7 +95,6 @@ fi
 
 if [[ "${DO_OCN}" == "YES" ]]; then
     declare -rx COMIN_OCEAN_ANALYSIS="${COMIN}/analysis/ocean"
-    declare -rx COMIN_OCEAN_INPUT="${COMIN}/model/ocean/input"
     declare -rx COMOUT_MED_RESTART="${COMOUT}/model/med/restart"
     declare -rx COMOUT_OCEAN_RESTART="${COMOUT}/model/ocean/restart"
     declare -rx COMOUT_OCEAN_HISTORY="${COMOUT}/model/ocean/history"
@@ -104,7 +103,6 @@ if [[ "${DO_OCN}" == "YES" ]]; then
 fi
 
 if [[ "${DO_ICE}" == "YES" ]]; then
-    declare -rx COMIN_ICE_INPUT="${COMIN}/model/ice/input"
     declare -rx COMIN_ICE_ANALYSIS="${COMIN}/analysis/ice"
     declare -rx COMOUT_ICE_RESTART="${COMOUT}/model/ice/restart"
     declare -rx COMOUT_ICE_HISTORY="${COMOUT}/model/ice/history"

dev/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE

@@ -38,7 +38,6 @@ declare -rx COMOUT_CONF="${ROTDIR}/${RUN}.${PDY}/${cyc}/conf"
 
 mkdir -m 775 -p "${COMOUT_OCEAN_ANALYSIS}"
 mkdir -m 775 -p "${COMOUT_ICE_ANALYSIS}"
-mkdir -m 775 -p "${COMOUT_ICE_RESTART}"
 mkdir -m 775 -p "${COMOUT_CONF}"
 
 ###############################################################

dev/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF

@@ -55,7 +55,6 @@ declare -rx COMOUT_ICE_LETKF="${ROTDIR}/${RUN}.${PDY}/${cyc}/analysis/ice/letkf"
 
 declare -rx COMOUT_CONF="${ROTDIR}/${RUN}.${PDY}/${cyc}/ensstat/conf"
 
-if [[ ! -d "${COMOUT_OCEAN_LETKF}" ]]; then mkdir -p "${COMOUT_OCEAN_LETKF}"; fi
 if [[ ! -d "${COMOUT_ICE_LETKF}" ]]; then mkdir -p "${COMOUT_ICE_LETKF}"; fi
 if [[ ! -d ${COMOUT_CONF} ]]; then mkdir -p "${COMOUT_CONF}"; fi
 ##############################################

dev/jobs/JGLOBAL_WAVE_POST_SBS

@@ -30,8 +30,6 @@ declare -rx COMIN_WAVE_PREP="${COMIN}/model/wave/prep"
 declare -rx COMIN_WAVE_HISTORY="${COMIN}/model/wave/history"
 declare -rx COMOUT_WAVE_GRID="${COMOUT}/products/wave/gridded"
 
-mkdir -p "${COMOUT_WAVE_GRID}"
-
 if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then
     for grdID in ${wavepostGRD} ${waveinterpGRD}; do
         process_grdID "${grdID}"

dev/scripts/exglobal_analysis_stats.py

@@ -41,7 +41,7 @@
     # Initialize JEDI variational analysis
     if 'atmos_gsi' in config.STAT_ANALYSES:
         AnlStats.convert_gsi_diags()
-    AnlStats.initialize()
     for anl in config.STAT_ANALYSES:
+        AnlStats.initialize(anl)
         AnlStats.execute(anl)
         AnlStats.finalize(anl)

dev/scripts/exglobal_atmos_sfcanl.sh

@@ -182,7 +182,7 @@ for hr in "${!gcycle_dates[@]}"; do
 
     # Copy outputs from DATA to COMOUT
     for ((nn = 1; nn <= ntiles; nn++)); do
-        cpfs "${DATA}/sfc_data_cycle.00${nn}" "${COMOUT_ATMOS_RESTART}/${datestr}.sfcanl_data.tile${nn}.nc"
+        "${HOMEglobal}/ush/cpfsd.sh" "${DATA}/sfc_data_cycle.00${nn}" "${COMOUT_ATMOS_RESTART}/${datestr}.sfcanl_data.tile${nn}.nc"
     done
 
 done

dev/scripts/exglobal_enkf_sfc.sh

@@ -188,7 +188,7 @@ if [[ "${DOIAU}" == "YES" ]]; then
             fi
 
             if [[ ${TILE_NUM} -eq 1 ]]; then
-                mkdir -p "${COMOUT_ATMOS_RESTART_MEM}"
+                if [[ ! -d "${COMOUT_ATMOS_RESTART_MEM}" ]]; then mkdir -p "${COMOUT_ATMOS_RESTART_MEM}"; fi
             fi
             cpreq "${sfcdata_dir}/${bPDY}.${bcyc}0000.${snow_prefix}sfc_data.tile${n}.nc" \
                 "${DATA}/sfc_data_cycle.${cmem}"
@@ -223,7 +223,7 @@ if [[ "${DOIAU}" == "YES" ]]; then
             declare -x COMOUT_ATMOS_RESTART_MEM=${ROTDIR}/${RUN}.${PDY}/${cyc}/${memchar}/model/atmos/restart
 
             if [[ ${TILE_NUM} -eq 1 ]]; then
-                mkdir -p "${COMOUT_ATMOS_RESTART_MEM}"
+                if [[ ! -d "${COMOUT_ATMOS_RESTART_MEM}" ]]; then mkdir -p "${COMOUT_ATMOS_RESTART_MEM}"; fi
             fi
 
             cpfs "${DATA}/sfc_data_cycle.${cmem}" "${COMOUT_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc"

parm/archive/gcafs_arcdir.yaml.j2

@@ -11,7 +11,7 @@
 
 mkdir:
     - "{{ ARCDIR }}"
-{% if DO_FIT2OBS == True %}
+{% if RUN == "gcafs" and DO_FIT2OBS == True %}
     - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}"
 {% endif %}
 

parm/archive/gfs_arcdir.yaml.j2

@@ -11,7 +11,7 @@
 
 mkdir:
     - "{{ ARCDIR }}"
-{% if DO_FIT2OBS == True %}
+{% if RUN == "gfs" and DO_FIT2OBS == True %}
     - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}"
 {% endif %}
 

ush/cpfsd.sh

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash
+
+#---------------------------------------------------------
+# cpfsd.sh
+#
+# A wrapper around cpfs that automatically creates the
+# destination directory if it does not already exist
+# before copying the file.
+#
+# USAGE: cpfsd <source> <destination>
+#   source      - path to the source file
+#   destination - path to the destination file
+#---------------------------------------------------------
+
+dest="${!#}"
+dest_dir=$(dirname "${dest}")
+if [[ ! -d "${dest_dir}" ]]; then
+    echo "INFO: Directory ${dest_dir} does not exist, creating..."
+    mkdir -p "${dest_dir}"
+    err=$?
+    if [[ ${err} -ne 0 ]]; then
+        echo "ERROR: Failed to create destination directory ${dest_dir}"
+        exit "${err}"
+    fi
+fi
+cpfs "$@"
+err=$?
+if [[ ${err} -ne 0 ]]; then
+    echo "ERROR: cpfs failed to copy to ${dest}"
+    exit "${err}"
+fi

ush/forecast_postdet.sh

@@ -361,6 +361,7 @@ EOF
         FV3_RESTART_FH="$(seq -s ' ' "${restart_interval_start}" "${restart_interval}" "${restart_interval_end}")"
     fi
     export FV3_RESTART_FH
+    if [[ -n "${FV3_RESTART_FH}" ]]; then mkdir -p "${DATArestart}/FV3_RESTART"; fi
     #============================================================================
 }
 
@@ -432,13 +433,20 @@ FV3_out() {
             done
         done
 
-        "${USHglobal}/run_mpmd.sh" "${cmdfile}" && true
-        export err=$?
-        if [[ ${err} -ne 0 ]]; then
-            err_exit "run_mpmd.sh failed to copy FV3 restart files!"
-        fi
+        if [[ -s "${cmdfile}" ]]; then
+            if [[ ! -d "${COMOUT_ATMOS_RESTART}" ]]; then
+                echo "INFO: Directory ${COMOUT_ATMOS_RESTART} does not exist, creating..."
+                mkdir -p "${COMOUT_ATMOS_RESTART}"
+            fi
 
-        echo "SUB ${FUNCNAME[0]}: Output data for FV3 copied"
+            "${USHglobal}/run_mpmd.sh" "${cmdfile}" && true
+            export err=$?
+            if [[ ${err} -ne 0 ]]; then
+                err_exit "run_mpmd.sh failed to copy FV3 restart files!"
+            fi
+
+            echo "SUB ${FUNCNAME[0]}: Output data for FV3 copied"
+        fi
     fi
 }
 
@@ -592,6 +600,11 @@ WW3_out() {
     fi
 
     if [[ -s "${cmdfile}" ]]; then
+        if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then
+            echo "INFO: Directory ${COMOUT_WAVE_RESTART} does not exist, creating..."
+            mkdir -p "${COMOUT_WAVE_RESTART}"
+        fi
+
         "${USHglobal}/run_mpmd.sh" "${cmdfile}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
@@ -604,6 +617,10 @@ WW3_out() {
 CPL_out() {
     echo "SUB ${FUNCNAME[0]}: Copying output data for general cpl fields"
     if [[ "${esmf_profile:-.false.}" == ".true." ]]; then
+        if [[ ! -d "${COMOUT_ATMOS_HISTORY}" ]]; then
+            echo "INFO: Directory ${COMOUT_ATMOS_HISTORY} does not exist, creating..."
+            mkdir -p "${COMOUT_ATMOS_HISTORY}"
+        fi
         cpfs "${DATA}/ESMF_Profile.summary" "${COMOUT_ATMOS_HISTORY}/ESMF_Profile.summary"
     fi
 }
@@ -760,6 +777,11 @@ MOM6_out() {
     esac
 
     if [[ -s "${cmdfile}" ]]; then
+        if [[ ! -d "${COMOUT_OCEAN_RESTART}" ]]; then
+            echo "INFO: Directory ${COMOUT_OCEAN_RESTART} does not exist, creating..."
+            mkdir -p "${COMOUT_OCEAN_RESTART}"
+        fi
+
         "${USHglobal}/run_mpmd.sh" "${cmdfile}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
@@ -886,6 +908,11 @@ CICE_out() {
     esac
 
     if [[ -s "${cmdfile}" ]]; then
+        if [[ ! -d "${COMOUT_ICE_RESTART}" ]]; then
+            echo "INFO: Directory ${COMOUT_ICE_RESTART} does not exist, creating..."
+            mkdir -p "${COMOUT_ICE_RESTART}"
+        fi
+
         "${USHglobal}/run_mpmd.sh" "${cmdfile}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
@@ -966,6 +993,7 @@ GOCART_out() {
 
     for fhr in $(GOCART_output_fh); do
         vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H)
+
         for file_type in "${file_types[@]}"; do
             if [[ -e "${DATA}/gocart.${file_type}.${vdate:0:8}_${vdate:8:2}00z.nc4" ]]; then
                 echo "cpfs ${DATA}/gocart.${file_type}.${vdate:0:8}_${vdate:8:2}00z.nc4 ${COMOUT_CHEM_HISTORY}/gocart.${file_type}.${vdate:0:8}_${vdate:8:2}00z.nc4" >> "${cmdfile}"
@@ -974,6 +1002,11 @@ GOCART_out() {
     done
 
     if [[ -s "${cmdfile}" ]]; then
+        if [[ ! -d "${COMOUT_CHEM_HISTORY}" ]]; then
+            echo "INFO: Directory ${COMOUT_CHEM_HISTORY} does not exist, creating..."
+            mkdir -p "${COMOUT_CHEM_HISTORY}"
+        fi
+
         "${USHglobal}/run_mpmd.sh" "${cmdfile}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
@@ -1094,6 +1127,8 @@ CMEPS_out() {
     esac
 
     if [[ -s "${cmdfile}" ]]; then
+        if [[ ! -d "${COMOUT_MED_RESTART}" ]]; then mkdir -p "${COMOUT_MED_RESTART}"; fi
+
         "${USHglobal}/run_mpmd.sh" "${cmdfile}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then

ush/forecast_predet.sh

@@ -146,10 +146,11 @@ FV3_predet() {
     echo "SUB ${FUNCNAME[0]}: Defining variables for FV3"
 
     if [[ ! -d "${COMOUT_ATMOS_HISTORY}" ]]; then mkdir -p "${COMOUT_ATMOS_HISTORY}"; fi
-    if [[ ! -d "${COMOUT_ATMOS_MASTER}" ]]; then mkdir -p "${COMOUT_ATMOS_MASTER}"; fi
-    if [[ ! -d "${COMOUT_ATMOS_RESTART}" ]]; then mkdir -p "${COMOUT_ATMOS_RESTART}"; fi
     if [[ ! -d "${DATAoutput}/FV3ATM_OUTPUT" ]]; then mkdir -p "${DATAoutput}/FV3ATM_OUTPUT"; fi
     if [[ ! -d "${DATArestart}/FV3_RESTART" ]]; then mkdir -p "${DATArestart}/FV3_RESTART"; fi
+    if [[ "${WRITE_DOPOST:-}" == ".true." ]] && [[ ! -d "${COMOUT_ATMOS_MASTER}" ]]; then
+        mkdir -p "${COMOUT_ATMOS_MASTER}"
+    fi
 
     # The $DATA/RESTART directory is used for writing FV3 restart files (it is hard-wired in the model)
     # Link the output and restart directories to the DATA directory
@@ -609,7 +610,6 @@ WW3_predet() {
     echo "SUB ${FUNCNAME[0]}: WW3 before run type determination"
 
     if [[ ! -d "${COMOUT_WAVE_HISTORY}" ]]; then mkdir -p "${COMOUT_WAVE_HISTORY}"; fi
-    if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}"; fi
     if [[ ! -d "${DATAoutput}/WW3_OUTPUT" ]]; then mkdir -p "${DATAoutput}/WW3_OUTPUT"; fi
     if [[ ! -d "${DATArestart}/WW3_RESTART" ]]; then mkdir -p "${DATArestart}/WW3_RESTART"; fi
 
@@ -665,8 +665,6 @@ CICE_predet() {
     echo "SUB ${FUNCNAME[0]}: CICE before run type determination"
 
     if [[ ! -d "${COMOUT_ICE_HISTORY}" ]]; then mkdir -p "${COMOUT_ICE_HISTORY}"; fi
-    if [[ ! -d "${COMOUT_ICE_RESTART}" ]]; then mkdir -p "${COMOUT_ICE_RESTART}"; fi
-    if [[ ! -d "${COMIN_ICE_INPUT}" ]]; then mkdir -p "${COMIN_ICE_INPUT}"; fi
     if [[ ! -d "${DATAoutput}/CICE_OUTPUT" ]]; then mkdir -p "${DATAoutput}/CICE_OUTPUT"; fi
     if [[ ! -d "${DATArestart}/CICE_RESTART" ]]; then mkdir -p "${DATArestart}/CICE_RESTART"; fi
 
@@ -690,8 +688,6 @@ MOM6_predet() {
     echo "SUB ${FUNCNAME[0]}: MOM6 before run type determination"
 
     if [[ ! -d "${COMOUT_OCEAN_HISTORY}" ]]; then mkdir -p "${COMOUT_OCEAN_HISTORY}"; fi
-    if [[ ! -d "${COMOUT_OCEAN_RESTART}" ]]; then mkdir -p "${COMOUT_OCEAN_RESTART}"; fi
-    if [[ ! -d "${COMIN_OCEAN_INPUT}" ]]; then mkdir -p "${COMIN_OCEAN_INPUT}"; fi
     if [[ ! -d "${DATAoutput}/MOM6_OUTPUT" ]]; then mkdir -p "${DATAoutput}/MOM6_OUTPUT"; fi
     if [[ ! -d "${DATArestart}/MOM6_RESTART" ]]; then mkdir -p "${DATArestart}/MOM6_RESTART"; fi
 
@@ -745,7 +741,6 @@ EOF
 CMEPS_predet() {
     echo "SUB ${FUNCNAME[0]}: CMEPS before run type determination"
 
-    if [[ ! -d "${COMOUT_MED_RESTART}" ]]; then mkdir -p "${COMOUT_MED_RESTART}"; fi
     if [[ ! -d "${DATArestart}/CMEPS_RESTART" ]]; then mkdir -p "${DATArestart}/CMEPS_RESTART"; fi
 
     # Link the restart directory to the DATA directory
@@ -756,8 +751,6 @@ CMEPS_predet() {
 GOCART_predet() {
     echo "SUB ${FUNCNAME[0]}: GOCART before run type determination"
 
-    if [[ ! -d "${COMOUT_CHEM_HISTORY}" ]]; then mkdir -p "${COMOUT_CHEM_HISTORY}"; fi
-
     # FHMAX gets modified when IAU is on, so keep origianl value for GOCART output
     GOCART_MAX=${FHMAX}
 

ush/python/pygfs/jedi/jedi.py

@@ -499,7 +499,8 @@ def save_obsbiasout(self, comout: str, archive_name: str) -> None:
                 logger.info(f"Adding temperature lapse rate file {tlapfile_rename} to {tarball}")
                 bcor.add(tlapfile, arcname=os.path.basename(tlapfile_rename))
 
-        # Copy files to COM
+        # Always copy the tarball to COM; it is always created above and required by the archive step
+        FileHandler({'mkdir': [comout]}).sync()
         FileHandler({'copy_opt': [[tarball, comout]]}).sync()
 
     @staticmethod