diff --git a/echopype/calibrate/api.py b/echopype/calibrate/api.py index e06a0b8a0..754743a6e 100644 --- a/echopype/calibrate/api.py +++ b/echopype/calibrate/api.py @@ -11,6 +11,7 @@ "EK60": CalibrateEK60, "EK80": CalibrateEK80, "AZFP": CalibrateAZFP, + "AZFP6": CalibrateAZFP, "ES70": CalibrateEK60, "ES80": CalibrateEK80, "EA640": CalibrateEK80, @@ -36,7 +37,7 @@ def _compute_cal( if waveform_mode is None or encode_mode is None: raise ValueError("waveform_mode and encode_mode must be specified for EK80 calibration") check_input_args_combination(waveform_mode=waveform_mode, encode_mode=encode_mode) - elif echodata.sonar_model in ("EK60", "AZFP"): + elif echodata.sonar_model in ("EK60", "AZFP", "AZFP6"): if waveform_mode is not None and waveform_mode != "CW": logger.warning( "This sonar model transmits only narrowband signals (waveform_mode='CW'). " diff --git a/echopype/calibrate/cal_params.py b/echopype/calibrate/cal_params.py index 684dc0370..9c19e4fdd 100644 --- a/echopype/calibrate/cal_params.py +++ b/echopype/calibrate/cal_params.py @@ -355,8 +355,13 @@ def get_cal_params_AZFP(beam: xr.DataArray, vend: xr.DataArray, user_dict: dict) if p == "equivalent_beam_angle": out_dict[p] = beam[p] # has only channel dim + elif p == "Sv_offset" and p in vend: + out_dict[p] = vend[ + p + ] # Sv_offset may not be echodata, will be calculated in calibrateAZFP + # Params from Vendor_specific group - elif p in ["EL", "DS", "TVR", "VTX0", "Sv_offset"]: + elif p in ["EL", "DS", "TVR", "VTX0"]: out_dict[p] = vend[p] # these params only have the channel dimension return out_dict diff --git a/echopype/calibrate/calibrate_azfp.py b/echopype/calibrate/calibrate_azfp.py index 1299be8bb..cada21948 100644 --- a/echopype/calibrate/calibrate_azfp.py +++ b/echopype/calibrate/calibrate_azfp.py @@ -1,11 +1,119 @@ import numpy as np +import xarray +from scipy.interpolate import LinearNDInterpolator from ..echodata import EchoData +from ..utils.log import _init_logger from .cal_params import get_cal_params_AZFP from .calibrate_ek import CalibrateBase from .env_params import get_env_params_AZFP from .range import compute_range_AZFP +logger = _init_logger(__name__) + +# Common Sv_offset values for frequency > 38 kHz +SV_OFFSET_HF = { + 150: 1.4, + 200: 1.4, + 250: 1.3, + 300: 1.1, + 500: 0.8, + 700: 0.5, + 900: 0.3, + 1000: 0.3, +} +SV_OFFSET_LF = { + 500: 1.1, + 1000: 0.7, +} +SV_OFFSET = { + 38000.0: {**SV_OFFSET_LF}, + 67000.0: { + **SV_OFFSET_HF, + 500: 1.1, + }, + 120000.0: { + **SV_OFFSET_HF, + 150: 1.4, + 250: 1.3, + }, + 125000.0: { + **SV_OFFSET_HF, + 150: 1.4, + 250: 1.3, + }, + 130000.0: { + **SV_OFFSET_HF, + 150: 1.4, + 250: 1.3, + }, + 200000.0: { + **SV_OFFSET_HF, + 150: 1.4, + 250: 1.3, + }, + 417000.0: { + **SV_OFFSET_HF, + }, + 455000.0: { + **SV_OFFSET_HF, + 250: 1.3, + }, + 769000.0: { + **SV_OFFSET_HF, + 150: 1.4, + }, +} + + +def _calc_azfp_Sv_offset( + frequency: float | int, + pulse_length: float | int, +) -> float: + """ + + Based on code provided by @IanTBlack + + Linearly interpolate an SV offset for a given pulse length and frequency. + The predefined offsets used in this function can be found in Table 3 of + the AZFP Operator's Manual. + + :param pulse_length: A pulse length value (in microseconds) from an + AZFP config file for a given frequency channel. + :param frequency: The AZFP frequency channel. + :return: Either the known Sv offset if it is predefined, or an interpolated + value for less common pulse lengths. + """ + + pulse_length = int(pulse_length) # Convert to an integer for consistency. + frequency = int(frequency) + + # Check if the specified freq is known values + if frequency in SV_OFFSET.keys() and pulse_length in SV_OFFSET[frequency]: + return SV_OFFSET[frequency][pulse_length] + + # convert dict to a grid + xs, ys, zs = [], [], [] + for x_val, inner in SV_OFFSET.items(): # frequencies + for y_val, z_val in inner.items(): # phases + xs.append(x_val) + ys.append(y_val) + zs.append(z_val) # known offsets + xs, ys, zs = np.array(xs), np.array(ys), np.array(zs) + points = np.column_stack([xs, ys]) + interp = LinearNDInterpolator(points, zs) + zq = interp(frequency, pulse_length) + + # Check if outside of calibration grid + if np.isnan(zq): + raise ValueError( + f"Pulse lengths less than 150 or greater than 1000 usecs for {frequency}kHz " + "are not supported. Set cal_params={'Sv_offset' : VALUES} " + "to provide your own Sv_offset." + ) + + return np.round(zq, 1) + class CalibrateAZFP(CalibrateBase): def __init__( @@ -14,7 +122,7 @@ def __init__( super().__init__(echodata, env_params, cal_params, ecs_file) # Set sonar_type - self.sonar_type = "AZFP" + self.sonar_type = "AZFP" # ULS5 and ULS6 use the same calculations currently # Screen for ECS file: currently not support if self.ecs_file is not None: @@ -30,6 +138,37 @@ def __init__( # self.range_meter computed under self._cal_power_samples() # because the implementation is different for Sv and TS + self.compute_Sv_offset() + + def compute_Sv_offset(self): + """ + If Sv_offset isn't in cal_params or the echodata then calculate it. + """ + if self.cal_params["Sv_offset"] is None: + Sv_offset = [] + for freq, pulse_len in zip( + self.echodata["Vendor_specific"]["frequency_nominal"].values, + self.echodata["Vendor_specific"]["XML_transmit_duration_nominal"].values[0], + ): + try: + Sv_offset.append(_calc_azfp_Sv_offset(freq, pulse_len * 1e6)) + except ValueError: + logger.warning( + f"The Sv for {freq}kHz and pulse length {pulse_len}us " + "is uncalibrated (Sv_offset=0.0)" + ) + Sv_offset.append(0.0) + + Sv_offset = xarray.DataArray( + Sv_offset, + coords=[("channel", self.echodata["Vendor_specific"].coords["channel"].values)], + dims=["channel"], + name="Sv_offset", + ) + Sv_offset.channel.attrs = ( + self.echodata["Vendor_specific"].coords["channel"].attrs.copy() + ) + self.cal_params["Sv_offset"] = Sv_offset def compute_echo_range(self, cal_type): """Calculate range (``echo_range``) in meter using AZFP formula. @@ -53,8 +192,8 @@ def _cal_power_samples(self, cal_type, **kwargs): the GU-100-AZFP-01-R50 Operator's Manual. Note a Sv_offset factor that varies depending on frequency is used in the calibration as documented on p.90. - See calc_Sv_offset() in convert/azfp.py """ + # Compute range in meters # range computation different for Sv and TS per AZFP matlab code self.compute_echo_range(cal_type=cal_type) diff --git a/echopype/convert/__init__.py b/echopype/convert/__init__.py index 78c4e2859..462886bb9 100644 --- a/echopype/convert/__init__.py +++ b/echopype/convert/__init__.py @@ -11,11 +11,11 @@ # flake8: noqa from .parse_ad2cp import ParseAd2cp -from .parse_azfp import ParseAZFP -from .parse_azfp6 import ParseAZFP6 from .parse_base import ParseBase from .parse_ek60 import ParseEK60 from .parse_ek80 import ParseEK80 +from .parse_uls5 import ParseULS5 +from .parse_uls6 import ParseULS6 from .set_groups_ad2cp import SetGroupsAd2cp from .set_groups_azfp import SetGroupsAZFP from .set_groups_azfp6 import SetGroupsAZFP6 diff --git a/echopype/convert/api.py b/echopype/convert/api.py index 91207c74f..3a28db823 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -371,13 +371,13 @@ def open_raw( - ``ES70``: Kongsberg Simrad ES70 echosounder - ``EK80``: Kongsberg Simrad EK80 echosounder - ``EA640``: Kongsberg EA640 echosounder - - ``AZFP``: ASL Environmental Sciences AZFP echosounder + - ``AZFP``: ASL Environmental Sciences AZFP echosounder (ULS5) - ``AZFP6``: ASL Environmental Sciences AZFP echosounder (ULS6) - ``AD2CP``: Nortek Signature series ADCP (tested with Signature 500 and Signature 1000) xml_path : str - path to XML config file used by AZFP + path to XML config file used by AZFP (ULS5 only) include_bot : bool, default `False` Include bottom depth file in parsing. Only used by EK60/EK80. include_index : bool, default `False` diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index 2cc8c8c62..a31aeff8c 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -1,17 +1,13 @@ -import os -import xml.etree.ElementTree as ET +import abc from collections import defaultdict -from datetime import datetime as dt from struct import unpack -import fsspec import numpy as np from ..utils.log import _init_logger -from ..utils.misc import camelcase2snakecase from .parse_base import ParseBase -FILENAME_DATETIME_AZFP = "\\w+.01A" +logger = _init_logger(__name__) # Common Sv_offset values for frequency > 38 kHz SV_OFFSET_HF = { @@ -51,6 +47,10 @@ 250: 1.3, **SV_OFFSET_HF, }, + 417000.0: { + **SV_OFFSET_HF, + 68: 0, # NOTE: Not official offset, Matlab code defaults to 0 in this scenario + }, 455000.0: { 250: 1.3, **SV_OFFSET_HF, @@ -61,65 +61,9 @@ }, } -HEADER_FIELDS = ( - ("profile_flag", "u2"), - ("profile_number", "u2"), - ("serial_number", "u2"), - ("ping_status", "u2"), - ("burst_int", "u4"), - ("year", "u2"), # Year - ("month", "u2"), # Month - ("day", "u2"), # Day - ("hour", "u2"), # Hour - ("minute", "u2"), # Minute - ("second", "u2"), # Second - ("hundredths", "u2"), # Hundredths of a second - ("dig_rate", "u2", 4), # Digitalization rate for each channel - ("lock_out_index", "u2", 4), # Lockout index for each channel - ("num_bins", "u2", 4), # Number of bins for each channel - ( - "range_samples_per_bin", - "u2", - 4, - ), # Range samples per bin for each channel - ("ping_per_profile", "u2"), # Number of pings per profile - ("avg_pings", "u2"), # Flag indicating whether the pings average in time - ("num_acq_pings", "u2"), # Pings acquired in the burst - ("ping_period", "u2"), # Ping period in seconds - ("first_ping", "u2"), - ("last_ping", "u2"), - ( - "data_type", - "u1", - 4, - ), # Datatype for each channel 1=Avg unpacked_data (5bytes), 0=raw (2bytes) - ("data_error", "u2"), # Error number is an error occurred - ("phase", "u1"), # Phase number used to acquire this profile - ("overrun", "u1"), # 1 if an overrun occurred - ("num_chan", "u1"), # 1, 2, 3, or 4 - ("gain", "u1", 4), # gain channel 1-4 - ("spare_chan", "u1"), # spare channel - ("pulse_len", "u2", 4), # Pulse length chan 1-4 uS - ("board_num", "u2", 4), # The board the data came from channel 1-4 - ("frequency", "u2", 4), # frequency for channel 1-4 in kHz - ( - "sensor_flag", - "u2", - ), # Flag indicating if pressure sensor or temperature sensor is available - ("ancillary", "u2", 5), # Tilt-X, Y, Battery, Pressure, Temperature - ("ad", "u2", 2), # AD channel 6 and 7 -) - -logger = _init_logger(__name__) - -class ParseAZFP(ParseBase): - """Class for converting data from ASL Environmental Sciences AZFP echosounder.""" - - # Instrument specific constants - HEADER_SIZE = 124 - HEADER_FORMAT = ">HHHHIHHHHHHHHHHHHHHHHHHHHHHHHHHHHHBBBBHBBBBBBBBHHHHHHHHHHHHHHHHHHHH" - FILE_TYPE = 64770 +class ParseAZFP(ParseBase, abc.ABC): + """Base class for converting data from ASL Environmental Sciences AZFP echosounder.""" def __init__( self, @@ -130,108 +74,17 @@ def __init__( **kwargs, ): super().__init__(file, storage_options, sonar_model) - # Parent class attributes - # regex pattern used to grab datetime embedded in filename - self.timestamp_pattern = FILENAME_DATETIME_AZFP - self.xml_path = file_meta - # Class attributes self.parameters = defaultdict(list) self.unpacked_data = defaultdict(list) - self.sonar_type = "AZFP" + @abc.abstractmethod def load_AZFP_xml(self): """ Parses the AZFP XML file. """ - xmlmap = fsspec.get_mapper(self.xml_path, **self.storage_options) - phase_number = None - for event, child in ET.iterparse(xmlmap.fs.open(xmlmap.root), events=("start", "end")): - if event == "end" and child.tag == "Phases": - phase_number = None - if event == "start": - if len(child.tag) > 3 and not child.tag.startswith("VTX"): - camel_case_tag = camelcase2snakecase(child.tag) - else: - camel_case_tag = child.tag - - if len(child.attrib) > 0: - for key, val in child.attrib.items(): - attrib_tag = camel_case_tag + "_" + camelcase2snakecase(key) - if phase_number is not None and camel_case_tag != "phase": - attrib_tag += f"_phase{phase_number}" - self.parameters[attrib_tag].append(val) - if child.tag == "Phase": - phase_number = val - - if all(char == "\n" for char in child.text): - continue - else: - try: - val = int(child.text) - except ValueError: - val = float(child.text) - if phase_number is not None and camel_case_tag != "phase": - camel_case_tag += f"_phase{phase_number}" - self.parameters[camel_case_tag].append(val) - - # Handling the case where there is only one value for each parameter - for key, val in self.parameters.items(): - if len(val) == 1: - self.parameters[key] = val[0] - - def _compute_temperature(self, ping_num, is_valid): - """ - Compute temperature in celsius. - - Parameters - ---------- - ping_num - ping number - is_valid - whether the associated parameters have valid values - """ - if not is_valid: - return np.nan - - counts = self.unpacked_data["ancillary"][ping_num][4] - v_in = 2.5 * (counts / 65535) - R = (self.parameters["ka"] + self.parameters["kb"] * v_in) / (self.parameters["kc"] - v_in) - - # fmt: off - T = 1 / ( - self.parameters["A"] - + self.parameters["B"] * (np.log(R)) - + self.parameters["C"] * (np.log(R) ** 3) - ) - 273 - # fmt: on - return T - - def _compute_tilt(self, ping_num, xy, is_valid): - """ - Compute instrument tilt. - - Parameters - ---------- - ping_num - ping number - xy - either "X" or "Y" - is_valid - whether the associated parameters have valid values - """ - if not is_valid: - return np.nan - else: - idx = 0 if xy == "X" else 1 - N = self.unpacked_data["ancillary"][ping_num][idx] - a = self.parameters[f"{xy}_a"] - b = self.parameters[f"{xy}_b"] - c = self.parameters[f"{xy}_c"] - d = self.parameters[f"{xy}_d"] - return a + b * N + c * N**2 + d * N**3 - + @abc.abstractmethod def _compute_battery(self, ping_num, battery_type): """ Compute battery voltage. @@ -243,160 +96,22 @@ def _compute_battery(self, ping_num, battery_type): type either "main" or "tx" """ - USL5_BAT_CONSTANT = (2.5 / 65536.0) * (86.6 + 475.0) / 86.6 - - if battery_type == "main": - N = self.unpacked_data["ancillary"][ping_num][2] - elif battery_type == "tx": - N = self.unpacked_data["ad"][ping_num][0] - - return N * USL5_BAT_CONSTANT - - def _compute_pressure(self, ping_num, is_valid): - """ - Compute pressure in decibar - Parameters - ---------- - ping_num - ping number - is_valid - whether the associated parameters have valid values - """ - if not is_valid or self.parameters["sensors_flag_pressure_sensor_installed"] == "no": - return np.nan - - counts = self.unpacked_data["ancillary"][ping_num][3] - v_in = 2.5 * (counts / 65535) - P = v_in * self.parameters["a1"] + self.parameters["a0"] - 10.125 - return P + @abc.abstractmethod + def _parse_header(self, file): + """Parse header of raw data file.""" + @abc.abstractmethod def parse_raw(self): """ Parse raw data file from AZFP echosounder. """ - # Read xml file into dict - self.load_AZFP_xml() - fmap = fsspec.get_mapper(self.source_file, **self.storage_options) - - # Set flags for presence of valid parameters for temperature and tilt - def _test_valid_params(params): - if all([np.isclose(self.parameters[p], 0) for p in params]): - return False - else: - return True - - temperature_is_valid = _test_valid_params(["ka", "kb", "kc"]) - pressure_is_valid = _test_valid_params(["a0", "a1"]) - tilt_x_is_valid = _test_valid_params(["X_a", "X_b", "X_c"]) - tilt_y_is_valid = _test_valid_params(["Y_a", "Y_b", "Y_c"]) - - with fmap.fs.open(fmap.root, "rb") as file: - ping_num = 0 - eof = False - while not eof: - header_chunk = file.read(self.HEADER_SIZE) - if header_chunk: - header_unpacked = unpack(self.HEADER_FORMAT, header_chunk) - # Reading will stop if the file contains an unexpected flag - if self._split_header(file, header_unpacked): - # Appends the actual 'data values' to unpacked_data - self._add_counts(file, ping_num) - if ping_num == 0: - # Display information about the file that was loaded in - self._print_status() - # Compute temperature from unpacked_data[ii]['ancillary][4] - self.unpacked_data["temperature"].append( - self._compute_temperature(ping_num, temperature_is_valid) - ) - # Compute pressure from unpacked_data[ii]['ancillary'][3] - self.unpacked_data["pressure"].append( - self._compute_pressure(ping_num, pressure_is_valid) - ) - # compute x tilt from unpacked_data[ii]['ancillary][0] - self.unpacked_data["tilt_x"].append( - self._compute_tilt(ping_num, "X", tilt_x_is_valid) - ) - # Compute y tilt from unpacked_data[ii]['ancillary][1] - self.unpacked_data["tilt_y"].append( - self._compute_tilt(ping_num, "Y", tilt_y_is_valid) - ) - # Compute cos tilt magnitude from tilt x and y values - self.unpacked_data["cos_tilt_mag"].append( - np.cos( - ( - np.sqrt( - self.unpacked_data["tilt_x"][ping_num] ** 2 - + self.unpacked_data["tilt_y"][ping_num] ** 2 - ) - ) - * np.pi - / 180 - ) - ) - # Calculate voltage of main battery pack - self.unpacked_data["battery_main"].append( - self._compute_battery(ping_num, battery_type="main") - ) - # If there is a Tx battery pack - self.unpacked_data["battery_tx"].append( - self._compute_battery(ping_num, battery_type="tx") - ) - else: - break - else: - # End of file - eof = True - ping_num += 1 - self._check_uniqueness() - self._get_ping_time() - - # Explicitly cast frequency to a float in accordance with the SONAR-netCDF4 convention - self.unpacked_data["frequency"] = self.unpacked_data["frequency"].astype(np.float64) - - # cast unpacked_data values to np arrays, so they are easier to reference - for key, val in self.unpacked_data.items(): - # if it is not a nested list, make the value into a ndarray - if isinstance(val, list) and (not isinstance(val[0], list)): - self.unpacked_data[key] = np.asarray(val) - - # cast all list parameter values to np array, so they are easier to reference - for key, val in self.parameters.items(): - if isinstance(val, list): - self.parameters[key] = np.asarray(val) - - # Get frequency values - freq_old = self.unpacked_data["frequency"] - - # Obtain sorted frequency indices - self.freq_ind_sorted = freq_old.argsort() - - # Obtain sorted frequencies - self.freq_sorted = freq_old[self.freq_ind_sorted] * 1000.0 - - # Build Sv offset - self.Sv_offset = np.zeros_like(self.freq_sorted) - for ind, ich in enumerate(self.freq_ind_sorted): - self.Sv_offset[ind] = self._calc_Sv_offset( - self.freq_sorted[ind], self.unpacked_data["pulse_len"][ich] - ) - + @abc.abstractmethod def _print_status(self): """Prints message to console giving information about the raw file being parsed.""" - filename = os.path.basename(self.source_file) - timestamp = dt( - self.unpacked_data["year"][0], - self.unpacked_data["month"][0], - self.unpacked_data["day"][0], - self.unpacked_data["hour"][0], - self.unpacked_data["minute"][0], - int(self.unpacked_data["second"][0] + self.unpacked_data["hundredths"][0] / 100), - ) - timestr = timestamp.strftime("%Y-%b-%d %H:%M:%S") - pathstr, xml_name = os.path.split(self.xml_path) - logger.info(f"parsing file {filename} with {xml_name}, " f"time of first ping: {timestr}") + @abc.abstractmethod def _split_header(self, raw, header_unpacked): """Splits the header information into a dictionary. Modifies self.unpacked_data @@ -412,49 +127,19 @@ def _split_header(self, raw, header_unpacked): ------- True or False depending on whether the unpacking was successful """ - if ( - header_unpacked[0] != self.FILE_TYPE - ): # first field should match hard-coded FILE_TYPE from manufacturer - check_eof = raw.read(1) - if check_eof: - logger.error("Unknown file type") - return False - header_byte_cnt = 0 - - # fields with num_freq data still takes 4 bytes, - # the extra bytes contain random numbers - firmware_freq_len = 4 - - field_w_freq = ( - "dig_rate", - "lock_out_index", - "num_bins", - "range_samples_per_bin", # fields with num_freq data - "data_type", - "gain", - "pulse_len", - "board_num", - "frequency", - ) - for field in HEADER_FIELDS: - if field[0] in field_w_freq: # fields with num_freq data - self.unpacked_data[field[0]].append( - header_unpacked[header_byte_cnt : header_byte_cnt + self.parameters["num_freq"]] - ) - header_byte_cnt += firmware_freq_len - elif len(field) == 3: # other longer fields ('ancillary' and 'ad') - self.unpacked_data[field[0]].append( - header_unpacked[header_byte_cnt : header_byte_cnt + field[2]] - ) - header_byte_cnt += field[2] - else: - self.unpacked_data[field[0]].append(header_unpacked[header_byte_cnt]) - header_byte_cnt += 1 - return True - def _add_counts(self, raw, ping_num): + def _add_counts(self, raw, ping_num, endian): """Unpacks the echosounder raw data. Modifies self.unpacked_data.""" vv_tmp = [[]] * self.unpacked_data["num_chan"][ping_num] + + # TODO: this is a bit hacky, convert the parameters to a numpy array and make a extra dim? + if self.unpacked_data["num_chan"][ping_num] == 1: + self.unpacked_data["num_bins"][ping_num] = [self.unpacked_data["num_bins"][ping_num]] + self.unpacked_data["data_type"][ping_num] = [self.unpacked_data["data_type"][ping_num]] + self.unpacked_data["range_samples_per_bin"][ping_num] = [ + self.unpacked_data["range_samples_per_bin"][ping_num] + ] + for freq_ch in range(self.unpacked_data["num_chan"][ping_num]): counts_byte_size = self.unpacked_data["num_bins"][ping_num][freq_ch] if self.unpacked_data["data_type"][ping_num][freq_ch]: @@ -466,10 +151,10 @@ def _add_counts(self, raw, ping_num): else: divisor = self.unpacked_data["range_samples_per_bin"][ping_num][freq_ch] ls = unpack( - ">" + "I" * counts_byte_size, raw.read(counts_byte_size * 4) + endian + "I" * counts_byte_size, raw.read(counts_byte_size * 4) ) # Linear sum lso = unpack( - ">" + "B" * counts_byte_size, raw.read(counts_byte_size * 1) + endian + "B" * counts_byte_size, raw.read(counts_byte_size * 1) ) # linear sum overflow v = (np.array(ls) + np.array(lso) * 4294967295) / divisor v = (np.log10(v) - 2.5) * (8 * 65535) * self.parameters["DS"][freq_ch] @@ -477,78 +162,127 @@ def _add_counts(self, raw, ping_num): vv_tmp[freq_ch] = v else: counts_chunk = raw.read(counts_byte_size * 2) - counts_unpacked = unpack(">" + "H" * counts_byte_size, counts_chunk) + counts_unpacked = unpack(endian + "H" * counts_byte_size, counts_chunk) vv_tmp[freq_ch] = counts_unpacked self.unpacked_data["counts"].append(vv_tmp) - def _check_uniqueness(self): + def _check_uniqueness(self, profile_flag): """Check for ping-by-ping consistency of sampling parameters and reduce if identical.""" + if not self.unpacked_data: - self.parse_raw() + raise ValueError("Possibly corrupted AZFP file.") - if np.array(self.unpacked_data["profile_flag"]).size != 1: # Only check uniqueness once. + if np.array(self.unpacked_data[profile_flag]).size != 1: # Only check uniqueness once. # fields with num_freq data - field_w_freq = ( - "dig_rate", - "lock_out_index", - "num_bins", - "range_samples_per_bin", - "data_type", - "gain", - "pulse_len", - "board_num", - "frequency", - ) - # fields to reduce size if the same for all pings - field_include = ( - "profile_flag", - "serial_number", - "burst_int", - "ping_per_profile", - "avg_pings", - "ping_period", - "phase", - "num_chan", - "spare_chan", - ) - for field in field_w_freq: + + for field in self.field_w_freq: + if field not in self.unpacked_data: + if field not in self.parameters: + continue + if not isinstance(self.parameters[field], (list, tuple)): + self.parameters[field] = [self.parameters[field]] + continue + uniq = np.unique(self.unpacked_data[field], axis=0) if uniq.shape[0] == 1: - self.unpacked_data[field] = uniq.squeeze() + uniq = uniq.squeeze() + if len(uniq.shape) == 0: + self.unpacked_data[field] = np.asarray([uniq]) + else: + self.unpacked_data[field] = uniq else: raise ValueError(f"Header value {field} is not constant for each ping") - for field in field_include: + + for field in self.field_reduce: + if field not in self.unpacked_data: + continue + uniq = np.unique(self.unpacked_data[field]) if uniq.shape[0] == 1: self.unpacked_data[field] = uniq.squeeze() - else: - raise ValueError(f"Header value {field} is not constant for each ping") + # else: + # raise ValueError(f"Header value {field} is not constant for each ping") + @abc.abstractmethod def _get_ping_time(self): """Assemble ping time from parsed values.""" - if not self.unpacked_data: - self.parse_raw() - - ping_time = [] - for ping_num, year in enumerate(self.unpacked_data["year"]): - ping_time.append( - np.datetime64( - dt( - year, - self.unpacked_data["month"][ping_num], - self.unpacked_data["day"][ping_num], - self.unpacked_data["hour"][ping_num], - self.unpacked_data["minute"][ping_num], - int( - self.unpacked_data["second"][ping_num] - + self.unpacked_data["hundredths"][ping_num] / 100 - ), - ).replace(tzinfo=None), - "[ns]", - ) - ) - self.ping_time = ping_time + def _compute_analog_temperature(self, ping_num, is_valid): + """ + Compute temperature in celsius from analog sensor. + + Parameters + ---------- + ping_num + ping number + is_valid + whether the associated parameters have valid values + """ + if not is_valid: + return np.nan + + counts = self.unpacked_data["ancillary"][ping_num][4] + v_in = 2.5 * (counts / 65535) + + # Sept 2007, use linear equation if ka < -98 for use with linear sensors + if self.parameters["ka"] < -98: + return self.parameters["A"] * v_in + self.parameters["B"] + + R = (self.parameters["ka"] + self.parameters["kb"] * v_in) / (self.parameters["kc"] - v_in) + if R <= 0: + return -99.0 + + # fmt: off + T = 1 / ( + self.parameters["A"] + + self.parameters["B"] * (np.log(R)) + + self.parameters["C"] * (np.log(R) ** 3) + ) - 273 + # fmt: on + return T + + def _compute_tilt(self, ping_num, xy, is_valid): + """ + Compute instrument tilt. + + Parameters + ---------- + ping_num + ping number + xy + either "X" or "Y" + is_valid + whether the associated parameters have valid values + """ + if not is_valid: + return np.nan + else: + idx = 0 if xy == "X" else 1 + N = self.unpacked_data["ancillary"][ping_num][idx] + a = self.parameters[f"{xy}_a"] + b = self.parameters[f"{xy}_b"] + c = self.parameters[f"{xy}_c"] + d = self.parameters[f"{xy}_d"] + return a + b * N + c * N**2 + d * N**3 + + def _compute_analog_pressure(self, ping_num, is_valid): + """ + Compute pressure in decibar from analog sensor + + Parameters + ---------- + ping_num + ping number + is_valid + whether the associated parameters have valid values + """ + if not is_valid or self.parameters["sensors_flag_pressure_sensor_installed"] == "no": + return np.nan + + counts = self.unpacked_data["ancillary"][ping_num][3] + v_in = 2.5 * (counts / 65535) + P = v_in * self.parameters["a1"] + self.parameters["a0"] - 10.125 + return P @staticmethod def _calc_Sv_offset(freq, pulse_len): diff --git a/echopype/convert/parse_azfp6.py b/echopype/convert/parse_azfp6.py deleted file mode 100644 index 37919eb83..000000000 --- a/echopype/convert/parse_azfp6.py +++ /dev/null @@ -1,698 +0,0 @@ -import os -import xml.etree.ElementTree as ET -from collections import defaultdict -from datetime import datetime as dt -from io import BytesIO -from struct import unpack - -import fsspec -import numpy as np - -from ..utils.log import _init_logger -from ..utils.misc import camelcase2snakecase -from .parse_base import ParseBase - -FILENAME_DATETIME_AZFP = "\\w+_\\w+.azfp" - -# NOTE: These values may change once the new AZFP details are finalized - -# Common Sv_offset values for frequency > 38 kHz -SV_OFFSET_HF = { - 300: 1.1, - 500: 0.8, - 700: 0.5, - 900: 0.3, - 1000: 0.3, -} -SV_OFFSET_LF = { - 500: 1.1, - 1000: 0.7, -} -SV_OFFSET = { - 38000.0: {**SV_OFFSET_LF}, - 67000.0: { - 500: 1.1, - **SV_OFFSET_HF, - }, - 125000.0: { - 150: 1.4, - 250: 1.3, - **SV_OFFSET_HF, - }, - 200000.0: { - 150: 1.4, - 250: 1.3, - **SV_OFFSET_HF, - }, - 417000.0: { - **SV_OFFSET_HF, - 68: 0, # NOTE: Not official offset, Matlab code defaults to 0 in this scenario - }, - 455000.0: { - 250: 1.3, - **SV_OFFSET_HF, - }, - 769000.0: { - 150: 1.4, - **SV_OFFSET_HF, - }, -} - -HEADER_FIELDS = [ - "FirstHeaderRecord", - "HeaderBytes", - "HeaderNumRecords", - "ProfileNumber", - "SerialNumber", - "Date", - "AcqStatus", - "BurstInt", - "BaseTime", - "PingPeriod", - "PingPeriodCounts", - "PingPerProfile", - "AvgPings", - "NumAcqPings", - "FirstPing", - "LastPing", - "DataError", - "OverRun", - "Phase", - "NumChan", - "DigRate", - "LockOutIndex", - "NumBins", - "RangeSamplesPerBin", - "DataType", - "PulseLen", - "BoardNum", - "Frequency", - "NumSensors", - "SensorStatus", - "Ancillary", - "GpsDateTime", - "GpsLatLon", - # 0X70 to 0x7F CUSTOM 0 Set by user Set by user Variable values - "Custom", - "LastHeaderRecord", -] - -OPTIONAL_HEADER_FIELDS = ["gps_date_time", "gps_lat_lon", "custom"] - -logger = _init_logger(__name__) - - -class ParseAZFP6(ParseBase): - """Class for converting data from ASL Environmental Sciences AZFP echosounder.""" - - # Instrument specific constants - XML_FILE_TYPE = 0xF044CC11 # Also the start flag - XML_END_FLAG = 0xE088DD66 - DATA_START_FLAG = 0xFF01AA00 - HEADER_START_FLAG = 0xBCD0 - HEADER_END_FLAG = 0xABC1 - DATA_END_FLAG = 0xEF02BB66 - - RECORD_DATA_TYPE_MASK = 0x00E0 - ARRAY_BITS_MASK = 0x001F - CODE_BITS_MASK = 0x7F00 - TYPE_BITS_MASK = 0x00E0 - REQUIRED_BITS_MASK = 0x8000 - - def __init__( - self, - file, - file_meta, - storage_options={}, - sonar_model="AZFP6", - **kwargs, - ): - super().__init__(file, storage_options, sonar_model) - # Parent class attributes - # regex pattern used to grab datetime embedded in filename - self.timestamp_pattern = FILENAME_DATETIME_AZFP - - # Class attributes - self.parameters = defaultdict(list) - self.unpacked_data = defaultdict(list) - self.sonar_type = "AZFP6" - - def load_AZFP_xml(self, raw): - """ - Parses the AZFP XML file embedded in the AZFP file. - - Updates self.parameters - """ - xml_byte_size = unpack(" 3 and not child.tag.startswith("VTX"): - camel_case_tag = camelcase2snakecase(child.tag) - else: - camel_case_tag = child.tag - - if len(child.attrib) > 0: - for key, val in child.attrib.items(): - attrib_tag = camel_case_tag + "_" + camelcase2snakecase(key) - if phase_number is not None and camel_case_tag != "phase": - attrib_tag += f"_phase{phase_number}" - self.parameters[attrib_tag].append(val) - if child.tag == "Phase": - phase_number = val - - if all(char == "\n" for char in child.text): - continue - - try: - val = int(child.text) - except ValueError: - try: - val = float(child.text) - except: - val = child.text - - if phase_number is not None and camel_case_tag != "phase": - camel_case_tag += f"_phase{phase_number}" - - # print(camel_case_tag, val) - self.parameters[camel_case_tag].append(val) - - # Handling the case where there is only one value for each parameter - for key, val in self.parameters.items(): - if len(val) == 1 and key != "phase_number": - self.parameters[key] = val[0] - - self.parameters["phase_number"] = [str(n + 1) for n in range(self.parameters["num_phases"])] - # Gain was removed, for backward compatibility adding in a Gain=1 field - for phase in range(self.parameters["num_phases"]): - self.parameters[f"gain_phase{phase + 1}"] = [1] * self.parameters["num_freq"] - - # from pprint import pprint as pp - # pp(self.parameters) - - def _compute_temperature(self, ping_num, is_valid): - """ - Compute temperature in celsius. - - Parameters - ---------- - ping_num - ping number - is_valid - whether the associated parameters have valid values - """ - if not is_valid: - return np.nan - - counts = self.unpacked_data["ancillary"][ping_num][4] - v_in = 2.5 * (counts / 65535) - R = (self.parameters["ka"] + self.parameters["kb"] * v_in) / (self.parameters["kc"] - v_in) - - # fmt: off - T = 1 / ( - self.parameters["A"] - + self.parameters["B"] * (np.log(R)) - + self.parameters["C"] * (np.log(R) ** 3) - ) - 273 - # fmt: on - return T - - def _compute_tilt(self, ping_num, xy, is_valid): - """ - Compute instrument tilt. - - Parameters - ---------- - ping_num - ping number - xy - either "X" or "Y" - is_valid - whether the associated parameters have valid values - """ - if not is_valid: - return np.nan - else: - idx = 0 if xy == "X" else 1 - N = self.unpacked_data["ancillary"][ping_num][idx] - a = self.parameters[f"{xy}_a"] - b = self.parameters[f"{xy}_b"] - c = self.parameters[f"{xy}_c"] - d = self.parameters[f"{xy}_d"] - return a + b * N + c * N**2 + d * N**3 - - def _compute_battery(self, ping_num, battery_type): - """ - Compute battery voltage. - - Parameters - ---------- - ping_num - ping number - type - either "main" or "tx" - """ - USL6_BAT_CONSTANT = (2.5 / 65535.0) * (86.6 + 475.0) / 86.6 - - if battery_type == "main": - N = self.unpacked_data["ancillary"][ping_num][2] - elif battery_type == "tx": - N = self.unpacked_data["ancillary"][ping_num][-2] - - return N * USL6_BAT_CONSTANT - - def _compute_pressure(self, ping_num, is_valid): - """ - Compute pressure in decibar - - Parameters - ---------- - ping_num - ping number - is_valid - whether the associated parameters have valid values - """ - if not is_valid or self.parameters["sensors_flag_pressure_sensor_installed"] == "no": - return np.nan - - counts = self.unpacked_data["ancillary"][ping_num][3] - v_in = 2.5 * (counts / 65535) - P = v_in * self.parameters["a1"] + self.parameters["a0"] # - 10.125 - return P - - def parse_raw(self): - """ - Parse raw data file from AZFP echosounder. - """ - - # Read xml file into dict - fmap = fsspec.get_mapper(self.source_file, **self.storage_options) - - # Set flags for presence of valid parameters for temperature and tilt - def _test_valid_params(params): - if all([np.isclose(self.parameters[p], 0) for p in params]): - return False - else: - return True - - # Initialize all *_is_valid flags to False - temperature_is_valid = False - pressure_is_valid = False - tilt_x_is_valid = False - tilt_y_is_valid = False - - with fmap.fs.open(fmap.root, "rb") as file: - - if ( - unpack(" 3 and not child.tag.startswith("VTX"): + camel_case_tag = camelcase2snakecase(child.tag) + else: + camel_case_tag = child.tag + + if len(child.attrib) > 0: + for key, val in child.attrib.items(): + attrib_tag = camel_case_tag + "_" + camelcase2snakecase(key) + if phase_number is not None and camel_case_tag != "phase": + attrib_tag += f"_phase{phase_number}" + self.parameters[attrib_tag].append(val) + if child.tag == "Phase": + phase_number = val + + if all(char == "\n" for char in child.text): + continue + else: + try: + val = int(child.text) + except ValueError: + val = float(child.text) + if phase_number is not None and camel_case_tag != "phase": + camel_case_tag += f"_phase{phase_number}" + self.parameters[camel_case_tag].append(val) + + # Handling the case where there is only one value for each parameter + for key, val in self.parameters.items(): + if len(val) == 1: + self.parameters[key] = val[0] + + # The last phase can be a Repeat phase, which switches back to the first phase + phase_number = self.parameters["phase_number"][-1] + if self.parameters[f"phase_type_svalue_phase{phase_number}"] == "Repeat": + items = list(self.parameters.items()) + for k, v in items: + if "_phase1" in k: + new_key = k.replace("_phase1", f"_phase{phase_number}") + if new_key not in self.parameters.keys(): + self.parameters[new_key] = v + + def _parse_header(self, file): + header_chunk = file.read(self.HEADER_SIZE) + if header_chunk: + header_unpacked = unpack(self.HEADER_FORMAT, header_chunk) + + # Reading will stop if the file contains an unexpected flag + return self._split_header(file, header_unpacked) + return False + + def parse_raw(self): + """ + Parse raw data file from AZFP echosounder. + """ + + # Read xml file into dict + self.load_AZFP_xml() + fmap = fsspec.get_mapper(self.source_file, **self.storage_options) + + # Set flags for presence of valid parameters for temperature and tilt + def _test_valid_params(params): + if all([np.isclose(self.parameters[p], 0) for p in params]): + return False + else: + return True + + with fmap.fs.open(fmap.root, "rb") as file: + + temperature_is_valid = _test_valid_params(["ka", "kb", "kc"]) + pressure_is_valid = _test_valid_params(["a0", "a1"]) + tilt_x_is_valid = _test_valid_params(["X_a", "X_b", "X_c"]) + tilt_y_is_valid = _test_valid_params(["Y_a", "Y_b", "Y_c"]) + + ping_num = 0 + eof = False + while not eof: + if self._parse_header(file): + # Appends the actual 'data values' to unpacked_data + self._add_counts(file, ping_num, endian=">") + + if ping_num == 0: + # Display information about the file that was loaded in + self._print_status() + + # Compute temperature from unpacked_data[ii]['ancillary][4] or using Paros + self.unpacked_data["temperature"].append( + self._compute_analog_temperature(ping_num, temperature_is_valid) + ) + # Compute pressure from unpacked_data[ii]['ancillary'][3] + self.unpacked_data["pressure"].append( + self._compute_analog_pressure(ping_num, pressure_is_valid) + ) + + # compute x tilt from unpacked_data[ii]['ancillary][0] + self.unpacked_data["tilt_x"].append( + self._compute_tilt(ping_num, "X", tilt_x_is_valid) + ) + # Compute y tilt from unpacked_data[ii]['ancillary][1] + self.unpacked_data["tilt_y"].append( + self._compute_tilt(ping_num, "Y", tilt_y_is_valid) + ) + # Compute cos tilt magnitude from tilt x and y values + self.unpacked_data["cos_tilt_mag"].append( + np.cos( + ( + np.sqrt( + self.unpacked_data["tilt_x"][ping_num] ** 2 + + self.unpacked_data["tilt_y"][ping_num] ** 2 + ) + ) + * np.pi + / 180 + ) + ) + # Calculate voltage of main battery pack + self.unpacked_data["battery_main"].append( + self._compute_battery(ping_num, battery_type="main") + ) + # If there is a Tx battery pack + self.unpacked_data["battery_tx"].append( + self._compute_battery(ping_num, battery_type="tx") + ) + + # print(f"ping {ping_num} ends at", file.tell()) + else: + # End of file + eof = True + ping_num += 1 + self._check_uniqueness("profile_flag") + self._get_ping_time() + + # Explicitly cast frequency to a float in accordance with the SONAR-netCDF4 convention + self.unpacked_data["frequency"] = self.unpacked_data["frequency"].astype(np.float64) + + # cast unpacked_data values to np arrays, so they are easier to reference + for key, val in self.unpacked_data.items(): + # if it is not a nested list, make the value into a ndarray + if isinstance(val, (list, tuple)) and (not isinstance(val[0], (tuple, list))): + self.unpacked_data[key] = np.asarray(val) + + # cast all list parameter values to np array, so they are easier to reference + for key, val in self.parameters.items(): + if isinstance(val, (list, tuple)): + self.parameters[key] = np.asarray(val) + + # Get frequency values + freq_old = self.unpacked_data["frequency"] + # Obtain sorted frequency indices + self.freq_ind_sorted = freq_old.argsort() + + # Obtain sorted frequencies + self.freq_sorted = freq_old[self.freq_ind_sorted] * 1000.0 + + # Build Sv offset + """ + self.Sv_offset = np.zeros_like(self.freq_sorted) + for ind, ich in enumerate(self.freq_ind_sorted): + self.Sv_offset[ind] = self._calc_Sv_offset( + self.freq_sorted[ind], self.unpacked_data["pulse_len"][ich] + ) + """ + + def _compute_battery(self, ping_num, battery_type): + """ + Compute battery voltage. + + Parameters + ---------- + ping_num + ping number + type + either "main" or "tx" + """ + if battery_type == "main": + N = self.unpacked_data["ancillary"][ping_num][2] + elif battery_type == "tx": + N = self.unpacked_data["ad"][ping_num][0] + + return N * self.BAT_CONSTANT + + def _print_status(self): + """Prints message to console giving information about the raw file being parsed.""" + filename = os.path.basename(self.source_file) + timestamp = dt( + self.unpacked_data["year"][0], + self.unpacked_data["month"][0], + self.unpacked_data["day"][0], + self.unpacked_data["hour"][0], + self.unpacked_data["minute"][0], + int(self.unpacked_data["second"][0] + self.unpacked_data["hundredths"][0] / 100), + ) + timestr = timestamp.strftime("%Y-%b-%d %H:%M:%S") + pathstr, xml_name = os.path.split(self.xml_path) + logger.info(f"parsing file {filename} with {xml_name}, " f"time of first ping: {timestr}") + + def _split_header(self, raw, header_unpacked): + """Splits the header information into a dictionary. + Modifies self.unpacked_data + + Parameters + ---------- + raw + open binary file + header_unpacked + output of struct unpack of raw file + + Returns + ------- + True or False depending on whether the unpacking was successful + """ + if ( + header_unpacked[0] != self.FILE_TYPE + ): # first field should match hard-coded FILE_TYPE from manufacturer + check_eof = raw.read(1) + if check_eof: + logger.error("Unknown file type") + return False + header_byte_cnt = 0 + + # fields with num_freq data still takes 4 bytes, + # the extra bytes contain random numbers + firmware_freq_len = 4 + + field_w_freq = ( + "dig_rate", + "lock_out_index", + "num_bins", + "range_samples_per_bin", # fields with num_freq data + "data_type", + "gain", + "pulse_len", + "board_num", + "frequency", + ) + for field in HEADER_FIELDS: + if field[0] in field_w_freq: # fields with num_freq data + self.unpacked_data[field[0]].append( + header_unpacked[header_byte_cnt : header_byte_cnt + self.parameters["num_freq"]] + ) + header_byte_cnt += firmware_freq_len + elif len(field) == 3: # other longer fields ('ancillary' and 'ad') + self.unpacked_data[field[0]].append( + header_unpacked[header_byte_cnt : header_byte_cnt + field[2]] + ) + header_byte_cnt += field[2] + else: + self.unpacked_data[field[0]].append(header_unpacked[header_byte_cnt]) + header_byte_cnt += 1 + return True + + def _get_ping_time(self): + """Assemble ping time from parsed values.""" + + ping_time = [] + for ping_num, year in enumerate(self.unpacked_data["year"]): + ping_time.append( + np.datetime64( + dt( + year, + self.unpacked_data["month"][ping_num], + self.unpacked_data["day"][ping_num], + self.unpacked_data["hour"][ping_num], + self.unpacked_data["minute"][ping_num], + int( + self.unpacked_data["second"][ping_num] + + self.unpacked_data["hundredths"][ping_num] / 100 + ), + ).replace(tzinfo=None), + "[ns]", + ) + ) + self.ping_time = ping_time diff --git a/echopype/convert/parse_uls6.py b/echopype/convert/parse_uls6.py new file mode 100644 index 000000000..6ae074157 --- /dev/null +++ b/echopype/convert/parse_uls6.py @@ -0,0 +1,620 @@ +import os +import xml.etree.ElementTree as ET +from datetime import datetime as dt +from io import BytesIO +from struct import unpack + +import fsspec +import numpy as np + +from ..utils.log import _init_logger +from ..utils.misc import camelcase2snakecase +from .parse_azfp import ParseAZFP + +FILENAME_DATETIME_AZFP = "\\w+_\\w+.[azfp|aps6]" + +MAXIMUM_CODES = 127 +HEADER_MAXIMUM_BYTES = (MAXIMUM_CODES * 2) + 512 + +# Latest header field codes +HEADER_CODES = dict( + START_FLAG=0xBCD0, # Start of header block + END_FLAG=0xABC1, # End of header block + FIRST_HEADER_RECORD=0xAA20, # Always the first record written + LAST_HEADER_RECORD=0xFE20, # Always the last record written + HEADER_BYTES=0xBB20, # The number of bytes of the header # always second + HEADER_NUM_RECORDS=0xCC20, # The number of variables # always first + BURST_NUMBER=0x8060, # Burst number + PROFILE_NUMBER=0x8060, # Backwards naming convention with ULS5 + SERIAL_NUMBER=0x8160, # Instrument Serial Number 2024 01 30 changed from 2 bytes to 4 bytes + DATE_TIME=0x8226, # date and time + ACQ_STATUS=0x8320, # Acquisition status + BURST_INT=0x8460, # Burst interval + BASE_TIME=0x85C0, # Base time + PING_PERIOD=0x86C0, # Ping period + PING_PERIOD_COUNTS=0x8720, # Ping period counts + PING_PER_BURST=0x8820, # Pings per burst + PING_PER_PROFILE=0x8820, # Backwards naming compatibility with ULS5 + AVERAGE_BURST_PINGS=0x8920, # Average burst pings + AVG_PINGS=0x8920, # Backwards naming compatibility with ULS5 + NUM_ACQUIRED_BURST_PINGS=0x8A20, # Number of acquired burst pings + NUM_ACQ_PINGS=0x8A20, # Backwards naming compatibility with ULS5 + FIRST_PING=0x8B60, # First ping number + LAST_PING=0x8C60, # Last acquired ping + DATA_ERROR=0x8D20, # Data error flag + OVER_RUN=0x8E20, # Overrun flag + PHASE=0x8F20, # The phase the data was collected from + NUM_CHAN=0x9020, # Number of stored frequencies + DIG_RATE=0x9120, # Digitization rate of stored frequencies + LOCK_OUT_INDEX=0x9220, # Lockout index + NUM_BINS=0x9320, # Number of bins + RANGE_SAMPLES_PER_BIN=0x9420, # Ranged + DATA_TYPE=0x9520, # The data type for each frequency + PULSE_LEN=0x9620, # The Pulse Length each frequency + BOARD_NUM=0x9720, # The Board number frequency + FREQUENCY=0x9820, # The frequency for each channel + NUM_SENSORS=0x9920, # The number of analog to digital sensor records summed + SENSOR_STATUS=0x9A20, # The Sensor status indicates available sensors + SENSOR_DATA=0x9B26, # The sensor data this is the analog sensor data + ANCILLARY=0x9B26, # Backwards naming compatibility with ULS5 + GPS_DATE_TIME=0x2026, # GPS date + GPS_LAT_LONG=0x21C1, # GPS Latitude longitude + PAROS_DATE_TIME=0x2226, # PAROS DATE + PAROS_PRESS_TEMP_RAW=0x2361, # PAROS PRESSURE & TEMPERATURE Raw Reading + PAROS_PRESS_TEMP_ENG=0x24C1, # PAROS PRESSURE & TEMPERATURE Engineering units + CUSTOM=0x5000, # Custom value set by user. values 50 to 5F can be custom values +) +HEADER_LOOKUP = {v: k for k, v in HEADER_CODES.items()} + +# Some earlier versions of ULS6 AZFP files have different codes +OLDER_CODES = dict( + SERIAL_NUMBER=0x8120, # Instrument Serial Number 2024 01 30 changed from 2 bytes to 4 bytes + DIG_RATE=0x9123, # Digitization rate of stored frequencies + LOCK_OUT_INDEX=0x9223, # Lockout index + NUM_BINS=0x9323, # Number of bins + RANGE_SAMPLES_PER_BIN=0x9423, # Ranged + DATA_TYPE=0x9523, # The data type for each frequency + PULSE_LEN=0x9623, # The Pulse Length each frequency + BOARD_NUM=0x9723, # The Board number frequency + FREQUENCY=0x9823, # The frequency for each channel +) +OLDER_HEADER_LOOKUP = {v: k for k, v in OLDER_CODES.items()} +HEADER_LOOKUP = {**HEADER_LOOKUP, **OLDER_HEADER_LOOKUP} + + +logger = _init_logger(__name__) + + +class ParseULS6(ParseAZFP): + """Class for converting data from ASL Environmental Sciences AZFP echosounder.""" + + field_w_freq = ( + "dig_rate", + "lock_out_index", + "num_bins", + "range_samples_per_bin", + "data_type", + "pulse_len", + "board_num", + "frequency", + "BP", # for single channel data, these are stored as scalars + "DS", + "EL", + "TVR", + "VTX0", + "VTX1", + "VTX2", + "VTX3", + ) + + # fields to reduce size if the same for all pings + field_reduce = ( + "base_time", + "ping_period_counts", + "serial_number", + "burst_int", + "ping_per_profile", + "avg_pings", + "ping_period", + "phase", + "num_chan", + ) + + # Instrument specific constants + XML_FILE_TYPE = 0xF044CC11 # Also the start flag + XML_END_FLAG = 0xE088DD66 + DATA_START_FLAG = 0xFF01AA00 + DATA_END_FLAG = 0xEF02BB66 + + RECORD_DATA_TYPE_MASK = 0x00E0 + ARRAY_BITS_MASK = 0x001F + CODE_BITS_MASK = 0x7F00 + TYPE_BITS_MASK = 0x00E0 + REQUIRED_BITS_MASK = 0x8000 + + BAT_CONSTANT = (2.5 / 65536.0) * (86.6 + 475.0) / 86.6 + + def __init__( + self, + file, + file_meta, + storage_options={}, + sonar_model="AZFP6", + **kwargs, + ): + super().__init__(file, file_meta, storage_options, sonar_model, **kwargs) + + self.sonar_model = "AZFP6" + self.sonar_firmware = "ULS6" + + def load_AZFP_xml(self, raw): + """ + Parses the AZFP XML file embedded in the AZFP file. + + Updates self.parameters + + """ + xml_byte_size = unpack(" 3 and not child.tag.startswith("VTX"): + camel_case_tag = camelcase2snakecase(child.tag) + else: + camel_case_tag = child.tag + + if len(child.attrib) > 0: + for key, val in child.attrib.items(): + attrib_tag = camel_case_tag + "_" + camelcase2snakecase(key) + if phase_number is not None and camel_case_tag != "phase": + attrib_tag += f"_phase{phase_number}" + self.parameters[attrib_tag].append(val) + if child.tag == "Phase": + phase_number = val + + if child.text is None or all(char == "\n" for char in child.text): + continue + + try: + val = int(child.text) + except ValueError: + try: + val = float(child.text) + except: + val = child.text + + if phase_number is not None and camel_case_tag != "phase": + camel_case_tag += f"_phase{phase_number}" + + self.parameters[camel_case_tag].append(val) + + # Handling the case where there is only one value for each parameter + for key, val in self.parameters.items(): + if len(val) == 1 and key != "phase_number": + self.parameters[key] = val[0] + + self.parameters["phase_number"] = [str(n + 1) for n in range(self.parameters["num_phases"])] + # Gain was removed, for backward compatibility adding in a Gain=1 field + for phase in range(self.parameters["num_phases"]): + self.parameters[f"gain_phase{phase + 1}"] = [1] * self.parameters["num_freq"] + + def _parse_header(self, file): + """Reads the first bytes of the header to get the header flag and number of data bytes + Calls _split_header where the header block is parsed. + + Modifies self.unpacked_data + + Parameters + ---------- + raw + open binary file + + Returns + ------- + True or False depending on whether the unpacking was successful + """ + try: + header_flag, num_data_bytes = unpack(" xr.Dataset: """Set the Environment group.""" @@ -130,46 +179,13 @@ def set_env(self) -> xr.Dataset: }, ) - # Additional variables, if present - temp_press = dict() - if not np.isnan(self.parser_obj.unpacked_data["temperature"]).all(): - temp_press["temperature"] = ( - ["time1"], - self.parser_obj.unpacked_data["temperature"], - { - "long_name": "Water temperature", - "standard_name": "sea_water_temperature", - "units": "deg_C", - }, - ) - if not np.isnan(self.parser_obj.unpacked_data["pressure"]).all(): - temp_press["pressure"] = ( - ["time1"], - self.parser_obj.unpacked_data["pressure"], - { - "long_name": "Sea water pressure", - "standard_name": "sea_water_pressure_due_to_sea_water", - "units": "dbar", - }, + additional_fields, coords = self._firmware_specific_env_fields() + if len(additional_fields) > 0: + ds_added = xr.Dataset( + additional_fields, + coords=coords, ) - - if len(temp_press) > 0: - ds_temp_press = xr.Dataset( - temp_press, - coords={ - "time1": ( - ["time1"], - self.parser_obj.ping_time, - { - "axis": "T", - "long_name": "Timestamp of each ping", - "standard_name": "time", - "comment": "Time coordinate corresponding to environmental variables.", - }, - ) - }, - ) - ds = xr.merge([ds, ds_temp_press], combine_attrs="override") + ds = xr.merge([ds, ds_added], combine_attrs="override") return set_time_encodings(ds) @@ -195,14 +211,30 @@ def set_sonar(self) -> xr.Dataset: return ds + def _firmware_specific_platform_fields(self, unpacked_data): + # Create nan time coordinate for lat/lon (lat/lon do not exist in AZFP 01A data) + time1 = [np.nan] + + variable_fields = { + "latitude": ( + ["time1"], + [np.nan], + self._varattrs["platform_var_default"]["latitude"], + ), + "longitude": ( + ["time1"], + [np.nan], + self._varattrs["platform_var_default"]["longitude"], + ), + } + + return variable_fields, {"time1": time1} + def set_platform(self) -> xr.Dataset: """Set the Platform group.""" platform_dict = {"platform_name": "", "platform_type": "", "platform_code_ICES": ""} unpacked_data = self.parser_obj.unpacked_data - # Create nan time coordinate for lat/lon (lat/lon do not exist in AZFP 01A data) - time1 = [np.nan] - # If tilt_x and/or tilt_y are all nan, create single-value time2 dimension # and single-value (np.nan) tilt_x and tilt_y tilt_x = [np.nan] if np.isnan(unpacked_data["tilt_x"]).all() else unpacked_data["tilt_x"] @@ -212,22 +244,16 @@ def set_platform(self) -> xr.Dataset: else: time2 = self.parser_obj.ping_time + firmware_variables, coords = self._firmware_specific_platform_fields(unpacked_data) + time1 = coords["time1"] + # Handle potential nan timestamp for time1 and time2 time1 = self._nan_timestamp_handler(time1) time2 = self._nan_timestamp_handler(time2) # should not be nan; but add for completeness ds = xr.Dataset( { - "latitude": ( - ["time1"], - [np.nan], - self._varattrs["platform_var_default"]["latitude"], - ), - "longitude": ( - ["time1"], - [np.nan], - self._varattrs["platform_var_default"]["longitude"], - ), + **firmware_variables, "pitch": ( ["time2"], [np.nan] * len(time2), @@ -339,13 +365,20 @@ def set_beam(self) -> List[xr.Dataset]: dig_rate = unpacked_data["dig_rate"][self.parser_obj.freq_ind_sorted] # dim: freq ping_time = self.parser_obj.ping_time + if self.parser_obj.sonar_firmware == "ULS5": + key = "year" + gain = np.array( + unpacked_data["gain"][self.parser_obj.freq_ind_sorted], dtype=np.float64 + ) + else: + key = "date_time" + gain = [1.0] * len(self.parser_obj.freq_ind_sorted) # Build variables in the output xarray Dataset N = [] # for storing backscatter_r values for each frequency + for ich in self.parser_obj.freq_ind_sorted: N.append( - np.array( - [unpacked_data["counts"][p][ich] for p in range(len(unpacked_data["year"]))] - ) + np.array([unpacked_data["counts"][p][ich] for p in range(len(unpacked_data[key]))]) ) # Largest number of counts along the range dimension among the different channels @@ -435,9 +468,7 @@ def set_beam(self) -> List[xr.Dataset]: ), "gain_correction": ( ["channel"], - np.array( - unpacked_data["gain"][self.parser_obj.freq_ind_sorted], dtype=np.float64 - ), + gain, {"long_name": "Gain correction", "units": "dB"}, ), "sample_interval": ( @@ -538,35 +569,61 @@ def set_beam(self) -> List[xr.Dataset]: return [set_time_encodings(ds)] + def _firmware_specific_vendor_fields(self, unpacked_data, anc=None): + """Adjust for different naming conventions and fields between firmware versions""" + ad_len = list(range(len(unpacked_data["ad"][0]))) + ad_channels = unpacked_data["ad"] + ancillary_len = list(range(len(unpacked_data["ancillary"][0]))) + + variable_fields = { + "ping_status": (["ping_time"], unpacked_data["ping_status"]), + "sensors_flag": (["ping_time"], unpacked_data["sensor_flag"]), + "ad_channels": ( + ["ping_time", "ad_len"], + ad_channels, + {"long_name": "AD channel 6 and 7"}, + ), + "spare_channel": ([], unpacked_data["spare_chan"], {"long_name": "Spare channel"}), + } + + return variable_fields, {"ancillary_len": ancillary_len, "ad_len": ad_len} + def set_vendor(self) -> xr.Dataset: """Set the Vendor_specific group.""" unpacked_data = self.parser_obj.unpacked_data parameters = self.parser_obj.parameters ping_time = self.parser_obj.ping_time - Sv_offset = self.parser_obj.Sv_offset - phase_params = ["burst_interval", "pings_per_burst", "average_burst_pings"] - phase_freq_params = [ - "dig_rate", - "range_samples", - "range_averaging_samples", - "lock_out_index", - "gain", - "storage_format", - ] + + phase_params = self.phase_params + phase_freq_params = self.phase_freq_params + tdn = [] for num in parameters["phase_number"]: - tdn.append(parameters[f"pulse_len_phase{num}"][self.parser_obj.freq_ind_sorted] / 1e6) + try: + tdn.append( + parameters[f"pulse_len_phase{num}"][self.parser_obj.freq_ind_sorted] / 1e6 + ) + except: + tdn.append([np.nan] * len(self.parser_obj.freq_ind_sorted)) + tdn = np.array(tdn) for param in phase_freq_params: for num in parameters["phase_number"]: - parameters[param].append( - parameters[f"{param}_phase{num}"][self.parser_obj.freq_ind_sorted] - ) + try: + parameters[param].append( + parameters[f"{param}_phase{num}"][self.parser_obj.freq_ind_sorted] + ) + except: + parameters[param].append([np.nan] * len(self.parser_obj.freq_ind_sorted)) + for param in phase_params: for num in parameters["phase_number"]: - parameters[param].append(parameters[f"{param}_phase{num}"]) + p = parameters[f"{param}_phase{num}"] + parameters[param].append(np.nan if isinstance(p, list) else p) anc = np.array(unpacked_data["ancillary"]) # convert to np array for easy slicing + firmware_variables, coords = self._firmware_specific_vendor_fields(unpacked_data, anc) + ds = xr.Dataset( { "frequency_nominal": ( @@ -619,7 +676,6 @@ def set_vendor(self) -> xr.Dataset: "0=raw (2bytes)" }, ), - "ping_status": (["ping_time"], unpacked_data["ping_status"]), "number_of_acquired_pings": ( ["ping_time"], unpacked_data["num_acq_pings"], @@ -632,17 +688,11 @@ def set_vendor(self) -> xr.Dataset: unpacked_data["data_error"], {"long_name": "Error number if an error occurred"}, ), - "sensors_flag": (["ping_time"], unpacked_data["sensor_flag"]), "ancillary": ( ["ping_time", "ancillary_len"], unpacked_data["ancillary"], {"long_name": "Tilt-X, Y, Battery, Pressure, Temperature"}, ), - "ad_channels": ( - ["ping_time", "ad_len"], - unpacked_data["ad"], - {"long_name": "AD channel 6 and 7"}, - ), "battery_main": (["ping_time"], unpacked_data["battery_main"]), "battery_tx": (["ping_time"], unpacked_data["battery_tx"]), "profile_number": (["ping_time"], unpacked_data["profile_number"]), @@ -677,7 +727,6 @@ def set_vendor(self) -> xr.Dataset: unpacked_data["avg_pings"], {"long_name": "Flag indicating whether the pings average in time"}, ), - "spare_channel": ([], unpacked_data["spare_chan"], {"long_name": "Spare channel"}), "ping_period": ( [], unpacked_data["ping_period"], @@ -693,6 +742,8 @@ def set_vendor(self) -> xr.Dataset: unpacked_data["num_chan"], {"long_name": "Number of channels (1, 2, 3, or 4)"}, ), + # firmware specific variables/field names + **firmware_variables, # parameters with channel dimension from XML file "XML_transmit_duration_nominal": ( ["phase_number", "channel"], @@ -767,7 +818,6 @@ def set_vendor(self) -> xr.Dataset: parameters["VTX3"][self.parser_obj.freq_ind_sorted], {"long_name": "Amplified voltage 3 sent to the transducer"}, ), - "Sv_offset": (["channel"], Sv_offset), "number_of_samples_digitized_per_pings": ( ["phase_number", "channel"], parameters["range_samples"], @@ -848,9 +898,9 @@ def set_vendor(self) -> xr.Dataset: ), "ancillary_len": ( ["ancillary_len"], - list(range(len(unpacked_data["ancillary"][0]))), + coords["ancillary_len"], ), - "ad_len": (["ad_len"], list(range(len(unpacked_data["ad"][0])))), + "ad_len": (["ad_len"], coords["ad_len"]), "phase_number": ( ["phase_number"], sorted([int(num) for num in parameters["phase_number"]]), diff --git a/echopype/convert/set_groups_azfp6.py b/echopype/convert/set_groups_azfp6.py index b98de5806..73960e419 100644 --- a/echopype/convert/set_groups_azfp6.py +++ b/echopype/convert/set_groups_azfp6.py @@ -2,20 +2,31 @@ Class to save unpacked echosounder data to appropriate groups in netcdf or zarr. """ -from typing import List - import numpy as np import xarray as xr -from ..utils.coding import set_time_encodings - -# from .set_groups_base import SetGroupsBase from .set_groups_azfp import SetGroupsAZFP class SetGroupsAZFP6(SetGroupsAZFP): """Class for saving groups to netcdf or zarr from AZFP6 data files.""" + phase_params = [ + "burst_interval", + "pings_per_burst", + "average_burst_pings", + "base_time", + "ping_period_counts", + ] + phase_freq_params = [ + "dig_rate", + "range_samples", + "range_averaging_samples", + "lock_out_index", + "gain", + "storage_format", + ] + def set_sonar(self) -> xr.Dataset: """Set the Sonar group.""" @@ -30,29 +41,17 @@ def set_sonar(self) -> xr.Dataset: "sonar_manufacturer": "ASL Environmental Sciences", "sonar_model": self.sonar_model, "sonar_serial_number": int(self.parser_obj.unpacked_data["serial_number"]), - "sonar_software_name": "AZFP6", - "sonar_software_version": "alpha version based on 2.0.07 version", + "sonar_software_name": "ULS6", + "sonar_software_version": "beta version", "sonar_type": "echosounder", } ds = ds.assign_attrs(sonar_attr_dict) return ds - def set_platform(self) -> xr.Dataset: - """Set the Platform group.""" - platform_dict = {"platform_name": "", "platform_type": "", "platform_code_ICES": ""} - unpacked_data = self.parser_obj.unpacked_data + def _firmware_specific_platform_fields(self, unpacked_data): + gps_latlon = np.array(unpacked_data["gps_lat_long"]) - # If tilt_x and/or tilt_y are all nan, create single-value time2 dimension - # and single-value (np.nan) tilt_x and tilt_y - tilt_x = [np.nan] if np.isnan(unpacked_data["tilt_x"]).all() else unpacked_data["tilt_x"] - tilt_y = [np.nan] if np.isnan(unpacked_data["tilt_y"]).all() else unpacked_data["tilt_y"] - if (len(tilt_x) == 1 and np.isnan(tilt_x)) and (len(tilt_y) == 1 and np.isnan(tilt_y)): - time2 = [self.parser_obj.ping_time[0]] - else: - time2 = self.parser_obj.ping_time - - gps_latlon = np.array(unpacked_data["gps_lat_lon"]) lat = ( [np.nan] if np.isnan(gps_latlon[:, 0]).all() or not np.any(gps_latlon[:, 0]) @@ -69,678 +68,131 @@ def set_platform(self) -> xr.Dataset: # time1 = time2 if not np.any(time1) else time1 time1 = [np.nan] if len(lat) != len(time1) else time1 - # Handle potential nan timestamp for time1 and time2 - time1 = self._nan_timestamp_handler(time1) - time2 = self._nan_timestamp_handler(time2) # should not be nan; but add for completeness - - ds = xr.Dataset( - { - "latitude": ( - ["time1"], - lat, - self._varattrs["platform_var_default"]["latitude"], - ), - "longitude": ( - ["time1"], - lon, - self._varattrs["platform_var_default"]["longitude"], - ), - "pitch": ( - ["time2"], - [np.nan] * len(time2), - self._varattrs["platform_var_default"]["pitch"], - ), - "roll": ( - ["time2"], - [np.nan] * len(time2), - self._varattrs["platform_var_default"]["roll"], - ), - "vertical_offset": ( - ["time2"], - [np.nan] * len(time2), - self._varattrs["platform_var_default"]["vertical_offset"], - ), - "water_level": ( - [], - np.nan, - self._varattrs["platform_var_default"]["water_level"], - ), - "tilt_x": ( - ["time2"], - tilt_x, - { - "long_name": "Tilt X", - "units": "arc_degree", - }, - ), - "tilt_y": ( - ["time2"], - tilt_y, - { - "long_name": "Tilt Y", - "units": "arc_degree", - }, - ), - **{ - var: ( - ["channel"], - [np.nan] * len(self.channel_ids_sorted), - self._varattrs["platform_var_default"][var], - ) - for var in [ - "transducer_offset_x", - "transducer_offset_y", - "transducer_offset_z", - ] + variable_fields = { + "latitude": ( + ["time1"], + lat, + self._varattrs["platform_var_default"]["latitude"], + ), + "longitude": ( + ["time1"], + lon, + self._varattrs["platform_var_default"]["longitude"], + ), + } + return variable_fields, {"time1": time1} + + def _firmware_specific_vendor_fields(self, unpacked_data, anc): + """Adjust for different naming conventions and fields between firmware versions""" + ad_len = list(range(anc[:, -2:].shape[-1])) + ad_channels = anc[:, -2:] + ancillary_len = list(range(anc.shape[-1])) + if len(unpacked_data["custom"]) == 0: + unpacked_data["custom"] = 0 + + variable_fields = { + "acq_status": (["ping_time"], unpacked_data["acq_status"]), + "sensor_status": (["ping_time"], unpacked_data["sensor_status"]), + "ad_channels": ( + ["ping_time", "ad_len"], + ad_channels, + {"long_name": "AD channel 6 and 7"}, + ), + "custom": ([], unpacked_data["custom"], {"long_name": "Spare/custom channel"}), + } + coords = {"ancillary_len": ancillary_len, "ad_len": ad_len} + + paros_time = self.parser_obj._get_paros_time() + paros_raw = np.asarray(unpacked_data["paros_press_temp_raw"]) + if len(paros_raw) > 0: + paros_fields = { + "paros_pressure_counts": ( + ["paros_time"], + paros_raw[:, 0], + {"long_name": "Raw counts for Paros pressure"}, + ), + "paros_temperature_counts": ( + ["paros_time"], + paros_raw[:, 1], + {"long_name": "Raw counts for Paros temperature"}, + ), + } + coords["paros_time"] = paros_time + variable_fields = {**variable_fields, **paros_fields} + + return variable_fields, coords + + def _firmware_specific_env_fields(self): + # Additional variables, if present + temp_press = dict() + if not np.isnan(self.parser_obj.unpacked_data["paros_temperature"]).all(): + temp_press["temperature"] = ( + ["paros_time"], + self.parser_obj.unpacked_data["paros_temperature"], + { + "long_name": "Water temperature", + "standard_name": "sea_water_temperature", + "units": "deg_C", }, - **{ - var: ([], np.nan, self._varattrs["platform_var_default"][var]) - for var in [ - "MRU_offset_x", - "MRU_offset_y", - "MRU_offset_z", - "MRU_rotation_x", - "MRU_rotation_y", - "MRU_rotation_z", - "position_offset_x", - "position_offset_y", - "position_offset_z", - ] + ) + if not np.isnan(self.parser_obj.unpacked_data["paros_pressure"]).all(): + temp_press["pressure"] = ( + ["paros_time"], + self.parser_obj.unpacked_data["paros_pressure"], + { + "long_name": "Sea water pressure", + "standard_name": "sea_water_pressure_due_to_sea_water", + "units": "dbar", }, - "frequency_nominal": ( - ["channel"], - self.parser_obj.freq_sorted, - { - "units": "Hz", - "long_name": "Transducer frequency", - "valid_min": 0.0, - "standard_name": "sound_frequency", - }, - ), - }, - coords={ - "channel": ( - ["channel"], - self.channel_ids_sorted, - self._varattrs["beam_coord_default"]["channel"], - ), - "time1": ( - ["time1"], - # xarray and probably CF don't accept time coordinate variable with Nan values - time1, - { - **self._varattrs["platform_coord_default"]["time1"], - "comment": "Time coordinate corresponding to GPS position data.", - }, - ), - "time2": ( - ["time2"], - time2, + ) + + if len(temp_press) > 0: + coords = { + "paros_time": ( + ["paros_time"], + self.parser_obj._get_paros_time(), { "axis": "T", - "long_name": "Timestamps for platform motion and orientation data", + "long_name": "Timestamp of each Paros measurement", "standard_name": "time", - "comment": "Time coordinate corresponding to platform motion and " - "orientation data.", + "comment": "Time coordinate corresponding to Paros sensor variables.", }, - ), - }, - ) - ds = ds.assign_attrs(platform_dict) - return set_time_encodings(ds) - - def set_beam(self) -> List[xr.Dataset]: - """Set the Beam group.""" - unpacked_data = self.parser_obj.unpacked_data - parameters = self.parser_obj.parameters - dig_rate = unpacked_data["dig_rate"][self.parser_obj.freq_ind_sorted] # dim: freq - ping_time = self.parser_obj.ping_time - - # Build variables in the output xarray Dataset - N = [] # for storing backscatter_r values for each frequency - for ich in self.parser_obj.freq_ind_sorted: - N.append( - np.array( - [ - unpacked_data["counts"][p][ich] for p in range(len(unpacked_data["date"])) - ] # year ) + } + return temp_press, coords + + if not np.isnan(self.parser_obj.unpacked_data["temperature"]).all(): + temp_press["temperature"] = ( + ["time1"], + self.parser_obj.unpacked_data["temperature"], + { + "long_name": "Water temperature", + "standard_name": "sea_water_temperature", + "units": "deg_C", + }, ) - - # Largest number of counts along the range dimension among the different channels - longest_range_sample = np.max(unpacked_data["num_bins"]) - range_sample = np.arange(longest_range_sample) - - # Pad power data - if any(unpacked_data["num_bins"] != longest_range_sample): - N_tmp = np.full((len(N), len(ping_time), longest_range_sample), np.nan) - for i, n in enumerate(N): - N_tmp[i, :, : n.shape[1]] = n - N = N_tmp - del N_tmp - - tdn = ( - unpacked_data["pulse_len"][self.parser_obj.freq_ind_sorted] / 1e6 - ) # Convert microseconds to seconds - range_samples_per_bin = unpacked_data["range_samples_per_bin"][ - self.parser_obj.freq_ind_sorted - ] # from data header - - # Calculate sample interval in seconds - if len(dig_rate) == len(range_samples_per_bin): - # TODO: below only correct if range_samples_per_bin=1, - # need to implement p.86 for the case when averaging is used - sample_int = range_samples_per_bin / dig_rate - else: - # TODO: not sure what this error means - raise ValueError("dig_rate and range_samples not unique across frequencies") - - ds = xr.Dataset( - { - "frequency_nominal": ( - ["channel"], - self.parser_obj.freq_sorted, - { - "units": "Hz", - "long_name": "Transducer frequency", - "valid_min": 0.0, - "standard_name": "sound_frequency", - }, - ), - "beam_type": ( - ["channel"], - [0] * len(self.channel_ids_sorted), - { - "long_name": "Beam type", - "flag_values": [0, 1], - "flag_meanings": [ - "Single beam", - "Split aperture beam", - ], - }, - ), - **{ - f"beam_direction_{var}": ( - ["channel"], - [np.nan] * len(self.channel_ids_sorted), - { - "long_name": f"{var}-component of the vector that gives the pointing " - "direction of the beam, in sonar beam coordinate " - "system", - "units": "1", - "valid_range": (-1.0, 1.0), - }, - ) - for var in ["x", "y", "z"] + if not np.isnan(self.parser_obj.unpacked_data["pressure"]).all(): + temp_press["pressure"] = ( + ["time1"], + self.parser_obj.unpacked_data["pressure"], + { + "long_name": "Sea water pressure", + "standard_name": "sea_water_pressure_due_to_sea_water", + "units": "dbar", }, - "backscatter_r": ( - ["channel", "ping_time", "range_sample"], - np.array(N, dtype=np.float32), - { - "long_name": self._varattrs["beam_var_default"]["backscatter_r"][ - "long_name" - ], - "units": "count", - }, - ), - "equivalent_beam_angle": ( - ["channel"], - parameters["BP"][self.parser_obj.freq_ind_sorted], - { - "long_name": "Equivalent beam angle", - "units": "sr", - "valid_range": (0.0, 4 * np.pi), - }, - ), - # "gain_correction": ( - # ["channel"], - # np.array( - # unpacked_data["gain"][self.parser_obj.freq_ind_sorted], dtype=np.float64 - # ), - # {"long_name": "Gain correction", "units": "dB"}, - # ), - "sample_interval": ( - ["channel"], - sample_int, - { - "long_name": "Interval between recorded raw data samples", - "units": "s", - "valid_min": 0.0, - }, - ), - "transmit_duration_nominal": ( - ["channel"], - tdn, - { - "long_name": "Nominal bandwidth of transmitted pulse", - "units": "s", - "valid_min": 0.0, - }, - ), - "transmit_frequency_start": ( - ["channel"], - self.parser_obj.freq_sorted, - self._varattrs["beam_var_default"]["transmit_frequency_start"], - ), - "transmit_frequency_stop": ( - ["channel"], - self.parser_obj.freq_sorted, - self._varattrs["beam_var_default"]["transmit_frequency_stop"], - ), - "transmit_type": ( - [], - "CW", - { - "long_name": "Type of transmitted pulse", - "flag_values": ["CW"], - "flag_meanings": [ - "Continuous Wave – a pulse nominally of one frequency", - ], - }, - ), - "beam_stabilisation": ( - [], - np.array(0, np.byte), - { - "long_name": "Beam stabilisation applied (or not)", - "flag_values": [0, 1], - "flag_meanings": ["not stabilised", "stabilised"], - }, - ), - "non_quantitative_processing": ( - [], - np.array(0, np.int16), - { - "long_name": "Presence or not of non-quantitative processing applied" - " to the backscattering data (sonar specific)", - "flag_values": [0], - "flag_meanings": ["None"], - }, - ), - "sample_time_offset": ( - [], - 0.0, - { - "long_name": "Time offset that is subtracted from the timestamp" - " of each sample", - "units": "s", - }, - ), - }, - coords={ - "channel": ( - ["channel"], - self.channel_ids_sorted, - self._varattrs["beam_coord_default"]["channel"], - ), - "ping_time": ( - ["ping_time"], - ping_time, - self._varattrs["beam_coord_default"]["ping_time"], - ), - "range_sample": ( - ["range_sample"], - range_sample, - self._varattrs["beam_coord_default"]["range_sample"], - ), - }, - attrs={ - "beam_mode": "", - "conversion_equation_t": "type_4", - }, - ) - - # Manipulate some Dataset dimensions to adhere to convention - self.beam_groups_to_convention( - ds, self.beam_only_names, self.beam_ping_time_names, self.ping_time_only_names - ) - - return [set_time_encodings(ds)] - - def set_vendor(self) -> xr.Dataset: - """Set the Vendor_specific group.""" - unpacked_data = self.parser_obj.unpacked_data - parameters = self.parser_obj.parameters - ping_time = self.parser_obj.ping_time - Sv_offset = self.parser_obj.Sv_offset - phase_params = [ - "burst_interval", - "pings_per_burst", - "average_burst_pings", - "base_time", - "ping_period_counts", - ] - phase_freq_params = [ - "dig_rate", - "range_samples", - "range_averaging_samples", - "lock_out_index", - "gain", - "storage_format", - ] - - tdn = [] - for num in parameters["phase_number"]: - try: - tdn.append( - parameters[f"pulse_len_phase{num}"][self.parser_obj.freq_ind_sorted] / 1e6 - ) - except: - tdn.append([np.nan] * len(self.parser_obj.freq_ind_sorted)) - tdn = np.array(tdn) - for param in phase_freq_params: - for num in parameters["phase_number"]: - try: - parameters[param].append( - parameters[f"{param}_phase{num}"][self.parser_obj.freq_ind_sorted] - ) - except: - parameters[param].append([np.nan] * len(self.parser_obj.freq_ind_sorted)) - - for param in phase_params: - for num in parameters["phase_number"]: - p = parameters[f"{param}_phase{num}"] - parameters[param].append(np.nan if isinstance(p, list) else p) - anc = np.array(unpacked_data["ancillary"]) # convert to np array for easy slicing + ) - ds = xr.Dataset( - { - "frequency_nominal": ( - ["channel"], - self.parser_obj.freq_sorted, - { - "units": "Hz", - "long_name": "Transducer frequency", - "valid_min": 0.0, - "standard_name": "sound_frequency", - }, - ), - # unpacked ping by ping data from 01A file - "digitization_rate": ( - ["channel"], - unpacked_data["dig_rate"][self.parser_obj.freq_ind_sorted], - { - "long_name": "Number of samples per second in kHz that is processed by the " - "A/D converter when digitizing the returned acoustic signal" - }, - ), - "lock_out_index": ( - ["channel"], - unpacked_data["lock_out_index"][self.parser_obj.freq_ind_sorted], - { - "long_name": "The distance, rounded to the nearest Bin Size after the " - "pulse is transmitted that over which AZFP will ignore echoes" - }, - ), - "number_of_bins_per_channel": ( - ["channel"], - unpacked_data["num_bins"][self.parser_obj.freq_ind_sorted], - {"long_name": "Number of bins per channel"}, - ), - "number_of_samples_per_average_bin": ( - ["channel"], - unpacked_data["range_samples_per_bin"][self.parser_obj.freq_ind_sorted], - {"long_name": "Range samples per bin for each channel"}, - ), - "board_number": ( - ["channel"], - unpacked_data["board_num"][self.parser_obj.freq_ind_sorted], - {"long_name": "The board the data came from channel 1-4"}, - ), - "data_type": ( - ["channel"], - unpacked_data["data_type"][self.parser_obj.freq_ind_sorted], - { - "long_name": "Datatype for each channel 1=Avg unpacked_data (5bytes), " - "0=raw (2bytes)" - }, - ), - "ping_status": (["ping_time"], unpacked_data["acq_status"]), - "number_of_acquired_pings": ( - ["ping_time"], - unpacked_data["num_acq_pings"], - {"long_name": "Pings acquired in the burst"}, - ), - "first_ping": (["ping_time"], unpacked_data["first_ping"]), - "last_ping": (["ping_time"], unpacked_data["last_ping"]), - "data_error": ( - ["ping_time"], - unpacked_data["data_error"], - {"long_name": "Error number if an error occurred"}, - ), - "sensors_flag": (["ping_time"], unpacked_data["sensor_status"]), - "ancillary": ( - ["ping_time", "ancillary_len"], - unpacked_data["ancillary"], - {"long_name": "Tilt-X, Y, Battery, Pressure, Temperature"}, - ), - "ad_channels": ( - ["ping_time", "ad_len"], - anc[:, -2:], # compatibility with Callable[[str], None]: @@ -46,16 +54,16 @@ def inner(test_ext: str): "xml": True, "accepts_bot": False, "accepts_idx": False, - "parser": ParseAZFP, + "parser": ParseULS5, "parsed2zarr": None, "set_groups": SetGroupsAZFP, }, "AZFP6": { - "validate_ext": validate_ext(".azfp"), + "validate_ext": validate_azfp_ext, "xml": False, "accepts_bot": False, "accepts_idx": False, - "parser": ParseAZFP6, + "parser": ParseULS6, "parsed2zarr": None, "set_groups": SetGroupsAZFP6, }, diff --git a/echopype/tests/calibrate/test_cal_params_integration.py b/echopype/tests/calibrate/test_cal_params_integration.py index 6168646f1..de067fc42 100644 --- a/echopype/tests/calibrate/test_cal_params_integration.py +++ b/echopype/tests/calibrate/test_cal_params_integration.py @@ -2,6 +2,7 @@ import numpy as np import xarray as xr +from xarray.testing import assert_identical import echopype as ep @@ -45,11 +46,23 @@ def test_cal_params_intake_AZFP(azfp_path): ) # Check cal params ingested from both ways - assert cal_obj.cal_params["EL"].identical(cal_params_manual["EL"]) + assert_identical(cal_obj.cal_params["EL"], cal_params_manual["EL"]) # Check against the final cal params in the calibration output ds_Sv = ep.calibrate.compute_Sv(ed, cal_params={"EL": EL_ext}, env_params=env_ext) - assert ds_Sv["EL"].identical(cal_params_manual["EL"]) + assert_identical(ds_Sv["EL"], cal_params_manual["EL"]) + + # Check passing Sv_offset values manually + SV_ext = xr.DataArray([1., 2., 3., 4.], dims=["channel"], coords={"channel": chan}, name="Sv_offset") + cal_params_manual = ep.calibrate.cal_params.get_cal_params_AZFP( + beam=ed["Sonar/Beam_group1"], vend=ed["Vendor_specific"], user_dict={"Sv_offset": SV_ext} + ) + cal_obj = ep.calibrate.calibrate_azfp.CalibrateAZFP( + echodata=ed, cal_params={"Sv_offset" : SV_ext}, env_params=env_ext + ) + ds_Sv = ep.calibrate.compute_Sv(ed, cal_params={"Sv_offset" : SV_ext}, env_params=env_ext) + assert_identical(ds_Sv["Sv_offset"], cal_params_manual["Sv_offset"]) + assert_identical(ds_Sv["Sv_offset"], cal_obj.cal_params["Sv_offset"]) def test_cal_params_intake_EK60(ek60_path): diff --git a/echopype/tests/calibrate/test_calibrate.py b/echopype/tests/calibrate/test_calibrate.py index 98aa333b3..f53a650dd 100644 --- a/echopype/tests/calibrate/test_calibrate.py +++ b/echopype/tests/calibrate/test_calibrate.py @@ -6,6 +6,7 @@ from echopype.calibrate.env_params_old import EnvParams from echopype.calibrate.cal_params import get_vend_cal_params_power import xarray as xr +from xarray.testing import assert_identical import dask.array as da @@ -13,6 +14,9 @@ def azfp_path(test_path): return test_path['AZFP'] +@pytest.fixture +def azfp6_path(test_path): + return test_path['AZFP6'] @pytest.fixture def ek60_path(test_path): @@ -186,6 +190,7 @@ def check_output(base_path, ds_cmp, cal_type): ds_cmp.echo_range.isel(channel=fidx, ping_time=0).values[None, :] == ds_base['Output'][0]['Range'][fidx] ) + assert np.allclose( ds_cmp[cal_type_in_ds_cmp[cal_type]].isel(channel=fidx).values, ds_base['Output'][0][cal_type][fidx], @@ -199,6 +204,75 @@ def check_output(base_path, ds_cmp, cal_type): # Check TS check_output(base_path=azfp_matlab_TS_path, ds_cmp=ds_TS, cal_type='TS') +def test_compute_Sv_offset_azfp(azfp_path): + # Test frequency/pulse length combinations not in known Sv_offset dict + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(70000.0, 500) == 1.1 + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(38000.0, 700) == 0.9 + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(38000.0, 900) == 0.8 + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(120000.0, 175) == 1.4 + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(120000.0, 400) == 1.0 + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(125000.0, 190) == 1.4 + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(769000.0, 1000) == 0.3 + assert ep.calibrate.calibrate_azfp._calc_azfp_Sv_offset(769000.0, 150) == 1.4 + + env_params = { + 'temperature': 20., + 'salinity': 27.9, + 'pressure': 59, + } + + #Check loading an ULS5 file that has and pulse length not in dictionary + azfp_01a_path = str(azfp_path.joinpath('Sv_offset/23110713_2ping.01A')) + azfp_xml_path = str(azfp_path.joinpath('Sv_offset/23110713.XML')) + echodata = ep.open_raw( + raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path + ) + + chan = echodata["Sonar/Beam_group1"]["channel"] + SV_ext = xr.DataArray([0.4, 0.4, 0., 0.], dims=["channel"], coords={"channel": chan}, name="Sv_offset") + ds_Sv = ep.calibrate.compute_Sv(echodata=echodata, env_params=env_params) + assert_identical(ds_Sv["Sv_offset"], SV_ext) + +def test_compute_sv_azfp6_matlab(azfp6_path): + azfp_asp_path = str(azfp6_path.joinpath('25040412_01A_2ping.azfp')) + azfp_matlab_sv_path = str( + azfp6_path.joinpath('from_matlab', '25040412_01A_2ping_sv.mat') + ) + azfp_matlab_range_path = str( + azfp6_path.joinpath('from_matlab', '25040412_01A_2ping_range.mat') + ) + + # convert to .nc file + echodata = ep.open_raw( + raw_file=azfp_asp_path, sonar_model='azfp6' + ) + + # calibrate using identical env params as in matlab parametersazfp.m + # azfp matlab code uses average temperature + env_params = { + 'temperature': 15, + 'salinity': 30, + 'pressure': 54.7, + } + + ds_sv = ep.calibrate.compute_Sv(echodata=echodata, env_params=env_params) + + ds_base = loadmat(azfp_matlab_sv_path) + ds_range = loadmat(azfp_matlab_range_path) + + assert np.allclose( + ds_sv.echo_range.isel(channel=0, ping_time=0).values[None, :], + ds_range['range'], + atol=1e-13, + ) + + assert np.allclose( + ds_sv["Sv"].isel(channel=0).values, + ds_base["sv"], + atol=1e-13, + ) + + def test_compute_Sv_ek80_CW_complex(ek80_path): """Test calibrate CW mode data encoded as complex samples.""" diff --git a/echopype/tests/calibrate/test_env_params_integration.py b/echopype/tests/calibrate/test_env_params_integration.py index 1d3630b01..13ef85a20 100644 --- a/echopype/tests/calibrate/test_env_params_integration.py +++ b/echopype/tests/calibrate/test_env_params_integration.py @@ -2,6 +2,7 @@ import numpy as np import xarray as xr +from xarray.testing import assert_identical import echopype as ep @@ -45,8 +46,8 @@ def test_env_params_intake_AZFP(azfp_path): ds_Sv = ep.calibrate.compute_Sv(ed, env_params=env_ext) assert ds_Sv["formula_sound_speed"] == "AZFP" assert ds_Sv["formula_absorption"] == "AZFP" - assert ds_Sv["sound_speed"].identical(env_params_manual["sound_speed"]) - assert ds_Sv["sound_absorption"].identical(env_params_manual["sound_absorption"]) + assert_identical(ds_Sv["sound_speed"], env_params_manual["sound_speed"]) + assert_identical(ds_Sv["sound_absorption"], env_params_manual["sound_absorption"]) def test_env_params_intake_EK60_with_input(ek60_path): diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index 4ff718797..b98f6e322 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -10,7 +10,7 @@ from scipy.io import loadmat from echopype import open_raw import pytest -from echopype.convert.parse_azfp import ParseAZFP +from echopype.convert.parse_uls5 import ParseULS5 @pytest.fixture @@ -154,7 +154,7 @@ def test_convert_azfp_01a_different_ranges(azfp_path): check_platform_required_scalar_vars(echodata) -@pytest.mark.skip(reason="required pulse length not in Sv offset dictionary") +#@pytest.mark.skip(reason="required pulse length not in Sv offset dictionary") def test_convert_azfp_01a_no_temperature_pressure_tilt(azfp_path): """Test converting file with no valid temperature, pressure and tilt data.""" @@ -209,7 +209,7 @@ def test_convert_azfp_UNB_glider_130kHz(azfp_path): def test_load_parse_azfp_xml(azfp_path): azfp_xml_path = azfp_path / '23081211.XML' - parseAZFP = ParseAZFP(None, str(azfp_xml_path)) + parseAZFP = ParseULS5(None, str(azfp_xml_path)) parseAZFP.load_AZFP_xml() expected_params = ['instrument_type_string', 'instrument_type', 'major', 'minor', 'date', 'program_name', 'program', 'CPU', 'serial_number', 'board_version', diff --git a/echopype/tests/convert/test_convert_azfp6.py b/echopype/tests/convert/test_convert_azfp6.py index 29122a0bb..af71dcaf8 100644 --- a/echopype/tests/convert/test_convert_azfp6.py +++ b/echopype/tests/convert/test_convert_azfp6.py @@ -11,7 +11,7 @@ from scipy.io import loadmat from echopype import open_raw import pytest -from echopype.convert.parse_azfp6 import ParseAZFP6 +from echopype.convert.parse_uls6 import ParseULS6 @pytest.fixture @@ -40,6 +40,10 @@ def check_platform_required_scalar_vars(echodata): def test_convert_azfp_02a_xml_parsing(azfp_path): """Compare the embedded XML parsed data with Matlab loaded XML file.""" +@pytest.mark.skip(reason="No test files with valid Paros data available yet.") +def test_convert_azfp_paros_calculations(azfp_path): + """Check the Paros temperature and pressure calculations.""" + def test_convert_azfp_01a_matlab_raw(azfp_path): """Compare parsed raw data with Matlab outputs.""" azfp_01a_path = azfp_path / '24052113_01A.azfp' @@ -60,46 +64,35 @@ def test_convert_azfp_01a_matlab_raw(azfp_path): assert np.array_equal( ds_matlab['Data']['Freq'][0][0][:, 0], echodata["Sonar/Beam_group1"].frequency_nominal / 1000, - ) # matlab file in kHz + ), "Failed Beam_group1 check: Frequency" # matlab file in kHz # backscatter count assert np.array_equal( np.array( [ds_matlab_output['Output'][0]['N'][fidx] for fidx in range(4)] ), echodata["Sonar/Beam_group1"].backscatter_r.values, - ) + ), "Failed Beam_group1 check: Backscatter count" # Test vendor group # Test temperature assert np.array_equal( np.array([d[4] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), echodata["Vendor_specific"].ancillary.isel(ancillary_len=4), - ) - assert np.allclose( - ds_matlab_output['Output']['BatteryTx'][0][0].squeeze(), - echodata["Vendor_specific"].battery_tx, - rtol=1e-08 - ) - assert np.allclose( - ds_matlab_output['Output']['BatteryMain'][0][0].squeeze(), - echodata["Vendor_specific"].battery_main, - atol=1e-12 - ) + ), "Failed Vendor_specific check: Temperature" # tilt x-y assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), echodata["Vendor_specific"].tilt_x_count, - ) + ), "Failed Vendor_specific check: Tilt X" assert np.array_equal( np.array([d[1] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), echodata["Vendor_specific"].tilt_y_count, - ) + ), "Failed Vendor_specific check: Tilt Y" # check convention-required variables in the Platform group check_platform_required_scalar_vars(echodata) - def test_convert_azfp_01a_matlab_derived(azfp_path): """Compare variables derived from raw parsed data with Matlab outputs.""" @@ -136,9 +129,10 @@ def test_convert_azfp_01a_matlab_derived(azfp_path): #NOTE: Only require a second precision accuracy as Matlab has some rounding errors at the nsec level if # the *.mat is run on Windows f = lambda x: np.datetime64(datetime.fromordinal(int(x)) + timedelta(days=x%1) - timedelta(days = 366), "[s]") + ping_time_s = echodata["Vendor_specific"].ping_time.astype('datetime64[s]') assert np.array_equal( np.vectorize(f)(ds_matlab_output["Output"]['Date'][0][0].squeeze()), - echodata["Vendor_specific"].ping_time + ping_time_s, ) check_platform_required_scalar_vars(echodata) @@ -166,3 +160,72 @@ def test_convert_azfp_02a_gps_lat_long(azfp_path): ) check_platform_required_scalar_vars(echodata) + +def test_convert_azfp_battery(azfp_path): + """Compare battery voltages derived from Matlab.""" + azfp_01a_path = azfp_path / '25040412_01A_2ping.azfp' + echodata = open_raw( + raw_file=azfp_01a_path, sonar_model='AZFP6' + ) + + # Test battery voltages + assert np.allclose( + np.array([13.6176, 0.0336]), + echodata["Vendor_specific"].battery_tx.to_numpy(), + rtol=1e-02 + ), "Failed Vendor_specific check: BatteryTx" + + + assert np.allclose( + np.array([7.089492762611865, 11.618574816415272]), + echodata["Vendor_specific"].battery_main.to_numpy(), + atol=1e-03, + ), "Failed Vendor_specific check: BatteryMain" + + check_platform_required_scalar_vars(echodata) + +def test_convert_azfp_nano_raw_open(azfp_path): + """ Test AZFP Nano file""" + + azfp_path = azfp_path / '25040412_01A_2ping.azfp' + + echodata = open_raw( + raw_file=azfp_path, sonar_model='AZFP6', + ) + + #Verify single channel is loaded + assert echodata["Sonar/Beam_group1"].backscatter_r.coords["channel"].size == 1 + + #Verify backscatter range shape + assert echodata["Sonar/Beam_group1"].backscatter_r.sel(channel='69001-200-1').dropna( + 'range_sample' + ).shape == (2, 2250) + + # Pressure variable is not present in the Environment group + assert "pressure" not in echodata["Environment"] + + # Temperature variable is present in the Environment group + assert "temperature" not in echodata["Environment"] + + check_platform_required_scalar_vars(echodata) + +def test_convert_azfp_aps6_paros_temp_pressure(azfp_path): + """Test converting file with Paros temperature, pressure data.""" + + azfp_path = azfp_path / "25111807_01A_61213_2ping.aps6" + + echodata = open_raw( + raw_file=azfp_path, sonar_model='AZFP6', + ) + + # Temperature and pressure variables are present in the Environment group and not all null + assert "temperature" in echodata["Environment"] + assert not echodata["Environment"]["temperature"].isnull().all() + + assert "pressure" in echodata["Environment"] + assert not echodata["Environment"]["pressure"].isnull().all() + + assert "paros_temperature_counts" in echodata["Vendor_specific"] + assert "paros_pressure_counts" in echodata["Vendor_specific"] + + check_platform_required_scalar_vars(echodata) \ No newline at end of file