diff --git a/lib/Makefile b/lib/Makefile index 5f10480..54076bd 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -20,3 +20,11 @@ prepublish: # poetry publish -r test-pypi # poetry config pypi-token.pypi TOKEN # poetry publish +build_docker_py3: + docker build -t bbpb_dev_py3 -f tests/Dockerfile.py3-dev . +run_docker_py3: + docker run -it --rm -v ${PWD}:/app/lib/ bbpb_dev_py3 /bin/bash +build_docker_py2: + docker build -t bbpb_dev_py2 -f tests/Dockerfile.py2-dev . +run_docker_py2: + docker run -it --rm -v ${PWD}:/app/lib/ bbpb_dev_py2 /bin/bash diff --git a/lib/blackboxprotobuf/lib/api.py b/lib/blackboxprotobuf/lib/api.py index de9c6ec..f107d5d 100644 --- a/lib/blackboxprotobuf/lib/api.py +++ b/lib/blackboxprotobuf/lib/api.py @@ -46,24 +46,26 @@ import blackboxprotobuf.lib.types.type_maps from blackboxprotobuf.lib.config import default as default_config from blackboxprotobuf.lib.exceptions import ( + BlackboxProtobufException, TypedefException, EncoderException, DecoderException, ) from blackboxprotobuf.lib.typedef import TypeDef +from blackboxprotobuf.lib import payloads if six.PY3: import typing # Circular imports on Config if we don't check here if typing.TYPE_CHECKING: - from typing import Dict, List, Optional + from typing import Dict, List, Tuple, Optional, ByteString from blackboxprotobuf.lib.pytypes import Message, TypeDefDict, FieldDefDict from blackboxprotobuf.lib.config import Config def decode_message(buf, message_type=None, config=None): - # type: (bytes, Optional[str | TypeDefDict], Optional[Config]) -> tuple[Message, TypeDefDict] + # type: (bytes, Optional[str | TypeDefDict | TypeDef], Optional[Config]) -> tuple[Message, TypeDefDict] """Decode a protobuf message and return a python dictionary representing the message. @@ -92,26 +94,16 @@ def decode_message(buf, message_type=None, config=None): if isinstance(buf, bytearray): buf = bytes(buf) buf = six.ensure_binary(buf) - if message_type is None: - message_type = {} - elif isinstance(message_type, six.string_types): - if message_type not in config.known_types: - message_type = {} - else: - message_type = config.known_types[message_type] - if not isinstance(message_type, dict): - raise DecoderException( - "Decode message received an invalid typedef type. Typedef should be a string with a message name, a dictionary, or None" - ) + typedef = _resolve_typedef(message_type, config) value, typedef, _, _ = blackboxprotobuf.lib.types.length_delim.decode_message( - buf, config, TypeDef.from_dict(message_type) + buf, config, typedef ) return value, typedef.to_dict() def encode_message(value, message_type, config=None): - # type: (Message, str | TypeDefDict, Optional[Config]) -> bytes + # type: (Message, str | TypeDefDict | TypeDef, Optional[Config]) -> bytes """Re-encode a python dictionary as a binary protobuf message. Args: @@ -131,37 +123,21 @@ def encode_message(value, message_type, config=None): if config is None: config = default_config - if message_type is None: - raise EncoderException( - "Encode message must have valid type definition. message_type cannot be None" - ) - - if isinstance(message_type, six.string_types): - if message_type not in config.known_types: - raise EncoderException( - "The provided message type name (%s) is not known. Encoding requires a valid type definition" - % message_type - ) - message_type = config.known_types[message_type] - - if not isinstance(message_type, dict): - raise EncoderException( - "Encode message received an invalid typedef type. Typedef should be a string with a message name or a dictionary." - ) + typedef = _resolve_typedef(message_type, config) + if typedef.is_empty() and len(value) > 0: + raise TypedefException("A typedef is required to encoded non-empty messages") return bytes( - blackboxprotobuf.lib.types.length_delim.encode_message( - value, config, TypeDef.from_dict(message_type) - ) + blackboxprotobuf.lib.types.length_delim.encode_message(value, config, typedef) ) def protobuf_to_json(buf, message_type=None, config=None): - # type: (bytes | list[bytes], Optional[str | TypeDefDict], Optional[Config]) -> tuple[str, TypeDefDict] - """Decode a protobuf messages and return a JSON string representing the - messages. + # type: (bytes | list[bytes], Optional[str | TypeDefDict | TypeDef], Optional[Config]) -> tuple[str, TypeDefDict] + """Decode one or more protobuf messages and return a JSON string + representing the messages. Args: - buf: One or more bytes representing encoded protobuf messages + buf: One or more byte strings representing encoded protobuf messages message_type: Optional type to use as the base for decoding. Allows for customizing field types or names. Can be a python dictionary or a message type name which maps to the `known_types` dictionary in the @@ -181,35 +157,33 @@ def protobuf_to_json(buf, message_type=None, config=None): provided, but may add additional fields if new fields were encountered during decoding. """ + if config is None: + config = default_config values = [] bufs = buf if isinstance(buf, list) else [buf] if len(bufs) == 0: raise DecoderException("No protobuf bytes were provided") + typedef_dict = _resolve_typedef(message_type, config).to_dict() + for data in bufs: - value, message_type = decode_message(data, message_type, config) - value = _json_safe_transform(value, message_type, False, config=config) - value = _sort_output(value, message_type, config=config) + value, typedef_dict = decode_message(data, typedef_dict, config) + value = _json_safe_transform(value, typedef_dict, False, config=config) + value = _sort_output(value, typedef_dict, config=config) values.append(value) - if not isinstance(message_type, dict): - # Shouldn't happen because of len(bufs) check, but make the type checker happy and verify edge cases - raise DecoderException( - "Error decoding to json: Could not find valid message_type type (dict). Found: %s" - % type(message_type) - ) - _annotate_typedef(message_type, values[0]) - message_type = sort_typedef(message_type) + _annotate_typedef(typedef_dict, values[0]) + typedef_dict = sort_typedef(typedef_dict) if not isinstance(buf, list) and len(values) == 1: - return json.dumps(values[0], indent=2), message_type + return json.dumps(values[0], indent=2), typedef_dict else: - return json.dumps(values, indent=2), message_type + return json.dumps(values, indent=2), typedef_dict def protobuf_from_json(json_str, message_type, config=None): - # type: (str, str | TypeDefDict, Optional[Config]) -> bytes | list[bytes] + # type: (str, str | TypeDefDict | TypeDef, Optional[Config]) -> bytes | list[bytes] """Re-encode a JSON string as a binary protobuf message. Args: @@ -228,27 +202,20 @@ def protobuf_from_json(json_str, message_type, config=None): """ if config is None: config = default_config - if isinstance(message_type, six.string_types): - if message_type not in config.known_types: - raise EncoderException( - 'protobuf_from_json must have valid type definition. message_type "%s" is not known' - % message_type - ) - message_type = config.known_types[message_type] - if not isinstance(message_type, dict): - raise EncoderException( - "Encode message received an invalid typedef type. Typedef should be a string with a message name or a dictionary." - ) + + typedef = _resolve_typedef(message_type, config) value = json.loads(json_str) values = value if isinstance(value, list) else [value] + if typedef.is_empty() and any([len(value) > 0 for value in values]): + raise TypedefException("A typedef is required to encoded non-empty messages") - _strip_typedef_annotations(message_type) - values = [_json_safe_transform(message, message_type, True) for message in values] + typedef_dict = typedef.to_dict() + values = [_json_safe_transform(message, typedef_dict, True) for message in values] payloads = [] for message in values: - payloads.append(encode_message(message, message_type, config)) + payloads.append(encode_message(message, typedef, config)) if not isinstance(value, list) and len(payloads) == 1: return payloads[0] @@ -256,6 +223,242 @@ def protobuf_from_json(json_str, message_type, config=None): return payloads +def decode_wrapped_message(buf, message_type=None, encoding=None, config=None): + # type: (bytes, Optional[str | TypeDefDict | TypeDef], Optional[str], Optional[Config]) -> tuple[List[Message], TypeDefDict, str] + """Decode a protobuf message which may be wrapped in an additional encoding, such as gRPC or gzip. + Args: + value: byte buffer containing the raw protobuf payload + message_type: Optional type definition used as the base for decoding, + which allows field types to be customized. If `buf` contains + multiple messages, the same typedef will be used for all messages. + encoding: The outer encoding around the protobuf payload. Valid values are: + - None: encoder will be guessed through trial and error. Specifying + an encoding should always be preferred when possible. + - 'none' - No extra encoding, same will be treated as raw protobuf + - 'gzip' - Single protobuf message compressed with gzip + - 'grpc' - One or more protobuf messages with a gRPC header. + Compressed gRPC is not supported. Once compression + support is added, it will likely be a variation of + `grpc`, such as `grpc-gzip`. + config: Optional `blackboxprotobuf.lib.config.Config` object which can + change default decoding behaviors + Returns: + A tuple containing: + - List of decoded protobuf messages. This list will contain only + a single element for `none` and `gzip` encodings, but `grpc` + may product multiple messages. + - Type definition for re-encoding the messages + - name of the encoding algorithm that was used + """ + if config is None: + config = default_config + + if isinstance(buf, bytearray): + buf = bytes(buf) + + buf = six.ensure_binary(buf) + typedef = _resolve_typedef(message_type, config) + + if encoding is None: + decoders = payloads.find_decoders(buf) + for decoder in decoders: + try: + protobuf_datas, encoding = decoder(buf) + except BlackboxProtobufException: + # Error while decoding wrapper, skip to next alg + continue + try: + values = [] + # Don't override typedef + decoder_typedef = typedef + for protobuf_data in protobuf_datas: + # If there are multiple messages, we assume they have the same + # message type and reuse the typedef + ( + value, + decoder_typedef, + _, + _, + ) = blackboxprotobuf.lib.types.length_delim.decode_message( + protobuf_data, config, decoder_typedef + ) + values.append(value) + return values, decoder_typedef.to_dict(), encoding + except BlackboxProtobufException as exc: + # If we hit an error decoding, we have to assume we have the + # wrong payload wrapper unless we are already using 'none' + if encoding == "none": + six.raise_from( + DecoderException( + "Unable to decode protobuf message with any encoding algorithm" + ), + exc, + ) + continue + # Should not hit this due to the raise on "none" encoding alg + raise DecoderException( + "Unable to decode protobuf message with any encoding algorithm" + ) + else: + protobuf_datas, encoding = payloads.decode_payload(buf, encoding) + values = [] + for protobuf_data in protobuf_datas: + # If there are multiple messages, we assume they have the same + # message type and reuse the typedef + ( + value, + typedef, + _, + _, + ) = blackboxprotobuf.lib.types.length_delim.decode_message( + protobuf_data, config, typedef + ) + values.append(value) + + return values, typedef.to_dict(), encoding + + +def encode_wrapped_message(messages, message_type, encoding, config=None): + # type: (List[Message], str | TypeDefDict, str, Optional[Config]) -> bytes + """This function re-encodes one or more messages using the provided + typedef and outer encoding algorithm, such as grpc or gzip. + + Args: + messages - List with one or more decoded protobuf messages. + message_type - Type definition for re-encoding the message. Should + generatlly be the type definition returned by a decoding function. + encoding - String representing the outer encoding algorithm. This + should generally be the value returned by `decode_wrapped_message`. + Valid values are: + - 'none' - Raw protobuf message, no outer encoding + - 'gzip' - gzip compressed message + - 'grpc' - One or more protobuf messages encoded with a gRPC + header. gRPC compression is not currently supported. + + Returns: + A bytearray containing the encoded protobuf message. + """ + if config is None: + config = default_config + + typedef = _resolve_typedef(message_type, config) + if typedef.is_empty() and any([len(message) > 0 for message in messages]): + raise TypedefException("A typedef is requiredto encode non-empty messages") + + values = [] + + for message in messages: + value = blackboxprotobuf.lib.types.length_delim.encode_message( + message, config, typedef + ) + values.append(bytes(value)) + wrapped_payload = payloads.encode_payload(values, encoding) + return wrapped_payload + + +def wrapped_protobuf_to_json(buf, message_type=None, encoding=None, config=None): + # type: (ByteString, Optional[str | TypeDefDict | TypeDef], Optional[str], Optional[Config]) -> Tuple[str, TypeDefDict, str] + """Decode a protobuf message, which may be encoded with gRPC or gzip, and + return a JSON string representing the messages. + + Args: + buf: A bytestring representing an encoded protobuf message + message_type: Optional type to use as the base for decoding. Allows for + customizing field types or names. Can be a python dictionary or a + message type name which maps to the `known_types` dictionary in the + config. Defaults to an empty definition '{}'. + encoding: A string identifying the encoding type. If not provided, or + set to `None`, the encoding will be guessed through trial and + error. Valid values are: + - "gRPC" - gRPC header. Can decode to multiple messages + - "gzip" - gzip encoding + - "none" - no encoding + config: `blackboxprotobuf.lib.config.Config` object which allows + customizing default types for wire types and holds the + `known_types` array. Defaults to + `blackboxprotobuf.lib.config.default` if not provided. + Returns: + A tuple containing a JSON string representing the messages, a type + definition for re-encoding the messages and the wrapper encoding. + + The JSON string and type definition are annotated and sorted for + readability. + + The type definition is based on the `message_type` argument if one was + provided, but may add additional fields if new fields were encountered + during decoding. + """ + if config is None: + config = default_config + if encoding is None: + decoders = payloads.find_decoders(buf) + for decoder in decoders: + try: + protobuf_datas, encoding = decoder(buf) + except BlackboxProtobufException: + # Error while decoding wrapper, skip to next alg + continue + try: + message, typedef = protobuf_to_json( + protobuf_datas, message_type, config + ) + return message, typedef, encoding + except BlackboxProtobufException as exc: + # If we hit an error decoding, we have to assume we have the + # wrong payload wrapper unless we are already using 'none' + if encoding == "none": + six.raise_from( + DecoderException( + "Unable to decode protobuf message with any encoding algorithm" + ), + exc, + ) + continue + # Should not hit this due to the raise on "none" encoding alg + raise DecoderException( + "Unable to decode protobuf message with any encoding algorithm" + ) + else: + protobuf_datas, encoding = payloads.decode_payload(buf, encoding) + message, typedef = protobuf_to_json(protobuf_datas, message_type, config) + + return message, typedef, encoding + + +def wrapped_protobuf_from_json(json_str, message_type, encoding, config=None): + # type: (str, str | TypeDefDict | TypeDef, str, Optional[Config]) -> bytes + """Re-encode a JSON string as a binary protobuf message with optional + additional encoding, such as gRPC or gzip. + + Args: + json_str: JSON string to re-encode to protobuf message bytes. This + should usually be a modified version of the value returned by + `protobuf_to_json`. + message_type: Type definition to use to re-encode the message. This + will should generally be the type definition returned from the + original `protobuf_to_json` call. + encoding: "Outer" encoding mechanisms for protobuf payload. The + encoding returned by `wrapped_protobuf_to_json` should generally be + used here. + Valid algorithms are: + - "gRPC" - gRPC header with length. Can encode multiple messages + - "gzip" + - "none" + config: `blackboxprotobuf.lib.config.Config` object which allows + customizing default types for wire types and holds the + `known_types` array. Defaults to + `blackboxprotobuf.lib.config.default` if not provided. + Returns: + A bytearray containing the encoded protobuf message. + """ + if config is None: + config = default_config + + values = protobuf_from_json(json_str, message_type, config) + wrapped_payload = payloads.encode_payload(values, encoding) + return wrapped_payload + + def export_protofile(message_types, output_filename): # type: (Dict[str, TypeDefDict], str) -> None """This function attempts to export a set of message type definitions to a @@ -781,3 +984,28 @@ def _strip_typedef_annotations(typedef): del field_def["example_value_ignored"] if "message_typedef" in field_def: _strip_typedef_annotations(field_def["message_typedef"]) + + +def _resolve_typedef(message_type, config): + # type: (Optional[str | TypeDefDict | TypeDef], Config) -> TypeDef + # Takes a message_type which is either None, a dictionary representing a typedef, or a string referencing `Config`, and return the correct typedef + # Raises an exception if message_type is str and not in Config + # Returns an empty typedef if message_type is None or empty string + + if isinstance(message_type, TypeDef): + return message_type + elif message_type is None or message_type == "": + return TypeDef() + elif isinstance(message_type, dict): + return TypeDef.from_dict(message_type) + elif isinstance(message_type, six.string_types): + if message_type in config.known_types: + return TypeDef.from_dict(config.known_types[message_type]) + else: + raise TypedefException( + "message_type (%s) is not in config.known_types" % message_type + ) + else: + raise TypedefException( + "message_type is not a valid type definition: %s" % message_type + ) diff --git a/lib/blackboxprotobuf/lib/payloads/__init__.py b/lib/blackboxprotobuf/lib/payloads/__init__.py index 96f2b8a..3354611 100644 --- a/lib/blackboxprotobuf/lib/payloads/__init__.py +++ b/lib/blackboxprotobuf/lib/payloads/__init__.py @@ -35,13 +35,19 @@ # to decode as a protobuf. This should minimize the chance of a false positive # on any decoders def find_decoders(buf): - # type: (bytes) -> List[Callable[[bytes], Tuple[bytes | list[bytes], str]]] + # type: (bytes) -> List[Callable[[bytes], Tuple[list[bytes], str]]] # In the future, we can take into account content-type too, such as for # grpc, but we risk false negatives - decoders = [] # type: List[Callable[[bytes], Tuple[bytes | list[bytes], str]]] + decoders = [] # type: List[Callable[[bytes], Tuple[list[bytes], str]]] if gzip.is_gzip(buf): - decoders.append(gzip.decode_gzip) + + def decode_gzip_list(buf): + # type: (bytes) -> Tuple[list[bytes], str] + value, encoding = gzip.decode_gzip(buf) + return [value], encoding + + decoders.append(decode_gzip_list) if grpc.is_grpc(buf): decoders.append(grpc.decode_grpc) @@ -51,22 +57,23 @@ def find_decoders(buf): def _none_decoder(buf): - # type: (bytes) -> Tuple[bytes, str] - return buf, "none" + # type: (bytes) -> Tuple[list[bytes], str] + return [buf], "none" # Decoder by name def decode_payload(buf, decoder): - # type: (bytes, Optional[str]) -> Tuple[bytes | list[bytes], str] + # type: (bytes, Optional[str]) -> Tuple[list[bytes], str] if decoder is None: - return buf, "none" + return [buf], "none" decoder = decoder.lower() if decoder == "none": - return buf, "none" + return [buf], "none" elif decoder.startswith("grpc"): return grpc.decode_grpc(buf) elif decoder == "gzip": - return gzip.decode_gzip(buf) + payload, encoding = gzip.decode_gzip(buf) + return [payload], encoding else: raise BlackboxProtobufException("Unknown decoder: " + decoder) @@ -80,9 +87,11 @@ def encode_payload(buf, encoder): encoder = encoder.lower() if encoder == "none": if isinstance(buf, list): - raise BlackboxProtobufException( - "Cannot encode multiple buffers with none/missing encoding" - ) + if len(buf) > 1: + raise BlackboxProtobufException( + "Cannot encode multiple buffers with none/missing encoding" + ) + buf = buf[0] return buf elif encoder.startswith("grpc"): return grpc.encode_grpc(buf, encoder) diff --git a/lib/blackboxprotobuf/lib/payloads/grpc.py b/lib/blackboxprotobuf/lib/payloads/grpc.py index 143c8d6..34b4ff3 100644 --- a/lib/blackboxprotobuf/lib/payloads/grpc.py +++ b/lib/blackboxprotobuf/lib/payloads/grpc.py @@ -51,7 +51,7 @@ def is_grpc(payload): def decode_grpc(payload): - # type: (bytes) -> Tuple[bytes | list[bytes], str] + # type: (bytes) -> Tuple[list[bytes], str] """Decode GRPC. Return the protobuf data""" if six.PY2 and isinstance(payload, bytearray): payload = bytes(payload) @@ -93,10 +93,7 @@ def decode_grpc(payload): "Error decoding GRPC. Payload length does not match encoded gRPC lengths" ) - if len(payloads) > 1: - return payloads, "grpc" - else: - return payloads[0], "grpc" + return payloads, "grpc" def encode_grpc(data, encoding="grpc"): @@ -115,4 +112,4 @@ def encode_grpc(data, encoding="grpc"): payload.extend(struct.pack(">I", len(data))) # Length payload.extend(data) - return payload + return bytes(payload) diff --git a/lib/blackboxprotobuf/lib/payloads/gzip.py b/lib/blackboxprotobuf/lib/payloads/gzip.py index f762bea..dec1dac 100644 --- a/lib/blackboxprotobuf/lib/payloads/gzip.py +++ b/lib/blackboxprotobuf/lib/payloads/gzip.py @@ -47,8 +47,11 @@ def decode_gzip(buf): def encode_gzip(buf): # type: (bytes | list[bytes]) -> bytes if isinstance(buf, list): - raise BlackboxProtobufException( - "Cannot encode as gzip: multiple buffers are not supported" - ) + if len(buf) > 1: + raise BlackboxProtobufException( + "Cannot encode as gzip: multiple buffers are not supported" + ) + else: + buf = buf[0] compressor = zlib.compressobj(-1, zlib.DEFLATED, 31) return compressor.compress(buf) + compressor.flush() diff --git a/lib/blackboxprotobuf/lib/typedef.py b/lib/blackboxprotobuf/lib/typedef.py index 385843d..b882b0f 100644 --- a/lib/blackboxprotobuf/lib/typedef.py +++ b/lib/blackboxprotobuf/lib/typedef.py @@ -83,6 +83,10 @@ def lookup_fielddef_number(self, field_id): return field_id, self._fields[field_id] return None + def is_empty(self): + # type: (TypeDef) -> bool + return len(self._fields) == 0 + class MutableTypeDef(TypeDef): def set_fielddef(self, field_number, fielddef): diff --git a/lib/tests/Dockerfile.py2-dev b/lib/tests/Dockerfile.py2-dev new file mode 100644 index 0000000..ea205d0 --- /dev/null +++ b/lib/tests/Dockerfile.py2-dev @@ -0,0 +1,12 @@ +# Dockerfile for python2 tests +# Installs dependencies and expects the `lib/` folder to be mapped to `/app` + +FROM python:2.7 + +WORKDIR /app/lib + +RUN apt update && \ + apt install -y protobuf-compiler + +COPY tests/requirements-python2-dev.txt ./ +RUN pip install -r requirements-python2-dev.txt diff --git a/lib/tests/Dockerfile.py3-dev b/lib/tests/Dockerfile.py3-dev new file mode 100644 index 0000000..ba88dc0 --- /dev/null +++ b/lib/tests/Dockerfile.py3-dev @@ -0,0 +1,18 @@ +# Dockerfile for python3 tests +# Installs dependencies and expects the `lib/` folder to be mapped to `/app` + +# Requires python 3.10 for type checking +FROM python:3.10 + +WORKDIR /app/lib + +RUN apt update && \ + apt install -y protobuf-compiler && \ + pip install poetry + +COPY pyproject.toml poetry.lock ./ +RUN poetry install --no-root --with dev + +# Could add types-six to dependencies, but I think that would require bumping the main python version +RUN poetry env use python3 && \ + poetry run pip install types-six diff --git a/lib/tests/py_test/test_api.py b/lib/tests/py_test/test_api.py new file mode 100644 index 0000000..0bf3003 --- /dev/null +++ b/lib/tests/py_test/test_api.py @@ -0,0 +1,55 @@ +"""Tests focused on common API behavior""" + +import pytest + +import blackboxprotobuf +from blackboxprotobuf.lib.exceptions import TypedefException + + +def test_encode_empty_typedef(): + # Only allow empty typedef for empty message for an encoder + + empty_typedefs = [{}, "", None] + for typedef in empty_typedefs: + typedef = {} + message = {} + payload = blackboxprotobuf.encode_message(message, typedef) + assert len(payload) == 0 + + payload = blackboxprotobuf.encode_wrapped_message([message], typedef, "none") + assert len(payload) == 0 + + payload = blackboxprotobuf.protobuf_from_json("{}", typedef) + assert len(payload) == 0 + + payload = blackboxprotobuf.wrapped_protobuf_from_json("{}", typedef, "none") + assert len(payload) == 0 + + message = {"1": 0} + with pytest.raises(TypedefException): + payload = blackboxprotobuf.encode_message(message, typedef) + with pytest.raises(TypedefException): + payload = blackboxprotobuf.protobuf_from_json('{"1": 0}', typedef) + with pytest.raises(TypedefException): + payload = blackboxprotobuf.encode_wrapped_message( + [message], typedef, "none" + ) + with pytest.raises(TypedefException): + payload = blackboxprotobuf.wrapped_protobuf_from_json( + '{"1": 0}', typedef, "none" + ) + + +def test_invalid_typedef_string(): + # String typedefs must exist in config + + message = {} + typedef = "test123" + with pytest.raises(TypedefException): + payload = blackboxprotobuf.encode_message(message, typedef) + with pytest.raises(TypedefException): + payload = blackboxprotobuf.protobuf_from_json("{}", typedef) + with pytest.raises(TypedefException): + payload = blackboxprotobuf.encode_wrapped_message([message], typedef, "none") + with pytest.raises(TypedefException): + payload = blackboxprotobuf.wrapped_protobuf_from_json("{}", typedef, "none") diff --git a/lib/tests/py_test/test_payloads.py b/lib/tests/py_test/test_payloads.py index 0df008e..b8013b6 100644 --- a/lib/tests/py_test/test_payloads.py +++ b/lib/tests/py_test/test_payloads.py @@ -23,7 +23,9 @@ import strategies import pytest +import blackboxprotobuf from blackboxprotobuf.lib import payloads +from blackboxprotobuf.lib.config import Config from blackboxprotobuf.lib.payloads import grpc, gzip from blackboxprotobuf.lib.exceptions import BlackboxProtobufException @@ -31,7 +33,7 @@ def test_grpc(): message = bytearray([0x00, 0x00, 0x00, 0x00, 0x01, 0xAA]) data, encoding = grpc.decode_grpc(message) - assert data == bytearray([0xAA]) + assert data == [bytearray([0xAA])] assert encoding == "grpc" # Compression flag @@ -57,7 +59,7 @@ def test_grpc(): # Empty message = bytearray([0x00, 0x00, 0x00, 0x00, 0x00]) data, encoding = grpc.decode_grpc(message) - assert len(data) == 0 + assert len(data[0]) == 0 assert encoding == "grpc" @@ -91,7 +93,7 @@ def test_grpc_inverse(data): encoded = grpc.encode_grpc(data) decoded, encoding_out = grpc.decode_grpc(encoded) - assert data == decoded + assert data == decoded[0] assert encoding == encoding_out @@ -113,9 +115,77 @@ def test_find_payload_inverse(data, alg): for decoder in decoders: try: decoded, decoder_alg = decoder(encoded) - valid_decoders[decoder_alg] = decoded + valid_decoders[decoder_alg] = decoded[0] except: pass assert "none" in valid_decoders assert alg in valid_decoders assert valid_decoders[alg] == data + + +@given( + x=strategies.gen_message(anon=True), + chosen_encoding=st.sampled_from(["grpc", "gzip", "none"]), +) +def test_wrapped_message(x, chosen_encoding): + config = Config() + + original_typedef, message = x + protobuf_data = blackboxprotobuf.encode_message(message, original_typedef, config) + data = payloads.encode_payload(protobuf_data, chosen_encoding) + + messages, typedef, encoding = blackboxprotobuf.decode_wrapped_message( + data, encoding=chosen_encoding, config=config + ) + assert encoding == chosen_encoding + + messages, typedef, encoding = blackboxprotobuf.decode_wrapped_message( + data, config=config + ) + assert encoding == chosen_encoding + + payload = blackboxprotobuf.encode_wrapped_message( + messages, typedef, encoding, config + ) + assert isinstance(payload, bytes) + + new_protobuf_data, encoding = payloads.decode_payload(payload, chosen_encoding) + # can't check against protobuf_data because of field ordering + new_message, _ = blackboxprotobuf.decode_message( + new_protobuf_data[0], original_typedef, config + ) + assert message == new_message + + +@given( + x=strategies.gen_message(anon=True), + chosen_encoding=st.sampled_from(["grpc", "gzip", "none"]), +) +def test_wrapped_message_json(x, chosen_encoding): + config = Config() + + original_typedef, message = x + protobuf_data = blackboxprotobuf.encode_message(message, original_typedef, config) + data = payloads.encode_payload(protobuf_data, chosen_encoding) + + messages, typedef, encoding = blackboxprotobuf.wrapped_protobuf_to_json( + data, encoding=chosen_encoding, config=config + ) + assert encoding == chosen_encoding + + messages, typedef, encoding = blackboxprotobuf.wrapped_protobuf_to_json( + data, config=config + ) + assert encoding == chosen_encoding + + payload = blackboxprotobuf.wrapped_protobuf_from_json( + messages, typedef, encoding, config + ) + assert isinstance(payload, bytes) + + new_protobuf_data, encoding = payloads.decode_payload(payload, chosen_encoding) + # can't check against protobuf_data because of field ordering + new_message, _ = blackboxprotobuf.decode_message( + new_protobuf_data[0], original_typedef, config + ) + assert message == new_message diff --git a/lib/tests/py_test/test_protobuf.py b/lib/tests/py_test/test_protobuf.py index 0f48762..1e40161 100644 --- a/lib/tests/py_test/test_protobuf.py +++ b/lib/tests/py_test/test_protobuf.py @@ -79,7 +79,6 @@ def test_decode(x): encoded = message.SerializeToString() decoded, typedef = blackboxprotobuf.decode_message(encoded, testMessage_typedef) blackboxprotobuf.validate_typedef(typedef) - hypothesis.note("Decoded: %r" % decoded) for key in decoded.keys(): assert x[key] == decoded[key] @@ -122,9 +121,6 @@ def test_modify(x, modify_num): elif isinstance(decoded[modify_key], float): mod_func = lambda x: 10 else: - hypothesis.note( - "Failed to modify key: %s (%r)" % (modify_key, type(decoded[modify_key])) - ) assert False decoded[modify_key] = mod_func(decoded[modify_key]) @@ -155,13 +151,7 @@ def test_decode_json(x): encoded, testMessage_typedef ) blackboxprotobuf.validate_typedef(typedef_json) - hypothesis.note("Encoded JSON:") - hypothesis.note(decoded_json) decoded = json.loads(decoded_json) - hypothesis.note("Original value:") - hypothesis.note(x) - hypothesis.note("Decoded valuec:") - hypothesis.note(decoded) for key in decoded.keys(): if key == "testBytes": decoded[key] = six.ensure_binary(decoded[key], encoding="latin1") @@ -176,27 +166,15 @@ def test_encode_json(x): x["testBytes"] = x["testBytes"].decode("latin1") json_str = json.dumps(x) - hypothesis.note("JSON Str Input:") - hypothesis.note(json_str) - hypothesis.note(json.loads(json_str)) - encoded = blackboxprotobuf.protobuf_from_json(json_str, testMessage_typedef) assert not isinstance(encoded, list) - hypothesis.note("BBP decoding:") test_decode, _ = blackboxprotobuf.decode_message(encoded, testMessage_typedef) - hypothesis.note(test_decode) message = Test_pb2.TestMessage() message.ParseFromString(encoded) - hypothesis.note("Message:") - hypothesis.note(message) for key in x.keys(): - hypothesis.note("Message value") - hypothesis.note(type(getattr(message, key))) - hypothesis.note("Original value") - hypothesis.note(type(x[key])) if key == "testBytes": x[key] = six.ensure_binary(x[key], encoding="latin1") assert getattr(message, key) == x[key] @@ -230,9 +208,6 @@ def test_modify_json(x, modify_num): elif isinstance(decoded[modify_key], float): mod_func = lambda x: 10 else: - hypothesis.note( - "Failed to modify key: %s (%r)" % (modify_key, type(decoded[modify_key])) - ) assert False decoded[modify_key] = mod_func(decoded[modify_key]) @@ -246,10 +221,6 @@ def test_modify_json(x, modify_num): message.ParseFromString(encoded) for key in decoded.keys(): - hypothesis.note("Message value:") - hypothesis.note(type(getattr(message, key))) - hypothesis.note("Orig value:") - hypothesis.note((x[key])) if key == "testBytes": x[key] = six.ensure_binary(x[key], encoding="latin1") assert getattr(message, key) == x[key]