Implement Wrapped Payload API Functions

lib/Makefile

@@ -20,3 +20,11 @@ prepublish:
 	# poetry publish -r test-pypi
 	# poetry config pypi-token.pypi TOKEN
 	# poetry publish
+build_docker_py3:
+	docker build -t bbpb_dev_py3 -f tests/Dockerfile.py3-dev .
+run_docker_py3:
+	docker run -it --rm -v ${PWD}:/app/lib/ bbpb_dev_py3 /bin/bash
+build_docker_py2:
+	docker build -t bbpb_dev_py2 -f tests/Dockerfile.py2-dev .
+run_docker_py2:
+	docker run -it --rm -v ${PWD}:/app/lib/ bbpb_dev_py2 /bin/bash

lib/blackboxprotobuf/lib/payloads/__init__.py

@@ -35,13 +35,19 @@
 # to decode as a protobuf. This should minimize the chance of a false positive
 # on any decoders
 def find_decoders(buf):
-    # type: (bytes) -> List[Callable[[bytes], Tuple[bytes | list[bytes], str]]]
+    # type: (bytes) -> List[Callable[[bytes], Tuple[list[bytes], str]]]
     # In the future, we can take into account content-type too, such as for
     # grpc, but we risk false negatives
-    decoders = []  # type: List[Callable[[bytes], Tuple[bytes | list[bytes], str]]]
+    decoders = []  # type: List[Callable[[bytes], Tuple[list[bytes], str]]]
 
     if gzip.is_gzip(buf):
-        decoders.append(gzip.decode_gzip)
+
+        def decode_gzip_list(buf):
+            # type: (bytes) -> Tuple[list[bytes], str]
+            value, encoding = gzip.decode_gzip(buf)
+            return [value], encoding
+
+        decoders.append(decode_gzip_list)
 
     if grpc.is_grpc(buf):
         decoders.append(grpc.decode_grpc)
@@ -51,22 +57,23 @@ def find_decoders(buf):
 
 
 def _none_decoder(buf):
-    # type: (bytes) -> Tuple[bytes, str]
-    return buf, "none"
+    # type: (bytes) -> Tuple[list[bytes], str]
+    return [buf], "none"
 
 
 # Decoder by name
 def decode_payload(buf, decoder):
-    # type: (bytes, Optional[str]) -> Tuple[bytes | list[bytes], str]
+    # type: (bytes, Optional[str]) -> Tuple[list[bytes], str]
     if decoder is None:
-        return buf, "none"
+        return [buf], "none"
     decoder = decoder.lower()
     if decoder == "none":
-        return buf, "none"
+        return [buf], "none"
     elif decoder.startswith("grpc"):
         return grpc.decode_grpc(buf)
     elif decoder == "gzip":
-        return gzip.decode_gzip(buf)
+        payload, encoding = gzip.decode_gzip(buf)
+        return [payload], encoding
     else:
         raise BlackboxProtobufException("Unknown decoder: " + decoder)
 
@@ -80,9 +87,11 @@ def encode_payload(buf, encoder):
     encoder = encoder.lower()
     if encoder == "none":
         if isinstance(buf, list):
-            raise BlackboxProtobufException(
-                "Cannot encode multiple buffers with none/missing encoding"
-            )
+            if len(buf) > 1:
+                raise BlackboxProtobufException(
+                    "Cannot encode multiple buffers with none/missing encoding"
+                )
+            buf = buf[0]
         return buf
     elif encoder.startswith("grpc"):
         return grpc.encode_grpc(buf, encoder)

lib/blackboxprotobuf/lib/payloads/grpc.py

@@ -51,7 +51,7 @@ def is_grpc(payload):
 
 
 def decode_grpc(payload):
-    # type: (bytes) -> Tuple[bytes | list[bytes], str]
+    # type: (bytes) -> Tuple[list[bytes], str]
     """Decode GRPC. Return the protobuf data"""
     if six.PY2 and isinstance(payload, bytearray):
         payload = bytes(payload)
@@ -93,10 +93,7 @@ def decode_grpc(payload):
             "Error decoding GRPC. Payload length does not match encoded gRPC lengths"
         )
 
-    if len(payloads) > 1:
-        return payloads, "grpc"
-    else:
-        return payloads[0], "grpc"
+    return payloads, "grpc"
 
 
 def encode_grpc(data, encoding="grpc"):
@@ -115,4 +112,4 @@ def encode_grpc(data, encoding="grpc"):
         payload.extend(struct.pack(">I", len(data)))  # Length
         payload.extend(data)
 
-    return payload
+    return bytes(payload)

lib/blackboxprotobuf/lib/payloads/gzip.py

@@ -47,8 +47,11 @@ def decode_gzip(buf):
 def encode_gzip(buf):
     # type: (bytes | list[bytes]) -> bytes
     if isinstance(buf, list):
-        raise BlackboxProtobufException(
-            "Cannot encode as gzip: multiple buffers are not supported"
-        )
+        if len(buf) > 1:
+            raise BlackboxProtobufException(
+                "Cannot encode as gzip: multiple buffers are not supported"
+            )
+        else:
+            buf = buf[0]
     compressor = zlib.compressobj(-1, zlib.DEFLATED, 31)
     return compressor.compress(buf) + compressor.flush()

lib/blackboxprotobuf/lib/typedef.py

@@ -83,6 +83,10 @@ def lookup_fielddef_number(self, field_id):
             return field_id, self._fields[field_id]
         return None
 
+    def is_empty(self):
+        # type: (TypeDef) -> bool
+        return len(self._fields) == 0
+
 
 class MutableTypeDef(TypeDef):
     def set_fielddef(self, field_number, fielddef):

lib/tests/Dockerfile.py2-dev

@@ -0,0 +1,12 @@
+# Dockerfile for python2 tests
+# Installs dependencies and expects the `lib/` folder to be mapped to `/app`
+
+FROM python:2.7
+
+WORKDIR /app/lib
+
+RUN apt update && \
+    apt install -y protobuf-compiler
+
+COPY tests/requirements-python2-dev.txt ./
+RUN pip install -r requirements-python2-dev.txt

lib/tests/Dockerfile.py3-dev

@@ -0,0 +1,18 @@
+# Dockerfile for python3 tests
+# Installs dependencies and expects the `lib/` folder to be mapped to `/app`
+
+# Requires python 3.10 for type checking
+FROM python:3.10
+
+WORKDIR /app/lib
+
+RUN apt update && \
+    apt install -y protobuf-compiler && \
+    pip install poetry
+
+COPY pyproject.toml poetry.lock ./
+RUN poetry install --no-root --with dev
+
+#  Could add types-six to dependencies, but I think that would require bumping the main python version
+RUN poetry env use python3 && \
+    poetry run pip install types-six

lib/tests/py_test/test_api.py

@@ -0,0 +1,55 @@
+"""Tests focused on common API behavior"""
+
+import pytest
+
+import blackboxprotobuf
+from blackboxprotobuf.lib.exceptions import TypedefException
+
+
+def test_encode_empty_typedef():
+    # Only allow empty typedef for empty message for an encoder
+
+    empty_typedefs = [{}, "", None]
+    for typedef in empty_typedefs:
+        typedef = {}
+        message = {}
+        payload = blackboxprotobuf.encode_message(message, typedef)
+        assert len(payload) == 0
+
+        payload = blackboxprotobuf.encode_wrapped_message([message], typedef, "none")
+        assert len(payload) == 0
+
+        payload = blackboxprotobuf.protobuf_from_json("{}", typedef)
+        assert len(payload) == 0
+
+        payload = blackboxprotobuf.wrapped_protobuf_from_json("{}", typedef, "none")
+        assert len(payload) == 0
+
+        message = {"1": 0}
+        with pytest.raises(TypedefException):
+            payload = blackboxprotobuf.encode_message(message, typedef)
+        with pytest.raises(TypedefException):
+            payload = blackboxprotobuf.protobuf_from_json('{"1": 0}', typedef)
+        with pytest.raises(TypedefException):
+            payload = blackboxprotobuf.encode_wrapped_message(
+                [message], typedef, "none"
+            )
+        with pytest.raises(TypedefException):
+            payload = blackboxprotobuf.wrapped_protobuf_from_json(
+                '{"1": 0}', typedef, "none"
+            )
+
+
+def test_invalid_typedef_string():
+    # String typedefs must exist in config
+
+    message = {}
+    typedef = "test123"
+    with pytest.raises(TypedefException):
+        payload = blackboxprotobuf.encode_message(message, typedef)
+    with pytest.raises(TypedefException):
+        payload = blackboxprotobuf.protobuf_from_json("{}", typedef)
+    with pytest.raises(TypedefException):
+        payload = blackboxprotobuf.encode_wrapped_message([message], typedef, "none")
+    with pytest.raises(TypedefException):
+        payload = blackboxprotobuf.wrapped_protobuf_from_json("{}", typedef, "none")

lib/tests/py_test/test_payloads.py

@@ -23,15 +23,17 @@
 import strategies
 import pytest
 
+import blackboxprotobuf
 from blackboxprotobuf.lib import payloads
+from blackboxprotobuf.lib.config import Config
 from blackboxprotobuf.lib.payloads import grpc, gzip
 from blackboxprotobuf.lib.exceptions import BlackboxProtobufException
 
 
 def test_grpc():
     message = bytearray([0x00, 0x00, 0x00, 0x00, 0x01, 0xAA])
     data, encoding = grpc.decode_grpc(message)
-    assert data == bytearray([0xAA])
+    assert data == [bytearray([0xAA])]
     assert encoding == "grpc"
 
     # Compression flag
@@ -57,7 +59,7 @@ def test_grpc():
     # Empty
     message = bytearray([0x00, 0x00, 0x00, 0x00, 0x00])
     data, encoding = grpc.decode_grpc(message)
-    assert len(data) == 0
+    assert len(data[0]) == 0
     assert encoding == "grpc"
 
 
@@ -91,7 +93,7 @@ def test_grpc_inverse(data):
     encoded = grpc.encode_grpc(data)
     decoded, encoding_out = grpc.decode_grpc(encoded)
 
-    assert data == decoded
+    assert data == decoded[0]
     assert encoding == encoding_out
 
 
@@ -113,9 +115,77 @@ def test_find_payload_inverse(data, alg):
     for decoder in decoders:
         try:
             decoded, decoder_alg = decoder(encoded)
-            valid_decoders[decoder_alg] = decoded
+            valid_decoders[decoder_alg] = decoded[0]
         except:
             pass
     assert "none" in valid_decoders
     assert alg in valid_decoders
     assert valid_decoders[alg] == data
+
+
+@given(
+    x=strategies.gen_message(anon=True),
+    chosen_encoding=st.sampled_from(["grpc", "gzip", "none"]),
+)
+def test_wrapped_message(x, chosen_encoding):
+    config = Config()
+
+    original_typedef, message = x
+    protobuf_data = blackboxprotobuf.encode_message(message, original_typedef, config)
+    data = payloads.encode_payload(protobuf_data, chosen_encoding)
+
+    messages, typedef, encoding = blackboxprotobuf.decode_wrapped_message(
+        data, encoding=chosen_encoding, config=config
+    )
+    assert encoding == chosen_encoding
+
+    messages, typedef, encoding = blackboxprotobuf.decode_wrapped_message(
+        data, config=config
+    )
+    assert encoding == chosen_encoding
+
+    payload = blackboxprotobuf.encode_wrapped_message(
+        messages, typedef, encoding, config
+    )
+    assert isinstance(payload, bytes)
+
+    new_protobuf_data, encoding = payloads.decode_payload(payload, chosen_encoding)
+    # can't check against protobuf_data because of field ordering
+    new_message, _ = blackboxprotobuf.decode_message(
+        new_protobuf_data[0], original_typedef, config
+    )
+    assert message == new_message
+
+
+@given(
+    x=strategies.gen_message(anon=True),
+    chosen_encoding=st.sampled_from(["grpc", "gzip", "none"]),
+)
+def test_wrapped_message_json(x, chosen_encoding):
+    config = Config()
+
+    original_typedef, message = x
+    protobuf_data = blackboxprotobuf.encode_message(message, original_typedef, config)
+    data = payloads.encode_payload(protobuf_data, chosen_encoding)
+
+    messages, typedef, encoding = blackboxprotobuf.wrapped_protobuf_to_json(
+        data, encoding=chosen_encoding, config=config
+    )
+    assert encoding == chosen_encoding
+
+    messages, typedef, encoding = blackboxprotobuf.wrapped_protobuf_to_json(
+        data, config=config
+    )
+    assert encoding == chosen_encoding
+
+    payload = blackboxprotobuf.wrapped_protobuf_from_json(
+        messages, typedef, encoding, config
+    )
+    assert isinstance(payload, bytes)
+
+    new_protobuf_data, encoding = payloads.decode_payload(payload, chosen_encoding)
+    # can't check against protobuf_data because of field ordering
+    new_message, _ = blackboxprotobuf.decode_message(
+        new_protobuf_data[0], original_typedef, config
+    )
+    assert message == new_message