added large fixtures to enable full extractor integration tests

Author: 0x6861746366574

block/block/extractor/extract.py

@@ -11,12 +11,10 @@
 import sys
 
 import msgpack
-import pandas as pd
 from tqdm import tqdm
 
 from block.extractor.body import deserialize_footer, deserialize_header
 from block.extractor.format import DB_OFFSET_BYTES, HEADER_LEN, SUBCACHE_MERKLE_ROOT_FORMAT, TX_HASH_FORMAT, TX_HASH_LEN
-from block.extractor.process import get_block_stats
 from block.extractor.state import XYMStateMap
 from block.extractor.statements import deserialize_statements, get_statement_paths
 from block.extractor.util import fmt_unpack
@@ -110,14 +108,6 @@ def main(args):
     print('statement data extraction complete!\n')
     print(f'statement data written to {os.path.join(args.output,args.statement_save_path)}')
 
-    # TODO: remove header output now that it is redundant with the process script?
-    header_df = pd.DataFrame.from_records([get_block_stats(x) for x in blocks])
-    header_df['dateTime'] = pd.to_datetime(header_df['timestamp'], origin=pd.to_datetime('2021-03-16 00:06:25'), unit='ms')
-    header_df = header_df.set_index('dateTime').sort_index(axis=0)
-    header_df.to_csv(os.path.join(args.output, args.header_save_path))
-
-    print(f'header data written to {os.path.join(args.output,args.header_save_path)}')
-
     state_map.to_msgpack(os.path.join(args.output, args.state_save_path))
 
     print(f'state data written to {os.path.join(args.output,args.state_save_path)}')
@@ -226,7 +216,6 @@ def parse_args(argv):
     parser.add_argument('--block_save_path', type=str, default='block_data.msgpack', help='file to write the extracted block data to')
     parser.add_argument('--statement_save_path', type=str, default='stmt_data.msgpack', help='file to write extracted statement data to')
     parser.add_argument('--state_save_path', type=str, default='state_map.msgpack', help='file to write the extracted chain state data to')
-    parser.add_argument('--header_save_path', type=str, default='block_header_df.csv', help='file to write the extracted data to')
     parser.add_argument('--block_extension', type=str, default='.dat', help='extension of block files; must be unique')
     parser.add_argument('--statement_extension', type=str, default='.stmt', help='extension of block files; must be unique')
     parser.add_argument('--db_offset_bytes', type=int, default=DB_OFFSET_BYTES, help='padding bytes at start of storage files')

block/block/extractor/test/conftest.py

@@ -5,9 +5,6 @@
 
 from block.extractor.format import DB_OFFSET_BYTES
 
-# TODO: maybe better to build raw fixtures by re-encoding parsed fixtures?
-# TODO: fixtures need to cover all tx types, payload variations
-
 
 @pytest.fixture
 def fixture_raw_header():
@@ -149,7 +146,7 @@ def fixture_tx_stats():
             'timestamp': 0,
             'recipient_address': 'NAOTO55CLDBWQ4UG47RLCFMTKDZHUX2LGFZVJ7A',
             'message_size': 0,
-            'mosaics': "[{'mosaic_id': 7777031834025731064, 'amount': 99780000}]",
+            'mosaics': '[{"mosaic_id": 7777031834025731064, "amount": 99780000}]',
             'message': '681d3777a258c3687286e7e2b1159350f27a5f4b317354fc0000010000000000f82302a23f91ed6ba085f20500000000'
         }
     ]
@@ -163,7 +160,6 @@ def fixture_extract_args():
     args.block_save_path = 'block_data.msgpack'
     args.statement_save_path = 'stmt_data.msgpack'
     args.state_save_path = 'state_map.msgpack'
-    args.header_save_path = 'block_header_df.csv'
     args.block_extension = '.dat'
     args.statement_extension = '.stmt'
     args.db_offset_bytes = DB_OFFSET_BYTES

block/block/extractor/test/fixture_data/00000.dat

@@ -10,11 +10,12 @@ def test_deserialize_header(fixture_raw_header, fixture_parsed_header):
 def test_deserialize_footer(fixture_raw_footer, fixture_parsed_header, fixture_parsed_footer):
     footer = deserialize_footer(fixture_raw_footer, fixture_parsed_header)
     for key, value in footer:
-        # should recurse for transactions list?
-        # if key == 'transactions':
-        #     for txn, fix_txn in zip(value, fixture_parsed_footer[key]):
-        #         pass
-        assert fixture_parsed_footer[key] == value
+        if key == 'transactions':
+            for txn, fix_txn in zip(value, fixture_parsed_footer[key]):
+                for txn_value, fix_txn_value in zip(txn.values(), fix_txn.values()):
+                    assert txn_value == fix_txn_value
+        else:
+            assert fixture_parsed_footer[key] == value
 
 
 def test_deserialize_tx_payload(fixture_parsed_payload):

block/block/extractor/test/fixture_data/00000.stmt

@@ -19,11 +19,12 @@ def test_get_tx_stats(fixture_block, fixture_tx_stats):
 def test_decode_msgpack(fixture_parsed_footer, fixture_packed_footer):
     decoded = decode_msgpack(fixture_packed_footer)
     for key, value in decoded:
-        # should recurse for transactions list?
-        # if key == 'transactions':
-        #     for txn, fix_txn in zip(value, fixture_parsed_footer[key]):
-        #         pass
-        assert fixture_parsed_footer[key] == value
+        if key == 'transactions':
+            for txn, fix_txn in zip(value, fixture_parsed_footer[key]):
+                for txn_value, fix_txn_value in zip(txn.values(), fix_txn.values()):
+                    assert txn_value == fix_txn_value
+        else:
+            assert fixture_parsed_footer[key] == value
 
 
 def test_main(fixture_process_args):