Swap pandas with fireduck

Author: devinmatte

mbta-performance/chalicelib/gtfs.py

@@ -1,9 +1,9 @@
 from datetime import date
 from tempfile import TemporaryDirectory
-import pandas as pd
 from typing import Iterable
-import boto3
 
+import boto3
+import fireducks.pandas as pd
 from mbta_gtfs_sqlite import MbtaGtfsArchive
 from mbta_gtfs_sqlite.models import StopTime, Trip
 from sqlalchemy import or_

mbta-performance/chalicelib/historic/gtfs_archive.py

@@ -1,9 +1,10 @@
 import datetime
-import pandas as pd
 import pathlib
 import shutil
 import urllib.request
 
+import fireducks.pandas as pd
+
 from ..date import to_dateint
 
 MAIN_DIR = pathlib.Path("./data/gtfs_archives/")

mbta-performance/chalicelib/historic/process.py

@@ -1,17 +1,19 @@
-import pandas as pd
-import uuid
 import pathlib
-from .constants import HISTORIC_COLUMNS_PRE_LAMP as HISTORIC_COLUMNS
+import uuid
+from datetime import datetime
+
+import fireducks.pandas as pd
+
 from .constants import (
     CSV_FIELDS,
     arrival_field_mapping,
     departure_field_mapping,
+    inbound_outbound,
     station_mapping,
     unofficial_ferry_labels_map,
-    inbound_outbound,
 )
+from .constants import HISTORIC_COLUMNS_PRE_LAMP as HISTORIC_COLUMNS
 from .gtfs_archive import add_gtfs_headways
-from datetime import datetime
 
 
 def process_events(input_csv: str, outdir: str, nozip: bool = False, columns: list = HISTORIC_COLUMNS):

mbta-performance/chalicelib/lamp/backfill/main.py

@@ -1,8 +1,9 @@
-import pandas as pd
-from ..ingest import fetch_pq_file_from_remote, ingest_pq_file, upload_to_s3
-from ... import parallel
 from datetime import date, timedelta
 
+import fireducks.pandas as pd
+
+from ... import parallel
+from ..ingest import fetch_pq_file_from_remote, ingest_pq_file, upload_to_s3
 
 _parallel_upload = parallel.make_parallel(upload_to_s3)
 

mbta-performance/chalicelib/lamp/ingest.py

@@ -2,7 +2,7 @@
 from datetime import date
 from typing import Tuple
 
-import pandas as pd
+import fireducks.pandas as pd
 import requests
 
 from .. import parallel, s3

mbta-performance/chalicelib/lamp/tests/test_ingest.py

@@ -1,13 +1,12 @@
-from datetime import date
 import io
 import os
 import unittest
+from datetime import date
 from unittest import mock
 
-import pandas as pd
+import fireducks.pandas as pd
 
-from .. import ingest
-from .. import constants
+from .. import constants, ingest
 
 # The sample file attached here is 10k events sampled from Feb 7th, 2024.
 # These rows contain real-world inconsistencies in their data!

mbta-performance/chalicelib/parallel.py

@@ -1,6 +1,6 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
-import pandas as pd
+import fireducks.pandas as pd
 
 
 def make_parallel(single_func, THREAD_COUNT=5):

mbta-performance/chalicelib/s3.py

@@ -1,8 +1,9 @@
-import boto3
 import io
-import pandas as pd
-import zlib
 import time
+import zlib
+
+import boto3
+import fireducks.pandas as pd
 
 s3 = boto3.client("s3")
 cloudfront = boto3.client("cloudfront")

pyproject.toml

@@ -8,10 +8,10 @@ license = "MIT"
 dependencies = [
     "requests>=2.32.3,<3",
     "boto3~=1.34.162",
-    "pandas~=2.2.3",
     "datadog_lambda==6.109.0",
-    "pyarrow~=17.0.0",
+    "pyarrow~=21.0.0",
     "mbta-gtfs-sqlite~=1.1.1",
+    "fireducks~=1.4.3",
 ]
 
 [dependency-groups]
@@ -28,7 +28,3 @@ package = false
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
-
-[tool.black]
-line-length = 120
-target-version = ['py312']