add file_name param

Author: aspeddro

python-package/basedosdados/upload/table.py

@@ -509,6 +509,7 @@ def create_from_pandas(
         self,
         df: pd.DataFrame,
         partition_columns: Optional[list[str]] = None,
+        file_name: str = "data",
         source_format: str = "csv",
         **kwargs,
     ) -> None:
@@ -532,6 +533,7 @@ def create_from_pandas(
             partition_columns: A list of column names to partition the data by.
                 These columns are removed from the data files and encoded in the
                 storage path following the Hive partitioning scheme.
+            file_name: Name of file without extension. Defaults to `data`.
             source_format: The format used to save the data. Only 'csv',
                 'parquet' and 'avro' are supported. Defaults to 'csv'.
             **kwargs: Additional keyword arguments forwarded to `Table.create`
@@ -559,7 +561,7 @@ def create_from_pandas(
         missing_columns = [
             col for col in partition_columns if col not in df.columns
         ]
-        if missing_columns:
+        if len(missing_columns) > 0:
             raise BaseDosDadosException(
                 f"Partition columns not found in the DataFrame: {missing_columns}"
             )
@@ -572,16 +574,17 @@ def create_from_pandas(
                 tmpdir=tmpdir,
             )
 
-            if partition_columns:
+            if len(partition_columns) > 0:
                 to_partitions(
                     data=df,
                     partition_columns=partition_columns,
                     savepath=tmppath,
+                    file_name=file_name,
                     file_format=source_format,
                 )
                 path = tmppath
             else:
-                path = tmppath / f"data.{source_format}"
+                path = tmppath / f"{file_name}.{source_format}"
                 if source_format == "csv":
                     df.to_csv(path, index=False)
                 elif source_format == "parquet":

python-package/basedosdados/upload/utils.py

@@ -13,13 +13,15 @@ def to_partitions(
     data: pd.DataFrame,
     partition_columns: List[str],
     savepath: str,
+    file_name: str = "data",
     file_format: str = "csv",
 ):
     """Save data in to hive patitions schema, given a dataframe and a list of partition columns.
     Args:
         data (pandas.core.frame.DataFrame): Dataframe to be partitioned.
         partition_columns (list): List of columns to be used as partitions.
         savepath (str, pathlib.PosixPath): folder path to save the partitions
+        file_name: Name of file without extension.
         file_format (str): The file format to save the partitioned data in.
             Only 'csv', 'parquet' and 'avro' are supported. Defaults to 'csv'.
     Exemple:
@@ -66,16 +68,17 @@ def to_partitions(
             filter_save_path.mkdir(parents=True, exist_ok=True)
 
             if file_format == "csv":
-                file_filter_save_path = Path(filter_save_path) / "data.csv"
-                # append data to csv
+                file_filter_save_path = (
+                    Path(filter_save_path) / f"{file_name}.csv"
+                )
                 df_filter.to_csv(
                     file_filter_save_path,
                     index=False,
-                    mode="a",
-                    header=not file_filter_save_path.exists(),
                 )
             elif file_format == "parquet":
-                file_filter_save_path = Path(filter_save_path) / "data.parquet"
+                file_filter_save_path = (
+                    Path(filter_save_path) / f"{file_name}.parquet"
+                )
                 df_filter.to_parquet(file_filter_save_path, index=False)
             elif file_format == "avro":
                 try:
@@ -85,7 +88,9 @@ def to_partitions(
                         "Optional dependencies for handling AVRO files are not installed. "
                         'Please install basedosdados with the "avro" extra'
                     ) from exc
-                file_filter_save_path = Path(filter_save_path) / "data.avro"
+                file_filter_save_path = (
+                    Path(filter_save_path) / f"{file_name}.avro"
+                )
                 pandavro.to_avro(str(file_filter_save_path), df_filter)
             else:
                 raise NotImplementedError(