Skip to content

Commit e64880d

Browse files
Data Transfer: Enable no partitioning (#216)
* Enable no partitioning when downloading data from the data transfer service. * Update default behaviour for partitioning to None instead of time. * PEP8
1 parent 2f78c63 commit e64880d

File tree

1 file changed

+10
-3
lines changed

1 file changed

+10
-3
lines changed

fink_client/scripts/fink_datatransfer.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,8 @@ def poll(process_id, nconsumers, queue, schema, kafka_config, rng, args):
208208
partitioning = ["tnsclass"]
209209
elif args.partitionby == "classId":
210210
partitioning = ["classId"]
211+
else:
212+
partitioning = None
211213

212214
table = pa.Table.from_pandas(pdf)
213215

@@ -288,8 +290,8 @@ def main():
288290
parser.add_argument(
289291
"-partitionby",
290292
type=str,
291-
default="time",
292-
help="Partition data by `time` (year=YYYY/month=MM/day=DD), or `finkclass` (finkclass=CLASS), or `tnsclass` (tnsclass=CLASS). `classId` is also available for ELASTiCC data. Default is time.",
293+
default=None,
294+
help="Partition data by `time` (year=YYYY/month=MM/day=DD), or `finkclass` (finkclass=CLASS), or `tnsclass` (tnsclass=CLASS). `classId` is also available for ELASTiCC data. Default is None, that is no partitioning is applied.",
293295
)
294296
parser.add_argument(
295297
"-batchsize",
@@ -332,7 +334,12 @@ def main():
332334
)
333335
args = parser.parse_args(None)
334336

335-
if args.partitionby not in ["time", "finkclass", "tnsclass", "classId"]:
337+
if args.partitionby is not None and args.partitionby not in [
338+
"time",
339+
"finkclass",
340+
"tnsclass",
341+
"classId",
342+
]:
336343
_LOG.error(
337344
"{} is an unknown partitioning. `-partitionby` should be in ['time', 'finkclass', 'tnsclass', 'classId']".format(
338345
args.partitionby

0 commit comments

Comments
 (0)