Data Transformation
with Python at Scale

>>> import dask.dataframe as dd    
>>> df = dd.read_parquet(
...     "s3://bucket/mydata.parquet",
... )

>>> df = df[df.balance < 0]
>>> df.to_parquet("s3://...")

>>> from optuna.integration.dask import DaskStorage
>>> def objective(...):
...     ...
>>> study = optuna.create_study(storage=DaskStorage())
>>> study.optimize(objective, ...)

>>> study.best_params
{"n_layers": ..., "optimizer": ...,  ...}

>>> futures = []
>>> for fn in filenames:
...     future = client.submit(process, fn)
...     futures.append(future)

>>> results = client.gather(futures)