AbstractDataSet

kedro-viz

AbstractDataSet

from typing import Dict
from pathlib import Path, PurePosixPath
import dask.dataframe as dd
from <http://kedro.io|kedro.io> import AbstractDataSet

class DaskCSVDataSet(AbstractDataSet):
    def __init__(self, filepath, param1, param2=True):
        self._filepath = PurePosixPath(filepath)
        self._param1 = param1
        self._param2 = param2

    def _load(self) -> Dict[str, dd.DataFrame]:
        # Implement logic to load multiple CSV files as Dask DataFrame
        first_row = dd.read_csv(self._filepath, include_path_column=True, sep=';').head(n=0)
        headers = dd.read_csv(self._filepath, include_path_column=True, sep=';', skiprows=[0]).head(n=2)
        ddf = dd.read_csv(self._filepath, include_path_column=True, sep=';', skiprows=[0,1,3])
        return {"first_row": first_row, "headers": headers, "ddf": ddf}

    def _save(self, data: dd.DataFrame) -> None:
        raise NotImplementedError("Saving Dask DataFrame as CSV is not supported")

    def _describe(self):
        return "hello csv"

    def _exists(self) -> bool:
        return Path(self._filepath.as_posix()).exists()

_describe