Felipe Monroy
02/02/2024, 1:31 PMdatajoely
02/02/2024, 1:35 PMFelipe Monroy
02/02/2024, 1:40 PMclass DataCatalogParamsFactory(DataCatalog):
"""DataCatalog that also resolves factories in the parameters."""
@overrides
def _get_dataset(
self,
dataset_name: str,
version: Version | None = None,
suggest: bool = True,
) -> AbstractDataset:
params_pattern = {
k: v
for k, v in self._datasets.items()
if k.startswith("params:") and self._is_pattern(k)
}
matched_pattern = self._match_pattern(params_pattern, dataset_name)
if dataset_name not in self._datasets and matched_pattern:
# If the dataset is a patterned dataset, materialise it and add it to
# the catalog
config_copy = copy.deepcopy(params_pattern[matched_pattern])
dataset_config = self._resolve_config(
dataset_name, matched_pattern, config_copy
)
# This only works with MemoryDataset, all the params corresponds to this.
if isinstance(dataset_config, MemoryDataset):
self.add(dataset_name, dataset_config)
return dataset_config
return super()._get_dataset(dataset_name, version, suggest)
Ankita Katiyar
02/02/2024, 1:40 PMJulian Nowak
03/06/2024, 9:17 AMdatajoely
03/06/2024, 9:18 AM