Mathilde Lavacquery
07/07/2023, 12:11 PMregion
from raw file, and then I apply the same modular pipeline to each region, using namespace=region
def create_preprocess_modular_pipeline(region) -> Pipeline:
def filter_scope_node(raw_national):
return filter_scope(raw_national, region=region)
return pipeline(
[
node(
func=filter_scope_node,
inputs=["raw_national"],
outputs="scope_filtered",
),
]
)
def create_preprocess_pipeline(**kwargs) -> Pipeline:
regions = kwargs.get("regions")
preprocess_pipeline = Pipeline([])
for region in regions:
preprocess_pipeline += pipeline(
pipe=create_preprocess_modular_pipeline(region=region),
namespace=region,
# inputs for which we don't have namespaces
inputs={"raw_national": "raw_national"},
)
return preprocess_pipeline
Current behaviour: it seems that only the last region is used in filter_scope_node
for all regions (seems to be an instantiation problem of this node)
How can I pass dynamically my namespace to some nodes ?
How to dynamically instantiate this filter_scope_node ?