Leonardo David Treiger Herszenhaut Brettas
08/09/2024, 6:10 PMdef update_dataset_with_silver_layer(**kwargs):
"""
Merge the dataset with the silver layer
Last dict item is always the pipeline name
The second to last dict item is always the silver managed table
"""
_, pipeline_name = kwargs.popitem()
master_dataset_name, master_dataset = kwargs.popitem()
_, bronze_dataset = kwargs.popitem()
print(type(master_dataset))
(
master_dataset.alias("silver").merge(
bronze_dataset.alias("bronze"),
mount_join_keys_string(
GRAPHS["bronze_to_silver"][pipeline_name][master_dataset_name]["pk"],
),
)
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute()
)
Nok Lam Chan
08/09/2024, 9:43 PMLeonardo David Treiger Herszenhaut Brettas
08/09/2024, 10:33 PMdatajoely
08/11/2024, 7:29 AMLeonardo David Treiger Herszenhaut Brettas
08/11/2024, 7:26 PM