Alice Cima
04/06/2026, 11:29 PMSaurabh Mishra
04/08/2026, 10:57 AM_save_overwrite behavior to unblock myself.
def _patched_save_overwrite(self, data: DataFrame) -> None:
"""Patched overwrite with schema auto-merge and dynamic partition overwrite.
Replaces the default ManagedTableDataset._save_overwrite to:
- Use Delta schema auto-merge instead of overwriteSchema
- Use dynamic partition overwrite mode (only overwrite partitions in the DataFrame)
"""
writer = (
data.write.format(self._table.format)
.mode("overwrite")
.option("partitionOverwriteMode", "dynamic")
)
if self._table.partition_columns:
writer.partitionBy(
*self._table.partition_columns
if isinstance(self._table.partition_columns, list)
else [self._table.partition_columns]
)
if self._table.location:
writer.option("path", self._table.location)
writer.saveAsTable(self._table.full_table_location() or "")Ankita Katiyar
04/08/2026, 2:41 PM