I have one pipeline sequence where it will run multiple dataflow's with interdependence. I have designed a job in such a way that on each successful completion of one dataflow-1 another will trigger and incase of failure I need to send a mail-notification to the corresponding team-members about the stats of the Error like failed dataflow/pipeline name and the reason for the failure of the dataflow.
I did a workaround to capture the errors in Notebook where I connected all the on-fail connections from all the dataflows to single NoteBook and I connected back this notebook to outlook. Here Iam using the following scirpt to capture the errros and show in the Outlook but its not working as I desired. Please let me know if anyone has come across such a scenario in your projects.
from datetime import datetime
from notebookutils import mssparkutils
# -------------------------------
# CONFIG: List of dataflows to check
dataflows = [
"df_dataflow-1", "df_dataflow-2", "df_dataflow-3",
"df_dataflow-5", "df_dataflow-4", "df_dataflow-6",
"df_dataflow-8", "df_dataflow-7", "df_dataflow-9",
"df_dataflow-10"
]
failed_flows = []
# -------------------------------
# LOOP: Check status and error for each flow
for flow in dataflows:
try:
output = mssparkutils.pipeline.getActivityOutput(flow)
status = output.get("status", "Unknown")
if status != "Succeeded":
error_detail = output.get("error", {}).get("message", "No error message available")
failed_flows.append((flow, error_detail))
except Exception as e:
failed_flows.append((flow, f"⚠️ Status unknown or error occurred: {str(e)}"))
# -------------------------------
# COMPOSE HTML MESSAGE
if failed_flows:
rows = "".join(
f"<tr><td>{flow}</td><td>{error}</td></tr>"
for flow, error in failed_flows
)
message = f"""
<html>
<body>
<h2 style='color:red;'>🚨 Pipeline Failure Alert</h2>
<p>The following dataflows failed in the pipeline:</p>
<table border='1' cellpadding='5' cellspacing='0' style='border-collapse: collapse;'>
<tr style='background-color: #f2f2f2;'><th>Dataflow Name</th><th>Error Message</th></tr>
{rows}
</table>
<p><b>Timestamp:</b> {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC</p>
</body>
</html>
"""
else:
message = f"""
<html>
<body>
<h2 style='color:green;'>✅ All Dataflows Succeeded</h2>
<p>No errors detected in the pipeline execution.</p>
<p><b>Timestamp:</b> {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC</p>
</body>
</html>
"""
# -------------------------------
# OUTPUT HTML MESSAGE TO PIPELINE
mssparkutils.notebook.exit(message)
