Send Email using Python /PySpark Notebook

Ashwath_Bala_S I don't understand the rationale behind not using the pipeline, which already has send email activity. I'm sure you can find a solution like below (I just googled it), but what is the point, why to reinvent the wheel?

from pyspark.sql import SparkSession
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
import os

# ---------------------------
# 1. Initialize Spark Session
# ---------------------------
spark = SparkSession.builder \
    .appName("PySparkEmailExample") \
    .getOrCreate()

# ---------------------------
# 2. Create a sample DataFrame
# ---------------------------
data = [("Alice", 25), ("Bob", 30), ("Charlie", 35)]
df = spark.createDataFrame(data, ["Name", "Age"])

# Save DataFrame to CSV (driver local path)
csv_path = "/tmp/sample_data.csv"
df.coalesce(1).toPandas().to_csv(csv_path, index=False)  # coalesce to 1 file for attachment

# ---------------------------
# 3. Email sending function
# ---------------------------
def send_email_with_attachment(
    smtp_server, smtp_port, sender_email, sender_password,
    recipient_email, subject, body, attachment_path=None
):
    try:
        # Create the email
        msg = MIMEMultipart()
        msg["From"] = sender_email
        msg["To"] = recipient_email
        msg["Subject"] = subject

        # Add body text
        msg.attach(MIMEText(body, "plain"))

        # Add attachment if provided
        if attachment_path and os.path.exists(attachment_path):
            with open(attachment_path, "rb") as f:
                part = MIMEBase("application", "octet-stream")
                part.set_payload(f.read())
            encoders.encode_base64(part)
            part.add_header(
                "Content-Disposition",
                f"attachment; filename={os.path.basename(attachment_path)}"
            )
            msg.attach(part)

        # Connect to SMTP server and send
        with smtplib.SMTP(smtp_server, smtp_port) as server:
            server.starttls()  # Secure connection
            server.login(sender_email, sender_password)
            server.send_message(msg)

        print("✅ Email sent successfully.")

    except Exception as e:
        print(f"❌ Failed to send email: {e}")

# ---------------------------
# 4. Call the function
# ---------------------------
# Example: Gmail SMTP (requires app password if 2FA enabled)
send_email_with_attachment(
    smtp_server="smtp.gmail.com",
    smtp_port=587,
    sender_email="[email protected]",
    sender_password="your_app_password",  # Use environment variable in production
    recipient_email="[email protected]",
    subject="PySpark Data Report",
    body="Hello,\n\nPlease find attached the latest data report.\n\nRegards,\nPySpark Job",
    attachment_path=csv_path
)

# ---------------------------
# 5. Stop Spark
# ---------------------------
spark.stop()