The Data Engineering Community

Grow your skills in Data Engineering

Join a community of data professionals exploring in-depth technical content and practical challenges in modern data and AI engineering.

data_pipeline.py

# PySpark Example
from pyspark.sql.functions import col, when, expr

# Read data from source
customers_df = (
    spark.read.format("csv")
      .option("header", "true")
      .option("inferSchema", "true")
      .load("s3://data-lake/customers.csv")
)

# Transform data
enriched_df = customers_df.withColumn(
    "full_name", 
    expr("concat(first_name, ' ', last_name)")
).withColumn(
    "is_active",
    when(col("last_activity_date") > "2026-01-01", True).otherwise(False)
)

# Write to destination
(
    enriched_df.write
      .format("parquet")
      .mode("overwrite")
      .partitionBy("region")
      .save("s3://analytics/customer_data/")
)
print(f"Processed {enriched_df.count()} customer records")