Big Data

# Word Count — the Hello World of MapReduce
# Map phase: emit (word, 1) for each word
def mapper(line):
    words = line.strip().split()
    for word in words:
        print(f"{word.lower()}\t1")

# Reduce phase: sum counts for each word
def reducer(word, counts):
    total = sum(counts)
    print(f"{word}\t{total}")

# Input: "hello world hello hadoop"
# Map outputs:
#   hello  1
#   world  1
#   hello  1
#   hadoop 1
# Shuffle & sort:
#   hadoop [1]
#   hello  [1, 1]
#   world  [1]
# Reduce outputs:
#   hadoop  1
#   hello   2
#   world   1

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, avg, count, when

# Initialize Spark
spark = SparkSession.builder \
    .appName("DataScience") \
    .getOrCreate()

# Read data
df = spark.read.csv("s3://bucket/sales.csv", header=True, inferSchema=True)
print(f"Partitions: {df.rdd.getNumPartitions()}")
print(f"Schema: {df.printSchema()}")

# Transformations (lazy — only computed on action)
daily_sales = df.groupBy("date").agg(
    count("*").alias("order_count"),
    avg("amount").alias("avg_amount"),
    sum("amount").alias("total_revenue")
)

# Action triggers computation
daily_sales.show(10)
daily_sales.write.parquet("output/daily_sales.parquet")

# Spark SQL
df.createOrReplaceTempView("sales")
result = spark.sql("""
    SELECT
        region,
        SUM(amount) as revenue
    FROM sales
    WHERE year = 2024
    GROUP BY region
    ORDER BY revenue DESC
""")
result.show()

# MLlib — distributed ML
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.clustering import KMeans