Statistics for Data Science

import numpy as np
from scipy import stats

data = [12, 15, 14, 10, 18, 20, 14, 16, 13, 15]

mean = np.mean(data)
median = np.median(data)
mode = stats.mode(data, keepdims=True)
variance = np.var(data, ddof=1)  # sample variance
std = np.std(data, ddof=1)
q1, q3 = np.percentile(data, [25, 75])
iqr = q3 - q1

print(f"Mean: {mean:.2f}")
print(f"Median: {median}")
print(f"Mode: {mode.mode[0]}")
print(f"Std: {std:.2f}")
print(f"IQR: {iqr:.2f}")
# Skewness & Kurtosis
print(f"Skew: {stats.skew(data):.2f}")
print(f"Kurtosis: {stats.kurtosis(data):.2f}")

import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

# Normal distribution
mu, sigma = 0, 1
norm_dist = stats.norm(mu, sigma)
x = np.linspace(-4, 4, 100)
pdf = norm_dist.pdf(x)       # probability density
cdf = norm_dist.cdf(x)       # cumulative prob

# Binomial distribution
n, p = 10, 0.5
binom_dist = stats.binom(n, p)
prob_6 = binom_dist.pmf(6)   # P(X = 6)

# Poisson distribution
lam = 3
poisson_dist = stats.poisson(lam)
prob_5 = poisson_dist.pmf(5) # P(X = 5) events

# Percentiles (z-score)
z = norm_dist.ppf(0.975)  # 1.96 — 95% confidence
print(f"95% CI z-score: {z:.3f}")

# Random sampling
samples = norm_dist.rvs(1000, random_state=42)
print(f"Sample mean: {samples.mean():.3f}")

from scipy import stats
import numpy as np

# One-sample t-test: is mean different from 100?
data = [98, 102, 97, 105, 99, 101, 96, 103]
t_stat, p_val = stats.ttest_1samp(data, 100)
print(f"t = {t_stat:.3f}, p = {p_val:.4f}")
# p < 0.05 → reject null hypothesis

# Two-sample t-test
group_a = np.random.normal(100, 10, 30)
group_b = np.random.normal(105, 10, 30)
t_stat, p_val = stats.ttest_ind(group_a, group_b)
print(f"Two-sample t = {t_stat:.3f}, p = {p_val:.4f}")

# Chi-squared test
observed = [[10, 20], [15, 25]]
chi2, p, dof, expected = stats.chi2_contingency(observed)
print(f"Chi2 = {chi2:.3f}, p = {p:.4f}")

# p-value interpretation:
# p < 0.05 → statistically significant (95% confidence)
# p < 0.01 → highly significant (99% confidence)
# p > 0.05 → not significant

import numpy as np
from scipy import stats

# Pearson correlation
x = np.array([1, 2, 3, 4, 5])
y = np.array([2, 4, 5, 4, 6])
r, p_val = stats.pearsonr(x, y)
print(f"Pearson r = {r:.3f}, p = {p_val:.4f}")

# Spearman rank correlation (non-parametric)
rho, p = stats.spearmanr(x, y)
print(f"Spearman rho = {rho:.3f}")

# Common pitfalls:
# - Spurious correlation: ice cream sales ← temp → drownings
# - Confounding variable: Z causes both X and Y
# - Reversed causation: does X cause Y or Y cause X?
# - Selection bias: cherry-picking data points