NumPy & pandas Essentials¶

import numpy as np

# ── Creation ──────────────────────────────────────────────────────
arr = np.array([1.0, 2.0, 3.0, 4.0, 5.0])   # from Python list
zeros = np.zeros((3, 4))                      # 3×4 matrix of 0.0
ones  = np.ones((2, 3), dtype=np.int32)       # 2×3 matrix of 1 (int)
rng   = np.arange(0, 10, 2)                   # [0, 2, 4, 6, 8]
linsp = np.linspace(0, 1, 5)                  # [0, 0.25, 0.5, 0.75, 1.0]
rand  = np.random.default_rng(seed=42).standard_normal((1000,))

# ── Shape and dtype ───────────────────────────────────────────────
matrix = np.array([[1, 2, 3], [4, 5, 6]])
print(matrix.shape)   # (2, 3) — 2 rows, 3 columns
print(matrix.dtype)   # int64
print(matrix.ndim)    # 2

# ── Indexing and slicing ──────────────────────────────────────────
arr[0]         # 1.0  — first element
arr[-1]        # 5.0  — last element
arr[1:4]       # array([2.0, 3.0, 4.0])
matrix[0, :]   # first row: array([1, 2, 3])
matrix[:, 1]   # second column: array([2, 5])

# Boolean indexing — selects elements where condition is True
big = arr[arr > 3.0]   # array([4.0, 5.0])

import numpy as np, time

prices = np.random.default_rng(0).uniform(10, 500, size=1_000_000)

# Slow: Python loop — 1 million iterations, Python overhead per element
t0 = time.perf_counter()
discounted_loop = [p * 0.9 for p in prices]
print(f"Loop:   {time.perf_counter() - t0:.3f}s")

# Fast: vectorised operation — single C-level call
t0 = time.perf_counter()
discounted_vec = prices * 0.9
print(f"NumPy:  {time.perf_counter() - t0:.3f}s")

import numpy as np

# Normalise a (100, 5) feature matrix to zero mean, unit variance per column
features = np.random.default_rng(1).standard_normal((100, 5))

col_means = features.mean(axis=0)    # shape (5,)  — mean of each column
col_stds  = features.std(axis=0)     # shape (5,)

# Broadcasting: (100, 5) - (5,) → NumPy broadcasts (5,) across 100 rows
normalised = (features - col_means) / col_stds   # shape (100, 5)

print(normalised.mean(axis=0).round(10))   # ≈ [0, 0, 0, 0, 0]
print(normalised.std(axis=0).round(10))    # ≈ [1, 1, 1, 1, 1]

import pandas as pd

# ── Creating DataFrames ───────────────────────────────────────────
df = pd.DataFrame({
    "user_id":  ["u1", "u2", "u3", "u1", "u2"],
    "event":    ["click", "view", "click", "purchase", "click"],
    "value":    [5, 2, 7, 120, 3],
    "country":  ["IN", "US", "IN", "IN", "US"],
})

# Load from CSV (the most common source)
# df = pd.read_csv("events.csv", parse_dates=["timestamp"])

# ── Inspection ───────────────────────────────────────────────────
df.shape           # (5, 4)
df.dtypes          # dtypes of each column
df.head(3)         # first 3 rows
df.info()          # non-null counts + dtypes summary
df.describe()      # min, max, mean, std for numeric columns

# ── Column selection ──────────────────────────────────────────────
df["value"]              # Series (single column)
df[["user_id", "value"]] # DataFrame (multiple columns)

# ── Filtering (boolean indexing, same concept as NumPy) ───────────
clicks = df[df["event"] == "click"]
high_value = df[df["value"] > 10]
india_clicks = df[(df["country"] == "IN") & (df["event"] == "click")]

import pandas as pd
import numpy as np

# ── 1. Load ───────────────────────────────────────────────────────
df = pd.read_csv(
    "raw_events.csv",
    parse_dates=["timestamp"],
    dtype={"user_id": "string", "event": "string", "country": "string"},
)
print(f"Loaded: {len(df):,} rows, {df.isna().sum().sum()} nulls")

# ── 2. Clean ──────────────────────────────────────────────────────
# Drop rows with missing critical fields
df = df.dropna(subset=["user_id", "event", "value"])

# Clip outliers at the 99th percentile (common for event value data)
p99 = df["value"].quantile(0.99)
df["value"] = df["value"].clip(upper=p99)

# Normalise string columns
df["country"] = df["country"].str.upper().str.strip()
df["event"]   = df["event"].str.lower().str.strip()

# Add derived columns
df["date"] = df["timestamp"].dt.date
df["hour"] = df["timestamp"].dt.hour

# ── 3. Aggregate: revenue by country and event type ───────────────
summary = (
    df
    .groupby(["country", "event"])
    .agg(
        sessions    = ("user_id", "nunique"),
        total_value = ("value",   "sum"),
        avg_value   = ("value",   "mean"),
        events      = ("event",   "count"),
    )
    .round(2)
    .reset_index()
    .sort_values("total_value", ascending=False)
)

# ── 4. Merge with a reference table ──────────────────────────────
country_meta = pd.read_csv("country_metadata.csv")  # has "country", "region"
summary = pd.merge(summary, country_meta, on="country", how="left")

# ── 5. Export ─────────────────────────────────────────────────────
summary.to_csv("summary_report.csv", index=False)
summary.to_parquet("summary_report.parquet", index=False)   # columnar, fast
print(summary.head(10).to_string(index=False))