Sample-Size Napkin Math

Sample size scales as 1/MDE². Halving the detectable effect requires 4× the sample.

Sample size calculator in Python

import numpy as np
from statsmodels.stats.power import NormalIndPower
from statsmodels.stats.proportion import proportion_effectsize

def sample_size_comparison(p_baseline, mde_abs):
    # 1. Pocket formula
    n_pocket = int(16 * p_baseline * (1 - p_baseline) / (mde_abs ** 2))

    # 2. Exact NormalIndPower calculation (alpha=0.05, power=0.80)
    effect_size = proportion_effectsize(p_baseline, p_baseline + mde_abs)
    n_exact = int(np.ceil(NormalIndPower().solve_power(
        effect_size=effect_size,
        alpha=0.05,
        power=0.80,
        ratio=1.0
    )))

    return n_pocket, n_exact

# Compare for baseline 5%, MDE = +1pp (0.01)
pocket_n, exact_n = sample_size_comparison(p_baseline=0.05, mde_abs=0.01)
print(f"Baseline: 5.0% -> 6.0% (MDE = +1.0pp)")
print(f"Pocket Formula N per arm:    {pocket_n:,}")
print(f"Exact Power Model N per arm: {exact_n:,}")
print(f"Total Experiment Traffic:    {exact_n * 2:,} visitors")

Takeaway

Always sanity-check sample requirements before launching.