"""
06_normalization_outliers.py
标准化与异常值处理模块：稳健 Min-Max、Z-score+CDF、Winsorization
Normalization & Outlier Treatment Module

用途 / Purpose:
    对插补后的面板数据（05 输出）进行标准化前处理：
    1. Winsorization（1/99 分位裁剪）处理极端异常值
       （如土耳其 2022 年 inflation_cpi=72.3%，希腊央行债务占比 GDP>200%）；
    2. 对数尺度差异悬殊的变量（如专利数）做 log1p 变换；
    3. 并行实施两种标准化方案：
       (a) 稳健 Min-Max（基于 1/99 分位而非绝对极值）
       (b) Z-score 经标准正态 CDF 映射至 [0,1] 区间
    两种标准化结果均保留，供 04_validity_robustness 阶段做敏感性对比。

依赖 / Dependencies: pandas, numpy, scipy
"""

import logging
from pathlib import Path

import numpy as np
import pandas as pd
from scipy.stats import norm

logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(message)s")
log = logging.getLogger("06_normalization")

DATA_DIR = Path("./data")

LOG1P_COLUMNS = ["patents_residents", "patents_nonresidents"]


def winsorize(series: pd.Series, lower_q: float = 0.01, upper_q: float = 0.99) -> pd.Series:
    lo, hi = series.quantile([lower_q, upper_q])
    return series.clip(lower=lo, upper=hi)


def robust_minmax(series: pd.Series, lower_q: float = 0.01, upper_q: float = 0.99) -> pd.Series:
    lo, hi = series.quantile([lower_q, upper_q])
    if hi == lo:
        return pd.Series(0.5, index=series.index)
    return ((series.clip(lo, hi) - lo) / (hi - lo)).clip(0, 1)


def zscore_cdf(series: pd.Series) -> pd.Series:
    mu, sigma = series.mean(), series.std(ddof=0)
    if sigma == 0 or np.isnan(sigma):
        return pd.Series(0.5, index=series.index)
    z = (series - mu) / sigma
    return pd.Series(norm.cdf(z), index=series.index)


def main():
    panel = pd.read_csv(DATA_DIR / "panel_imputed.csv")
    id_cols = ["iso3", "year"]
    value_cols = [c for c in panel.columns if c not in id_cols]

    # Step 1: log1p transform for heavy-tailed count variables
    for col in LOG1P_COLUMNS:
        if col in panel.columns:
            panel[col] = np.log1p(panel[col].clip(lower=0))
            log.info("Applied log1p transform to %s", col)

    # Step 2: Winsorization (1/99 quantile clipping) across all value columns
    for col in value_cols:
        panel[col] = winsorize(panel[col])
    log.info("Winsorization (1/99 quantiles) applied to %d columns", len(value_cols))

    # Step 3a: robust Min-Max normalization
    minmax_cols = {}
    for col in value_cols:
        minmax_cols[f"{col}__minmax"] = robust_minmax(panel[col])
    panel_minmax = pd.concat([panel[id_cols], pd.DataFrame(minmax_cols)], axis=1)
    panel_minmax.to_csv(DATA_DIR / "panel_normalized_minmax.csv", index=False)

    # Step 3b: Z-score + CDF mapping normalization (alternative scheme)
    cdf_cols = {}
    for col in value_cols:
        cdf_cols[f"{col}__zcdf"] = zscore_cdf(panel[col])
    panel_zcdf = pd.concat([panel[id_cols], pd.DataFrame(cdf_cols)], axis=1)
    panel_zcdf.to_csv(DATA_DIR / "panel_normalized_zcdf.csv", index=False)

    log.info(
        "Normalization complete: minmax -> data/panel_normalized_minmax.csv, "
        "zcdf -> data/panel_normalized_zcdf.csv"
    )


if __name__ == "__main__":
    main()