"""
18_external_validation.py
外部效度检验模块：与第三方治理/竞争力指数的相关性验证
External Validation Module: correlation with third-party governance/competitiveness indices

用途 / Purpose:
    将本文构建的 re_score 与外部已发表的参照指数（03 号脚本获取的
    V-Dem 自由民主指数、Fraser Economic Freedom of the World 指数、
    WIPO Global Innovation Index 代理变量）做 Pearson/Spearman 相关，
    作为外部效度证据：若 RE 分数与这些独立构建的指数存在合理方向、
    合理强度（既不过低导致质疑构念、也不过高导致重复测量）的相关性，
    则支持 RE 指数的构念效度。

依赖 / Dependencies: pandas, numpy, scipy
"""

import logging
from pathlib import Path

import pandas as pd
from scipy.stats import pearsonr, spearmanr

logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(message)s")
log = logging.getLogger("18_external_validation")

DATA_DIR = Path("./data")

REFERENCE_COLUMNS = {
    "vdem_polyarchy": "V-Dem 自由民主指数 / V-Dem Liberal Democracy Index",
    "fraser_efw_score": "Fraser 经济自由度指数 / Fraser Economic Freedom of the World",
    "gii_proxy_score": "WIPO 全球创新指数代理变量 / WIPO GII proxy",
}


def main():
    scores = pd.read_csv(DATA_DIR / "panel_re_scores.csv")
    try:
        reference = pd.read_csv(DATA_DIR / "reference_comparison_variables.csv")
    except FileNotFoundError:
        log.warning(
            "data/reference_comparison_variables.csv not found "
            "(produced by 03_data_acquisition_reference_sources.py); "
            "using empty placeholder for structure documentation."
        )
        reference = pd.DataFrame(columns=["iso3", "year"] + list(REFERENCE_COLUMNS.keys()))

    merged = scores.merge(reference, on=["iso3", "year"], how="left")

    results = []
    for col, desc in REFERENCE_COLUMNS.items():
        if col not in merged.columns:
            continue
        sub = merged[["re_score", col]].dropna()
        if len(sub) < 3:
            log.warning("Insufficient overlapping observations for %s; skipping", col)
            continue
        pearson_r, pearson_p = pearsonr(sub["re_score"], sub[col])
        spearman_r, spearman_p = spearmanr(sub["re_score"], sub[col])
        results.append({
            "reference_variable": col,
            "description": desc,
            "n_obs": len(sub),
            "pearson_r": round(pearson_r, 4),
            "pearson_p": round(pearson_p, 6),
            "spearman_rho": round(spearman_r, 4),
            "spearman_p": round(spearman_p, 6),
        })
        log.info("%s: Pearson r=%.4f (p=%.4g), Spearman rho=%.4f", col, pearson_r, pearson_p, spearman_r)

    out = pd.DataFrame(results)
    out.to_csv(DATA_DIR / "external_validation_report.csv", index=False)
    log.info("External validation report saved -> data/external_validation_report.csv")


if __name__ == "__main__":
    main()