"""
03_data_acquisition_reference_sources.py
参照/对比数据源获取模块（WIPO GII、V-Dem、QoG、Fraser EFW）
Reference & Comparison Data Sources Acquisition Module

用途 / Purpose:
    下载用于外部效度检验（external validation）的参照数据集：
    - WIPO Global Innovation Index（全球创新指数）
    - V-Dem v14（民主多样性项目核心变量）
    - Quality of Government (QoG) Standard Dataset（治理质量标准数据集）
    - Fraser Institute Economic Freedom of the World (EFW)

    这些数据集不直接进入 RE 指数的构建方程，而是用于 21 章"外部效度
    检验"中与 RE 综合得分的相关性分析。

依赖 / Dependencies: requests, pandas
"""

import logging
from pathlib import Path

import pandas as pd

try:
    import requests
except ImportError:  # pragma: no cover
    requests = None

logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(message)s")
log = logging.getLogger("03_reference_sources")

OUTPUT_DIR = Path("./data")
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

SOURCES = {
    "qog_std_ts": "https://www.qogdata.pol.gu.se/data/qog_std_ts_jan25.csv",
    "vdem_embedded_in_qog": None,  # V-Dem core indicators embedded in QoG (v_dem_* columns)
    "fraser_efw_embedded_in_qog": None,  # EFW indicators embedded in QoG (fi_* columns)
    "wipo_gii": "https://www.wipo.int/edocs/pubdocs/en/wipo-pub-gii-2024-tables.csv",  # reference only
}


def download_qog_std(dest: Path) -> bool:
    """Download the QoG Standard Time-Series dataset (large file, ~59 MB)."""
    if requests is None:
        log.warning("requests not installed; skipping live download.")
        return False
    url = SOURCES["qog_std_ts"]
    try:
        with requests.get(url, stream=True, timeout=120) as resp:
            resp.raise_for_status()
            with open(dest, "wb") as fh:
                for chunk in resp.iter_content(chunk_size=1 << 20):
                    fh.write(chunk)
        log.info("Downloaded QoG standard dataset -> %s", dest)
        return True
    except Exception as exc:  # noqa: BLE001
        log.warning("QoG download failed: %s", exc)
        return False


def extract_relevant_columns(qog_path: Path, iso3_list: list, years: range) -> pd.DataFrame:
    """
    Extract the subset of QoG columns relevant to RE Index construction:
    governance quality proxies, V-Dem accountability indicators, and the
    embedded Fraser EFW regulatory-freedom sub-indicators.
    """
    relevant_cols = [
        "ccodealp", "year",
        # V-Dem core (embedded in QoG)
        "vdem_polyarchy", "vdem_libdem", "vdem_gender",
        # Fraser EFW (embedded in QoG, prefix fi_)
        "fi_efw", "fi_reg", "fi_sound_money",
        # ICRG composite risk indicators (embedded in QoG)
        "icrg_qog", "icrg_bureau",
    ]
    try:
        df = pd.read_csv(qog_path, usecols=lambda c: c in relevant_cols, low_memory=False)
    except (FileNotFoundError, ValueError) as exc:
        log.warning("QoG extraction skipped (%s); returning empty frame.", exc)
        return pd.DataFrame(columns=relevant_cols)

    df = df[df["ccodealp"].isin(iso3_list) & df["year"].isin(list(years))]
    return df.rename(columns={"ccodealp": "iso3"})


def main():
    qog_dest = OUTPUT_DIR / "qog_std_ts_jan25.csv"
    downloaded = download_qog_std(qog_dest)

    oecd38 = [
        "AUS", "AUT", "BEL", "CAN", "CHE", "CHL", "COL", "CRI", "CZE", "DEU",
        "DNK", "ESP", "EST", "FIN", "FRA", "GBR", "GRC", "HUN", "IRL", "ISL",
        "ISR", "ITA", "JPN", "KOR", "LTU", "LUX", "LVA", "MEX", "NLD", "NOR",
        "NZL", "POL", "PRT", "SVK", "SVN", "SWE", "TUR", "USA",
    ]
    years = range(2000, 2024)

    subset = extract_relevant_columns(qog_dest, oecd38, years)
    subset.to_csv(OUTPUT_DIR / "reference_comparison_variables.csv", index=False)
    log.info(
        "Reference/comparison extraction done (QoG downloaded=%s): %d rows -> "
        "data/reference_comparison_variables.csv",
        downloaded, len(subset),
    )


if __name__ == "__main__":
    main()