"""
Scraper de dados meteorológicos de superfície - ICEA/DECEA
Site: https://pesquisa.icea.decea.mil.br/superficie_list/

Uso standalone:
    python scraper_meteorologia.py --aerodrome SBGR --all-years
    python scraper_meteorologia.py --aerodrome SBGR --start-year 2020 --end-year 2025
    python scraper_meteorologia.py --aerodrome SBSP --start-year 2023 --end-year 2023 --no-headless
    python scraper_meteorologia.py --fetch-catalog          # grava catálogo de aeródromos no banco

Importável pelo pipeline:
    from scraper_meteorologia import make_driver, scrape_year, fetch_aerodrome_catalog, SITE_MIN_DATE
"""

import argparse
import calendar
import csv
import io
import os
import re
import sys
import time
from datetime import date
from typing import Callable, Optional

import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager

BASE_URL         = "https://pesquisa.icea.decea.mil.br/superficie_list/"
SITE_MIN_DATE    = date(1947, 12, 1)
SITE_MAX_DATE    = date.today()        # dinâmico: nunca maior que hoje
CHUNK_MONTHS     = 3
STOP_EMPTY_YEARS = 2


def fetch_site_max_date(driver: webdriver.Chrome) -> date:
    """
    Lê o atributo data-date-end-date do campo de período no site e atualiza
    SITE_MAX_DATE globalmente. Retorna date.today() como fallback.
    """
    global SITE_MAX_DATE
    try:
        val = driver.execute_script(
            "var el = document.getElementById('dtPeriodo');"
            "return el ? el.getAttribute('data-date-end-date') : null;"
        )
        if val:
            d, m, y = val.strip().split("/")
            SITE_MAX_DATE = date(int(y), int(m), int(d))
            return SITE_MAX_DATE
    except Exception:
        pass
    return SITE_MAX_DATE

TABS = [
    ("tab_cgt",   "CGT"),
    ("tab_nuvem", "Nuvem"),
    ("tab_prec",  "Precipitação"),
    ("tab_pres",  "Pressão"),
    ("tab_rvr",   "RVR"),
    ("tab_temp",  "Temperatura"),
    ("tab_teto",  "Teto"),
    ("tab_vent",  "Vento"),
    ("tab_visib", "Visibilidade"),
]


# ---------------------------------------------------------------------------
# WebDriver
# ---------------------------------------------------------------------------

def make_driver(headless: bool = True) -> webdriver.Chrome:
    opts = webdriver.ChromeOptions()
    if headless:
        opts.add_argument("--headless=new")
    opts.add_argument("--no-sandbox")
    opts.add_argument("--disable-dev-shm-usage")
    opts.add_argument("--window-size=1920,1080")
    opts.add_argument("--disable-blink-features=AutomationControlled")
    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
    service = Service(ChromeDriverManager().install())
    driver  = webdriver.Chrome(service=service, options=opts)
    driver.set_page_load_timeout(300)
    driver.set_script_timeout(120)
    try:
        driver.command_executor.client_config.timeout = 300
    except Exception:
        pass
    return driver


# ---------------------------------------------------------------------------
# Formulário
# ---------------------------------------------------------------------------

def select_aerodrome(driver: webdriver.Chrome, wait: WebDriverWait, code: str) -> str:
    """Seleciona aeródromo no Bootstrap Select (JS-first, fallback visual)."""
    result = driver.execute_script(f"""
        var candidates = document.querySelectorAll(
            'select.selectpicker, select[name*="localidade"], select[name*="aerodrome"], select'
        );
        for (var s = 0; s < candidates.length; s++) {{
            var sel = candidates[s];
            for (var i = 0; i < sel.options.length; i++) {{
                if (sel.options[i].text.indexOf('{code}') !== -1) {{
                    sel.selectedIndex = i;
                    var txt = sel.options[i].text;
                    if (typeof jQuery !== 'undefined') {{
                        try {{ jQuery(sel).selectpicker('val', sel.options[i].value); }} catch(e) {{}}
                        jQuery(sel).trigger('change');
                    }} else {{
                        sel.dispatchEvent(new Event('change', {{bubbles: true}}));
                    }}
                    return txt;
                }}
            }}
        }}
        return null;
    """)
    if result:
        return result.strip()

    btn = wait.until(EC.element_to_be_clickable(
        (By.CSS_SELECTOR, "div.bootstrap-select > button.dropdown-toggle")
    ))
    btn.click()
    time.sleep(0.4)
    try:
        search = driver.find_element(By.CSS_SELECTOR, ".bootstrap-select .bs-searchbox input")
        search.clear()
        search.send_keys(code)
        time.sleep(0.5)
    except Exception:
        pass
    option_link = wait.until(EC.element_to_be_clickable((
        By.XPATH,
        f"//ul[contains(@class,'dropdown-menu inner')]"
        f"//li[not(contains(@class,'hidden')) and not(contains(@class,'d-none'))]"
        f"//span[contains(text(),'{code}')]/..",
    )))
    try:
        text = option_link.find_element(By.CSS_SELECTOR, "span.text").text.strip()
    except Exception:
        text = code
    option_link.click()
    time.sleep(0.3)
    return text


def fetch_aerodrome_catalog(headless: bool = True) -> list[dict]:
    """
    Acessa o site ICEA e extrai a lista completa de aeródromos disponíveis.
    Retorna lista de dicts com chaves 'icao', 'nome', 'uf'.
    """
    driver = make_driver(headless=headless)
    try:
        driver.get(BASE_URL)
        WebDriverWait(driver, 30).until(
            EC.presence_of_element_located(
                (By.CSS_SELECTOR, "select.selectpicker, select")
            )
        )
        time.sleep(1.5)  # aguarda Bootstrap Select renderizar todas as opções
        raw_options = driver.execute_script("""
            var candidates = document.querySelectorAll(
                'select.selectpicker, select[name*="localidade"], select[name*="aerodrome"], select'
            );
            var result = [];
            for (var s = 0; s < candidates.length; s++) {
                var sel = candidates[s];
                if (sel.options.length > 5) {
                    for (var i = 0; i < sel.options.length; i++) {
                        var txt = sel.options[i].text.trim();
                        if (txt) result.push(txt);
                    }
                    break;
                }
            }
            return result;
        """)
    finally:
        driver.quit()

    catalog = []
    for text in (raw_options or []):
        if not text or len(text) < 4:
            continue
        icao = text[:4].upper()
        if not re.match(r"[A-Z]{4}", icao):
            continue
        # formato típico: "SBGR - GUARULHOS / CUMBICA (SP)"
        nome = text[4:].lstrip(" -–—").strip()
        uf_match = re.search(r"\(([A-Z]{2})\)\s*$", nome)
        uf = uf_match.group(1) if uf_match else ""
        if uf_match:
            nome = nome[: uf_match.start()].strip().rstrip("(),- ")
        catalog.append({"icao": icao, "nome": nome, "uf": uf})
    return catalog


def set_period(driver: webdriver.Chrome, start: date, end: date) -> None:
    s = start.strftime("%d/%m/%Y")
    e = end.strftime("%d/%m/%Y")
    driver.execute_script(f"""
        var el = document.getElementById('dtPeriodo');
        if (!el) return;
        if (typeof jQuery !== 'undefined' && jQuery(el).data('daterangepicker')) {{
            jQuery(el).data('daterangepicker').setStartDate('{s}');
            jQuery(el).data('daterangepicker').setEndDate('{e}');
        }}
        el.value = '{s} - {e}';
        el.dispatchEvent(new Event('change', {{bubbles: true}}));
    """)


# ---------------------------------------------------------------------------
# Extração de dados
# ---------------------------------------------------------------------------

def show_all_rows(driver: webdriver.Chrome, tab_id: str) -> None:
    driver.execute_script(f"""
        try {{
            var dt = jQuery('#{tab_id} table').DataTable();
            dt.page.len(-1).draw();
        }} catch(e) {{
            var sel = document.querySelector('#{tab_id} select[name$="_length"]');
            if (sel) {{
                var best = 0;
                for (var i = 0; i < sel.options.length; i++) {{
                    var v = parseInt(sel.options[i].value);
                    if (v === -1 || v > parseInt(sel.options[best].value)) best = i;
                }}
                sel.selectedIndex = best;
                sel.dispatchEvent(new Event('change', {{bubbles: true}}));
            }}
        }}
    """)
    time.sleep(1.2)


def parse_html_table(html: str) -> Optional[pd.DataFrame]:
    soup = BeautifulSoup(html, "lxml")
    table = soup.find("table")
    if not table:
        return None
    headers = []
    thead = table.find("thead")
    if thead:
        headers = [th.get_text(strip=True) for th in thead.find_all(["th", "td"])]
    rows = []
    tbody = table.find("tbody")
    if tbody:
        for tr in tbody.find_all("tr"):
            cells = [td.get_text(strip=True) for td in tr.find_all(["th", "td"])]
            if any(c.strip() for c in cells):
                rows.append(cells)
    if not rows:
        return None
    return pd.DataFrame(rows, columns=headers if headers else None)


def get_full_table(driver: webdriver.Chrome, tab_id: str) -> Optional[pd.DataFrame]:
    frames: list[pd.DataFrame] = []
    seen: set[str] = set()
    while True:
        tab_elem = driver.find_element(By.ID, tab_id)
        html = tab_elem.get_attribute("innerHTML")
        fp = html[:800]
        if fp in seen:
            break
        seen.add(fp)
        df = parse_html_table(html)
        if df is not None:
            frames.append(df)
        try:
            next_btn = driver.find_element(
                By.CSS_SELECTOR, f"#{tab_id} .dataTables_paginate .next"
            )
            if "disabled" in (next_btn.get_attribute("class") or ""):
                break
            driver.execute_script("arguments[0].click();", next_btn)
            time.sleep(0.5)
        except Exception:
            break
    if not frames:
        return None
    if len(frames) == 1:
        return frames[0]
    return pd.concat(frames, ignore_index=True).drop_duplicates()


def extract_tab(driver: webdriver.Chrome, wait: WebDriverWait, tab_id: str) -> Optional[pd.DataFrame]:
    link = driver.find_element(By.CSS_SELECTOR, f"a[href='#{tab_id}']")
    driver.execute_script("arguments[0].click();", link)
    try:
        wait.until(lambda d: "active" in (
            d.find_element(By.ID, tab_id).get_attribute("class") or ""
        ))
    except Exception:
        time.sleep(1.5)
    show_all_rows(driver, tab_id)
    return get_full_table(driver, tab_id)


# ---------------------------------------------------------------------------
# Saída
# ---------------------------------------------------------------------------

def make_filename(tab_name: str, localidade: str, start: date, end: date) -> str:
    safe = re.sub(r'[<>:"/\\|?*\r\n]', '', localidade).strip()
    s = start.strftime("%d%m%Y")
    e = end.strftime("%d%m%Y")
    return f"Dados de Superfície {tab_name} - Localidade {safe} - Período {s}  -  {e}.csv"


def save_csv(df: pd.DataFrame, path: str) -> None:
    df.to_csv(path, index=False, encoding="utf-8-sig", quoting=csv.QUOTE_ALL)


# ---------------------------------------------------------------------------
# Lógica principal de scraping
# ---------------------------------------------------------------------------

def _quarter_chunks(year: int) -> list[tuple[date, date]]:
    """Divide o ano em chunks de CHUNK_MONTHS meses (do mais recente para o mais antigo)."""
    chunks = []
    month = 1
    while month <= 12:
        end_month = min(month + CHUNK_MONTHS - 1, 12)
        last_day  = calendar.monthrange(year, end_month)[1]
        chunks.append((date(year, month, 1), date(year, end_month, last_day)))
        month = end_month + 1
    return list(reversed(chunks))   # Q4 → Q3 → Q2 → Q1


def scrape_period(
    driver: webdriver.Chrome,
    wait: WebDriverWait,
    aerodrome: str,
    start: date,
    end: date,
    output_dir: str,
    log: Callable[[str], None] = print,
) -> bool:
    """
    Faz o scraping de um único período e salva os CSVs das 9 abas.
    Retorna True se pelo menos uma aba teve dados, False se tudo veio vazio.
    """
    log(f"  [{start.strftime('%d/%m/%Y')} -> {end.strftime('%d/%m/%Y')}]")

    driver.get(BASE_URL)
    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "div.bootstrap-select")))
    fetch_site_max_date(driver)   # atualiza SITE_MAX_DATE com o valor real do site
    time.sleep(1.5)

    localidade = select_aerodrome(driver, wait, aerodrome)
    if not localidade:
        log(f"  ERR  Aerodrome '{aerodrome}' nao encontrado.")
        return False
    log(f"  Localidade: {localidade}")

    set_period(driver, start, end)
    submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[type='submit']")))
    submit.click()
    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".nav-tabs")))
    time.sleep(3)

    found_any = False
    for tab_id, tab_name in TABS:
        try:
            df = extract_tab(driver, wait, tab_id)
            if df is not None and not df.empty:
                found_any = True
                fname = make_filename(tab_name, localidade, start, end)
                fpath = os.path.join(output_dir, fname)
                save_csv(df, fpath)
                log(f"    OK  {tab_name:<15s} {len(df):>6} linhas  ->  {fname}")
            else:
                log(f"    --  {tab_name:<15s} sem dados")
        except Exception as exc:
            log(f"    ERR {tab_name:<15s} {exc}")

    return found_any


def scrape_year(
    driver: webdriver.Chrome,
    wait: WebDriverWait,
    aerodrome: str,
    year: int,
    output_dir: str,
    stop_before: Optional[date] = None,
    log: Callable[[str], None] = print,
) -> int:
    """
    Itera os chunks trimestrais do ano (do mais recente para o mais antigo).
    Para se o trimestre for anterior a stop_before (data já coberta).
    Retorna o número de trimestres que tiveram dados.
    """
    non_empty = 0
    for chunk_start, chunk_end in _quarter_chunks(year):
        start = max(chunk_start, SITE_MIN_DATE)
        end   = min(chunk_end,   SITE_MAX_DATE)
        if start > end:
            continue
        # Para quando atingir dados já cobertos pelo arquivo existente
        if stop_before is not None and end <= stop_before:
            log(f"  [trimestre {start.strftime('%d/%m/%Y')} -> {end.strftime('%d/%m/%Y')} já coberto, parando]")
            break
        had_data = scrape_period(driver, wait, aerodrome, start, end, output_dir, log)
        if had_data:
            non_empty += 1
    return non_empty


# ---------------------------------------------------------------------------
# Entry point (CLI standalone)
# ---------------------------------------------------------------------------

def _fix_stdout() -> None:
    """Garante UTF-8 no terminal Windows ao rodar como script."""
    try:
        if hasattr(sys.stdout, "buffer"):
            sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
        if hasattr(sys.stderr, "buffer"):
            sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")
    except Exception:
        pass


def main() -> None:
    _fix_stdout()
    parser = argparse.ArgumentParser(
        description="Scraper de meteorologia de superfície — ICEA/DECEA",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
exemplos:
  python scraper_meteorologia.py --aerodrome SBGR --all-years
  python scraper_meteorologia.py --aerodrome SBGR --start-year 2020 --end-year 2025
  python scraper_meteorologia.py --aerodrome SBSP --start-year 2023 --end-year 2023 --no-headless
        """,
    )
    parser.add_argument("--aerodrome",     default="SBGR", metavar="ICAO",
                        help="Código ICAO do aeródromo (padrão: SBGR)")
    parser.add_argument("--all-years",     action="store_true",
                        help="Coleta todos os anos disponíveis (para automaticamente sem dados)")
    parser.add_argument("--start-year",    type=int, metavar="ANO",
                        help="Ano inicial (obrigatório sem --all-years)")
    parser.add_argument("--end-year",      type=int, metavar="ANO",
                        help="Ano final (obrigatório sem --all-years)")
    parser.add_argument("--output-dir",    default="dados", metavar="DIR",
                        help="Diretório de saída (padrão: dados/)")
    parser.add_argument("--no-headless",   action="store_true",
                        help="Abre janela do Chrome visível (útil para debug)")
    parser.add_argument("--fetch-catalog", action="store_true",
                        help="Baixa lista de aeródromos do site ICEA e grava no banco SQLite")
    parser.add_argument("--db-path",       default="dados/met.db", metavar="PATH",
                        help="Caminho do banco SQLite (usado com --fetch-catalog)")
    args = parser.parse_args()

    # ── fetch-catalog: ação independente ──────────────────────────────────────
    if args.fetch_catalog:
        from pathlib import Path
        import db as _db
        print("Acessando site ICEA para obter lista de aeródromos…")
        catalog = fetch_aerodrome_catalog(headless=not args.no_headless)
        print(f"  {len(catalog)} aeródromos encontrados.")
        db_path = Path(args.db_path)
        db_path.parent.mkdir(parents=True, exist_ok=True)
        conn = _db.get_connection(db_path)
        _db.ensure_schema(conn)
        n = _db.upsert_aerodromes(conn, catalog)
        conn.close()
        print(f"  {n} registros gravados em '{db_path}'.")
        return

    if not args.all_years and (args.start_year is None or args.end_year is None):
        parser.error("Use --all-years ou informe --start-year e --end-year")

    end_year   = args.end_year   if not args.all_years else date.today().year
    start_year = args.start_year if not args.all_years else SITE_MIN_DATE.year

    os.makedirs(args.output_dir, exist_ok=True)
    driver = make_driver(headless=not args.no_headless)
    wait   = WebDriverWait(driver, 60)

    try:
        consecutive_empty = 0
        for year in range(end_year, start_year - 1, -1):   # do mais recente para o mais antigo
            print(f"\n{'=' * 60}")
            print(f"Ano {year}  —  {args.aerodrome}")
            print(f"{'=' * 60}")
            non_empty = scrape_year(driver, wait, args.aerodrome, year, args.output_dir)
            if non_empty == 0:
                consecutive_empty += 1
                print(f"  [sem dados] ({consecutive_empty}/{STOP_EMPTY_YEARS} consecutivos)")
                if args.all_years and consecutive_empty >= STOP_EMPTY_YEARS:
                    print(f"\n{STOP_EMPTY_YEARS} anos consecutivos sem dados — parando busca.")
                    break
            else:
                consecutive_empty = 0
    except KeyboardInterrupt:
        print("\nInterrompido pelo usuário.")
    finally:
        driver.quit()
        print("\nFinalizado.")


if __name__ == "__main__":
    main()