V.1.0

softwares/test/meteorologia_aeroportos/README.md

@@ -8,8 +8,17 @@ Coleta, processa e visualiza dados meteorológicos horários de aeródromos bras
 
 Clone o repositório, entre na pasta do módulo e execute:
 
-**Windows**
+```bash
+cd softwares/test/meteorologia_aeroportos
 ```
+
+**Windows (PowerShell)**
+```powershell
+.\run\run.bat
+```
+
+**Windows (CMD)**
+```cmd
 run\run.bat
 ```
 
@@ -21,6 +30,8 @@ chmod +x run/run.sh && ./run/run.sh
 O script cria o ambiente virtual, instala as dependências e abre o dashboard em **http://localhost:8501**.
 
 > **Pré-requisito:** Python 3.10+ e Google Chrome instalados.
+>
+> No PowerShell o `.\` antes do caminho é obrigatório para executar arquivos `.bat` por caminho relativo.
 
 ---
 

softwares/test/meteorologia_aeroportos/_apps/collect_all.py

@@ -0,0 +1,457 @@
+"""
+Bulk data collection for all aerodromes in the ICEA surface-data catalog.
+
+Designed as a one-time migration script to bootstrap the database with the
+full available history for every aerodrome on the portal.  A JSON status
+file (``db/collect_all_status.json``) tracks progress so interrupted runs
+can be resumed with ``--resume``.
+
+Usage:
+    # First time: fetch catalog, then collect everything
+    python _apps/collect_all.py --fetch-catalog
+    python _apps/collect_all.py
+
+    # Resume after interruption (skip aerodromes already with data)
+    python _apps/collect_all.py --resume
+
+    # Force re-collection of all aerodromes
+    python _apps/collect_all.py --force
+
+    # Specific aerodromes only
+    python _apps/collect_all.py --aerodrome SBSP SBGL SBRJ
+
+    # Dry run / test (first 3 aerodromes only)
+    python _apps/collect_all.py --max-aerodromes 3
+
+    # Visible Chrome (useful for debugging portal issues)
+    python _apps/collect_all.py --no-headless --aerodrome SBSP
+"""
+
+import argparse
+import io
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable, Optional
+
+# ── Paths ─────────────────────────────────────────────────────────────────────
+_APPS_DIR   = Path(__file__).resolve().parent          # .../meteorologia_aeroportos/_apps/
+_BASE_DIR   = _APPS_DIR.parent                         # .../meteorologia_aeroportos/
+_REPO_ROOT  = _BASE_DIR.parents[2]                     # dataset/
+DADOS_DIR   = _BASE_DIR / "db" / "dados"
+DB_PATH     = _BASE_DIR / "db" / "met.db"
+PREPROC_DIR = _REPO_ROOT / "tabelas" / "preproc" / "meteorologia_aeroportos"
+STATUS_FILE = _BASE_DIR / "db" / "collect_all_status.json"
+DEFAULT_LOG = _BASE_DIR / "db" / "collect_all.log"
+
+
+# ── Status helpers ────────────────────────────────────────────────────────────
+
+def _load_status() -> dict:
+    """Loads the JSON status file, returning an empty dict on any error."""
+    if STATUS_FILE.exists():
+        try:
+            return json.loads(STATUS_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            return {}
+    return {}
+
+
+def _save_status(status: dict) -> None:
+    """Persists the status dict to the JSON status file.
+
+    Args:
+        status: Dict mapping ICAO codes to their collection state.
+    """
+    STATUS_FILE.parent.mkdir(parents=True, exist_ok=True)
+    STATUS_FILE.write_text(
+        json.dumps(status, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+
+
+# ── Logging ───────────────────────────────────────────────────────────────────
+
+def _make_logger(log_path: Optional[Path]) -> Callable[[str], None]:
+    """Returns a logger that writes to stdout and optionally to a file.
+
+    Args:
+        log_path: File to append log lines to, or ``None`` for stdout only.
+
+    Returns:
+        A callable that accepts a single string message.
+    """
+    def _log(msg: str) -> None:
+        print(msg)
+        if log_path:
+            try:
+                with log_path.open("a", encoding="utf-8") as f:
+                    f.write(msg + "\n")
+            except Exception:
+                pass
+    return _log
+
+
+# ── Catalog helpers ───────────────────────────────────────────────────────────
+
+def fetch_and_store_catalog(headless: bool = True, log: Callable = print) -> int:
+    """Downloads the aerodrome catalog from the ICEA site and upserts it.
+
+    Args:
+        headless: Run Chrome in headless mode.
+        log: Logging callback.
+
+    Returns:
+        Number of aerodrome records stored.
+    """
+    import db as _db
+    from scraper_meteorologia import fetch_aerodrome_catalog
+
+    log("Atualizando catálogo do site ICEA…")
+    catalog = fetch_aerodrome_catalog(headless=headless)
+    log(f"  {len(catalog)} aeródromos encontrados no portal.")
+
+    conn = _db.get_connection(DB_PATH)
+    _db.ensure_schema(conn)
+    n = _db.upsert_aerodromes(conn, catalog)
+    conn.close()
+
+    log(f"  {n} registros gravados no catálogo.")
+    return n
+
+
+def _get_catalog() -> list[dict]:
+    """Returns all entries from the local aerodrome catalog table.
+
+    Returns:
+        List of ``{"icao", "nome", "uf"}`` dicts.
+    """
+    import db as _db
+    conn = _db.get_connection(DB_PATH)
+    _db.ensure_schema(conn)
+    catalog = _db.list_all_aerodromes(conn)
+    conn.close()
+    return catalog
+
+
+def _aerodromes_with_data() -> set[str]:
+    """Returns the set of ICAO codes that have at least one observation row.
+
+    Returns:
+        Set of ICAO code strings.
+    """
+    import db as _db
+    conn = _db.get_connection(DB_PATH)
+    _db.ensure_schema(conn)
+    aeros = set(_db.list_aerodromes(conn))
+    conn.close()
+    return aeros
+
+
+# ── Core collection loop ──────────────────────────────────────────────────────
+
+def collect_all(
+    target_aerodromes: Optional[list[str]] = None,
+    resume: bool = True,
+    force: bool = False,
+    max_aerodromes: Optional[int] = None,
+    headless: bool = True,
+    n_samples: int = 5,
+    do_cleanup: bool = True,
+    log: Callable[[str], None] = print,
+    on_start: Optional[Callable[[str, int, int, str], None]] = None,
+    on_done: Optional[Callable[[str, str, dict], None]] = None,
+) -> dict:
+    """Collects full historical data for every aerodrome in the catalog.
+
+    For each aerodrome, the full pipeline (``all_years=True``) is run.
+    Progress is saved to ``db/collect_all_status.json`` after every aerodrome
+    so the run can be resumed if interrupted.
+
+    Args:
+        target_aerodromes: Specific ICAO codes to process, or ``None`` for all.
+        resume: Skip aerodromes that already have data in the database.
+        force: Re-collect aerodromes even if they already have data.
+            Overrides *resume*.
+        max_aerodromes: Cap the number of aerodromes processed (useful for
+            testing).
+        headless: Run Chrome without a visible window.
+        n_samples: Spot-check sample size per aerodrome (0 disables validation).
+        do_cleanup: Remove intermediate quarterly CSVs after processing.
+        log: Logging callback.
+        on_start: Optional callback fired before each aerodrome starts.
+            Signature: ``on_start(icao, idx, total, nome)``.
+        on_done: Optional callback fired after each aerodrome finishes.
+            Signature: ``on_done(icao, event, result)`` where *event* is one
+            of ``"completed"``, ``"skipped"``, ``"failed"``,
+            ``"interrupted"``.
+
+    Returns:
+        Summary dict with keys ``n_total``, ``n_done``, ``n_skipped``,
+        ``n_failed``, ``elapsed_s``.
+    """
+    from pipeline import run_pipeline
+
+    status  = _load_status()
+    catalog = _get_catalog()
+
+    if not catalog:
+        log(
+            "[collect_all] Catálogo vazio.\n"
+            "  Execute: python _apps/collect_all.py --fetch-catalog"
+        )
+        return {"n_total": 0, "n_done": 0, "n_skipped": 0, "n_failed": 0, "elapsed_s": 0}
+
+    # Filter to requested aerodromes
+    if target_aerodromes:
+        icao_set = {a.upper() for a in target_aerodromes}
+        catalog = [d for d in catalog if d["icao"] in icao_set]
+        missing = icao_set - {d["icao"] for d in catalog}
+        if missing:
+            log(f"[collect_all] AVISO — não encontrados no catálogo: {sorted(missing)}")
+
+    if max_aerodromes:
+        catalog = catalog[:max_aerodromes]
+
+    existing_data = _aerodromes_with_data() if (resume and not force) else set()
+
+    total   = len(catalog)
+    n_done  = 0
+    n_skip  = 0
+    n_fail  = 0
+    started = datetime.now(timezone.utc)
+
+    log(f"\n{'='*60}")
+    log(f"collect_all  —  {total} aeródromos  —  {started.strftime('%Y-%m-%d %H:%M UTC')}")
+    log(f"resume={resume}  force={force}  headless={headless}  n_samples={n_samples}")
+    log(f"{'='*60}\n")
+
+    for idx, aero_dict in enumerate(catalog, 1):
+        icao = aero_dict["icao"]
+        nome = aero_dict.get("nome", "")
+        label = f"{icao}  ({nome})" if nome else icao
+
+        log(f"\n[{idx:>4}/{total}]  {label}")
+        if on_start:
+            on_start(icao, idx, total, nome)
+
+        # Resume logic: skip aerodromes that already have data
+        if not force and icao in existing_data:
+            log(f"  SKIP  dados existentes no banco")
+            status[icao] = {"status": "skipped", "reason": "data_exists"}
+            _save_status(status)
+            if on_done:
+                on_done(icao, "skipped", {})
+            n_skip += 1
+            continue
+
+        # Mark as in-progress
+        status[icao] = {
+            "status":     "in_progress",
+            "started_at": datetime.now(timezone.utc).isoformat(),
+        }
+        _save_status(status)
+
+        try:
+            result = run_pipeline(
+                aerodrome   = icao,
+                dados_dir   = DADOS_DIR,
+                db_path     = DB_PATH,
+                preproc_dir = PREPROC_DIR,
+                all_years   = True,
+                headless    = headless,
+                n_samples   = n_samples,
+                do_validate = n_samples > 0,
+                do_cleanup  = do_cleanup,
+                log         = log,
+                progress    = lambda pct, msg: None,
+            )
+            rows  = result.get("rows", 0)
+            p_min = result.get("period_start", "")
+            p_max = result.get("period_end",   "")
+            log(f"  DONE  {rows:,} observações  |  {p_min} → {p_max}")
+            status[icao] = {
+                "status":       "completed",
+                "rows":         rows,
+                "period_start": p_min,
+                "period_end":   p_max,
+                "completed_at": datetime.now(timezone.utc).isoformat(),
+            }
+            if on_done:
+                on_done(icao, "completed", result)
+            n_done += 1
+
+        except KeyboardInterrupt:
+            log("\n[collect_all] Interrompido pelo usuário.")
+            status[icao]["status"] = "interrupted"
+            _save_status(status)
+            if on_done:
+                on_done(icao, "interrupted", {})
+            break
+
+        except Exception as exc:
+            err = str(exc)[:300]
+            log(f"  ERRO  {err}")
+            status[icao] = {
+                "status":    "failed",
+                "error":     err,
+                "failed_at": datetime.now(timezone.utc).isoformat(),
+            }
+            if on_done:
+                on_done(icao, "failed", {"error": err})
+            n_fail += 1
+
+        _save_status(status)
+
+    elapsed = (datetime.now(timezone.utc) - started).total_seconds()
+
+    log(f"\n{'='*60}")
+    log(
+        f"Resultado  |  total={total}  done={n_done}  "
+        f"skipped={n_skip}  failed={n_fail}  "
+        f"elapsed={elapsed / 3600:.1f}h"
+    )
+    log(f"Status gravado em: {STATUS_FILE}")
+    log(f"{'='*60}\n")
+
+    return {
+        "n_total":   total,
+        "n_done":    n_done,
+        "n_skipped": n_skip,
+        "n_failed":  n_fail,
+        "elapsed_s": elapsed,
+    }
+
+
+# ── Status report ─────────────────────────────────────────────────────────────
+
+def print_status_report() -> None:
+    """Prints a human-readable summary of the collect_all status file."""
+    status = _load_status()
+    if not status:
+        print("Nenhum status encontrado. Execute collect_all.py primeiro.")
+        return
+
+    from collections import Counter
+    counts: Counter = Counter(v.get("status", "unknown") for v in status.values())
+
+    print(f"\n=== collect_all status: {STATUS_FILE} ===\n")
+    for state, n in sorted(counts.items()):
+        print(f"  {state:<15} {n:>5}")
+    print(f"  {'TOTAL':<15} {sum(counts.values()):>5}")
+
+    failed = [(k, v) for k, v in status.items() if v.get("status") == "failed"]
+    if failed:
+        print(f"\n  Falhas ({len(failed)}):")
+        for icao, info in failed:
+            print(f"    {icao}: {info.get('error','?')[:80]}")
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    """Parses arguments and runs the bulk collection or a status report."""
+    try:
+        if hasattr(sys.stdout, "buffer"):
+            sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
+        if hasattr(sys.stderr, "buffer"):
+            sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")
+    except Exception:
+        pass
+
+    parser = argparse.ArgumentParser(
+        description="Bulk ICEA aerodrome data collection (one-time migration)",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+examples:
+  # Step 1 — populate catalog
+  python _apps/collect_all.py --fetch-catalog
+
+  # Step 2 — collect all (skip aerodromes that already have data)
+  python _apps/collect_all.py
+
+  # Resume after interruption
+  python _apps/collect_all.py --resume
+
+  # Re-collect everything from scratch
+  python _apps/collect_all.py --force
+
+  # Specific aerodromes only
+  python _apps/collect_all.py --aerodrome SBSP SBGL SBRJ
+
+  # Test run: first 3 aerodromes, visible Chrome
+  python _apps/collect_all.py --max-aerodromes 3 --no-headless
+
+  # Show collection status
+  python _apps/collect_all.py --status
+        """,
+    )
+    parser.add_argument(
+        "--fetch-catalog", action="store_true",
+        help="Fetch/update aerodrome catalog from the ICEA site before collecting",
+    )
+    parser.add_argument(
+        "--aerodrome", nargs="+", metavar="ICAO",
+        help="Collect only these ICAO codes (default: all catalog entries)",
+    )
+    parser.add_argument(
+        "--resume", action="store_true", default=True,
+        help="Skip aerodromes that already have data in the database (default: on)",
+    )
+    parser.add_argument(
+        "--no-resume", dest="resume", action="store_false",
+        help="Re-attempt aerodromes even if they already have data",
+    )
+    parser.add_argument(
+        "--force", action="store_true",
+        help="Re-collect aerodromes even if they already have data (stronger than --no-resume)",
+    )
+    parser.add_argument(
+        "--max-aerodromes", type=int, metavar="N",
+        help="Limit processing to the first N catalog entries (testing)",
+    )
+    parser.add_argument(
+        "--no-headless", action="store_true",
+        help="Open Chrome in visible mode (useful for debugging)",
+    )
+    parser.add_argument(
+        "--n-samples", type=int, default=5,
+        help="Spot-check validation sample size per aerodrome (0 = skip, default: 5)",
+    )
+    parser.add_argument(
+        "--no-cleanup", action="store_true",
+        help="Keep intermediate quarterly CSVs after processing",
+    )
+    parser.add_argument(
+        "--log", metavar="PATH", default=str(DEFAULT_LOG),
+        help=f"Log file path (default: {DEFAULT_LOG})",
+    )
+    parser.add_argument(
+        "--status", action="store_true",
+        help="Print a summary of the collect_all status file and exit",
+    )
+    args = parser.parse_args()
+
+    if args.status:
+        print_status_report()
+        return
+
+    log = _make_logger(Path(args.log))
+
+    if args.fetch_catalog:
+        fetch_and_store_catalog(headless=not args.no_headless, log=log)
+
+    collect_all(
+        target_aerodromes = args.aerodrome,
+        resume            = args.resume,
+        force             = args.force,
+        max_aerodromes    = args.max_aerodromes,
+        headless          = not args.no_headless,
+        n_samples         = args.n_samples,
+        do_cleanup        = not args.no_cleanup,
+        log               = log,
+    )
+
+
+if __name__ == "__main__":
+    main()

softwares/test/meteorologia_aeroportos/_apps/dashboard.py

@@ -1281,6 +1281,342 @@ with tab0:
                         with st.expander("Divergências"):
                             for e in r["errors"]: st.text(e)
 
+    # ── Atualização Total ─────────────────────────────────────────────────────
+    st.divider()
+    st.markdown("### 🌐 Atualização Total")
+    st.markdown(
+        "**O que faz:** verifica novos aeródromos no catálogo ICEA "
+        "e baixa **apenas os dados novos** para cada aeródromo que já "
+        "possui observações no banco. Recomendado a cada 2 meses.\n\n"
+        "> **Nota:** aeródromos que ainda **não têm dados** no banco não são "
+        "coletados aqui — apenas adicionados ao catálogo. "
+        "Para coletar o histórico completo de todos os aeródromos, "
+        "use o script `_apps/collect_all.py` (pode levar horas/dias)."
+    )
+
+    _at_c1, _at_c2, _ = st.columns([1, 1, 3])
+    with _at_c1:
+        _at_headless = not st.checkbox("Mostrar Chrome", value=False, key="at_headless")
+    with _at_c2:
+        _at_validate = st.checkbox("Validar amostras", value=False, key="at_validate")
+
+    _at_submitted = st.button(
+        "🌐 Iniciar Atualização Total",
+        type="primary",
+        help="Atualiza catálogo + coleta dados novos para todos os aeródromos com dados no banco",
+    )
+
+    if _at_submitted:
+        import db as _db_at
+
+        _at_conn_check = _db_at.get_connection(DB_PATH)
+        _db_at.ensure_schema(_at_conn_check)
+        _at_aeros = _db_at.list_aerodromes_with_data(_at_conn_check)
+        _at_conn_check.close()
+
+        if not _at_aeros:
+            st.warning(
+                "Nenhum aeródromo com dados no banco. "
+                "Use o formulário acima para coletar dados primeiro."
+            )
+        else:
+            _at_n_total  = len(_at_aeros)
+            _at_q: queue.Queue            = queue.Queue()
+            _at_results: list[dict]       = []
+            _at_log_lines: list[str]      = []
+
+            _at_prog_bar  = st.progress(0.0)
+            _at_stat_txt  = st.empty()
+            _at_log_box   = st.empty()
+            _at_table_box = st.empty()
+
+            _at_aero_list = [d["icao"] for d in _at_aeros]
+            _at_aero_meta = {d["icao"]: d for d in _at_aeros}
+
+            def _at_worker() -> None:
+                try:
+                    # Phase 1 — refresh aerodrome catalog
+                    _at_q.put(("status", "Fase 1 — Atualizando catálogo ICEA…"))
+                    from scraper_meteorologia import fetch_aerodrome_catalog as _fat
+                    import db as _dbi
+                    _new_cat = _fat(headless=_at_headless)
+                    _cat_conn = _dbi.get_connection(DB_PATH)
+                    _dbi.ensure_schema(_cat_conn)
+                    _n_new = _dbi.upsert_aerodromes(_cat_conn, _new_cat)
+                    _cat_conn.close()
+                    _at_q.put(("log",
+                        f"[catálogo] {len(_new_cat)} aeródromos no portal  "
+                        f"|  {_n_new} registros atualizados"
+                    ))
+
+                    # Phase 2 — forward-only update for each aerodrome
+                    for _i, _icao in enumerate(_at_aero_list):
+                        _at_q.put(("aero_start", _icao, _i))
+                        try:
+                            from pipeline import run_pipeline as _rp
+                            _res = _rp(
+                                aerodrome   = _icao,
+                                dados_dir   = DADOS_DIR,
+                                db_path     = DB_PATH,
+                                preproc_dir = PREPROC_DIR,
+                                all_years   = False,
+                                update_only = True,
+                                headless    = _at_headless,
+                                n_samples   = 5 if _at_validate else 0,
+                                do_validate = _at_validate,
+                                do_cleanup  = True,
+                                log         = lambda m: _at_q.put(("log", m)),
+                                progress    = lambda p, m: None,
+                            )
+                            _at_q.put(("aero_done", _icao, _res))
+                        except Exception as _exc:
+                            _at_q.put(("aero_error", _icao, str(_exc)[:120]))
+
+                except Exception as _exc:
+                    _at_q.put(("log", f"ERRO: {_exc}"))
+                finally:
+                    _at_q.put(("done",))
+
+            _at_t = threading.Thread(target=_at_worker, daemon=True)
+            _at_t.start()
+
+            _at_n_done = 0
+
+            while _at_t.is_alive() or not _at_q.empty():
+                try:
+                    _item = _at_q.get(timeout=0.4)
+                except queue.Empty:
+                    continue
+
+                if _item[0] == "done":
+                    break
+
+                elif _item[0] == "status":
+                    _at_stat_txt.caption(_item[1])
+
+                elif _item[0] == "log":
+                    _at_log_lines.append(_item[1])
+                    _at_log_box.code("\n".join(_at_log_lines[-60:]))
+
+                elif _item[0] == "aero_start":
+                    _, _icao, _i = _item
+                    _at_prog_bar.progress(_i / max(_at_n_total, 1))
+                    _at_stat_txt.caption(
+                        f"Fase 2 — {_icao}  ({_i + 1} / {_at_n_total})"
+                    )
+                    _meta = _at_aero_meta.get(_icao, {})
+                    _at_results.append({
+                        "ICAO":   _icao,
+                        "Nome":   _meta.get("nome", ""),
+                        "UF":     _meta.get("uf", ""),
+                        "Status": "⏳ Coletando…",
+                        "Novas linhas": "—",
+                    })
+                    _at_table_box.dataframe(
+                        _at_results, use_container_width=True, hide_index=True
+                    )
+
+                elif _item[0] == "aero_done":
+                    _, _icao, _res = _item
+                    _at_n_done += 1
+                    _new = _res.get("n_upserted", 0)
+                    for _row in _at_results:
+                        if _row["ICAO"] == _icao:
+                            _row["Status"]       = "✅ Concluído"
+                            _row["Novas linhas"] = f"{_new:,}"
+                            break
+                    _at_table_box.dataframe(
+                        _at_results, use_container_width=True, hide_index=True
+                    )
+
+                elif _item[0] == "aero_error":
+                    _, _icao, _err = _item
+                    for _row in _at_results:
+                        if _row["ICAO"] == _icao:
+                            _row["Status"]       = "❌ Erro"
+                            _row["Novas linhas"] = _err[:50]
+                            break
+                    _at_table_box.dataframe(
+                        _at_results, use_container_width=True, hide_index=True
+                    )
+
+            _at_t.join()
+            _at_prog_bar.progress(1.0)
+            st.cache_data.clear()
+            _at_stat_txt.empty()
+            st.success(
+                f"✅ Atualização Total concluída — "
+                f"{_at_n_done} / {_at_n_total} aeródromos atualizados."
+            )
+            st.rerun()
+
+    # ── Coleta Total — Histórico Completo ─────────────────────────────────────
+    st.divider()
+    st.markdown("### 🗄️ Coleta Total — Histórico Completo")
+    st.markdown(
+        "Coleta o **histórico completo** de todos os aeródromos do catálogo "
+        "que ainda **não têm dados** no banco. "
+        "Aeródromos com dados são ignorados automaticamente.\n\n"
+        "> ⚠️ Esta operação pode demorar **horas ou dias** dependendo do número "
+        "de aeródromos. Não feche o dashboard durante a coleta. "
+        "Em caso de interrupção, basta clicar novamente — os aeródromos já "
+        "coletados serão pulados automaticamente."
+    )
+
+    # Status summary
+    if _db_available():
+        import db as _db_ct_info
+        _ct_conn_info = _db_ct_info.get_connection(DB_PATH)
+        _db_ct_info.ensure_schema(_ct_conn_info)
+        _ct_catalog   = _db_ct_info.list_all_aerodromes(_ct_conn_info)
+        _ct_with_data = set(_db_ct_info.list_aerodromes(_ct_conn_info))
+        _ct_conn_info.close()
+        _ct_total     = len(_ct_catalog)
+        _ct_done      = len(_ct_with_data)
+        _ct_pending   = _ct_total - _ct_done
+    else:
+        _ct_total = _ct_done = _ct_pending = 0
+
+    _ct_info_col1, _ct_info_col2, _ct_info_col3 = st.columns(3)
+    _ct_info_col1.metric("No catálogo",    _ct_total)
+    _ct_info_col2.metric("Com dados",      _ct_done)
+    _ct_info_col3.metric("Aguardando coleta", _ct_pending)
+
+    _ct_opt1, _ct_opt2, _ct_opt3 = st.columns([1, 1, 2])
+    with _ct_opt1:
+        _ct_headless = not st.checkbox("Mostrar Chrome", value=False, key="ct_headless")
+    with _ct_opt2:
+        _ct_max = int(st.number_input(
+            "Limite de aeródromos (0 = todos)",
+            min_value=0, max_value=200, value=0, step=1, key="ct_max",
+            help="Útil para testes. 0 processa todos os pendentes.",
+        ))
+
+    if _ct_pending == 0 and _ct_total > 0:
+        st.info("✅ Todos os aeródromos do catálogo já têm dados no banco.")
+    elif _ct_total == 0:
+        st.warning(
+            "Catálogo vazio. Clique em **Atualizar catálogo ICEA** (acima) para "
+            "baixar a lista de aeródromos disponíveis."
+        )
+
+    _ct_submitted = st.button(
+        "🗄️ Iniciar Coleta Total",
+        type="primary",
+        disabled=(_ct_pending == 0),
+        help="Coleta o histórico completo para os aeródromos sem dados no banco",
+    )
+
+    if _ct_submitted and _ct_pending > 0:
+        _ct_q: queue.Queue       = queue.Queue()
+        _ct_results: list[dict]  = []
+        _ct_log_lines: list[str] = []
+
+        _ct_prog_bar  = st.progress(0.0)
+        _ct_stat_txt  = st.empty()
+        _ct_log_box   = st.empty()
+        _ct_table_box = st.empty()
+
+        # Build metadata dict for display
+        _ct_meta = {d["icao"]: d for d in _ct_catalog}
+        _ct_max_arg = _ct_max if _ct_max > 0 else None
+
+        def _ct_worker() -> None:
+            try:
+                from collect_all import collect_all as _ca
+
+                def _ct_on_start(icao: str, idx: int, total: int, nome: str) -> None:
+                    _ct_q.put(("aero_start", icao, idx, total, nome))
+
+                def _ct_on_done(icao: str, event: str, result: dict) -> None:
+                    _ct_q.put(("aero_done", icao, event, result))
+
+                _ca(
+                    resume          = True,
+                    max_aerodromes  = _ct_max_arg,
+                    headless        = _ct_headless,
+                    n_samples       = 0,
+                    do_cleanup      = True,
+                    log             = lambda m: _ct_q.put(("log", m)),
+                    on_start        = _ct_on_start,
+                    on_done         = _ct_on_done,
+                )
+            except Exception as _exc:
+                _ct_q.put(("log", f"ERRO: {_exc}"))
+            finally:
+                _ct_q.put(("done",))
+
+        _ct_t = threading.Thread(target=_ct_worker, daemon=True)
+        _ct_t.start()
+
+        _ct_n_done = 0
+        _ct_n_total_run = _ct_max_arg or _ct_pending
+
+        while _ct_t.is_alive() or not _ct_q.empty():
+            try:
+                _citem = _ct_q.get(timeout=0.4)
+            except queue.Empty:
+                continue
+
+            if _citem[0] == "done":
+                break
+
+            elif _citem[0] == "log":
+                _ct_log_lines.append(_citem[1])
+                _ct_log_box.code("\n".join(_ct_log_lines[-80:]))
+
+            elif _citem[0] == "aero_start":
+                _, _icao, _idx, _tot, _nome = _citem
+                _ct_prog_bar.progress((_idx - 1) / max(_tot, 1))
+                _ct_stat_txt.caption(f"Coletando {_icao} — {_idx} / {_tot}")
+                _meta = _ct_meta.get(_icao, {})
+                _ct_results.append({
+                    "ICAO":    _icao,
+                    "Nome":    _meta.get("nome", _nome),
+                    "UF":      _meta.get("uf", ""),
+                    "Status":  "⏳ Coletando…",
+                    "Período": "—",
+                    "Linhas":  "—",
+                })
+                _ct_table_box.dataframe(
+                    _ct_results, use_container_width=True, hide_index=True
+                )
+
+            elif _citem[0] == "aero_done":
+                _, _icao, _event, _res = _citem
+                _status_map = {
+                    "completed":   "✅ Concluído",
+                    "skipped":     "⏭️ Já tinha dados",
+                    "failed":      "❌ Erro",
+                    "interrupted": "⏸️ Interrompido",
+                }
+                _st = _status_map.get(_event, _event)
+                for _row in _ct_results:
+                    if _row["ICAO"] == _icao:
+                        _row["Status"]  = _st
+                        if _event == "completed":
+                            _ct_n_done += 1
+                            _row["Período"] = (
+                                f"{_res.get('period_start','')} → "
+                                f"{_res.get('period_end','')}"
+                            )
+                            _row["Linhas"] = f"{_res.get('rows', 0):,}"
+                        elif _event == "failed":
+                            _row["Período"] = _res.get("error", "")[:40]
+                        break
+                _ct_table_box.dataframe(
+                    _ct_results, use_container_width=True, hide_index=True
+                )
+
+        _ct_t.join()
+        _ct_prog_bar.progress(1.0)
+        st.cache_data.clear()
+        _ct_stat_txt.empty()
+        st.success(
+            f"✅ Coleta Total concluída — {_ct_n_done} aeródromos coletados."
+        )
+        st.rerun()
+
 # ── Visão Geral ───────────────────────────────────────────────────────────────
 with tab1:
     st.plotly_chart(chart_overview(anl_r), use_container_width=True)

softwares/test/meteorologia_aeroportos/_apps/pipeline.py

@@ -26,6 +26,8 @@ from typing import Callable, Optional
 import pandas as pd
 from selenium.webdriver.support.ui import WebDriverWait
 
+import sqlite3
+
 import db as _db
 from scraper_meteorologia import (
     SITE_MIN_DATE,
@@ -281,6 +283,52 @@ def _run_backward(
     return total
 
 
+# ---------------------------------------------------------------------------
+# CSV backup helper
+# ---------------------------------------------------------------------------
+
+def _write_csv_backup(
+    conn: sqlite3.Connection,
+    aerodrome: str,
+    preproc_dir: Path,
+    log: Callable[[str], None],
+) -> None:
+    """Writes the complete analytics history for *aerodrome* to its CSV backup.
+
+    Always queries the **full** history from the database so the backup reflects
+    the entire dataset, regardless of how much data was collected in this run.
+    Deletes any previous backup file before writing the updated one.
+
+    Args:
+        conn: Open SQLite connection.
+        aerodrome: ICAO code (e.g. ``"SBGR"``).
+        preproc_dir: Root directory for permanent analytics CSV backups.
+        log: Logging callback.
+    """
+    cov = _db.get_coverage(conn, aerodrome)
+    if not cov:
+        return
+
+    anl_dir = preproc_dir / aerodrome
+    anl_dir.mkdir(parents=True, exist_ok=True)
+
+    s_str = str(cov[0]).replace("-", "_")
+    e_str = str(cov[1]).replace("-", "_")
+    csv_backup = anl_dir / f"{aerodrome}_{s_str}_{e_str}.csv"
+
+    for old in anl_dir.glob(f"{aerodrome}_*.csv"):
+        old.unlink()
+
+    anl_full = _db.query_analytics(conn, aerodrome)
+    if anl_full.empty:
+        return
+
+    anl_out = anl_full.copy()
+    anl_out["_dt"] = anl_out["_dt"].dt.strftime("%Y-%m-%d %H:%M:%S")
+    anl_out.to_csv(csv_backup, index=False, encoding="utf-8-sig")
+    log(f"[pipeline] Backup CSV: {csv_backup.name} ({len(anl_full):,} linhas)")
+
+
 # ---------------------------------------------------------------------------
 # Main pipeline
 # ---------------------------------------------------------------------------
@@ -297,6 +345,7 @@ def run_pipeline(
     n_samples: int = 20,
     do_validate: bool = True,
     do_cleanup: bool = True,
+    update_only: bool = False,
     log: Callable[[str], None] = print,
     progress: Callable[[float, str], None] = lambda pct, msg: None,
 ) -> dict:
@@ -315,6 +364,9 @@ def run_pipeline(
         n_samples: Number of spot-check samples for validation.
         do_validate: Enable spot-check validation against source CSVs.
         do_cleanup: Remove quarterly CSVs after a successful validation.
+        update_only: When ``True``, skip the backward (historical) pass and only
+            download data newer than the latest existing record.  Aerodromes
+            without any existing data are skipped entirely.
         log: Logging callback receiving a single string.
         progress: Progress callback receiving ``(fraction: float, message: str)``.
 
@@ -347,9 +399,12 @@ def run_pipeline(
     try:
         if coverage:
             _run_forward(driver, wait, aerodrome, ex_max, str(base), log)
-            progress(_PROG_FORWARD_DONE, f"Histórico {aerodrome}")
-            _run_backward(driver, wait, aerodrome, ex_min, all_years,
-                          str(base), log, progress)
+            if not update_only:
+                progress(_PROG_FORWARD_DONE, f"Histórico {aerodrome}")
+                _run_backward(driver, wait, aerodrome, ex_min, all_years,
+                              str(base), log, progress)
+        elif update_only:
+            log(f"[pipeline] {aerodrome}: sem dados existentes e update_only=True — ignorado.")
         else:
             log(f"\n[pipeline] === Coleta inicial: {date.today().year} → {SITE_MIN_DATE.year} ===")
             end_yr   = end_year   if not all_years else date.today().year
@@ -386,11 +441,18 @@ def run_pipeline(
         if cov:
             s, e = str(cov[0]), str(cov[1])
             stats = _db.aerodrome_stats(conn, aerodrome)
+            # Regenerate CSV backup if it is missing (e.g. after a partial run)
+            anl_dir = Path(preproc_dir) / aerodrome
+            csv_exists = anl_dir.is_dir() and bool(list(anl_dir.glob(f"{aerodrome}_*.csv")))
+            if not csv_exists:
+                log("[pipeline] CSV de backup ausente — regenerando do banco…")
+                _write_csv_backup(conn, aerodrome, Path(preproc_dir), log)
             progress(_PROG_DONE, "Concluído")
-            return dict(rows=stats.get("n_obs", 0), period_start=s, period_end=e,
+            return dict(rows=stats.get("n_obs", 0), n_upserted=0,
+                        period_start=s, period_end=e,
                         n_ok=0, n_fail=0, errors=[], db_path=str(db_path))
         log("[pipeline] Nenhum dado disponível.")
-        return dict(rows=0, period_start="", period_end="",
+        return dict(rows=0, n_upserted=0, period_start="", period_end="",
                     n_ok=0, n_fail=0, errors=[], db_path=str(db_path))
 
     _, merged = build_aerodrome_table(new_files, log=log)
@@ -398,7 +460,7 @@ def run_pipeline(
     if merged.empty:
         log("[pipeline] Merged vazio.")
         progress(_PROG_DONE, "Concluído")
-        return dict(rows=0, period_start="", period_end="",
+        return dict(rows=0, n_upserted=0, period_start="", period_end="",
                     n_ok=0, n_fail=0, errors=[], db_path=str(db_path))
 
     anl = build_analytics(merged)
@@ -410,20 +472,8 @@ def run_pipeline(
     n_upserted = _db.upsert_analytics(conn, aerodrome, anl)
     log(f"\n[pipeline] {n_upserted} linhas upsertadas no SQLite ({db_path.name})")
 
-    # Write analytics CSV backup to tabelas/preproc/meteorologia_aeroportos/<ICAO>/
-    anl_dir = Path(preproc_dir) / aerodrome
-    anl_dir.mkdir(parents=True, exist_ok=True)
-    cov_final = _db.get_coverage(conn, aerodrome)
-    if cov_final:
-        s_str = str(cov_final[0]).replace("-", "_")
-        e_str = str(cov_final[1]).replace("-", "_")
-        csv_backup = anl_dir / f"{aerodrome}_{s_str}_{e_str}.csv"
-        for old in anl_dir.glob(f"{aerodrome}_*.csv"):
-            old.unlink()
-        anl_out = anl.copy()
-        anl_out["_dt"] = anl_out["_dt"].dt.strftime("%Y-%m-%d %H:%M:%S")
-        anl_out.to_csv(csv_backup, index=False, encoding="utf-8-sig")
-        log(f"[pipeline] Backup CSV: {csv_backup}")
+    # Write analytics CSV backup — full history from DB, not just the new records
+    _write_csv_backup(conn, aerodrome, Path(preproc_dir), log)
 
     progress(_PROG_VALIDATE, "Validando…")
 
@@ -460,6 +510,7 @@ def run_pipeline(
 
     return dict(
         rows         = stats.get("n_obs", len(anl)),
+        n_upserted   = n_upserted,
         period_start = str(cov_now[0]) if cov_now else "",
         period_end   = str(cov_now[1]) if cov_now else "",
         n_ok         = n_ok,
@@ -507,6 +558,9 @@ exemplos:
     parser.add_argument("--no-validate",  action="store_true")
     parser.add_argument("--n-samples",    type=int, default=20)
     parser.add_argument("--no-cleanup",   action="store_true")
+    parser.add_argument("--update-only",  action="store_true",
+                        help="Only download data newer than the last existing record "
+                             "(skip backward/historical pass)")
     args = parser.parse_args()
 
     if not args.all_years and (args.start_year is None or args.end_year is None):
@@ -524,6 +578,7 @@ exemplos:
         n_samples   = args.n_samples,
         do_validate = not args.no_validate,
         do_cleanup  = not args.no_cleanup,
+        update_only = args.update_only,
     )
 
     print(f"\n{'=' * 55}")

softwares/test/meteorologia_aeroportos/db/collect_all_status.json

@@ -0,0 +1,6 @@
+{
+  "SBAA": {
+    "status": "in_progress",
+    "started_at": "2026-06-02T11:28:18.831671+00:00"
+  }
+}