445 lines
16 KiB
Python
445 lines
16 KiB
Python
"""
|
|
Scraper de dados meteorológicos de superfície - ICEA/DECEA
|
|
Site: https://pesquisa.icea.decea.mil.br/superficie_list/
|
|
|
|
Uso standalone:
|
|
python scraper_meteorologia.py --aerodrome SBGR --all-years
|
|
python scraper_meteorologia.py --aerodrome SBGR --start-year 2020 --end-year 2025
|
|
python scraper_meteorologia.py --aerodrome SBSP --start-year 2023 --end-year 2023 --no-headless
|
|
|
|
Importável pelo pipeline:
|
|
from scraper_meteorologia import make_driver, scrape_year, SITE_MIN_DATE
|
|
"""
|
|
|
|
import argparse
|
|
import calendar
|
|
import csv
|
|
import io
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
from datetime import date
|
|
from typing import Callable, Optional
|
|
|
|
import pandas as pd
|
|
from bs4 import BeautifulSoup
|
|
from selenium import webdriver
|
|
from selenium.webdriver.chrome.service import Service
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
from webdriver_manager.chrome import ChromeDriverManager
|
|
|
|
BASE_URL = "https://pesquisa.icea.decea.mil.br/superficie_list/"
|
|
SITE_MIN_DATE = date(1947, 12, 1)
|
|
SITE_MAX_DATE = date.today() # dinâmico: nunca maior que hoje
|
|
CHUNK_MONTHS = 3
|
|
STOP_EMPTY_YEARS = 2
|
|
|
|
|
|
def fetch_site_max_date(driver: webdriver.Chrome) -> date:
|
|
"""
|
|
Lê o atributo data-date-end-date do campo de período no site e atualiza
|
|
SITE_MAX_DATE globalmente. Retorna date.today() como fallback.
|
|
"""
|
|
global SITE_MAX_DATE
|
|
try:
|
|
val = driver.execute_script(
|
|
"var el = document.getElementById('dtPeriodo');"
|
|
"return el ? el.getAttribute('data-date-end-date') : null;"
|
|
)
|
|
if val:
|
|
d, m, y = val.strip().split("/")
|
|
SITE_MAX_DATE = date(int(y), int(m), int(d))
|
|
return SITE_MAX_DATE
|
|
except Exception:
|
|
pass
|
|
return SITE_MAX_DATE
|
|
|
|
TABS = [
|
|
("tab_cgt", "CGT"),
|
|
("tab_nuvem", "Nuvem"),
|
|
("tab_prec", "Precipitação"),
|
|
("tab_pres", "Pressão"),
|
|
("tab_rvr", "RVR"),
|
|
("tab_temp", "Temperatura"),
|
|
("tab_teto", "Teto"),
|
|
("tab_vent", "Vento"),
|
|
("tab_visib", "Visibilidade"),
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# WebDriver
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def make_driver(headless: bool = True) -> webdriver.Chrome:
|
|
opts = webdriver.ChromeOptions()
|
|
if headless:
|
|
opts.add_argument("--headless=new")
|
|
opts.add_argument("--no-sandbox")
|
|
opts.add_argument("--disable-dev-shm-usage")
|
|
opts.add_argument("--window-size=1920,1080")
|
|
opts.add_argument("--disable-blink-features=AutomationControlled")
|
|
opts.add_experimental_option("excludeSwitches", ["enable-automation"])
|
|
service = Service(ChromeDriverManager().install())
|
|
driver = webdriver.Chrome(service=service, options=opts)
|
|
driver.set_page_load_timeout(300)
|
|
driver.set_script_timeout(120)
|
|
try:
|
|
driver.command_executor.client_config.timeout = 300
|
|
except Exception:
|
|
pass
|
|
return driver
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Formulário
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def select_aerodrome(driver: webdriver.Chrome, wait: WebDriverWait, code: str) -> str:
|
|
"""Seleciona aeródromo no Bootstrap Select (JS-first, fallback visual)."""
|
|
result = driver.execute_script(f"""
|
|
var candidates = document.querySelectorAll(
|
|
'select.selectpicker, select[name*="localidade"], select[name*="aerodrome"], select'
|
|
);
|
|
for (var s = 0; s < candidates.length; s++) {{
|
|
var sel = candidates[s];
|
|
for (var i = 0; i < sel.options.length; i++) {{
|
|
if (sel.options[i].text.indexOf('{code}') !== -1) {{
|
|
sel.selectedIndex = i;
|
|
var txt = sel.options[i].text;
|
|
if (typeof jQuery !== 'undefined') {{
|
|
try {{ jQuery(sel).selectpicker('val', sel.options[i].value); }} catch(e) {{}}
|
|
jQuery(sel).trigger('change');
|
|
}} else {{
|
|
sel.dispatchEvent(new Event('change', {{bubbles: true}}));
|
|
}}
|
|
return txt;
|
|
}}
|
|
}}
|
|
}}
|
|
return null;
|
|
""")
|
|
if result:
|
|
return result.strip()
|
|
|
|
btn = wait.until(EC.element_to_be_clickable(
|
|
(By.CSS_SELECTOR, "div.bootstrap-select > button.dropdown-toggle")
|
|
))
|
|
btn.click()
|
|
time.sleep(0.4)
|
|
try:
|
|
search = driver.find_element(By.CSS_SELECTOR, ".bootstrap-select .bs-searchbox input")
|
|
search.clear()
|
|
search.send_keys(code)
|
|
time.sleep(0.5)
|
|
except Exception:
|
|
pass
|
|
option_link = wait.until(EC.element_to_be_clickable((
|
|
By.XPATH,
|
|
f"//ul[contains(@class,'dropdown-menu inner')]"
|
|
f"//li[not(contains(@class,'hidden')) and not(contains(@class,'d-none'))]"
|
|
f"//span[contains(text(),'{code}')]/..",
|
|
)))
|
|
try:
|
|
text = option_link.find_element(By.CSS_SELECTOR, "span.text").text.strip()
|
|
except Exception:
|
|
text = code
|
|
option_link.click()
|
|
time.sleep(0.3)
|
|
return text
|
|
|
|
|
|
def set_period(driver: webdriver.Chrome, start: date, end: date) -> None:
|
|
s = start.strftime("%d/%m/%Y")
|
|
e = end.strftime("%d/%m/%Y")
|
|
driver.execute_script(f"""
|
|
var el = document.getElementById('dtPeriodo');
|
|
if (!el) return;
|
|
if (typeof jQuery !== 'undefined' && jQuery(el).data('daterangepicker')) {{
|
|
jQuery(el).data('daterangepicker').setStartDate('{s}');
|
|
jQuery(el).data('daterangepicker').setEndDate('{e}');
|
|
}}
|
|
el.value = '{s} - {e}';
|
|
el.dispatchEvent(new Event('change', {{bubbles: true}}));
|
|
""")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Extração de dados
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def show_all_rows(driver: webdriver.Chrome, tab_id: str) -> None:
|
|
driver.execute_script(f"""
|
|
try {{
|
|
var dt = jQuery('#{tab_id} table').DataTable();
|
|
dt.page.len(-1).draw();
|
|
}} catch(e) {{
|
|
var sel = document.querySelector('#{tab_id} select[name$="_length"]');
|
|
if (sel) {{
|
|
var best = 0;
|
|
for (var i = 0; i < sel.options.length; i++) {{
|
|
var v = parseInt(sel.options[i].value);
|
|
if (v === -1 || v > parseInt(sel.options[best].value)) best = i;
|
|
}}
|
|
sel.selectedIndex = best;
|
|
sel.dispatchEvent(new Event('change', {{bubbles: true}}));
|
|
}}
|
|
}}
|
|
""")
|
|
time.sleep(1.2)
|
|
|
|
|
|
def parse_html_table(html: str) -> Optional[pd.DataFrame]:
|
|
soup = BeautifulSoup(html, "lxml")
|
|
table = soup.find("table")
|
|
if not table:
|
|
return None
|
|
headers = []
|
|
thead = table.find("thead")
|
|
if thead:
|
|
headers = [th.get_text(strip=True) for th in thead.find_all(["th", "td"])]
|
|
rows = []
|
|
tbody = table.find("tbody")
|
|
if tbody:
|
|
for tr in tbody.find_all("tr"):
|
|
cells = [td.get_text(strip=True) for td in tr.find_all(["th", "td"])]
|
|
if any(c.strip() for c in cells):
|
|
rows.append(cells)
|
|
if not rows:
|
|
return None
|
|
return pd.DataFrame(rows, columns=headers if headers else None)
|
|
|
|
|
|
def get_full_table(driver: webdriver.Chrome, tab_id: str) -> Optional[pd.DataFrame]:
|
|
frames: list[pd.DataFrame] = []
|
|
seen: set[str] = set()
|
|
while True:
|
|
tab_elem = driver.find_element(By.ID, tab_id)
|
|
html = tab_elem.get_attribute("innerHTML")
|
|
fp = html[:800]
|
|
if fp in seen:
|
|
break
|
|
seen.add(fp)
|
|
df = parse_html_table(html)
|
|
if df is not None:
|
|
frames.append(df)
|
|
try:
|
|
next_btn = driver.find_element(
|
|
By.CSS_SELECTOR, f"#{tab_id} .dataTables_paginate .next"
|
|
)
|
|
if "disabled" in (next_btn.get_attribute("class") or ""):
|
|
break
|
|
driver.execute_script("arguments[0].click();", next_btn)
|
|
time.sleep(0.5)
|
|
except Exception:
|
|
break
|
|
if not frames:
|
|
return None
|
|
if len(frames) == 1:
|
|
return frames[0]
|
|
return pd.concat(frames, ignore_index=True).drop_duplicates()
|
|
|
|
|
|
def extract_tab(driver: webdriver.Chrome, wait: WebDriverWait, tab_id: str) -> Optional[pd.DataFrame]:
|
|
link = driver.find_element(By.CSS_SELECTOR, f"a[href='#{tab_id}']")
|
|
driver.execute_script("arguments[0].click();", link)
|
|
try:
|
|
wait.until(lambda d: "active" in (
|
|
d.find_element(By.ID, tab_id).get_attribute("class") or ""
|
|
))
|
|
except Exception:
|
|
time.sleep(1.5)
|
|
show_all_rows(driver, tab_id)
|
|
return get_full_table(driver, tab_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Saída
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def make_filename(tab_name: str, localidade: str, start: date, end: date) -> str:
|
|
safe = re.sub(r'[<>:"/\\|?*\r\n]', '', localidade).strip()
|
|
s = start.strftime("%d%m%Y")
|
|
e = end.strftime("%d%m%Y")
|
|
return f"Dados de Superfície {tab_name} - Localidade {safe} - Período {s} - {e}.csv"
|
|
|
|
|
|
def save_csv(df: pd.DataFrame, path: str) -> None:
|
|
df.to_csv(path, index=False, encoding="utf-8-sig", quoting=csv.QUOTE_ALL)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Lógica principal de scraping
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _quarter_chunks(year: int) -> list[tuple[date, date]]:
|
|
"""Divide o ano em chunks de CHUNK_MONTHS meses (do mais recente para o mais antigo)."""
|
|
chunks = []
|
|
month = 1
|
|
while month <= 12:
|
|
end_month = min(month + CHUNK_MONTHS - 1, 12)
|
|
last_day = calendar.monthrange(year, end_month)[1]
|
|
chunks.append((date(year, month, 1), date(year, end_month, last_day)))
|
|
month = end_month + 1
|
|
return list(reversed(chunks)) # Q4 → Q3 → Q2 → Q1
|
|
|
|
|
|
def scrape_period(
|
|
driver: webdriver.Chrome,
|
|
wait: WebDriverWait,
|
|
aerodrome: str,
|
|
start: date,
|
|
end: date,
|
|
output_dir: str,
|
|
log: Callable[[str], None] = print,
|
|
) -> bool:
|
|
"""
|
|
Faz o scraping de um único período e salva os CSVs das 9 abas.
|
|
Retorna True se pelo menos uma aba teve dados, False se tudo veio vazio.
|
|
"""
|
|
log(f" [{start.strftime('%d/%m/%Y')} -> {end.strftime('%d/%m/%Y')}]")
|
|
|
|
driver.get(BASE_URL)
|
|
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "div.bootstrap-select")))
|
|
fetch_site_max_date(driver) # atualiza SITE_MAX_DATE com o valor real do site
|
|
time.sleep(1.5)
|
|
|
|
localidade = select_aerodrome(driver, wait, aerodrome)
|
|
if not localidade:
|
|
log(f" ERR Aerodrome '{aerodrome}' nao encontrado.")
|
|
return False
|
|
log(f" Localidade: {localidade}")
|
|
|
|
set_period(driver, start, end)
|
|
submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[type='submit']")))
|
|
submit.click()
|
|
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".nav-tabs")))
|
|
time.sleep(3)
|
|
|
|
found_any = False
|
|
for tab_id, tab_name in TABS:
|
|
try:
|
|
df = extract_tab(driver, wait, tab_id)
|
|
if df is not None and not df.empty:
|
|
found_any = True
|
|
fname = make_filename(tab_name, localidade, start, end)
|
|
fpath = os.path.join(output_dir, fname)
|
|
save_csv(df, fpath)
|
|
log(f" OK {tab_name:<15s} {len(df):>6} linhas -> {fname}")
|
|
else:
|
|
log(f" -- {tab_name:<15s} sem dados")
|
|
except Exception as exc:
|
|
log(f" ERR {tab_name:<15s} {exc}")
|
|
|
|
return found_any
|
|
|
|
|
|
def scrape_year(
|
|
driver: webdriver.Chrome,
|
|
wait: WebDriverWait,
|
|
aerodrome: str,
|
|
year: int,
|
|
output_dir: str,
|
|
stop_before: Optional[date] = None,
|
|
log: Callable[[str], None] = print,
|
|
) -> int:
|
|
"""
|
|
Itera os chunks trimestrais do ano (do mais recente para o mais antigo).
|
|
Para se o trimestre for anterior a stop_before (data já coberta).
|
|
Retorna o número de trimestres que tiveram dados.
|
|
"""
|
|
non_empty = 0
|
|
for chunk_start, chunk_end in _quarter_chunks(year):
|
|
start = max(chunk_start, SITE_MIN_DATE)
|
|
end = min(chunk_end, SITE_MAX_DATE)
|
|
if start > end:
|
|
continue
|
|
# Para quando atingir dados já cobertos pelo arquivo existente
|
|
if stop_before is not None and end <= stop_before:
|
|
log(f" [trimestre {start.strftime('%d/%m/%Y')} -> {end.strftime('%d/%m/%Y')} já coberto, parando]")
|
|
break
|
|
had_data = scrape_period(driver, wait, aerodrome, start, end, output_dir, log)
|
|
if had_data:
|
|
non_empty += 1
|
|
return non_empty
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Entry point (CLI standalone)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _fix_stdout() -> None:
|
|
"""Garante UTF-8 no terminal Windows ao rodar como script."""
|
|
try:
|
|
if hasattr(sys.stdout, "buffer"):
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
|
if hasattr(sys.stderr, "buffer"):
|
|
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def main() -> None:
|
|
_fix_stdout()
|
|
parser = argparse.ArgumentParser(
|
|
description="Scraper de meteorologia de superfície — ICEA/DECEA",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
exemplos:
|
|
python scraper_meteorologia.py --aerodrome SBGR --all-years
|
|
python scraper_meteorologia.py --aerodrome SBGR --start-year 2020 --end-year 2025
|
|
python scraper_meteorologia.py --aerodrome SBSP --start-year 2023 --end-year 2023 --no-headless
|
|
""",
|
|
)
|
|
parser.add_argument("--aerodrome", default="SBGR", metavar="ICAO",
|
|
help="Código ICAO do aeródromo (padrão: SBGR)")
|
|
parser.add_argument("--all-years", action="store_true",
|
|
help="Coleta todos os anos disponíveis (para automaticamente sem dados)")
|
|
parser.add_argument("--start-year", type=int, metavar="ANO",
|
|
help="Ano inicial (obrigatório sem --all-years)")
|
|
parser.add_argument("--end-year", type=int, metavar="ANO",
|
|
help="Ano final (obrigatório sem --all-years)")
|
|
parser.add_argument("--output-dir", default="dados", metavar="DIR",
|
|
help="Diretório de saída (padrão: dados/)")
|
|
parser.add_argument("--no-headless", action="store_true",
|
|
help="Abre janela do Chrome visível (útil para debug)")
|
|
args = parser.parse_args()
|
|
|
|
if not args.all_years and (args.start_year is None or args.end_year is None):
|
|
parser.error("Use --all-years ou informe --start-year e --end-year")
|
|
|
|
end_year = args.end_year if not args.all_years else date.today().year
|
|
start_year = args.start_year if not args.all_years else SITE_MIN_DATE.year
|
|
|
|
os.makedirs(args.output_dir, exist_ok=True)
|
|
driver = make_driver(headless=not args.no_headless)
|
|
wait = WebDriverWait(driver, 60)
|
|
|
|
try:
|
|
consecutive_empty = 0
|
|
for year in range(end_year, start_year - 1, -1): # do mais recente para o mais antigo
|
|
print(f"\n{'=' * 60}")
|
|
print(f"Ano {year} — {args.aerodrome}")
|
|
print(f"{'=' * 60}")
|
|
non_empty = scrape_year(driver, wait, args.aerodrome, year, args.output_dir)
|
|
if non_empty == 0:
|
|
consecutive_empty += 1
|
|
print(f" [sem dados] ({consecutive_empty}/{STOP_EMPTY_YEARS} consecutivos)")
|
|
if args.all_years and consecutive_empty >= STOP_EMPTY_YEARS:
|
|
print(f"\n{STOP_EMPTY_YEARS} anos consecutivos sem dados — parando busca.")
|
|
break
|
|
else:
|
|
consecutive_empty = 0
|
|
except KeyboardInterrupt:
|
|
print("\nInterrompido pelo usuário.")
|
|
finally:
|
|
driver.quit()
|
|
print("\nFinalizado.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|