353 lines
14 KiB
Python
353 lines
14 KiB
Python
"""
|
|
Módulo SQLite para armazenamento de dados meteorológicos analíticos.
|
|
|
|
Tabelas:
|
|
observations — séries temporais analíticas (10 variáveis float por timestamp)
|
|
outlier_log — auditoria de valores tratados por limites físicos
|
|
aerodromes — catálogo de aeródromos disponíveis no site ICEA
|
|
|
|
Uso:
|
|
from db import get_connection, ensure_schema, upsert_analytics, query_analytics
|
|
conn = get_connection(Path("dados/met.db"))
|
|
ensure_schema(conn)
|
|
upsert_analytics(conn, "SBGR", anl_df)
|
|
df = query_analytics(conn, "SBGR", "2024-01-01", "2024-12-31")
|
|
"""
|
|
|
|
import sqlite3
|
|
from datetime import date, datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import pandas as pd
|
|
|
|
# ── Colunas analíticas ────────────────────────────────────────────────────────
|
|
ANL_COLS = ["T", "Td", "UR", "QNH", "WS", "WG", "WD", "VIS", "TETO", "PREC"]
|
|
|
|
# ── Limites físicos por variável ─────────────────────────────────────────────
|
|
PHYSICAL_LIMITS: dict[str, tuple[float, float]] = {
|
|
"T": (-25.0, 55.0), # °C — extremos possíveis no Brasil
|
|
"Td": (-30.0, 40.0), # °C — ponto de orvalho
|
|
"UR": ( 0.0, 100.0), # %
|
|
"QNH": (940.0, 1060.0), # hPa — recordes mundiais absolutos
|
|
"WS": ( 0.0, 200.0), # kt
|
|
"WG": ( 0.0, 250.0), # kt — rajada furacão Cat 5 ≈ 175 kt
|
|
"WD": ( 0.0, 360.0), # °
|
|
"VIS": ( 0.0, 9999.0), # dam
|
|
"TETO": ( 0.0, 9999.0), # dam
|
|
"PREC": ( 0.0, 500.0), # mm — recorde horário mundial ≈ 300 mm/h
|
|
}
|
|
|
|
# ── Schema SQL ────────────────────────────────────────────────────────────────
|
|
_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS observations (
|
|
aerodrome TEXT NOT NULL,
|
|
dt TEXT NOT NULL,
|
|
T REAL, Td REAL, UR REAL, QNH REAL,
|
|
WS REAL, WG REAL, WD REAL,
|
|
VIS REAL, TETO REAL, PREC REAL,
|
|
PRIMARY KEY (aerodrome, dt)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_aero_dt ON observations(aerodrome, dt);
|
|
|
|
CREATE TABLE IF NOT EXISTS outlier_log (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
aerodrome TEXT NOT NULL,
|
|
dt TEXT NOT NULL,
|
|
variable TEXT NOT NULL,
|
|
orig_value REAL NOT NULL,
|
|
treatment TEXT NOT NULL DEFAULT 'SET_NULL',
|
|
reason TEXT,
|
|
applied_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_outlier_aero ON outlier_log(aerodrome, variable);
|
|
|
|
CREATE TABLE IF NOT EXISTS aerodromes (
|
|
icao TEXT PRIMARY KEY,
|
|
nome TEXT,
|
|
uf TEXT,
|
|
updated_at TEXT
|
|
);
|
|
"""
|
|
|
|
|
|
# ── Conexão ───────────────────────────────────────────────────────────────────
|
|
def get_connection(db_path: Path) -> sqlite3.Connection:
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
conn = sqlite3.connect(str(db_path), check_same_thread=False)
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
conn.execute("PRAGMA cache_size=-32000")
|
|
return conn
|
|
|
|
|
|
def ensure_schema(conn: sqlite3.Connection) -> None:
|
|
conn.executescript(_SCHEMA)
|
|
conn.commit()
|
|
|
|
|
|
# ── Clip de limites físicos no DataFrame ─────────────────────────────────────
|
|
def _clip_to_limits(df: pd.DataFrame) -> pd.DataFrame:
|
|
"""Define como NaN valores fora dos limites físicos (sem registrar no log)."""
|
|
df = df.copy()
|
|
for col, (lo, hi) in PHYSICAL_LIMITS.items():
|
|
if col in df.columns:
|
|
mask = df[col].notna() & ((df[col] < lo) | (df[col] > hi))
|
|
df.loc[mask, col] = float("nan")
|
|
return df
|
|
|
|
|
|
# ── Upsert analytics ──────────────────────────────────────────────────────────
|
|
def upsert_analytics(
|
|
conn: sqlite3.Connection,
|
|
aerodrome: str,
|
|
anl_df: pd.DataFrame,
|
|
) -> int:
|
|
"""
|
|
Insere ou substitui linhas no banco.
|
|
Aplica limites físicos automaticamente antes da inserção.
|
|
Retorna número de linhas processadas.
|
|
"""
|
|
if anl_df.empty:
|
|
return 0
|
|
|
|
anl_df = _clip_to_limits(anl_df)
|
|
|
|
cols_present = [c for c in ANL_COLS if c in anl_df.columns]
|
|
placeholders = ", ".join(["?"] * (2 + len(cols_present)))
|
|
col_names = ", ".join(["aerodrome", "dt"] + cols_present)
|
|
sql = f"INSERT OR REPLACE INTO observations ({col_names}) VALUES ({placeholders})"
|
|
|
|
def _dt_str(v):
|
|
if isinstance(v, (datetime, pd.Timestamp)):
|
|
return v.strftime("%Y-%m-%d %H:%M:%S")
|
|
return str(v)[:19]
|
|
|
|
rows = []
|
|
for _, row in anl_df.iterrows():
|
|
vals = [aerodrome, _dt_str(row["_dt"])]
|
|
for c in cols_present:
|
|
v = row[c]
|
|
vals.append(None if pd.isna(v) else float(v))
|
|
rows.append(tuple(vals))
|
|
|
|
with conn:
|
|
conn.executemany(sql, rows)
|
|
return len(rows)
|
|
|
|
|
|
# ── Repair de outliers no banco ───────────────────────────────────────────────
|
|
def apply_physical_limits(
|
|
conn: sqlite3.Connection,
|
|
aerodrome: Optional[str] = None,
|
|
) -> int:
|
|
"""
|
|
Aplica limites físicos aos dados existentes, com auditoria completa em outlier_log.
|
|
Se aerodrome=None, processa todos os aeródromos.
|
|
Retorna total de outliers registrados e corrigidos.
|
|
"""
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
total = 0
|
|
aero_filter = "AND aerodrome=?" if aerodrome else ""
|
|
aero_params_suffix = (aerodrome,) if aerodrome else ()
|
|
|
|
for col, (lo, hi) in PHYSICAL_LIMITS.items():
|
|
# Registra valores abaixo do limite
|
|
rows = conn.execute(
|
|
f"SELECT aerodrome, dt, {col} FROM observations "
|
|
f"WHERE {col} IS NOT NULL AND {col} < ? {aero_filter}",
|
|
(lo,) + aero_params_suffix,
|
|
).fetchall()
|
|
if rows:
|
|
conn.executemany(
|
|
"INSERT INTO outlier_log"
|
|
"(aerodrome,dt,variable,orig_value,treatment,reason,applied_at)"
|
|
" VALUES(?,?,?,?,?,?,?)",
|
|
[(r[0], r[1], col, r[2], "SET_NULL", f"below_limit:{lo}", now)
|
|
for r in rows],
|
|
)
|
|
total += len(rows)
|
|
|
|
# Registra valores acima do limite
|
|
rows = conn.execute(
|
|
f"SELECT aerodrome, dt, {col} FROM observations "
|
|
f"WHERE {col} IS NOT NULL AND {col} > ? {aero_filter}",
|
|
(hi,) + aero_params_suffix,
|
|
).fetchall()
|
|
if rows:
|
|
conn.executemany(
|
|
"INSERT INTO outlier_log"
|
|
"(aerodrome,dt,variable,orig_value,treatment,reason,applied_at)"
|
|
" VALUES(?,?,?,?,?,?,?)",
|
|
[(r[0], r[1], col, r[2], "SET_NULL", f"above_limit:{hi}", now)
|
|
for r in rows],
|
|
)
|
|
total += len(rows)
|
|
|
|
# Aplica o UPDATE com CASE para todas as variáveis de uma vez
|
|
where_clause = f"WHERE aerodrome='{aerodrome}'" if aerodrome else ""
|
|
conn.execute(f"""
|
|
UPDATE observations SET
|
|
T = CASE WHEN T IS NOT NULL AND (T < -25 OR T > 55 ) THEN NULL ELSE T END,
|
|
Td = CASE WHEN Td IS NOT NULL AND (Td < -30 OR Td > 40 ) THEN NULL ELSE Td END,
|
|
UR = CASE WHEN UR IS NOT NULL AND (UR < 0 OR UR > 100 ) THEN NULL ELSE UR END,
|
|
QNH = CASE WHEN QNH IS NOT NULL AND (QNH < 940 OR QNH > 1060) THEN NULL ELSE QNH END,
|
|
WS = CASE WHEN WS IS NOT NULL AND (WS < 0 OR WS > 200 ) THEN NULL ELSE WS END,
|
|
WG = CASE WHEN WG IS NOT NULL AND (WG < 0 OR WG > 250 ) THEN NULL ELSE WG END,
|
|
WD = CASE WHEN WD IS NOT NULL AND (WD < 0 OR WD > 360 ) THEN NULL ELSE WD END,
|
|
VIS = CASE WHEN VIS IS NOT NULL AND (VIS < 0 OR VIS > 9999) THEN NULL ELSE VIS END,
|
|
TETO = CASE WHEN TETO IS NOT NULL AND (TETO < 0 OR TETO > 9999) THEN NULL ELSE TETO END,
|
|
PREC = CASE WHEN PREC IS NOT NULL AND (PREC < 0 OR PREC > 500 ) THEN NULL ELSE PREC END
|
|
{where_clause}
|
|
""")
|
|
conn.commit()
|
|
return total
|
|
|
|
|
|
# ── Consultas de auditoria ────────────────────────────────────────────────────
|
|
def get_outlier_summary(
|
|
conn: sqlite3.Connection,
|
|
aerodrome: Optional[str] = None,
|
|
) -> pd.DataFrame:
|
|
"""
|
|
Resumo de outliers por variável: contagem, min/max do valor original,
|
|
limite aplicado e data do último tratamento.
|
|
"""
|
|
aero_filter = "WHERE aerodrome=?" if aerodrome else ""
|
|
params = (aerodrome,) if aerodrome else ()
|
|
return pd.read_sql_query(
|
|
f"""
|
|
SELECT
|
|
variable,
|
|
aerodrome,
|
|
COUNT(*) AS n_outliers,
|
|
MIN(orig_value) AS min_orig,
|
|
MAX(orig_value) AS max_orig,
|
|
reason,
|
|
MAX(applied_at) AS last_applied
|
|
FROM outlier_log
|
|
{aero_filter}
|
|
GROUP BY variable, aerodrome, reason
|
|
ORDER BY variable, aerodrome
|
|
""",
|
|
conn, params=params,
|
|
)
|
|
|
|
|
|
def get_outlier_detail(
|
|
conn: sqlite3.Connection,
|
|
aerodrome: Optional[str] = None,
|
|
variable: Optional[str] = None,
|
|
limit: int = 500,
|
|
) -> pd.DataFrame:
|
|
"""Detalhe linha-a-linha dos outliers registrados."""
|
|
conditions = []
|
|
params: list = []
|
|
if aerodrome:
|
|
conditions.append("aerodrome=?")
|
|
params.append(aerodrome)
|
|
if variable:
|
|
conditions.append("variable=?")
|
|
params.append(variable)
|
|
where = ("WHERE " + " AND ".join(conditions)) if conditions else ""
|
|
return pd.read_sql_query(
|
|
f"SELECT aerodrome, dt, variable, orig_value, reason, applied_at "
|
|
f"FROM outlier_log {where} ORDER BY applied_at DESC, aerodrome, dt "
|
|
f"LIMIT {limit}",
|
|
conn, params=params,
|
|
)
|
|
|
|
|
|
def has_outlier_log(conn: sqlite3.Connection) -> bool:
|
|
"""Verifica se existe algum registro de outlier."""
|
|
n = conn.execute("SELECT COUNT(*) FROM outlier_log").fetchone()[0]
|
|
return n > 0
|
|
|
|
|
|
# ── Catálogo de aeródromos ────────────────────────────────────────────────────
|
|
def upsert_aerodromes(
|
|
conn: sqlite3.Connection,
|
|
aerodromes: list[dict],
|
|
) -> int:
|
|
"""
|
|
Insere ou atualiza lista de aeródromos.
|
|
Cada dict deve ter: icao, nome, uf.
|
|
Retorna número de registros processados.
|
|
"""
|
|
if not aerodromes:
|
|
return 0
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
with conn:
|
|
conn.executemany(
|
|
"INSERT OR REPLACE INTO aerodromes(icao, nome, uf, updated_at)"
|
|
" VALUES(?, ?, ?, ?)",
|
|
[(a["icao"], a.get("nome", ""), a.get("uf", ""), now)
|
|
for a in aerodromes if a.get("icao")],
|
|
)
|
|
return len(aerodromes)
|
|
|
|
|
|
def list_all_aerodromes(conn: sqlite3.Connection) -> list[dict]:
|
|
"""
|
|
Retorna todos os aeródromos cadastrados (tabela aerodromes).
|
|
Formato: [{"icao": "SBGR", "nome": "Guarulhos, SP", "uf": "SP"}, ...]
|
|
"""
|
|
rows = conn.execute(
|
|
"SELECT icao, nome, uf FROM aerodromes ORDER BY icao"
|
|
).fetchall()
|
|
return [{"icao": r[0], "nome": r[1], "uf": r[2]} for r in rows]
|
|
|
|
|
|
# ── Queries gerais ────────────────────────────────────────────────────────────
|
|
def query_analytics(
|
|
conn: sqlite3.Connection,
|
|
aerodrome: str,
|
|
start_dt: Optional[str] = None,
|
|
end_dt: Optional[str] = None,
|
|
) -> pd.DataFrame:
|
|
sql = "SELECT dt, T, Td, UR, QNH, WS, WG, WD, VIS, TETO, PREC FROM observations WHERE aerodrome=?"
|
|
params: list = [aerodrome]
|
|
if start_dt:
|
|
sql += " AND dt >= ?"; params.append(start_dt)
|
|
if end_dt:
|
|
sql += " AND dt <= ?"
|
|
params.append(end_dt + " 23:59:59" if len(end_dt) == 10 else end_dt)
|
|
sql += " ORDER BY dt"
|
|
df = pd.read_sql_query(sql, conn, params=params)
|
|
if df.empty:
|
|
return df
|
|
df["_dt"] = pd.to_datetime(df["dt"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
|
|
return df.drop(columns=["dt"]).dropna(subset=["_dt"]).reset_index(drop=True)
|
|
|
|
|
|
def get_coverage(
|
|
conn: sqlite3.Connection,
|
|
aerodrome: str,
|
|
) -> Optional[tuple[date, date]]:
|
|
row = conn.execute(
|
|
"SELECT MIN(dt), MAX(dt) FROM observations WHERE aerodrome=?",
|
|
(aerodrome,),
|
|
).fetchone()
|
|
if not row or row[0] is None:
|
|
return None
|
|
return date.fromisoformat(row[0][:10]), date.fromisoformat(row[1][:10])
|
|
|
|
|
|
def list_aerodromes(conn: sqlite3.Connection) -> list[str]:
|
|
rows = conn.execute(
|
|
"SELECT DISTINCT aerodrome FROM observations ORDER BY aerodrome"
|
|
).fetchall()
|
|
return [r[0] for r in rows]
|
|
|
|
|
|
def aerodrome_stats(conn: sqlite3.Connection, aerodrome: str) -> dict:
|
|
row = conn.execute(
|
|
"SELECT MIN(dt), MAX(dt), COUNT(*), COUNT(T), COUNT(VIS) "
|
|
"FROM observations WHERE aerodrome=?",
|
|
(aerodrome,),
|
|
).fetchone()
|
|
if not row or row[0] is None:
|
|
return {}
|
|
return {"min_dt": row[0], "max_dt": row[1],
|
|
"n_obs": row[2], "n_T": row[3], "n_VIS": row[4]}
|