get-sinapi 3.1.0__py3-none-any.whl

getsinapi/bootstrap.py

@@ -0,0 +1,422 @@
+"""Bootstrap histórico, incremental e idempotente da base oficial SINAPI."""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import sqlite3
+import threading
+import time
+import zipfile
+from pathlib import Path
+
+from curl_cffi import requests as browser_requests
+
+from getsinapi.collectors.caixa import CaixaDownloadError, download_monthly_xlsx
+from getsinapi.collectors.history import (
+    LEGACY_LAYOUT_END,
+    UNIFIED_LAYOUT_START,
+    discover_historical_xlsx_catalog,
+)
+from getsinapi.config import Settings
+from getsinapi.extractors.workbooks import extract_and_catalog
+from getsinapi.pipeline.normalize import normalize_reference_workbook
+from getsinapi.storage.sqlite import load_silver_reference
+
+log = logging.getLogger("getsinapi.bootstrap")
+_BOOTSTRAP_LOCK = threading.Lock()
+DISCOVERY_CACHE_TTL_SECONDS = 6 * 60 * 60
+REQUIRED_SHEETS = {"ISD", "ICD", "ISE", "Analítico"}
+
+
+def _month_shift(year: int, month: int, delta: int) -> tuple[int, int]:
+    """Desloca uma competência mensal sem depender de datas ou fuso horário."""
+    absolute_month = year * 12 + (month - 1) + delta
+    shifted_year, zero_based_month = divmod(absolute_month, 12)
+    return shifted_year, zero_based_month + 1
+
+
+def _database_path(settings: Settings) -> Path:
+    return settings.data_dir / "getsinapi.sqlite3"
+
+
+def database_is_valid(path: Path) -> bool:
+    if not path.is_file() or path.stat().st_size == 0:
+        return False
+    try:
+        with sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5) as db:
+            if db.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
+                return False
+            tables = {r[0] for r in db.execute(
+                "SELECT name FROM sqlite_master WHERE type='table'"
+            )}
+            required = {"source_references", "input_prices", "compositions", "composition_items"}
+            if not required <= tables:
+                return False
+            return db.execute("SELECT COUNT(*) FROM source_references").fetchone()[0] > 0
+    except (sqlite3.Error, OSError):
+        return False
+
+
+def _loaded_and_complete(database: Path, reference: str) -> bool:
+    """Verifica a presença dos dados sem executar integrity_check em todo o banco.
+
+    PRAGMA integrity_check é uma varredura integral e não deve ser repetida
+    antes/depois de cada competência durante a ingestão histórica.
+    A integridade completa é verificada uma vez no início/final do bootstrap.
+    """
+    if not database.is_file() or database.stat().st_size == 0:
+        return False
+    try:
+        with sqlite3.connect(f"file:{database}?mode=ro", uri=True, timeout=5) as db:
+            tables = {row[0] for row in db.execute(
+                "SELECT name FROM sqlite_master WHERE type='table'"
+            )}
+            required = {"source_references", "input_prices", "compositions", "composition_items"}
+            if not required <= tables:
+                return False
+            source = db.execute(
+                "SELECT 1 FROM source_references WHERE reference=?", (reference,)
+            ).fetchone()
+            if not source:
+                return False
+            prices, compositions, items = db.execute(
+                """SELECT
+                    (SELECT COUNT(*) FROM input_prices WHERE reference=?),
+                    (SELECT COUNT(*) FROM compositions WHERE reference=?),
+                    (SELECT COUNT(*) FROM composition_items WHERE reference=?)""",
+                (reference, reference, reference),
+            ).fetchone()
+            return prices > 0 and compositions > 0 and items > 0
+    except sqlite3.Error:
+        return False
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _find_archive(settings: Settings, reference: str) -> Path | None:
+    for path in sorted(settings.raw_dir.glob(f"SINAPI-{reference}-formato-xlsx-*.zip")):
+        if zipfile.is_zipfile(path):
+            return path
+    return None
+
+
+def _catalog_valid(settings: Settings, reference: str, archive: Path) -> tuple[dict, Path] | None:
+    digest = _sha256(archive)
+    catalog_path = settings.manifest_dir / reference / f"workbook-catalog-{digest[:12]}.json"
+    destination = settings.staging_dir / reference / digest[:12]
+    if not catalog_path.is_file():
+        return None
+    try:
+        catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
+        if catalog.get("reference") != reference or catalog.get("source_sha256") != digest:
+            return None
+        for entry in catalog.get("files", []):
+            file_path = destination / entry["path"]
+            if not file_path.is_file() or file_path.stat().st_size != entry["size_bytes"]:
+                return None
+        return catalog, destination
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+
+
+def _normalization_valid(settings: Settings, reference: str) -> bool:
+    folder = settings.processed_dir / reference
+    manifest_path = folder / "normalization-manifest.json"
+    try:
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        if manifest.get("reference") != reference:
+            return False
+        artifacts = {item["file"]: item for item in manifest["artifacts"]}
+        expected = {"input_prices.jsonl", "compositions.jsonl", "composition_items.jsonl"}
+        if set(artifacts) != expected:
+            return False
+        for name, metadata in artifacts.items():
+            path = folder / name
+            if not path.is_file() or path.stat().st_size != metadata["size_bytes"]:
+                return False
+            if _sha256(path) != metadata["sha256"]:
+                return False
+        return True
+    except (OSError, ValueError, KeyError, TypeError):
+        return False
+
+
+def _write_status(settings: Settings, *, reset: bool = False, **values) -> None:
+    path = settings.data_dir / "logs" / "bootstrap-state.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    current = {}
+    if not reset and path.is_file():
+        try:
+            current = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            pass
+    current.update(values)
+    current["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
+    temporary = path.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8")
+    temporary.replace(path)
+
+
+def _process_reference(
+    config: Settings,
+    database: Path,
+    reference: str,
+    source_url: str,
+    session,
+) -> dict:
+    if _loaded_and_complete(database, reference):
+        return {"reference": reference, "status": "already_loaded"}
+
+    archive = _find_archive(config, reference)
+    if archive is None:
+        _write_status(
+            config, stage="download", current_reference=reference,
+            message=f"Baixando pacote oficial {reference}.",
+        )
+        archive = download_monthly_xlsx(
+            reference, settings=config, source_url=source_url, session=session
+        ).path
+
+    catalog_result = _catalog_valid(config, reference, archive)
+    if catalog_result is None:
+        _write_status(
+            config, stage="extraction", current_reference=reference,
+            message=f"Extraindo e catalogando {reference}.",
+        )
+        extracted = extract_and_catalog(archive, reference, settings=config)
+        catalog = extracted["catalog"]
+        destination = config.staging_dir / reference / _sha256(archive)[:12]
+    else:
+        catalog, destination = catalog_result
+
+    workbook_path = None
+    for entry in catalog.get("workbooks", []):
+        sheet_names = {sheet["name"] for sheet in entry.get("sheets", [])}
+        if REQUIRED_SHEETS <= sheet_names:
+            candidate = destination / entry["relative_path"]
+            if candidate.is_file():
+                workbook_path = candidate
+                break
+    if workbook_path is None:
+        raise CaixaDownloadError(
+            f"O pacote {reference} não contém XLSX com as abas "
+            f"oficiais {sorted(REQUIRED_SHEETS)}."
+        )
+
+    if not _normalization_valid(config, reference):
+        _write_status(
+            config, stage="normalization", current_reference=reference,
+            message=f"Normalizando preços e composições de {reference}.",
+        )
+        normalized = normalize_reference_workbook(
+            workbook_path, reference=reference, settings=config
+        )
+    else:
+        normalized = json.loads(
+            (config.processed_dir / reference / "normalization-manifest.json").read_text(
+                encoding="utf-8"
+            )
+        )
+
+    if not _loaded_and_complete(database, reference):
+        _write_status(
+            config, stage="database_load", current_reference=reference,
+            message=f"Carregando {reference} no SQLite sem remover o histórico existente.",
+        )
+        def report_load_progress(phase: str, records: int) -> None:
+            _write_status(
+                config,
+                status="running",
+                stage="database_load",
+                current_reference=reference,
+                message=(
+                    f"Gravando {reference} no SQLite: "
+                    f"{phase} — {records:,} registros processados."
+                ),
+            )
+
+        loaded = load_silver_reference(
+            config.processed_dir,
+            reference=reference,
+            progress_callback=report_load_progress,
+        )
+    else:
+        loaded = {"reference": reference, "database_path": str(database)}
+
+    if not _loaded_and_complete(database, reference):
+        raise RuntimeError(f"A validação pós-carga de {reference} não foi aprovada.")
+
+    return {
+        "reference": reference,
+        "status": "loaded",
+        "counts": normalized.get("counts", {}),
+        "load": loaded,
+    }
+
+
+def _catalog_has_full_monthly_range(catalog: dict) -> bool:
+    """Reject stale/partial inventories that would hide earlier competencies."""
+    if UNIFIED_LAYOUT_START not in catalog:
+        return False
+    references = {str(reference) for reference in catalog}
+    last_reference = max(references)
+    year, month = 2025, 1
+    while f"{year:04d}-{month:02d}" <= last_reference:
+        if f"{year:04d}-{month:02d}" not in references:
+            return False
+        year, month = _month_shift(year, month, 1)
+    return True
+
+
+def _load_discovery_cache(config: Settings, *, force: bool) -> dict[str, str] | None:
+    path = config.manifest_dir / "historical-catalog.json"
+    if force or not path.is_file():
+        return None
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        age = time.time() - float(payload["checked_at"])
+        catalog = payload["catalog"]
+        if (
+            age <= DISCOVERY_CACHE_TTL_SECONDS
+            and isinstance(catalog, dict)
+            and catalog
+            and _catalog_has_full_monthly_range(catalog)
+        ):
+            return {str(key): str(value) for key, value in catalog.items()}
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+    return None
+
+
+def _save_discovery_cache(config: Settings, catalog: dict[str, str]) -> None:
+    path = config.manifest_dir / "historical-catalog.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(".json.tmp")
+    temporary.write_text(
+        json.dumps({"checked_at": time.time(), "catalog": catalog}, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    temporary.replace(path)
+
+
+def _run_bootstrap(settings: Settings | None = None, *, force: bool = False) -> dict:
+    config = settings or Settings.from_env()
+    config.create_data_directories()
+    database = _database_path(config)
+    had_database = database_is_valid(database)
+    _write_status(
+        config, reset=True, status="checking", stage="discovery", percent=0,
+        message="Descobrindo todas as competências XLSX publicadas no catálogo oficial da CAIXA.",
+        error=None,
+    )
+    try:
+        with browser_requests.Session(impersonate="chrome") as http_session:
+            catalog = _load_discovery_cache(config, force=force)
+            if catalog is None:
+                catalog = discover_historical_xlsx_catalog(
+                    settings=config, session=http_session
+                )
+                _save_discovery_cache(config, catalog)
+
+            references = sorted(catalog)
+            inventory = {
+                "schema_version": "1.0",
+                "source": "CAIXA",
+                "discovered_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+                "layout": "unified-xlsx",
+                "covered_from": UNIFIED_LAYOUT_START,
+                "legacy_layout_until": LEGACY_LAYOUT_END,
+                "references": [
+                    {"reference": reference, "url": catalog[reference]}
+                    for reference in references
+                ],
+            }
+            inventory_path = config.manifest_dir / "historical-inventory.json"
+            inventory_tmp = inventory_path.with_suffix(".json.tmp")
+            inventory_tmp.write_text(
+                json.dumps(inventory, ensure_ascii=False, indent=2), encoding="utf-8"
+            )
+            inventory_tmp.replace(inventory_path)
+
+            outcomes = []
+            failures = []
+            for index, reference in enumerate(references, start=1):
+                percent = round((index - 1) / max(len(references), 1) * 100)
+                _write_status(
+                    config, status="running", stage="historical_ingestion",
+                    percent=percent, current_reference=reference,
+                    completed=len(outcomes), total=len(references),
+                    message=f"Processando competência {reference} ({index}/{len(references)}).",
+                )
+                try:
+                    result = _process_reference(
+                        config, database, reference, catalog[reference], http_session
+                    )
+                    outcomes.append(result)
+                except Exception as exc:
+                    log.exception("Falha ao processar competência histórica %s", reference)
+                    failures.append({
+                        "reference": reference,
+                        "error": f"{type(exc).__name__}: {exc}",
+                    })
+                    _write_status(
+                        config, status="running", stage="historical_ingestion",
+                        current_reference=reference, failed=len(failures),
+                        failures=failures,
+                        message=f"Falha em {reference}; continuando as demais competências.",
+                    )
+
+        loaded_references = [
+            item["reference"] for item in outcomes
+            if item["status"] in {"loaded", "already_loaded"}
+        ]
+        legacy_pending = True
+        result = {
+            "status": "partial" if legacy_pending or failures else "complete",
+            "database": str(database),
+            "discovered": len(references),
+            "loaded_or_verified": len(loaded_references),
+            "failed": failures,
+            "references": loaded_references,
+            "covered_from": UNIFIED_LAYOUT_START,
+            "legacy_history_pending": legacy_pending,
+            "legacy_layout_until": LEGACY_LAYOUT_END,
+            "previous_database_preserved": had_database,
+        }
+        if failures:
+            result["status"] = "partial"
+        _write_status(
+            config, **result, stage="partial" if legacy_pending or failures else "complete",
+            percent=100, current_reference=None,
+            message=(
+                "Pacotes XLSX unificados processados. O histórico legado até 2024 "
+                "ainda exige ingestão por UF e parser próprio."
+            ),
+            error=None if not failures else "Uma ou mais competências não foram processadas.",
+        )
+        return result
+    except Exception as exc:
+        log.exception("Falha na descoberta/ingestão histórica SINAPI")
+        _write_status(
+            config, status="error", stage="error", percent=None,
+            message="Ingestão interrompida; os dados históricos já persistidos foram preservados.",
+            error=f"{type(exc).__name__}: {exc}",
+        )
+        raise
+
+
+def run_bootstrap(settings: Settings | None = None, *, force: bool = False) -> dict:
+    """Executa uma única atualização histórica por processo; chamadas concorrentes são coalescidas."""
+    if not _BOOTSTRAP_LOCK.acquire(blocking=False):
+        return {"status": "already_running", "message": "A atualização SINAPI já está em andamento."}
+    try:
+        return _run_bootstrap(settings, force=force)
+    finally:
+        _BOOTSTRAP_LOCK.release()

getsinapi/cli.py

@@ -0,0 +1,206 @@
+"""Interface de linha de comando do GET-SINAPI."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+
+from getsinapi import __version__
+from getsinapi.config import Settings
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="getsinapi",
+        description="Coleta, consulta e exporta dados SINAPI/CAIXA.",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    commands = parser.add_subparsers(dest="command")
+    commands.add_parser("check-config", help="Valida as configurações locais.")
+    commands.add_parser("export", help="Abre o menu interativo de exportação em lote.")
+
+    download = commands.add_parser("download", help="Baixa o pacote XLSX mensal oficial da CAIXA.")
+    download.add_argument("--reference", required=True, help="Competência no formato AAAA-MM")
+
+    extract = commands.add_parser("extract", help="Extrai ZIP e cataloga planilhas XLSX.")
+    extract.add_argument("--archive", required=True, type=Path, help="Caminho do ZIP já baixado.")
+    extract.add_argument("--reference", required=True, help="Competência no formato AAAA-MM")
+
+    parse = commands.add_parser("parse", help="Normaliza preços e composições para Silver.")
+    parse.add_argument("--workbook", required=True, type=Path, help="XLSX oficial de referência.")
+    parse.add_argument("--reference", required=True, help="Competência no formato AAAA-MM")
+
+    load = commands.add_parser("load", help="Carrega os artefatos Silver no SQLite.")
+    load.add_argument("--reference", required=True, help="Competência no formato AAAA-MM")
+    return parser
+
+
+def _ask_choice(prompt: str, options: dict[str, str], default: str | None = None) -> str:
+    while True:
+        suffix = f" [padrão {default}]" if default else ""
+        answer = input(f"{prompt}{suffix}: ").strip() or (default or "")
+        if answer in options:
+            return answer
+        print("Opção inválida. Escolha uma das opções exibidas.")
+
+
+def _export_menu(settings: Settings) -> int:
+    from getsinapi.exporter import available_references, export_catalog
+
+    desktop_data = Path(os.getenv("LOCALAPPDATA", Path.home())) / "NRTech" / "GET-SINAPI-V3" / "data"
+    candidates = [
+        Path(os.getenv("GETSINAPI_DATABASE_PATH", "")) if os.getenv("GETSINAPI_DATABASE_PATH") else None,
+        desktop_data / "getsinapi.sqlite3",
+        settings.data_dir / "getsinapi.sqlite3",
+    ]
+    database = next((path for path in candidates if path and path.is_file()),
+                    desktop_data / "getsinapi.sqlite3" if os.name == "nt" else settings.data_dir / "getsinapi.sqlite3")
+    if not database.is_file():
+        print(f"Banco SINAPI não encontrado: {database}")
+        print("Inicialize a base com os comandos download, extract, parse e load.")
+        return 2
+
+    print("\n" + "=" * 62)
+    print("GET-SINAPI | CENTRAL DE EXPORTAÇÃO EM LOTE")
+    print("=" * 62)
+    print("1. Exportar tudo (insumos, composições e componentes)")
+    print("2. Escolher categorias")
+    mode = _ask_choice("Selecione", {"1": "Tudo", "2": "Personalizado"}, "1")
+
+    categories = ["INSUMO", "COMPOSICAO", "COMPONENTE"]
+    if mode == "2":
+        print("\nCategorias disponíveis:")
+        print("1. Insumos   2. Composições   3. Componentes")
+        selected = input("Informe uma ou mais opções separadas por vírgula (ex.: 1,3): ").strip()
+        mapping = {"1": "INSUMO", "2": "COMPOSICAO", "3": "COMPONENTE"}
+        chosen = [mapping[item.strip()] for item in selected.split(",") if item.strip() in mapping]
+        if not chosen:
+            print("Nenhuma categoria válida foi selecionada.")
+            return 2
+        categories = list(dict.fromkeys(chosen))
+
+    refs = available_references(database)
+    if not refs:
+        print("A base SQLite não possui competências carregadas.")
+        return 2
+    print("\nCompetências disponíveis:")
+    for index, reference in enumerate(refs, start=1):
+        print(f"{index}. {reference}")
+    print("A. Todas as competências")
+    reference_choice = input("Competência [A]: ").strip().upper() or "A"
+    if reference_choice == "A":
+        references = None
+    elif reference_choice.isdigit() and 1 <= int(reference_choice) <= len(refs):
+        references = [refs[int(reference_choice) - 1]]
+    else:
+        print("Competência inválida.")
+        return 2
+
+    uf = input("\nUF (duas letras) ou ENTER para todas: ").strip().upper() or None
+    if uf and (len(uf) != 2 or not uf.isalpha()):
+        print("UF inválida.")
+        return 2
+
+    regime_choice = _ask_choice(
+        "Regime: 1 todos | 2 desonerado | 3 não desonerado",
+        {"1": "all", "2": "desonerado", "3": "nao_desonerado"},
+        "1",
+    )
+    regime = None if regime_choice == "1" else regime_choice
+
+    encargo_choice = _ask_choice(
+        "Encargos: 1 todos | 2 com encargos | 3 sem encargos",
+        {"1": "all", "2": "com_encargos", "3": "sem_encargos"},
+        "1",
+    )
+    encargo_mode = None if encargo_choice == "1" else encargo_choice
+
+    search = input("Pesquisar por código/descrição (ENTER para sem filtro): ").strip() or None
+    file_format = _ask_choice(
+        "Formato: 1 Excel (.xlsx) | 2 CSV | 3 Excel + CSV",
+        {"1": "xlsx", "2": "csv", "3": "both"},
+        "1",
+    )
+    default_output = database.parent / "exports"
+    output_text = input(f"Pasta de destino [{default_output}]: ").strip()
+    output_dir = Path(output_text).expanduser() if output_text else default_output
+
+    print("\nIniciando extração da base local...")
+    generated = export_catalog(
+        database,
+        output_dir,
+        file_format=file_format,
+        references=references,
+        uf=uf,
+        categories=categories,
+        regime=regime,
+        encargo_mode=encargo_mode,
+        search=search,
+    )
+    print(f"\nExportação concluída. Arquivos gerados: {len(generated)}")
+    for path in generated:
+        print(f"  - {path}")
+    return 0
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    settings = Settings.from_env()
+
+    if args.command in (None, "export"):
+        settings.create_data_directories()
+        return _export_menu(settings)
+
+    if args.command == "check-config":
+        settings.create_data_directories()
+        print(f"Configuração válida | ambiente={settings.environment}")
+        print(f"Diretório de dados: {settings.data_dir}")
+        return 0
+
+    if args.command == "download":
+        from getsinapi.collectors.caixa import download_monthly_xlsx
+        artifact = download_monthly_xlsx(args.reference, settings=settings)
+        print(f"Download validado: {artifact.path}")
+        print(f"SHA-256: {artifact.sha256}")
+        print(f"Tamanho: {artifact.size_bytes} bytes")
+        return 0
+
+    if args.command == "extract":
+        from getsinapi.extractors.workbooks import extract_and_catalog
+        result = extract_and_catalog(args.archive, args.reference, settings=settings)
+        print(f"Arquivos extraídos: {len(result['files'])}")
+        print(f"Planilhas catalogadas: {len(result['workbooks'])}")
+        print(f"Catálogo: {result['catalog_path']}")
+        for workbook in result["workbooks"]:
+            print(f"- {workbook['file']} | abas: {len(workbook['sheets'])}")
+        return 0
+
+    if args.command == "parse":
+        from getsinapi.pipeline.normalize import normalize_reference_workbook
+        result = normalize_reference_workbook(args.workbook, reference=args.reference, settings=settings)
+        print(f"Competência: {result['reference']}")
+        print(f"Preços: {result['counts']['input_prices']:,}")
+        print(f"Composições: {result['counts']['compositions']:,}")
+        print(f"Itens de composição: {result['counts']['composition_items']:,}")
+        print(f"Advertências: {result['warning_count']:,}")
+        print(f"Manifesto: {result['manifest_path']}")
+        return 0
+
+    if args.command == "load":
+        from getsinapi.storage.sqlite import load_silver_reference
+        result = load_silver_reference(settings.processed_dir, reference=args.reference)
+        print(f"Competência carregada: {result['reference']}")
+        print(f"Preços: {result['input_prices']:,}")
+        print(f"Composições: {result['compositions']:,}")
+        print(f"Itens: {result['composition_items']:,}")
+        print(f"Banco: {result['database_path']}")
+        return 0
+
+    parser.error("Comando não reconhecido.")
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

getsinapi/collectors/__init__.py

@@ -0,0 +1 @@
+"""Coletores de fontes oficiais."""