get-sinapi 3.1.0__py3-none-any.whl

getsinapi/extractors/reference_workbook.py

@@ -0,0 +1,68 @@
+"""Orquestra a leitura das tabelas oficiais de referência SINAPI."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from openpyxl import load_workbook
+
+from getsinapi.extractors.compositions import (
+    CompositionParseResult,
+    parse_analytic_compositions,
+)
+from getsinapi.extractors.price_tables import (
+    InputPriceParseResult,
+    parse_input_price_table,
+)
+
+PRICE_SHEETS = ("ISD", "ICD", "ISE")
+COMPOSITION_SHEET = "Analítico"
+
+
+@dataclass(frozen=True, slots=True)
+class ReferenceWorkbookResult:
+    reference: str
+    source_file: str
+    prices: tuple[InputPriceParseResult, ...]
+    compositions: CompositionParseResult
+
+
+def parse_reference_workbook(
+    workbook_path: str | Path,
+    *,
+    reference: str,
+) -> ReferenceWorkbookResult:
+    """Extrai preços ISD/ICD/ISE e estrutura Analítico do mesmo XLSX oficial.
+
+    A função não persiste nem transforma ausência de preço em zero; entrega
+    resultados tipados para a próxima etapa de validação e carga.
+    """
+    path = Path(workbook_path)
+    workbook = load_workbook(path, read_only=True, data_only=True)
+    try:
+        available = set(workbook.sheetnames)
+    finally:
+        workbook.close()
+
+    required = {*PRICE_SHEETS, COMPOSITION_SHEET}
+    missing = sorted(required - available)
+    if missing:
+        raise ValueError(
+            "Planilha de referência incompleta; abas ausentes: "
+            + ", ".join(missing)
+        )
+
+    prices = tuple(
+        parse_input_price_table(path, sheet_name, reference=reference)
+        for sheet_name in PRICE_SHEETS
+    )
+    compositions = parse_analytic_compositions(
+        path, COMPOSITION_SHEET, reference=reference
+    )
+    return ReferenceWorkbookResult(
+        reference=reference,
+        source_file=path.name,
+        prices=prices,
+        compositions=compositions,
+    )

getsinapi/extractors/sinapi_parser.py

@@ -0,0 +1,158 @@
+"""Parser configurável para planilhas SINAPI catalogadas.
+
+A detecção de colunas é conservadora: só mapeia cabeçalhos conhecidos.
+Nenhuma coluna ou regime é inferido apenas pela posição da coluna.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from openpyxl import load_workbook
+
+from getsinapi.domain import PriceObservation, Regime
+from getsinapi.normalizers import classify_price
+
+HEADER_ALIASES = {
+    "code": {"codigo", "cod", "codigo do insumo", "codigo da composicao"},
+    "description": {"descricao", "descricao do insumo", "descricao da composicao"},
+    "unit": {"unidade", "un"},
+    "price": {"preco", "preco mediano", "custo"},
+    "status": {"situacao", "observacao", "status", "nota"},
+}
+UF_PATTERN = re.compile(
+    r"\b(AC|AL|AP|AM|BA|CE|DF|ES|GO|MA|MT|MS|MG|PA|PB|PR|PE|PI|RJ|RN|RS|RO|RR|SC|SP|SE|TO)\b"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class SheetParseResult:
+    sheet_name: str
+    regime: Regime | None
+    uf: str | None
+    header_row: int | None
+    records: tuple[PriceObservation, ...]
+    warnings: tuple[str, ...]
+
+
+def normalize_header(value: Any) -> str:
+    if value is None:
+        return ""
+    text = unicodedata.normalize("NFKD", str(value))
+    text = "".join(char for char in text if not unicodedata.combining(char))
+    return re.sub(r"\s+", " ", re.sub(r"[^a-zA-Z0-9 ]", " ", text)).strip().lower()
+
+
+def detect_regime(sheet_name: str) -> Regime | None:
+    normalized = normalize_header(sheet_name)
+    if "nao desonerado" in normalized or "nao desonerada" in normalized:
+        return Regime.NAO_DESONERADO
+    if "desonerado" in normalized or "desonerada" in normalized:
+        return Regime.DESONERADO
+    return None
+
+
+def detect_uf(*values: Any) -> str | None:
+    text = " ".join(str(value) for value in values if value is not None).upper()
+    matches = UF_PATTERN.findall(text)
+    return matches[0] if len(set(matches)) == 1 else None
+
+
+def _column_map(row: tuple[Any, ...]) -> dict[str, int]:
+    normalized = [normalize_header(value) for value in row]
+    result: dict[str, int] = {}
+    for field, aliases in HEADER_ALIASES.items():
+        matches = [index for index, value in enumerate(normalized) if value in aliases]
+        if len(matches) == 1:
+            result[field] = matches[0]
+    return result
+
+
+def _cell(row: tuple[Any, ...], columns: dict[str, int], field: str) -> Any:
+    index = columns.get(field)
+    return row[index] if index is not None and index < len(row) else None
+
+
+def parse_worksheet(
+    workbook_path: str | Path,
+    sheet_name: str,
+    *,
+    reference: str,
+    source_file: str | None = None,
+) -> SheetParseResult:
+    path = Path(workbook_path)
+    workbook = load_workbook(path, read_only=True, data_only=True)
+    try:
+        if sheet_name not in workbook.sheetnames:
+            raise ValueError(f"Aba inexistente: {sheet_name}")
+        sheet = workbook[sheet_name]
+        regime = detect_regime(sheet_name)
+        rows = sheet.iter_rows(values_only=True)
+        header_row = None
+        columns: dict[str, int] = {}
+        uf: str | None = None
+        context_rows: list[Any] = []
+
+        for row_number, row in enumerate(rows, start=1):
+            context_rows.extend(row)
+            candidate = _column_map(tuple(row))
+            if {"code", "description", "unit", "price"}.issubset(candidate):
+                header_row = row_number
+                columns = candidate
+                uf = detect_uf(*context_rows)
+                break
+
+        if header_row is None:
+            return SheetParseResult(
+                sheet_name,
+                regime,
+                None,
+                None,
+                (),
+                ("Cabeçalho não reconhecido; layout requer mapeamento explícito.",),
+            )
+
+        records: list[PriceObservation] = []
+        warnings: list[str] = []
+        for row_number, row in enumerate(rows, start=header_row + 1):
+            values = tuple(row)
+            code = _cell(values, columns, "code")
+            description = _cell(values, columns, "description")
+            unit = _cell(values, columns, "unit")
+            raw_price = _cell(values, columns, "price")
+            raw_status = _cell(values, columns, "status")
+            if code is None and description is None:
+                continue
+            if code is None or description is None or unit is None:
+                warnings.append(f"Linha {row_number}: registro incompleto ignorado.")
+                continue
+            if uf is None or regime is None:
+                warnings.append(
+                    f"Linha {row_number}: UF ou regime não identificados; registro não emitido."
+                )
+                continue
+            price, status = classify_price(raw_price, raw_status)
+            records.append(
+                PriceObservation(
+                    reference=reference,
+                    uf=uf,
+                    regime=regime,
+                    code=str(code).strip(),
+                    description=str(description).strip(),
+                    unit=str(unit).strip(),
+                    price=price,
+                    status=status,
+                    source_file=source_file or path.name,
+                    source_sheet=sheet_name,
+                    source_row=row_number,
+                )
+            )
+        return SheetParseResult(
+            sheet_name, regime, uf, header_row, tuple(records), tuple(warnings)
+        )
+    finally:
+        workbook.close()

getsinapi/extractors/workbooks.py

@@ -0,0 +1,126 @@
+"""Extração segura e inventário de planilhas XLSX de pacotes SINAPI."""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from zipfile import BadZipFile
+
+from openpyxl import load_workbook
+
+from getsinapi.collectors.caixa import CaixaDownloadError, safe_extract_zip
+from getsinapi.config import Settings
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _json_value(value):
+    """Converte valores de células para tipos serializáveis sem alterar seu significado."""
+    if hasattr(value, "isoformat"):
+        return value.isoformat()
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    return str(value)
+
+
+def inspect_workbook(path: Path) -> dict:
+    """Retorna metadados e amostra técnica das abas, sem inferir conteúdo de negócio."""
+    try:
+        workbook = load_workbook(path, read_only=True, data_only=True)
+    except (BadZipFile, OSError, ValueError) as exc:
+        raise CaixaDownloadError(f"Planilha XLSX inválida: {path.name}: {exc}") from exc
+
+    sheets = []
+    try:
+        for worksheet in workbook.worksheets:
+            preview = []
+            for row in worksheet.iter_rows(
+                min_row=1,
+                max_row=min(worksheet.max_row or 0, 12),
+                min_col=1,
+                max_col=min(worksheet.max_column or 0, 20),
+                values_only=True,
+            ):
+                values = [_json_value(value) for value in row]
+                if any(value is not None and value != "" for value in values):
+                    preview.append(values)
+            sheets.append({
+                "name": worksheet.title,
+                "max_row": worksheet.max_row,
+                "max_column": worksheet.max_column,
+                "state": worksheet.sheet_state,
+                "preview_rows": preview,
+            })
+    finally:
+        workbook.close()
+
+    return {
+        "file": path.name,
+        "relative_path": path.name,
+        "size_bytes": path.stat().st_size,
+        "sha256": _sha256(path),
+        "sheets": sheets,
+    }
+
+
+def extract_and_catalog(
+    archive_path: Path,
+    reference: str,
+    settings: Settings | None = None,
+) -> dict:
+    """Extrai um pacote oficial e gera catálogo JSON reproduzível."""
+    if not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", reference):
+        raise ValueError("Competência inválida. Informe AAAA-MM.")
+
+    archive_path = Path(archive_path)
+    if not archive_path.is_file():
+        raise FileNotFoundError(f"Arquivo ZIP não encontrado: {archive_path}")
+    settings = settings or Settings.from_env()
+    settings.create_data_directories()
+
+    digest = _sha256(archive_path)
+    destination = settings.staging_dir / reference / digest[:12]
+    destination.mkdir(parents=True, exist_ok=True)
+
+    extracted = safe_extract_zip(archive_path, destination)
+    workbooks = []
+    for path in sorted(extracted):
+        if path.suffix.lower() == ".xlsx":
+            workbooks.append(inspect_workbook(path))
+
+    catalog = {
+        "schema_version": "1.0",
+        "source_archive": archive_path.name,
+        "source_sha256": digest,
+        "reference": reference,
+        "extracted_at": datetime.now(UTC).isoformat(),
+        "extracted_file_count": len(extracted),
+        "workbook_count": len(workbooks),
+        "files": [
+            {"path": path.relative_to(destination).as_posix(), "size_bytes": path.stat().st_size}
+            for path in sorted(extracted)
+        ],
+        "workbooks": workbooks,
+    }
+
+    catalog_dir = settings.manifest_dir / reference
+    catalog_dir.mkdir(parents=True, exist_ok=True)
+    catalog_path = catalog_dir / f"workbook-catalog-{digest[:12]}.json"
+    temporary = catalog_path.with_suffix(".json.part")
+    temporary.write_text(json.dumps(catalog, ensure_ascii=False, indent=2), encoding="utf-8")
+    temporary.replace(catalog_path)
+
+    return {
+        "files": extracted,
+        "workbooks": workbooks,
+        "catalog_path": catalog_path,
+        "catalog": catalog,
+    }

getsinapi/normalizers.py

@@ -0,0 +1,114 @@
+
+"""Normalização conservadora de valores e observações de preço SINAPI."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+from getsinapi.domain import PriceStatus
+
+
+def _normalize_text(value: Any) -> str:
+    """Normaliza texto para comparação, preservando o conteúdo semântico."""
+    text = str(value).strip().casefold()
+    text = unicodedata.normalize("NFKD", text)
+    text = "".join(
+        char for char in text
+        if not unicodedata.combining(char)
+    )
+    return re.sub(r"\s+", " ", text)
+
+
+_NOT_RESEARCHED = re.compile(
+    r"\b(?:"
+    r"nao pesquisad[oa]s?"
+    r"|sem pesquisa de preco"
+    r"|preco nao pesquisad[oa]"
+    r")\b",
+    flags=re.IGNORECASE,
+)
+
+
+def parse_decimal(value: Any) -> Decimal | None:
+    """Converte valores numéricos sem arredondamento; vazio permanece None.
+
+    Aceita Decimal, inteiros, floats e strings numéricas brasileiras.
+    Não interpreta textos de ausência como zero.
+    """
+    if value is None:
+        return None
+
+    if isinstance(value, bool):
+        raise TypeError("Booleano não é um valor monetário válido.")
+
+    if isinstance(value, Decimal):
+        result = value
+    elif isinstance(value, int):
+        result = Decimal(value)
+    elif isinstance(value, float):
+        result = Decimal(str(value))
+    else:
+        raw = str(value).strip()
+
+        if not raw:
+            return None
+
+        raw = raw.replace("R$", "").replace(" ", "")
+
+        if "," in raw and "." in raw:
+            if raw.rfind(",") > raw.rfind("."):
+                raw = raw.replace(".", "").replace(",", ".")
+            else:
+                raw = raw.replace(",", "")
+        elif "," in raw:
+            raw = raw.replace(".", "").replace(",", ".")
+
+        try:
+            result = Decimal(raw)
+        except InvalidOperation as exc:
+            raise ValueError(
+                f"Valor numérico inválido: {value!r}"
+            ) from exc
+
+    if not result.is_finite():
+        raise ValueError(
+            f"Valor numérico não finito: {value!r}"
+        )
+
+    return result
+
+
+def classify_price(
+    value: Any,
+    source_status: Any = None,
+) -> tuple[Decimal | None, PriceStatus]:
+    """Classifica preços preservando as distinções da fonte.
+
+    - NOT_RESEARCHED: indicação textual explícita da fonte.
+    - MISSING: valor ausente ou célula vazia.
+    - REPORTED_ZERO: preço numérico igual a zero.
+    - REPORTED: preço numérico diferente de zero.
+
+    A ausência de informação nunca é convertida automaticamente em zero.
+    """
+    status_text = (
+        ""
+        if source_status is None
+        else _normalize_text(source_status)
+    )
+
+    if _NOT_RESEARCHED.search(status_text):
+        return None, PriceStatus.NOT_RESEARCHED
+
+    parsed = parse_decimal(value)
+
+    if parsed is None:
+        return None, PriceStatus.MISSING
+
+    if parsed == Decimal(0):
+        return parsed, PriceStatus.REPORTED_ZERO
+
+    return parsed, PriceStatus.REPORTED

getsinapi/pipeline/__init__.py

@@ -0,0 +1 @@
+"""Orquestração das etapas de processamento SINAPI."""

getsinapi/pipeline/normalize.py

@@ -0,0 +1,114 @@
+"""Publica os resultados normalizados do XLSX oficial na camada Silver."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Iterable
+from dataclasses import asdict
+from decimal import Decimal
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+from getsinapi.config import Settings
+from getsinapi.extractors.reference_workbook import parse_reference_workbook
+
+
+def _json_default(value: Any) -> str:
+    if isinstance(value, Decimal):
+        return str(value)
+    if isinstance(value, Enum):
+        return value.value
+    raise TypeError(f"Tipo não serializável: {type(value).__name__}")
+
+
+def _write_jsonl(path: Path, records: Iterable[dict[str, Any]]) -> dict[str, Any]:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    digest = hashlib.sha256()
+    record_count = 0
+    with temporary.open("w", encoding="utf-8", newline="\n") as stream:
+        for record in records:
+            line = json.dumps(
+                record,
+                ensure_ascii=False,
+                sort_keys=True,
+                default=_json_default,
+                separators=(",", ":"),
+            ) + "\n"
+            stream.write(line)
+            digest.update(line.encode("utf-8"))
+            record_count += 1
+    os.replace(temporary, path)
+    return {
+        "file": path.name,
+        "records": record_count,
+        "sha256": digest.hexdigest(),
+        "size_bytes": path.stat().st_size,
+    }
+
+
+def normalize_reference_workbook(
+    workbook_path: str | Path,
+    *,
+    reference: str,
+    settings: Settings | None = None,
+) -> dict[str, Any]:
+    """Extrai tabelas e grava JSONL rastreável em data/processed/AAAA-MM."""
+    config = settings or Settings.from_env()
+    result = parse_reference_workbook(workbook_path, reference=reference)
+    output_dir = config.processed_dir / reference
+
+    # Não materializa cópias adicionais de centenas de milhares de registros
+    # como listas de dicionários; serializa cada registro diretamente em JSONL.
+    def iter_price_records():
+        for table in result.prices:
+            for record in table.records:
+                yield {**asdict(record), "table_type": table.table_type}
+
+    artifacts = [
+        _write_jsonl(output_dir / "input_prices.jsonl", iter_price_records()),
+        _write_jsonl(
+            output_dir / "compositions.jsonl",
+            (asdict(record) for record in result.compositions.compositions),
+        ),
+        _write_jsonl(
+            output_dir / "composition_items.jsonl",
+            (asdict(record) for record in result.compositions.items),
+        ),
+    ]
+    counts_by_file = {item["file"]: item["records"] for item in artifacts}
+
+    warnings = [
+        {"source": table.table_type, "message": warning}
+        for table in result.prices
+        for warning in table.warnings
+    ] + [
+        {"source": "Analítico", "message": warning}
+        for warning in result.compositions.warnings
+    ]
+
+    manifest = {
+        "reference": reference,
+        "source_file": result.source_file,
+        "source_path": str(Path(workbook_path)),
+        "artifacts": artifacts,
+        "counts": {
+            "input_prices": counts_by_file["input_prices.jsonl"],
+            "compositions": counts_by_file["compositions.jsonl"],
+            "composition_items": counts_by_file["composition_items.jsonl"],
+        },
+        "warning_count": len(warnings),
+        "warnings": warnings,
+    }
+    manifest_path = output_dir / "normalization-manifest.json"
+    temporary_manifest = manifest_path.with_suffix(".json.tmp")
+    temporary_manifest.write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2, default=_json_default) + "\n",
+        encoding="utf-8",
+    )
+    os.replace(temporary_manifest, manifest_path)
+    manifest["manifest_path"] = str(manifest_path)
+    return manifest

getsinapi/services/__init__.py

@@ -0,0 +1 @@
+"""Serviços de domínio e cálculos do GET-SINAPI."""