get-sinapi 3.1.0__py3-none-any.whl

getsinapi/domain.py

@@ -0,0 +1,74 @@
+"""Modelos de domínio para registros normalizados do SINAPI.
+
+Os modelos preservam a diferença entre preço zero informado, preço ausente
+e ausência de pesquisa explicitamente documentada pela fonte.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from decimal import Decimal
+from enum import StrEnum
+
+
+class Regime(StrEnum):
+    DESONERADO = "desonerado"
+    NAO_DESONERADO = "nao_desonerado"
+
+
+class EncargoMode(StrEnum):
+    """Tratamento de encargos sociais aplicado ao relatório."""
+
+    COM_ENCARGOS = "com_encargos"
+    SEM_ENCARGOS = "sem_encargos"
+
+
+class CompositionStatus(StrEnum):
+    """Situação da composição conforme a classificação da fonte oficial."""
+
+    COM_CUSTO = "com_custo"
+    COM_PRECO = "com_preco"
+    SEM_CUSTO = "sem_custo"
+    SEM_PRECO = "sem_preco"
+    EM_ESTUDO = "em_estudo"
+
+
+class PriceStatus(StrEnum):
+    REPORTED = "reported"
+    REPORTED_ZERO = "reported_zero"
+    MISSING = "missing"
+    NOT_RESEARCHED = "not_researched"
+
+
+@dataclass(frozen=True, slots=True)
+class PriceObservation:
+    reference: str
+    uf: str
+    regime: Regime
+    code: str
+    description: str
+    unit: str
+    price: Decimal | None
+    status: PriceStatus
+    source_file: str
+    source_sheet: str
+    source_row: int
+
+    def __post_init__(self) -> None:
+        if len(self.reference) != 7 or self.reference[4] != "-":
+            raise ValueError("Competência deve estar no formato AAAA-MM.")
+        if len(self.uf) != 2 or not self.uf.isalpha():
+            raise ValueError("UF deve conter a sigla de duas letras.")
+        if not self.code.strip():
+            raise ValueError("Código SINAPI não pode ser vazio.")
+        if self.source_row < 1:
+            raise ValueError("Linha de origem deve ser maior ou igual a 1.")
+        reported = {PriceStatus.REPORTED, PriceStatus.REPORTED_ZERO}
+        absent = {PriceStatus.MISSING, PriceStatus.NOT_RESEARCHED}
+        if self.status in reported and self.price is None:
+            raise ValueError("Preço informado exige valor numérico.")
+        if self.status == PriceStatus.REPORTED_ZERO and self.price != Decimal(0):
+            raise ValueError("Status reported_zero exige preço igual a zero.")
+        if self.status == PriceStatus.REPORTED and self.price == Decimal(0):
+            raise ValueError("Preço zero deve usar status reported_zero.")
+        if self.status in absent and self.price is not None:
+            raise ValueError("Preço ausente ou não pesquisado não pode conter valor.")

getsinapi/exporter.py

@@ -0,0 +1,189 @@
+"""Exportação em lote do catálogo SINAPI para Excel e CSV."""
+
+from __future__ import annotations
+
+import csv
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from openpyxl import Workbook
+
+CATEGORIES = {
+    "INSUMO": "Insumos",
+    "COMPOSICAO": "Composicoes",
+    "COMPONENTE": "Componentes",
+}
+EXCEL_MAX_DATA_ROWS = 1_048_575
+
+CATALOG_CTE = """
+WITH catalog AS (
+    SELECT reference, 'INSUMO' AS record_type, code, description, unit, uf,
+           regime, encargo_mode, table_type, price AS value, 'PRECO' AS value_type,
+           NULL AS composition_code, status, source_file, source_sheet, source_row
+    FROM input_prices
+    UNION ALL
+    SELECT reference, 'COMPOSICAO', code, description, unit,
+           NULL, NULL, NULL, 'Analitico', NULL, NULL, NULL,
+           status, source_file, source_sheet, source_row
+    FROM compositions
+    UNION ALL
+    SELECT reference, 'COMPONENTE', item_code, description, unit,
+           NULL, NULL, NULL, 'Analitico', coefficient, 'COEFICIENTE',
+           composition_code, status, source_file, source_sheet, source_row
+    FROM composition_items
+)
+"""
+
+COLUMNS = (
+    "reference", "record_type", "code", "description", "unit", "uf", "regime",
+    "encargo_mode", "table_type", "value", "value_type", "composition_code",
+    "status", "source_file", "source_sheet", "source_row",
+)
+
+
+def _query(
+    *,
+    references: list[str] | None,
+    uf: str | None,
+    categories: list[str],
+    regime: str | None,
+    encargo_mode: str | None,
+    search: str | None,
+) -> tuple[str, tuple[Any, ...]]:
+    clauses: list[str] = []
+    params: list[Any] = []
+    if references:
+        clauses.append("reference IN (" + ",".join("?" for _ in references) + ")")
+        params.extend(references)
+    if uf:
+        clauses.append("(uf = ? OR uf IS NULL)")
+        params.append(uf.upper())
+    if categories:
+        clauses.append("record_type IN (" + ",".join("?" for _ in categories) + ")")
+        params.extend(categories)
+    if regime:
+        clauses.append("(regime = ? OR regime IS NULL)")
+        params.append(regime)
+    if encargo_mode:
+        clauses.append("(encargo_mode = ? OR encargo_mode IS NULL)")
+        params.append(encargo_mode)
+    if search:
+        term = "%" + search.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
+        clauses.append(
+            "(code LIKE ? ESCAPE '\\' OR description LIKE ? ESCAPE '\\' "
+            "OR composition_code LIKE ? ESCAPE '\\')"
+        )
+        params.extend((term, term, term))
+    where = " WHERE " + " AND ".join(clauses) if clauses else ""
+    sql = CATALOG_CTE + " SELECT " + ", ".join(COLUMNS) + " FROM catalog" + where
+    sql += " ORDER BY reference DESC, record_type, code, uf"
+    return sql, tuple(params)
+
+
+def available_references(database_path: Path) -> list[str]:
+    with sqlite3.connect(database_path) as connection:
+        rows = connection.execute(
+            "SELECT reference FROM source_references ORDER BY reference DESC"
+        ).fetchall()
+    return [str(row[0]) for row in rows]
+
+
+def export_catalog(
+    database_path: Path,
+    output_dir: Path,
+    *,
+    file_format: str,
+    references: list[str] | None = None,
+    uf: str | None = None,
+    categories: list[str] | None = None,
+    regime: str | None = None,
+    encargo_mode: str | None = None,
+    search: str | None = None,
+) -> list[Path]:
+    """Exporta categorias individualizadas e uma visão consolidada."""
+    if file_format not in {"xlsx", "csv", "both"}:
+        raise ValueError("Formato deve ser xlsx, csv ou both.")
+    selected = categories or list(CATEGORIES)
+    invalid = set(selected) - set(CATEGORIES)
+    if invalid:
+        raise ValueError(f"Categorias inválidas: {', '.join(sorted(invalid))}.")
+    if not database_path.is_file():
+        raise FileNotFoundError(f"Banco SINAPI não encontrado: {database_path}")
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    base = output_dir / f"getsinapi_export_{stamp}"
+    sql, params = _query(
+        references=references, uf=uf, categories=selected, regime=regime,
+        encargo_mode=encargo_mode, search=search
+    )
+    generated: list[Path] = []
+
+    with sqlite3.connect(database_path) as connection:
+        connection.row_factory = sqlite3.Row
+        if file_format in {"xlsx", "both"}:
+            workbook = Workbook(write_only=True)
+            sheets: dict[str, Any] = {}
+
+            def new_sheet(label: str, index: int = 1):
+                suffix = f"_{index:02d}" if index > 1 else ""
+                sheet = workbook.create_sheet((label[:25] + suffix)[:31])
+                sheet.append(list(COLUMNS))
+                return sheet
+
+            sheets["CONSOLIDADO"] = new_sheet("CONSOLIDADO")
+            for category in selected:
+                sheets[category] = new_sheet(CATEGORIES[category])
+
+            cursor = connection.execute(sql, params)
+            counts = {key: 0 for key in sheets}
+            sheet_parts = {key: 1 for key in sheets}
+            while rows := cursor.fetchmany(2000):
+                for row in rows:
+                    values = [row[column] for column in COLUMNS]
+                    category = str(row["record_type"])
+                    for key in ("CONSOLIDADO", category):
+                        if key not in sheets:
+                            continue
+                        if counts[key] >= EXCEL_MAX_DATA_ROWS:
+                            sheet_parts[key] += 1
+                            label = "CONSOLIDADO" if key == "CONSOLIDADO" else CATEGORIES[key]
+                            sheets[key] = new_sheet(label, sheet_parts[key])
+                            counts[key] = 0
+                        sheets[key].append(values)
+                        counts[key] += 1
+            xlsx_path = base.with_suffix(".xlsx")
+            workbook.save(xlsx_path)
+            generated.append(xlsx_path)
+
+        if file_format in {"csv", "both"}:
+            csv_targets = {"CONSOLIDADO": base.with_name(base.name + "_consolidado.csv")}
+            csv_targets.update({
+                category: base.with_name(base.name + "_" + CATEGORIES[category].lower() + ".csv")
+                for category in selected
+            })
+            handles = {}
+            writers = {}
+            try:
+                for key, path in csv_targets.items():
+                    handle = path.open("w", newline="", encoding="utf-8-sig")
+                    handles[key] = handle
+                    writer = csv.writer(handle, delimiter=";", quoting=csv.QUOTE_MINIMAL)
+                    writer.writerow(COLUMNS)
+                    writers[key] = writer
+                cursor = connection.execute(sql, params)
+                while rows := cursor.fetchmany(2000):
+                    for row in rows:
+                        values = [row[column] for column in COLUMNS]
+                        writers["CONSOLIDADO"].writerow(values)
+                        category = str(row["record_type"])
+                        if category in writers:
+                            writers[category].writerow(values)
+            finally:
+                for handle in handles.values():
+                    handle.close()
+            generated.extend(csv_targets.values())
+
+    return generated

getsinapi/extractors/__init__.py

@@ -0,0 +1 @@
+"""Extratores de pacotes oficiais SINAPI."""

getsinapi/extractors/compositions.py

@@ -0,0 +1,213 @@
+"""Parser do relatório Analítico oficial de composições SINAPI.
+
+Layout auditado na referência 2026-09:
+- Linha 10: cabeçalho das colunas.
+- Linha 11 em diante: linha-pai (C e D vazias), COMPOSICAO e INSUMO.
+- B: código da composição; C: tipo do item; D: código do item;
+  E: descrição; F: unidade; G: coeficiente; H: situação.
+"""
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+from openpyxl import load_workbook
+
+from getsinapi.domain import CompositionStatus
+from getsinapi.normalizers import parse_decimal
+
+REFERENCE_PATTERN = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
+STATUS_MAP = {
+    "com custo": CompositionStatus.COM_CUSTO,
+    "com preco": CompositionStatus.COM_PRECO,
+    "sem custo": CompositionStatus.SEM_CUSTO,
+    "sem preco": CompositionStatus.SEM_PRECO,
+    "em estudo": CompositionStatus.EM_ESTUDO,
+}
+
+
+def _text(value: Any) -> str:
+    if value is None:
+        return ""
+    normalized = unicodedata.normalize("NFKD", str(value).strip().casefold())
+    normalized = "".join(char for char in normalized if not unicodedata.combining(char))
+    return re.sub(r"\s+", " ", normalized)
+
+
+def _code(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, float) and value.is_integer():
+        value = int(value)
+    raw = str(value).strip()
+    if raw.endswith(".0") and raw[:-2].isdigit():
+        raw = raw[:-2]
+    return raw
+
+
+@dataclass(frozen=True, slots=True)
+class CompositionRecord:
+    reference: str
+    code: str
+    description: str
+    unit: str
+    status: CompositionStatus | None
+    source_file: str
+    source_sheet: str
+    source_row: int
+
+
+@dataclass(frozen=True, slots=True)
+class CompositionItemRecord:
+    reference: str
+    composition_code: str
+    item_code: str
+    item_type: str
+    description: str
+    unit: str
+    coefficient: Decimal
+    source_file: str
+    source_sheet: str
+    source_row: int
+    status: CompositionStatus | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class CompositionParseResult:
+    compositions: tuple[CompositionRecord, ...]
+    items: tuple[CompositionItemRecord, ...]
+    warnings: tuple[str, ...]
+
+
+def parse_analytic_compositions(
+    workbook_path: str | Path,
+    sheet_name: str = "Analítico",
+    *,
+    reference: str,
+) -> CompositionParseResult:
+    """Extrai composições e seus componentes sem preencher dados ausentes."""
+    if not REFERENCE_PATTERN.fullmatch(reference):
+        raise ValueError("Competência inválida. Informe AAAA-MM.")
+
+    path = Path(workbook_path)
+    workbook = load_workbook(path, read_only=True, data_only=True)
+    try:
+        if sheet_name not in workbook.sheetnames:
+            raise ValueError(f"Aba inexistente: {sheet_name}")
+        sheet = workbook[sheet_name]
+
+        header = [_text(sheet.cell(row=10, column=column).value) for column in range(1, 9)]
+        required_headers = {1: "codigo", 2: "tipo", 4: "descricao", 5: "unidade", 6: "coeficiente"}
+        if any(not header[index] or expected not in header[index]
+               for index, expected in required_headers.items()) or not any(
+            "situacao" in value or "status" in value for value in header
+        ):
+            raise ValueError("Cabeçalho do relatório Analítico não reconhecido na linha 10.")
+
+        compositions: list[CompositionRecord] = []
+        items: list[CompositionItemRecord] = []
+        warnings: list[str] = []
+        known_compositions: set[str] = set()
+
+        rows = sheet.iter_rows(min_row=11, max_col=8, values_only=True)
+        for row_number, row in enumerate(rows, start=11):
+            values = tuple(row)
+            parent_code = _code(values[1] if len(values) > 1 else None)
+            row_type = _text(values[2] if len(values) > 2 else None)
+            item_code = _code(values[3] if len(values) > 3 else None)
+            description = str(values[4] or "").strip() if len(values) > 4 else ""
+            unit = str(values[5] or "").strip() if len(values) > 5 else ""
+            coefficient_value = values[6] if len(values) > 6 else None
+            status_value = _text(values[7] if len(values) > 7 else None)
+
+            if not parent_code and not row_type and not description:
+                continue
+
+            # Na planilha oficial, a linha-pai não possui tipo nem código de item.
+            # A descrição e a unidade identificam a composição; não há rótulo CABECALHO.
+            if not row_type and not item_code and parent_code and description and unit:
+                if not parent_code or not description or not unit:
+                    warnings.append(f"Linha {row_number}: cabeçalho de composição incompleto.")
+                    continue
+                status = STATUS_MAP.get(status_value)
+                if status_value and status is None:
+                    warnings.append(
+                        f"Linha {row_number}: situação não reconhecida: {status_value!r}."
+                    )
+                if parent_code in known_compositions:
+                    warnings.append(
+                        f"Linha {row_number}: composição duplicada {parent_code!r}."
+                    )
+                    continue
+                known_compositions.add(parent_code)
+                compositions.append(CompositionRecord(
+                    reference=reference,
+                    code=parent_code,
+                    description=description,
+                    unit=unit,
+                    status=status,
+                    source_file=path.name,
+                    source_sheet=sheet_name,
+                    source_row=row_number,
+                ))
+                continue
+
+            if row_type not in {"composicao", "insumo"}:
+                if row_type:
+                    warnings.append(
+                        f"Linha {row_number}: tipo de registro não reconhecido: {row_type!r}."
+                    )
+                continue
+
+            if not parent_code or not item_code or not description or not unit:
+                warnings.append(f"Linha {row_number}: componente incompleto ignorado.")
+                continue
+            try:
+                coefficient = parse_decimal(coefficient_value)
+            except (TypeError, ValueError):
+                warnings.append(f"Linha {row_number}: coeficiente inválido.")
+                continue
+            if coefficient is None:
+                warnings.append(f"Linha {row_number}: coeficiente ausente.")
+                continue
+            if coefficient < 0:
+                warnings.append(f"Linha {row_number}: coeficiente negativo ignorado.")
+                continue
+
+            item_status = STATUS_MAP.get(status_value)
+            if status_value and item_status is None:
+                warnings.append(
+                    f"Linha {row_number}: situação de componente não reconhecida: "
+                    f"{status_value!r}."
+                )
+
+            items.append(CompositionItemRecord(
+                reference=reference,
+                composition_code=parent_code,
+                item_code=item_code,
+                item_type="COMPOSICAO" if row_type == "composicao" else "INSUMO",
+                description=description,
+                unit=unit,
+                coefficient=coefficient,
+                source_file=path.name,
+                source_sheet=sheet_name,
+                source_row=row_number,
+                status=item_status,
+            ))
+
+        orphan_items = [item for item in items if item.composition_code not in known_compositions]
+        for item in orphan_items:
+            warnings.append(
+                f"Linha {item.source_row}: componente sem composição-pai "
+                f"{item.composition_code!r}."
+            )
+
+        return CompositionParseResult(
+            tuple(compositions), tuple(items), tuple(warnings)
+        )
+    finally:
+        workbook.close()

getsinapi/extractors/price_tables.py

@@ -0,0 +1,151 @@
+"""Parser específico para as tabelas matriciais de preços de insumos SINAPI.
+
+Layout auditado no XLSX oficial: cabeçalhos dos campos e das UFs na linha 10,
+com registros a partir da linha 11. As colunas F:AF representam as 27 UFs.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+from openpyxl import load_workbook
+
+from getsinapi.domain import EncargoMode, PriceStatus, Regime
+from getsinapi.normalizers import classify_price
+
+REFERENCE_PATTERN = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
+UF_CODES = {
+    "AC", "AL", "AP", "AM", "BA", "CE", "DF", "ES", "GO", "MA", "MT",
+    "MS", "MG", "PA", "PB", "PR", "PE", "PI", "RJ", "RN", "RS", "RO",
+    "RR", "SC", "SP", "SE", "TO",
+}
+PRICE_TABLES = {
+    "ISD": (Regime.NAO_DESONERADO, EncargoMode.COM_ENCARGOS),
+    "ICD": (Regime.DESONERADO, EncargoMode.COM_ENCARGOS),
+    "ISE": (None, EncargoMode.SEM_ENCARGOS),
+}
+
+
+@dataclass(frozen=True, slots=True)
+class InputPriceRecord:
+    reference: str
+    code: str
+    description: str
+    unit: str
+    uf: str
+    regime: Regime | None
+    encargo_mode: EncargoMode
+    price: Decimal | None
+    status: PriceStatus
+    price_origin: str | None
+    source_file: str
+    source_sheet: str
+    source_row: int
+
+
+@dataclass(frozen=True, slots=True)
+class InputPriceParseResult:
+    table_type: str
+    records: tuple[InputPriceRecord, ...]
+    warnings: tuple[str, ...]
+
+
+def _table_type(path: Path, sheet_name: str) -> str:
+    candidates = (path.stem.upper(), sheet_name.upper())
+    for candidate in candidates:
+        match = re.search(r"(?<![A-Z])(ISD|ICD|ISE)(?![A-Z])", candidate)
+        if match:
+            return match.group(1)
+    raise ValueError("Tipo de tabela não identificado; esperado ISD, ICD ou ISE.")
+
+
+def _uf(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip().upper()
+    match = re.search(r"\b([A-Z]{2})\b", text)
+    if match and match.group(1) in UF_CODES:
+        return match.group(1)
+    return None
+
+
+def parse_input_price_table(
+    workbook_path: str | Path,
+    sheet_name: str,
+    *,
+    reference: str,
+) -> InputPriceParseResult:
+    """Converte uma tabela ISD/ICD/ISE larga em observações por UF."""
+    if not REFERENCE_PATTERN.fullmatch(reference):
+        raise ValueError("Competência inválida. Informe AAAA-MM.")
+    path = Path(workbook_path)
+    table_type = _table_type(path, sheet_name)
+    regime, encargo_mode = PRICE_TABLES[table_type]
+    workbook = load_workbook(path, read_only=True, data_only=True)
+    try:
+        if sheet_name not in workbook.sheetnames:
+            raise ValueError(f"Aba inexistente: {sheet_name}")
+        sheet = workbook[sheet_name]
+
+        # No XLSX oficial, os rótulos dos campos e as UFs compartilham a linha 10.
+        # Streaming sequencial mantém compatibilidade com read_only=True.
+        rows = sheet.iter_rows(min_row=10, max_col=32, values_only=True)
+        header_row = next(rows, ())
+        uf_columns = [
+            (column, state)
+            for column, value in enumerate(header_row, start=1)
+            if column >= 6 and (state := _uf(value)) is not None
+        ]
+        if len(uf_columns) != 27 or len({state for _, state in uf_columns}) != 27:
+            raise ValueError(
+                f"Mapa de UFs inválido na linha 10: {len(uf_columns)} colunas reconhecidas."
+            )
+
+        headers = [str(value or "").strip().casefold() for value in header_row[:5]]
+        if not any("código" in value or "codigo" in value for value in headers):
+            raise ValueError("Cabeçalho da tabela não reconhecido na linha 10.")
+
+        records: list[InputPriceRecord] = []
+        warnings: list[str] = []
+        for row_number, row in enumerate(rows, start=11):
+            values = tuple(row)
+            code_value = values[1] if len(values) > 1 else None
+            description_value = values[2] if len(values) > 2 else None
+            unit_value = values[3] if len(values) > 3 else None
+            if code_value is None and description_value is None:
+                continue
+            if code_value is None or description_value is None or unit_value is None:
+                warnings.append(f"Linha {row_number}: registro incompleto ignorado.")
+                continue
+
+            code = str(code_value).strip()
+            description = str(description_value).strip()
+            unit = str(unit_value).strip()
+            origin_value = values[4] if len(values) > 4 else None
+            origin = str(origin_value).strip() if origin_value is not None else None
+
+            for column, state in uf_columns:
+                raw_price = values[column - 1] if len(values) >= column else None
+                price, status = classify_price(raw_price)
+                records.append(InputPriceRecord(
+                    reference=reference,
+                    code=code,
+                    description=description,
+                    unit=unit,
+                    uf=state,
+                    regime=regime,
+                    encargo_mode=encargo_mode,
+                    price=price,
+                    status=status,
+                    price_origin=origin,
+                    source_file=path.name,
+                    source_sheet=sheet_name,
+                    source_row=row_number,
+                ))
+
+        return InputPriceParseResult(table_type, tuple(records), tuple(warnings))
+    finally:
+        workbook.close()