pyflutter-app 0.3.0__tar.gz → 0.3.1__tar.gz

{pyflutter_app-0.3.0 → pyflutter_app-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyflutter_app
-Version: 0.3.0
+Version: 0.3.1
 Summary: Bibliotecas Python para conversão de documentos, interface e processamento de tabelas.
 Requires-Python: >=3.12
 Requires-Dist: opencv-python>=5.0.0.93

{pyflutter_app-0.3.0 → pyflutter_app-0.3.1}/cartas/common.py

@@ -10,6 +10,7 @@ class ConfigTOI:
     file_relatorio_toi: File
     file_relatorio_forms: File
     file_table_toi: File
+    file_master_forms: File  # NOVO
     dir_base_toi: Directory
     dir_log_excel: Directory
     dir_relatorios: Directory
@@ -66,6 +67,11 @@ class BuilderConfigTOI(object):
         d = self._builder_ocr.builder().dir_table
         return d.join_file("cartas-toi-lancadas.xlsx")
 
+    @property
+    def file_master_forms(self) -> File:
+        d = self._builder_ocr.builder().dir_table
+        return d.join_file("master-forms-toi.xlsx")
+
     @property
     def file_table_toi(self) -> File:
         d = self._builder_ocr.builder().dir_table
@@ -80,6 +86,7 @@ class BuilderConfigTOI(object):
             file_relatorio_forms=self.file_relatorio_forms,
             dir_log_excel=self.dir_log_excel,
             dir_relatorios=self.dir_relatorios,
+            file_master_forms=self.file_master_forms,
         )
 
     def builder(self) -> ConfigTOI:

{pyflutter_app-0.3.0 → pyflutter_app-0.3.1}/converter/page_forms/config.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 from dataclasses import dataclass, field
-
 from cartas import LocalEntregaCarta, TipoCarta
 
 FORMS_URL = (
@@ -31,11 +30,12 @@ class FormsSubmission:
     Contrato entre Model e Service: UMA resposta do formulário, já validada
     e com os textos finais das opções. O Service não conhece pandas nem PyFlutter.
     """
+
     uc: str
     toi: str
     local_label: str
     tipo_label: str
-    date: str            # dd/MM/yyyy
+    date: str  # dd/MM/yyyy
     files: list[str] = field(default_factory=list)
 
 
@@ -47,15 +47,15 @@ class FormsConfig:
     user_data_dir: str = ""
     screenshot_dir: str = ""
     headless: bool = False
-    slow_mo_ms: int = 150                # pausa automática entre CADA ação do Playwright
-    pause_between_rows_s: float = 1.5    # pausa entre uma carta e outra
-    browser_channel: str | None = None   # ex.: "chrome" para usar o Chrome instalado
+    slow_mo_ms: int = 150  # pausa automática entre CADA ação do Playwright
+    pause_between_rows_s: float = 1.5  # pausa entre uma carta e outra
+    browser_channel: str | None = None  # ex.: "chrome" para usar o Chrome instalado
 
     login_timeout_s: int = 900
     action_timeout_ms: int = 15000
     upload_timeout_ms: int = 120000
     wait_after_submit_ms: int = 1500
-    send_receipt: bool = True            # marcar "Enviar-me um email de confirmação"
+    send_receipt: bool = True  # marcar "Enviar-me um email de confirmação"
 
     # Regex (case-insensitive) que identifica o TÍTULO de cada pergunta.
     # O texto do bloco vem colado ("1.UCTexto de linha única."), por isso NÃO

{pyflutter_app-0.3.0 → pyflutter_app-0.3.1}/converter/page_forms/controller_forms.py

@@ -9,8 +9,15 @@ from pyflutter_ui.ui import ChangeNotifier, Provider
 from pyflutter_ui.scheduler import MainThreadScheduler
 
 from converter.config_app import AppConfigNotifier
-from converter.page_forms.common import MapRows, StatusForms
-from converter.page_forms.model_forms import ModelForms, FormsCallbacks
+from converter.page_forms.common import MapRows, StatusForms, HeaderForms
+
+from converter.page_forms.master_forms import ControllerMasterForms, make_key
+from converter.page_forms.model_forms import (
+    ModelForms,
+    FormsCallbacks,
+    ResultGenerateTable,
+)
+from table_stream import File, Directory, TableData, ArrayList
 
 from cartas import HeaderFilesTOI, BuilderCartas
 from table_stream import File, Directory, TableData
@@ -37,6 +44,9 @@ class ControllerForms(ChangeNotifier):
         self.progress: tuple[int, int] = (0, 0)
         self.send_receipt: bool = True
 
+        self.master = ControllerMasterForms()
+        self.generating: bool = False
+
         # Log em Excel gravado durante o lançamento (1 arquivo por execução).
         self.save_every: int = 2  # grava a cada N linhas concluídas (sucesso OU falha)
         self.result_file: File | None = None
@@ -66,16 +76,119 @@ class ControllerForms(ChangeNotifier):
         if self.fn_show_error is not None:
             _ui(lambda: self.fn_show_error(e))
 
+        # ------------------------------------------------------------------ master
+
+    def _apply_master(self, mp: MapRows) -> int:
+        """Marca como SKIPPED o que já está no master (key UC+TOI) ou repetido na tabela."""
+        keys = self.master.load_keys()
+        seen: set[str] = set()
+        skipped = 0
+
+        for idx in mp.header():
+            row = mp[idx]
+            if mp.get_status(idx) == StatusForms.SUCCESS:
+                seen.add(make_key(row.get(HeaderForms.UC), row.get(HeaderForms.TOI)))
+                continue
+
+            key = make_key(row.get(HeaderForms.UC), row.get(HeaderForms.TOI))
+            if key in keys:
+                mp.update_status(idx, StatusForms.SKIPPED, "Já lançada (master)")
+                skipped += 1
+            elif key in seen:
+                mp.update_status(idx, StatusForms.SKIPPED, "Duplicada na tabela")
+                skipped += 1
+            seen.add(key)
+        return skipped
+
     # ------------------------------------------------------------------ importação
     def set_map_rows(self, mp: MapRows) -> None:
         if mp.is_empty():
             self.show_error("Nenhuma linha TOI encontrada na tabela.")
             return
+
+        try:
+            skipped = self._apply_master(mp)
+        except Exception as e:
+            self.show_error(f"Falha ao ler o arquivo master:\n{e}")
+            return
+
         self.map_rows = mp
         self.progress = (0, 0)
-        self.status_text = f"{mp.size_header()} cartas importadas. Pronto para lançar."
+        pending = mp.size_header() - skipped
+        self.status_text = (
+            f"{mp.size_header()} cartas importadas | "
+            f"{skipped} já lançadas/ignoradas | {pending} a lançar."
+        )
+        self.notify_listeners()
+
+    # ------------------------------------------------------------------ gerar tabela (cache)
+    def generate_table(self, files: ArrayList[File]) -> None:
+        if self.running or self.generating:
+            self.show_info("Existe outra operação em andamento.")
+            return
+        if files.empty():
+            return
+
+        self.generating = True
+        self.progress = (0, 0)
+        self.status_text = (
+            f"Gerando tabela a partir de {files.size()} arquivos (somente cache)..."
+        )
+        self.notify_listeners()
+
+        try:
+            self.model.start_generate(
+                files,
+                on_progress=lambda d, t: _ui(lambda: self._on_gen_progress(d, t)),
+                on_done=lambda r, e: _ui(lambda: self._on_gen_done(r, e)),
+            )
+        except Exception as e:
+            self.generating = False
+            self.show_error(str(e))
+
+    def _on_gen_progress(self, done: int, total: int) -> None:
+        self.progress = (done, total)
+        self.status_text = f"Calculando hashes {done}/{total}"
+        if done == total or done % 25 == 0:  # evita piscar a tela
+            self.notify_listeners()
+
+    def _on_gen_done(
+        self, res: ResultGenerateTable | None, err: Exception | None
+    ) -> None:
+        self.generating = False
+        self.progress = (0, 0)
+
+        if err is not None:
+            self.status_text = f"Erro ao gerar tabela: {err}"
+            self.notify_listeners()
+            self.show_error(str(err))
+            return
+
+        if res is None or res.table is None:
+            self.status_text = "Nenhum arquivo encontrado no cache."
+            self.notify_listeners()
+            self.show_info(
+                "Nenhum dos arquivos está na tabela TOI em cache.\n"
+                "Aplique o OCR/relatório TOI antes de gerar a tabela."
+            )
+            return
+
+        self.status_text = f"Tabela gerada: {res.found}/{res.total} arquivos."
         self.notify_listeners()
 
+        msg = f"Tabela gerada com {res.found} de {res.total} arquivos.\n\n{res.file.absolute()}"
+        if res.not_found:
+            sample = "\n".join(res.not_found[:10])
+            more = (
+                f"\n... e mais {len(res.not_found) - 10}"
+                if len(res.not_found) > 10
+                else ""
+            )
+            msg += f"\n\nSem dados no cache ({len(res.not_found)}):\n{sample}{more}"
+        if res.invalid:
+            msg += f"\n\nIgnorados (não são PDF/imagem): {res.invalid}"
+        self.show_info(msg)
+
     def set_table_toi(self, tb: TableData[HeaderFilesTOI]) -> None:
         if tb.is_empty():
             self.show_error("Tabela TOI vazia")
@@ -122,17 +235,27 @@ class ControllerForms(ChangeNotifier):
 
     # ------------------------------------------------------------------ execução
     def start_forms(self) -> None:
-        if self.running:
-            self.show_info("Já existe um lançamento em andamento.")
+
+        if self.running or self.generating:
+            self.show_info("Já existe uma operação em andamento.")
             return
         if self.map_rows.is_empty():
             self.show_error("Importe uma tabela TOI antes de iniciar.")
             return
 
+        try:
+            self._apply_master(self.map_rows)  # o master pode ter mudado
+        except Exception as e:
+            self.show_error(f"Falha ao ler o arquivo master:\n{e}")
+            return
+
         jobs = {}
         for idx in self.map_rows.header():
-            if self.map_rows.get_status(idx) == StatusForms.SUCCESS:
-                continue  # já lançada em uma execução anterior desta sessão
+            if self.map_rows.get_status(idx) in (
+                StatusForms.SUCCESS,
+                StatusForms.SKIPPED,
+            ):
+                continue
             try:
                 jobs[idx] = self.model.build_submission(self.map_rows[idx])
                 self.map_rows.update_status(idx, StatusForms.PENDING)
@@ -140,10 +263,13 @@ class ControllerForms(ChangeNotifier):
                 self.map_rows.update_status(idx, StatusForms.FAILED, str(e))
 
         if not jobs:
-            self.status_text = "Nenhuma linha válida para lançar."
+            self.status_text = (
+                "Nenhuma carta nova para lançar (todas já lançadas ou inválidas)."
+            )
             self.notify_listeners()
             return
 
+        # ... resto idêntico ao atual
         config = self.model.create_config(
             self.get_config().value.work_dir, self.send_receipt
         )
@@ -184,6 +310,12 @@ class ControllerForms(ChangeNotifier):
     def _on_status(self, idx: int, status: StatusForms, msg: str) -> None:
         self.map_rows.update_status(idx, status, msg)
 
+        if status == StatusForms.SUCCESS:
+            try:
+                self.master.register(self.map_rows[idx])
+            except Exception as e:
+                self.show_error(f"Carta lançada, mas falhou ao gravar no master:\n{e}")
+
         if status in (StatusForms.SUCCESS, StatusForms.FAILED):
             self._since_save += 1
             if self._since_save >= self.save_every:
@@ -203,7 +335,9 @@ class ControllerForms(ChangeNotifier):
     def _on_done(self, err: Exception | None, stopped: bool) -> None:
         ok = self.map_rows.count_status(StatusForms.SUCCESS)
         bad = self.map_rows.count_status(StatusForms.FAILED)
-        summary = f"sucesso: {ok} | falha: {bad}"
+        skip = self.map_rows.count_status(StatusForms.SKIPPED)
+        summary = f"sucesso: {ok} | falha: {bad} | ignoradas: {skip}"
+        # summary = f"sucesso: {ok} | falha: {bad}"
 
         if err is not None:
             self.status_text = f"Erro: {err}"

pyflutter_app-0.3.1/converter/page_forms/master_forms.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+import os
+import threading
+from datetime import datetime
+from enum import StrEnum
+from typing import Any
+
+import pandas as pd
+
+from table_stream import RowH, TableData
+from cartas import BuilderConfigTOI
+
+from converter.page_forms.common import HeaderForms, clean_number, clean_text
+
+
+class HeaderMasterForms(StrEnum):
+    KEY = "key_uc_toi"
+    UC = HeaderForms.UC.value
+    TOI = HeaderForms.TOI.value
+    DATA_ENTREGA = HeaderForms.DATA_ENTREGA.value
+    LOCAL_ENTREGA = HeaderForms.LOCAL_ENTREGA.value
+    TIPO = HeaderForms.TIPO.value
+    HASH_FILE = HeaderForms.HASH_FILE.value
+    FULL_PATH = HeaderForms.FULL_PATH.value
+    DATA_LANCAMENTO = "data_lancamento"
+
+
+def make_key(uc: Any, toi: Any) -> str:
+    """Chave de uma carta: UC sozinha não identifica, usa UC+TOI."""
+    return f"{clean_number(uc)}_{clean_number(toi)}"
+
+
+class ControllerMasterForms(object):
+    """
+    Arquivo master do Forms: tudo que já foi lançado com sucesso.
+    Equivalente ao ControllerTOI().load_table_toi(), mas exclusivo do Forms.
+    """
+
+    COLUMNS: list[str] = [h.value for h in HeaderMasterForms]
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+
+    @property
+    def file_master(self):
+        return BuilderConfigTOI().builder().file_master_forms
+
+    # ---------------------------------------------------------------- leitura
+    def load_dataframe(self) -> pd.DataFrame:
+        f = self.file_master
+        if not f.exists():
+            return pd.DataFrame(columns=self.COLUMNS)
+
+        df = pd.read_excel(f.absolute(), dtype=str).fillna("")
+        for c in self.COLUMNS:
+            if c not in df.columns:
+                df[c] = ""
+        return df[self.COLUMNS]
+
+    def load_table_master(self) -> TableData[HeaderMasterForms]:
+        return TableData(self.load_dataframe())
+
+    def load_keys(self) -> set[str]:
+        df = self.load_dataframe()
+        return set(df[HeaderMasterForms.KEY.value].astype(str).tolist())
+
+    def contains(self, uc: Any, toi: Any) -> bool:
+        return make_key(uc, toi) in self.load_keys()
+
+    # ---------------------------------------------------------------- escrita
+    def register(self, row: RowH[HeaderForms, Any]) -> bool:
+        """
+        Grava uma carta lançada com sucesso. Retorna False se a chave já existia.
+        Escrita atômica (tmp + replace): nunca deixa .xlsx corrompido.
+        """
+        uc = clean_number(row.get(HeaderForms.UC))
+        toi = clean_number(row.get(HeaderForms.TOI))
+        key = make_key(uc, toi)
+
+        with self._lock:
+            df = self.load_dataframe()
+            if key in set(df[HeaderMasterForms.KEY.value].tolist()):
+                return False
+
+            record = {
+                HeaderMasterForms.KEY.value: key,
+                HeaderMasterForms.UC.value: uc,
+                HeaderMasterForms.TOI.value: toi,
+                HeaderMasterForms.DATA_ENTREGA.value: clean_text(
+                    row.get(HeaderForms.DATA_ENTREGA)
+                ),
+                HeaderMasterForms.LOCAL_ENTREGA.value: clean_text(
+                    row.get(HeaderForms.LOCAL_ENTREGA)
+                ),
+                HeaderMasterForms.TIPO.value: clean_text(row.get(HeaderForms.TIPO)),
+                HeaderMasterForms.HASH_FILE.value: clean_text(
+                    row.get(HeaderForms.HASH_FILE)
+                ),
+                HeaderMasterForms.FULL_PATH.value: clean_text(
+                    row.get(HeaderForms.FULL_PATH)
+                ),
+                HeaderMasterForms.DATA_LANCAMENTO.value: datetime.now().strftime(
+                    "%d/%m/%Y %H:%M:%S"
+                ),
+            }
+            new = pd.DataFrame([record], columns=self.COLUMNS)
+            final = new if df.empty else pd.concat([df, new], ignore_index=True)
+
+            path = self.file_master.absolute()
+            tmp = f"{path}-tmp.xlsx"
+            final.to_excel(tmp, index=False)
+            os.replace(tmp, path)
+            return True

{pyflutter_app-0.3.0 → pyflutter_app-0.3.1}/converter/page_forms/model_forms.py

@@ -1,13 +1,29 @@
 from __future__ import annotations
 import os
 import threading
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from datetime import datetime
+from cartas import BuilderConfigTOI
 from typing import Any, Callable
 
 import pandas as pd
 
-from table_stream import ArrayList, File, Directory, HashMapDict, RowH, load_file_hash
-from cartas import TableTOI, ControllerTOI, HeaderTOI
+from table_stream import (
+    ArrayList,
+    File,
+    Directory,
+    HashMapDict,
+    RowH,
+    TableData,
+    load_file_hash,
+)
+from cartas import (
+    TableTOI,
+    ControllerTOI,
+    HeaderEntregaTOI,
+    HeaderFilesTOI,
+    BuilderCartas,
+)
 
 from converter.page_forms.common import (
     HeaderForms,
@@ -43,10 +59,21 @@ class FormsCallbacks:
     on_done: Callable[[Exception | None, bool], None] = _noop  # (erro, interrompido)
 
 
+@dataclass
+class ResultGenerateTable:
+    table: TableData[HeaderFilesTOI] | None = None
+    file: File | None = None
+    total: int = 0
+    found: int = 0
+    invalid: int = 0
+    not_found: list[str] = field(default_factory=list)
+
+
 class ModelForms(object):
 
     def __init__(self) -> None:
         self._relatorio_toi: ControllerTOI | None = None
+        self.builder_carta = BuilderCartas()
         self._thread: threading.Thread | None = None
         self._stop = threading.Event()
 
@@ -66,21 +93,23 @@ class ModelForms(object):
             self._relatorio_toi = ControllerTOI()
         return self._relatorio_toi
 
-    def create_relatorio_toi(self, files: ArrayList[File]) -> TableTOI:
+    def create_relatorio_toi(self, files: ArrayList[File]) -> TableData[HeaderFilesTOI]:
         if files.empty():
-            return TableTOI()
+            return self.builder_carta.create_void()
 
         mp = HashMapDict()
         files.for_each(lambda f: mp.set_value(load_file_hash(f), f))
 
-        tb = self.relatorio_toi.load_from_map(mp)
+        tb: TableData[HeaderFilesTOI] | None = self.relatorio_toi.load_from_map(mp)
         if tb is None:
-            return TableTOI()
+            return self.builder_carta.create_void()
 
         df = tb.to_dataframe().astype(str)
-        df = df[df[HeaderTOI.UC] != TableTOI.NULL_STRING]
-        df = df[df[HeaderTOI.DATE] != TableTOI.NULL_STRING]
-        return TableTOI.create_from_data(df)
+
+        df = df[df[HeaderFilesTOI.UC] != TableTOI.NULL_STRING]
+        df = df[df[HeaderFilesTOI.TOI] != TableTOI.NULL_STRING]
+        df = df[df[HeaderFilesTOI.DATE] != TableTOI.NULL_STRING]
+        return self.builder_carta.from_data(df)
 
     # ------------------------------------------------------------------ validação/conversão
     @staticmethod
@@ -213,3 +242,88 @@ class ModelForms(object):
         finally:
             service.stop()
             cb.on_done(error, self._stop.is_set())
+
+    # ------------------------------------------------------------------
+    # gerar tabela (SOMENTE cache, sem OCR)
+    # ------------------------------------------------------------------
+    def is_generating(self) -> bool:
+        t = getattr(self, "_gen_thread", None)
+        return t is not None and t.is_alive()
+
+    def generate_table(
+        self,
+        files: ArrayList[File],
+        on_progress: Callable[[int, int], None] = _noop,
+    ) -> ResultGenerateTable:
+        """Calcula o hash, consulta ControllerTOI.find_on_cache() e NÃO aplica OCR."""
+        res = ResultGenerateTable()
+        valid = ArrayList[File]()
+
+        for f in files:
+            if f.is_image() or f.is_pdf():
+                valid.add(f)
+            else:
+                res.invalid += 1
+
+        mp = HashMapDict()
+        total = valid.size()
+        for n, f in enumerate(valid):
+            mp.set_value(load_file_hash(f), f)
+            on_progress(n + 1, total)
+
+        res.total = mp.size_header()
+        if mp.is_empty():
+            return res
+
+        tb: TableData[HeaderFilesTOI] = self.relatorio_toi.find_on_cache(mp)
+        found_hashes: set[str] = set()
+
+        if not tb.is_empty():
+            df = tb.to_dataframe().astype(str)
+            for h in (HeaderFilesTOI.UC, HeaderFilesTOI.TOI, HeaderFilesTOI.DATE):
+                df = df[df[h] != TableTOI.NULL_STRING]
+
+            if not df.empty:
+                found_hashes = set(df[HeaderFilesTOI.HASH_FILE].tolist())
+                res.table = self.builder_carta.from_data(df)
+
+        res.found = len(found_hashes)
+        res.not_found = [
+            mp.get_value(h).basename() for h in mp.header() if h not in found_hashes
+        ]
+        return res
+
+    def save_generated_table(self, tb: TableData[HeaderFilesTOI]) -> File:
+        """Grava em ConfigTOI.dir_relatorios (padrão do projeto)."""
+        cfg = BuilderConfigTOI().builder()
+        out = cfg.dir_relatorios.join_file(
+            f"tabela-toi-{datetime.now():%Y%m%d-%H%M%S}.xlsx"
+        )
+        tmp = f"{out.absolute()}-tmp.xlsx"
+        tb.to_dataframe().to_excel(tmp, index=False)
+        os.replace(tmp, out.absolute())
+        return out
+
+    def start_generate(
+        self,
+        files: ArrayList[File],
+        on_progress: Callable[[int, int], None],
+        on_done: Callable[[ResultGenerateTable | None, Exception | None], None],
+    ) -> None:
+        if self.is_generating():
+            raise RuntimeError("Já existe uma geração de tabela em andamento")
+
+        def work() -> None:
+            res: ResultGenerateTable | None = None
+            err: Exception | None = None
+            try:
+                res = self.generate_table(files, on_progress)
+                if res.table is not None:
+                    res.file = self.save_generated_table(res.table)
+            except Exception as e:
+                err = e
+            finally:
+                on_done(res, err)
+
+        self._gen_thread = threading.Thread(target=work, daemon=True, name="gen-table")
+        self._gen_thread.start()