pyflutter-app 0.2.3__tar.gz → 0.2.5__tar.gz

{pyflutter_app-0.2.3 → pyflutter_app-0.2.5}/.idea/pyflutter_app.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
     </content>
-    <orderEntry type="jdk" jdkName="~/GIT/pyflutter/.venv" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="uv (pyflutter)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyDocumentationSettings">

{pyflutter_app-0.2.3 → pyflutter_app-0.2.5}/PKG-INFO

@@ -1,12 +1,15 @@
 Metadata-Version: 2.5
 Name: pyflutter_app
-Version: 0.2.3
+Version: 0.2.5
 Summary: Bibliotecas Python para conversão de documentos, interface e processamento de tabelas.
 Requires-Python: >=3.12
 Requires-Dist: opencv-python>=5.0.0.93
 Requires-Dist: openpyxl>=3.1.5
 Requires-Dist: pandas-stubs~=3.0.3
 Requires-Dist: pandas>=3.0.0
+Requires-Dist: playwright>=1.63.0
+Requires-Dist: pymupdf>=1.28.2
+Requires-Dist: pytesseract>=0.3.13
 Description-Content-Type: text/markdown
 
 # pyflutter

pyflutter_app-0.2.5/cartas/__init__.py

@@ -0,0 +1,6 @@
+from .find_toi import extrair_uc_toi
+from .table_toi import HeaderTOI, TableTOI, LocalEntregaCarta, TipoCarta
+from .common import BuilderConfigTOI, ConfigTOI
+from .relatorio import (
+    CreateRelatorioTOI, TableTOI, ReadWriteTableTOI, load_files_toi
+)

pyflutter_app-0.2.5/cartas/common.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+from dataclasses import dataclass
+
+from table_stream import File, Directory, UserFileSystem
+
+
+@dataclass
+class ConfigTOI:
+
+    file_relatorio_toi: File
+    file_table_toi: File
+    dir_base_toi: Directory
+    work_dir: Directory 
+
+
+class BuilderConfigTOI(object):
+
+    _instance = None  # Atributo de classe para armazenar a instância singleton
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super(BuilderConfigTOI, cls).__new__(cls)
+        return cls._instance
+
+    def __init__(self) -> None:
+
+        # Garante que __init__ não será executado mais de uma vez
+        if hasattr(self, "_initialized") and self._initialized:
+            return
+
+        super().__init__()
+        self._initialized: bool = True
+        self._work_dir: Directory = (
+            UserFileSystem()
+            .get_user_documents()
+            .concat("RFA", True)
+            .concat("Cartas-TOI")
+        )
+
+        self._config = self._create()
+
+    @property
+    def dir_retorno_de_campo(self) -> Directory:
+        return self._work_dir.concat("Retorno-de-campo", True)
+
+    @property
+    def file_relatorio_toi(self) -> File:
+        return self._work_dir.concat("Relatorios", True).join_file("Relatorio-tois.xlsx")
+
+    @property
+    def file_table_toi(self) -> File:
+        return self._work_dir.concat("Relatorios", True).join_file("Tabela-tois.xlsx")
+    
+    def _create(self) -> ConfigTOI:
+        return ConfigTOI(
+                work_dir=self._work_dir,
+                dir_base_toi=self.dir_retorno_de_campo,
+                file_relatorio_toi=self.file_relatorio_toi,
+                file_table_toi=self.file_table_toi,
+            )
+    
+    def set_work_dir(self, d: Directory) -> BuilderConfigTOI:
+        self._work_dir = d
+        self._config = self._create()
+        return self
+
+    def builder(self) -> ConfigTOI:
+        return self._config

pyflutter_app-0.2.5/cartas/find_toi.py

@@ -0,0 +1,27 @@
+import re
+
+
+# ============================================================#
+# Extração de UC/TOI tolerante a falhas de OCR
+# ============================================================#
+#
+# UC: 1305934 TOI: 193166923 ...
+# UC: 1305934 TO! 193166923 ...   (TOI virou TO!, ":" some)
+# UC| 1305934 T0I: 193166923 ...  (0 no lugar do O)
+#
+
+UC_TOI_PATTERN = re.compile(
+    r"U[C0]\s*[:;|!]?\s*(\d{4,10})" r".{0,60}?" r"T[O0][I1L!l]?\s*[:;|!]?\s*(\d{4,10})",
+    re.IGNORECASE | re.DOTALL,
+)
+
+
+def extrair_uc_toi(texto: str) -> tuple[str, str] | None:
+    """
+        Retorna (uc, toi) a partir do texto OCR completo de um arquivo,
+    ou None se o padrão não for encontrado.
+    """
+    m = UC_TOI_PATTERN.search(texto)
+    if not m:
+        return None
+    return m.group(1), m.group(2)

pyflutter_app-0.2.5/cartas/relatorio.py

@@ -0,0 +1,290 @@
+from ocrlib import (
+    TableOcr, HeaderOcr, DocumentInputStream, ReadWriteTableOCR,
+    CreateRelatorioExcelOcr, ColumnV,
+)
+
+from whatsapp_files import TableWhatsApp, HeaderWhatsApp, CreateRelatorioWhatsApp
+
+from table_stream import (
+    File, Directory, HashMapDict, hashFile, ColumnNotFoundError,
+    ArrayList, PathElement, TableElements, RowH, load_file_hash,
+    concat_tables, create_table_from_file, create_void_table_files, BaseObjetct
+)
+
+import pandas as pd
+from cartas.table_toi import HeaderTOI, TableTOI, TipoCarta, LocalEntregaCarta
+from cartas.find_toi import extrair_uc_toi
+from cartas.common import ConfigTOI, BuilderConfigTOI
+from os import rename as rename_file
+
+
+class ReadWriteTableTOI(object):
+    """
+    Leitura e escrita de dados na tabela OCR.
+    """
+
+    _instance = None  # Atributo de classe para armazenar a instância singleton
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super(ReadWriteTableTOI, cls).__new__(cls)
+        return cls._instance
+
+    def __init__(self) -> None:
+
+        # Garante que __init__ não será executado mais de uma vez
+        if hasattr(self, "_initialized") and self._initialized:
+            return
+
+        super().__init__()
+        self._initialized: bool = True
+        self.builder: BuilderConfigTOI = BuilderConfigTOI()
+        
+    @property
+    def file_table_cache(self) -> File:
+        return BuilderConfigTOI().builder().file_table_toi
+    
+    def load_table_cache(self) -> TableTOI:
+
+        f = self.file_table_cache
+
+        if not f.exists():
+            tb = TableTOI()
+            tb.add(ColumnV(HeaderTOI.UC))
+            tb.add(ColumnV(HeaderTOI.TOI))
+            tb.add(ColumnV(HeaderTOI.YEAR))
+            tb.add(ColumnV(HeaderTOI.MON))
+            tb.add(ColumnV(HeaderTOI.DAY))
+            tb.add(ColumnV(HeaderTOI.DATE))
+            tb.add(ColumnV(HeaderTOI.LOCAL_ENTREGA))
+            tb.add(ColumnV(HeaderTOI.TIPO_DOC))
+            tb.add(ColumnV(HeaderTOI.HASH_FILE))
+            return tb
+
+        data = pd.read_excel(f.absolute())
+        return TableTOI.create_from_data(data.astype("str"))
+
+    def save_table_cache(self, tb: TableTOI) -> None:
+        data = tb.to_dataframe()
+        path = self.file_table_cache.absolute()
+        tmp_file = f"{path}-tmp.xlsx"
+
+        print(f"Salvando ... {path}")
+        data.to_excel(tmp_file, index=False)
+        rename_file(tmp_file, path)
+
+
+class CreateRelatorioTOI(object):
+    """
+        Cria uma tabela de Cartas TOI apartir dos chats do WhatsApp.
+    Conectando os arquivos das conversas aos textos extraídos via OCR.
+
+    TableWhatsApp() é usado para obter DATA e HORA do envio de um arquivo 
+    no chat, pois a data de envio no chat e a data de entrega de uma carta em campo.
+    """
+
+    MAP_OCR_TO_TOI: HashMapDict[HeaderOcr, HeaderTOI] = HashMapDict(
+        {
+            HeaderOcr.FILE_PATH: HeaderTOI.FULL_PATH,
+            HeaderOcr.FILE_EXTENSION: HeaderTOI.FILE_EXTENSION,
+            HeaderOcr.FILE_BASENAME: HeaderTOI.FILE_BASENAME,
+            HeaderOcr.HASH_FILE: HeaderTOI.HASH_FILE,
+        }
+    )
+
+    def __init__(self) -> None:
+
+        self.relatorio_ocr: CreateRelatorioExcelOcr = CreateRelatorioExcelOcr()
+        self.relatorio_whatsapp: CreateRelatorioWhatsApp = CreateRelatorioWhatsApp()
+
+    def load_table_whatsapp(self) -> TableWhatsApp:
+        return self.relatorio_whatsapp.load_cache_table()
+    
+    def find_on_cache(self, map: HashMapDict[hashFile, File]) -> TableTOI | None:
+        """
+            Gera uma tabela TOI apartir da tabela TOI em cache sem executar
+        o filtro de texto em OCR com expressão regular.
+        """
+        
+        if map.is_empty():
+            return None
+        
+        tb_cache = ReadWriteTableTOI().load_table_cache()
+        final = TableTOI()
+        
+        header = map.header()
+        h: hashFile
+        
+        for h in header:
+            f = map.get_value(h)
+            
+            content = tb_cache.seach(HeaderTOI.HASH_FILE, h)
+            if content.empty():
+                continue
+            
+            content_row = content.get_row_at(0)
+            content_row.set(HeaderTOI.FILE_EXTENSION, f.extension() or '.nan')
+            content_row.set(HeaderTOI.FILE_BASENAME, f.basename())
+            content_row.set(HeaderTOI.IS_FILE, "S" if f.is_file() else "N")
+            content_row.set(HeaderTOI.FULL_PATH, f.absolute())
+            content_row.set(HeaderTOI.SRC, f.parent().absolute())
+            final.add_row(content_row)
+            
+        return final
+
+    def create_from_map(self, map: HashMapDict[hashFile, File], *, ignore_null: bool = True) -> TableTOI | None:
+        """
+        I) Usa a HASH MD5 dos arquivos para buscar a data de envio do arquivo na
+        tabela whatsapp; 
+        
+        II) Gera um relatório OCR [CreateRelatorioExcelOcr()] com os metadados básicos 
+        de cada arquivo; 
+        
+        III) Obtem a UC e TOI de cada arquivo.
+        """
+
+        if map.is_empty():
+            return None
+
+        # Dados OCR completos de cada arquivo do parâmetro map, esses arquivos
+        # serão comparados com a tabela whatsapp para filtar a data de envio no chat.
+        table_ocr: TableOcr = self.relatorio_ocr.create_all(map)
+        table_whatsapp: TableWhatsApp = self.load_table_whatsapp()
+
+        if table_ocr is None:
+            return None
+
+        if table_ocr.empty():
+            return None
+
+        # Cada arquivo gera/extrai a sua própria tabela, ao final essa
+        # lista de tabelas serão concatenadas em uma tabela única.
+        list_tables_toi: ArrayList[TableTOI] = ArrayList()
+        keys = map.header()
+        total = keys.size()
+
+        print(f'Criando relatório TOI de {total} arquivos')
+        k: hashFile
+        f: File
+
+        for n, k in enumerate(keys):
+            f = map.get_value(k)
+
+            print(f"[FILTRANDO] [{n+1}/{total}] {f.basename()}")
+
+            current_table_ocr: TableOcr = table_ocr.seach(HeaderOcr.HASH_FILE, k)
+            text_ocr: str = None
+            
+            if not current_table_ocr.empty():
+                text_ocr = current_table_ocr.to_string(HeaderOcr.TEXT)
+
+            txt_uc_toi: tuple | None = extrair_uc_toi(text_ocr) if text_ocr is not None else None
+            
+            if txt_uc_toi is None:
+                if ignore_null:
+                    continue
+
+            uc = TableTOI.NULL_STRING
+            toi = TableTOI.NULL_STRING
+            year = TableTOI.NULL_STRING
+            mon = TableTOI.NULL_STRING
+            day = TableTOI.NULL_STRING
+            dt: str = TableTOI.NULL_STRING
+
+            if txt_uc_toi is not None:
+                uc = txt_uc_toi[0]
+                toi = txt_uc_toi[1]
+                
+            current_table_whatsapp = table_whatsapp.seach(HeaderWhatsApp.hash_md5, k)
+            
+            if not current_table_whatsapp.empty():
+                dt = current_table_whatsapp.get_row_at(0).get_date().get_first()
+                day = dt.split(' ')[0]
+                mon = dt.split(' ')[1]
+                year = dt.split(' ')[2]
+
+            tb_toi = TableTOI()
+            
+            tb_toi.add(ColumnV(HeaderTOI.UC, ArrayList([uc])))
+            tb_toi.add(ColumnV(HeaderTOI.TOI, ArrayList([toi])))
+            tb_toi.add(ColumnV(HeaderTOI.YEAR, ArrayList([year])))
+            tb_toi.add(ColumnV(HeaderTOI.MON, ArrayList([mon])))
+            tb_toi.add(ColumnV(HeaderTOI.DAY, ArrayList([day])))
+            tb_toi.add(ColumnV(HeaderTOI.DATE, ArrayList([dt])))
+            
+            tb_toi.add(ColumnV(HeaderTOI.FILE_EXTENSION, ArrayList([f.extension() or '.nan'])))
+            tb_toi.add(ColumnV(HeaderTOI.FILE_BASENAME, ArrayList([f.basename()])))
+            tb_toi.add(ColumnV(HeaderTOI.HASH_FILE, ArrayList([k])))
+            tb_toi.add(ColumnV(HeaderTOI.FULL_PATH, ArrayList([f.absolute()])))
+            tb_toi.add(ColumnV(HeaderTOI.IS_FILE, ArrayList(['S' if f.is_file() else "N"])))
+            tb_toi.add(ColumnV(HeaderTOI.SRC, ArrayList([f.dirname()])))
+            
+            tb_toi.add(ColumnV(HeaderTOI.TIPO_DOC, ArrayList([TipoCarta.CARTA_CALCULO])))
+            tb_toi.add(ColumnV(HeaderTOI.LOCAL_ENTREGA, ArrayList([LocalEntregaCarta.OUTRO])))
+            
+            list_tables_toi.add(tb_toi)
+    
+        if list_tables_toi.empty():
+            return None
+        
+        final = TableTOI.create_void_table()
+        list_tables_toi.for_each(lambda t: final.merge(t))
+        return final
+
+    @staticmethod
+    def filter_uc_toi(hf: hashFile, table_ocr_file: TableOcr) -> tuple[str, str] | None:
+        """
+        Filtra UC e TOI de uma carta apartir do texto OCR extraído.
+        """
+        tb_current_hash = table_ocr_file.seach(HeaderOcr.HASH_FILE, hf)
+        if tb_current_hash.empty():
+            return None
+
+        values = tb_current_hash.get(HeaderOcr.TEXT).elements
+
+        for v in values:
+            txt = extrair_uc_toi(v)
+            if txt is not None:
+                return txt
+        return None
+
+
+def load_files_toi(files: ArrayList[File]) -> TableTOI | None:
+    """
+        Analisa arquivos TOI apartir da HASH MD5, compara o texto na tabela OCR e inclui
+    UC e TOI na tabela TOI principal e ao final retorna os dados no formato de tabela TOI.
+    """
+    if files.empty():
+        print(f"[DEBUG]: load_files_toi() lista de arquivos vazia!")
+        return None
+    
+    map = HashMapDict[hashFile, File]()
+    files.for_each(lambda f: map.set_value(load_file_hash(f), f))
+    
+    content_table = CreateRelatorioTOI().create_from_map(map)
+    if content_table is None:
+        return None
+    if content_table.empty():
+        return None
+    
+    # Selecionar apenas algumas colunas para salvar em cache.
+    content_tb_merge = content_table.select_columns([
+        HeaderTOI.UC, HeaderTOI.TOI, HeaderTOI.YEAR, HeaderTOI.MON,
+        HeaderTOI.DAY, HeaderTOI.DATE, HeaderTOI.LOCAL_ENTREGA,
+        HeaderTOI.TIPO_DOC, HeaderTOI.HASH_FILE,
+    ])
+    
+    # Filtar UCs validas
+    df_merge = content_tb_merge.to_dataframe().astype(str)
+    df_merge = df_merge[df_merge[HeaderTOI.UC] != TableTOI.NULL_STRING]
+    df_merge.drop_duplicates(subset=[HeaderTOI.HASH_FILE], inplace=True)
+    
+    rw = ReadWriteTableTOI()
+    tb_cache_toi: TableTOI = rw.load_table_cache()
+    df_cache_toi = pd.DataFrame() if tb_cache_toi.empty() else tb_cache_toi.to_dataframe().astype(str)
+    
+    df_cache = pd.concat([df_cache_toi, df_merge.astype(str)]) if not df_cache_toi.empty else df_merge.astype(str)
+    df_cache.drop_duplicates(subset=[HeaderTOI.HASH_FILE]).astype(str)
+    
+    rw.save_table_cache(TableTOI.create_from_data(df_cache))
+    return content_table

pyflutter_app-0.2.5/cartas/table_toi.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+from enum import StrEnum
+from typing import Any, Callable, TypeVar
+
+from pandas import DataFrame
+
+from table_stream import (
+    ArrayList, HashMapDict, TableElements, ColumnV, RowH, PathElement, 
+    Directory, File, HeaderMetaData, concat_tables, ColumnNotFoundError,
+)
+
+from cartas.common import ConfigTOI, BuilderConfigTOI
+
+
+class HeaderTOI(StrEnum):
+
+    UC = "uc"
+    TOI = "toi"
+    DAY = "day"
+    MON = "mon"
+    YEAR = "year"
+    DATE = "data_entrega"
+    LOCAL_ENTREGA = "local_entrega"
+    TIPO_DOC = "tipo_carta"
+    IS_FILE = "arquivo?"
+    
+    FULL_PATH = HeaderMetaData.arquivo.value
+    SRC = HeaderMetaData.pasta_pai.value
+    FILE_EXTENSION = HeaderMetaData.extensao_arquivo.value
+    
+    HASH_FILE = HeaderMetaData.md5.value
+    FILE_BASENAME = "basename"
+    
+
+class TipoCarta(StrEnum):
+
+    CARTA_CALCULO = "CARTA-CALCULO"
+
+
+class LocalEntregaCarta(StrEnum):
+
+    OUTRO = "OUTRO"
+    MAOS = "MÃOS"
+    CORREIOS = "CORREIOS"
+    PORTAO = "PORTÃO"
+
+
+class TableTOI(TableElements[HeaderTOI]):
+    """
+        Tabela para representar dados de cartas TOI.
+    """
+    
+    def __init__(self, table_map: HashMapDict[HeaderTOI, ColumnV] | None = None) -> None:
+        super().__init__(table_map)
+
+    def rows(self) -> ArrayList[RowH[HeaderTOI, Any]]:
+        return super().rows()
+    
+    def select_columns(self, header: list[HeaderTOI]) -> TableTOI:
+        final = TableTOI()
+                
+        for h in header:
+            if not self.header().contains(h):
+                raise ColumnNotFoundError(f"Coluna não encontrada: {h}")
+            final.add(self.get(h))
+        return final
+
+    @staticmethod
+    def create_map_from_filename(filename: str) -> HashMapDict[HeaderTOI, str]:
+        data = HashMapDict()
+
+        try:
+            values = filename.split(" ")
+            if len(values) < 5:
+                return data
+        except:
+            return data
+        else:
+            for x in values:
+                if not x.isnumeric():
+                    return data
+
+            data[HeaderTOI.UC] = values[0]  # UC
+            data[HeaderTOI.TOI] = values[1]  # TOI
+            data[HeaderTOI.DAY] = values[2]  # DIA
+            data[HeaderTOI.MON] = values[3]  # MÊS
+            data[HeaderTOI.YEAR] = values[4]  # ANO
+            data[HeaderTOI.DATE] = f"{values[2]}/{values[3]}/{values[4]}"
+            return data
+
+    @classmethod
+    def create_table_from_path(cls, path: PathElement) -> TableTOI:
+        mp = cls.create_map_from_filename(path.name())
+        tb = cls.create_void_table()
+
+        for h in mp.header():
+            e = mp[h]
+            tb.get(h).add(e)
+
+        tb.get(HeaderTOI.FULL_PATH).add(path.absolute())
+        tb.get(HeaderTOI.IS_FILE).add("S" if path.is_file() else "N")
+        tb.get(HeaderTOI.TIPO_DOC).add(TipoCarta.CARTA_CALCULO.value)
+        tb.get(HeaderTOI.LOCAL_ENTREGA).add(LocalEntregaCarta.OUTRO.value)
+        tb.get(HeaderTOI.SRC).add(path.parent().absolute())
+        tb.get(HeaderTOI.FILE_EXTENSION).add(path.extension())
+        return tb
+
+    @classmethod
+    def create_void_table(cls) -> TableTOI:
+        """
+        Cria uma tabela toi vazia com todos os elementos de HeaderTOI
+        """
+        cols = [x for x in HeaderTOI]
+        columns_v = [ColumnV(c) for c in cols]
+        tb = cls()
+        
+        for c in columns_v:
+            tb.add(c)
+        return tb
+
+    @classmethod
+    def create_from_files(cls, elements: ArrayList[PathElement]) -> TableTOI:
+
+        _path_element: PathElement
+        list_tables = ArrayList[TableTOI]()
+
+        for _path_element in elements:
+            current_tb = cls.create_table_from_path(_path_element)
+            if not current_tb.empty():
+                list_tables.append(current_tb)
+
+        if list_tables.empty():
+            return cls()
+
+        final = concat_tables(list_tables)
+        return cls(final.tb_map)
+
+    @classmethod
+    def create_from_data(cls, data: DataFrame) -> TableTOI:
+        columns_df = ArrayList(data.columns.tolist())
+        columns_toi = [x for x in HeaderTOI]
+        tb = cls()
+
+        for col_toi in columns_toi:
+            if not columns_df.contains(col_toi):
+                continue
+
+            elements = data[col_toi].astype("str").values.tolist()
+            col_v = ColumnV(col_toi, ArrayList(elements))
+            tb.add(col_v)
+        return tb

pyflutter_app-0.2.5/converter/__init__.py

@@ -0,0 +1 @@
+from .run_app_convert import main as run_ui_app_convert

pyflutter_app-0.2.5/converter/__main__.py

@@ -0,0 +1,9 @@
+from converter import run_app_convert
+
+def main() -> None:
+    run_app_convert.main()
+
+
+if __name__ == '__main__':
+    main()
+    

{pyflutter_app-0.2.3 → pyflutter_app-0.2.5}/converter/home_page/_page_edit.py

@@ -1,4 +1,4 @@
-# converter/excel/page_edit.py
+# converter/home_page/_page_edit.py
 from __future__ import annotations
 
 from converter.util import (

{pyflutter_app-0.2.3 → pyflutter_app-0.2.5}/converter/home_page/view.py

@@ -207,13 +207,12 @@ class _HomePageState(StatelessWidget):
             Seta o DataFrame() da página atual como propriedade do ControllerForms()
         antes de carregar a página.
         """
-        
-        forms_controller: ControllerForms = context.read(ControllerForms)
-        forms_controller.set_data(controller.data_notifier.value)
-        
-        if not forms_controller.forms_state.data_is_valid():
-            messagebox.showerror("Erro", "Tabela TOI inválida")
-            return
+        #df = controller.data_notifier.value
+        #forms_controller: ControllerForms = context.read(ControllerForms)
+        #forms_controller.set_data(df)
+        #if not forms_controller.forms_state.data_is_valid():
+        #    messagebox.showerror("Erro", "Tabela TOI inválida")
+        #    return
         
         context.navigator.push(PageRespondsForms())