datamule 0.428__cp310-cp310-macosx_10_9_universal2.whl → 0.430__cp310-cp310-macosx_10_9_universal2.whl

datamule/document.py

@@ -17,6 +17,9 @@ class Document:
         self.data = None
         self.content = None
 
+    def _load_content(self):
+        self.content = load_file_content(self.path)
+
     def contains_string(self, pattern):
         """Currently only works for .htm, .html, and .txt files"""
         if self.path.suffix in ['.htm', '.html', '.txt']:

datamule/downloader/premiumdownloader.py

@@ -259,7 +259,8 @@ class PremiumDownloader:
                 keepalive_timeout=60
             )
 
-            async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=3600)) as session:
+            # timeout should be max 2 hours.
+            async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=7200)) as session:
                 tasks = [self.download_and_process(session, url, semaphore, decompression_pool, output_dir, processor) for url in urls]
                 await asyncio.gather(*tasks, return_exceptions=True)
 

datamule/parser/document_parsing/helper.py

@@ -62,7 +62,7 @@ def load_file_content(filename):
     elif filename.suffix in ['.html','.htm']:
         return load_html_content(filename)
     else:
-        raise ValueError(f"Unsupported file type: {filename}")
+        raise ValueError(f"Unsupported file type: {filename.suffix}")
 
 def clean_title(title: str) -> str:
     """Clean up section title by removing newlines, periods, and all whitespace, converting to lowercase."""

datamule/portfolio.py

@@ -1,78 +1,95 @@
 from pathlib import Path
 from tqdm import tqdm
-from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import ThreadPoolExecutor
 from .submission import Submission
 from .downloader.premiumdownloader import PremiumDownloader
 from .downloader.downloader import Downloader
 from .config import Config
+import os
 
 class Portfolio:
-    def create(cls, path):
-        # This method handles the process pool lifecycle
-        with ProcessPoolExecutor() as executor:
-            portfolio = cls(path, executor)
-            return portfolio
-
-    def __init__(self, path, executor=None):
+    def __init__(self, path):
         self.path = Path(path)
-        # check if path exists
+        self.submissions = []
+        self.MAX_WORKERS = os.cpu_count() - 1 
+        
         if self.path.exists():
-            folders = [f for f in self.path.iterdir() if f.is_dir()]
-            print(f"Loading {len(folders)} submissions")
-            
-            if executor is None:
-                # Fall back to sequential loading if no executor
-                self.submissions = [Submission(f) for f in tqdm(folders, desc="Loading submissions")]
-            else:
-                # Use provided executor for parallel loading
-                self.submissions = list(tqdm(
-                    executor.map(Submission, folders),
-                    total=len(folders),
-                    desc="Loading submissions"  
-                ))
-            
-        else:
-            pass
+            self._load_submissions()
+    
+    def _load_submissions(self):
+        folders = [f for f in self.path.iterdir() if f.is_dir()]
+        print(f"Loading {len(folders)} submissions")
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            self.submissions = list(tqdm(
+                executor.map(Submission, folders),
+                total=len(folders),
+                desc="Loading submissions"
+            ))
+
+    def process_submissions(self, callback):
+        """Process all submissions using a thread pool."""
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, self.submissions),
+                total=len(self.submissions),
+                desc="Processing submissions"
+            ))
+            return results
+
+    def process_documents(self, callback):
+        """Process all documents using a thread pool."""
+        documents = [doc for sub in self.submissions for doc in sub]
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, documents),
+                total=len(documents),
+                desc="Processing documents"
+            ))
+            return results
 
     def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None):
         if provider is None:
             config = Config()
             provider = config.get_default_source()
 
-        if provider == 'sec':
-            downloader = Downloader()
-        elif provider == 'datamule':
-            downloader = PremiumDownloader()
+        downloader = PremiumDownloader() if provider == 'datamule' else Downloader()
+        downloader.download_submissions(
+            output_dir=self.path,
+            cik=cik,
+            ticker=ticker,
+            submission_type=submission_type,
+            filing_date=filing_date
+        )
+        
+        # Reload submissions after download
+        self._load_submissions()
 
-        downloader.download_submissions(output_dir=self.path, cik=cik, ticker=ticker, submission_type=submission_type, filing_date=filing_date
-                                        )
     def __iter__(self):
         return iter(self.submissions)
     
     def document_type(self, document_types):
-        # Convert single document type to list for consistent handling
+        """Filter documents by type(s)."""
         if isinstance(document_types, str):
             document_types = [document_types]
             
         for submission in self.submissions:
             yield from submission.document_type(document_types)
 
-    def contains_string(self, pattern, document_types=None, executor=None):
+    def contains_string(self, pattern, document_types=None):
+        """Search for pattern in documents, with optional type filter."""
         def check_document(document):
             return document if document.contains_string(pattern) else None
         
-        documents = list(self.document_type(document_types) if document_types else (
-            doc for sub in tqdm(self.submissions, desc="Collecting documents") for doc in sub
-        ))
+        # Get documents, filtered by type if specified
+        documents = list(self.document_type(document_types)) if document_types else [
+            doc for sub in self.submissions for doc in sub
+        ]
         
-        if executor:
-            results = list(tqdm(
-                executor.map(check_document, documents),
-                total=len(documents),
-                desc=f"Searching for '{pattern}'"
-            ))
-            yield from (doc for doc in results if doc is not None)
-        else:
-            for document in tqdm(documents, desc=f"Searching for '{pattern}'"):
-                if document.contains_string(pattern):
-                    yield document
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = executor.map(check_document, documents)
+            
+            for doc in tqdm(results, total=len(documents), desc=f"Searching for '{pattern}'"):
+                if doc is not None:
+                    yield doc

{datamule-0.428.dist-info → datamule-0.430.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 0.428
+Version: 0.430
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -24,8 +24,8 @@ Requires-Dist: pandas; extra == "dataset-builder"
 Requires-Dist: google-generativeai; extra == "dataset-builder"
 Requires-Dist: psutil; extra == "dataset-builder"
 Provides-Extra: all
-Requires-Dist: openai; extra == "all"
-Requires-Dist: pandas; extra == "all"
+Requires-Dist: psutil; extra == "all"
 Requires-Dist: google-generativeai; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: flask; extra == "all"
-Requires-Dist: psutil; extra == "all"
+Requires-Dist: pandas; extra == "all"

{datamule-0.428.dist-info → datamule-0.430.dist-info}/RECORD

@@ -1,10 +1,10 @@
 datamule/__init__.py,sha256=qRl4jjMTyekdRV16KOyA4_DoqB-pBi8hMxaUDo88Vu8,2352
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
-datamule/document.py,sha256=GI2BbwSijRkzRp-qz94HzSkYTebGYKXXPwWMbqfepTM,5633
+datamule/document.py,sha256=LIUaDUS-mBigCghnecEespNnFwKsVj6pwpZIyusMq-M,5715
 datamule/helper.py,sha256=tr3AQWus9dHNZFKpLSglWjcb8zmm5qDXjOWACMhvMxQ,4594
 datamule/monitor.py,sha256=mRaM8v5NgcMF9DJ1s_YBzucjrbr-3yFwW422MVml-_Q,9114
 datamule/packageupdater.py,sha256=vEGqlTj6FudIeVHBVJltPh2eBDEqMG9HYmnyrRVKeSU,9595
-datamule/portfolio.py,sha256=Iq-MmOIMdV1VpOcPnOgh7xk1hqUzWWmHoy2iUziLjow,3116
+datamule/portfolio.py,sha256=CNGhNpk8JwL85CCX7RYheTevFB-eGB0_VK02Bo_t9Mc,3566
 datamule/submission.py,sha256=j-uIuwHIvRheXo5RcDf1GYVeYbdjq7yfoU1-GKTH2f8,2790
 datamule/data/company_former_names.csv,sha256=HE9cAv-_QKFX6jT-_-D0rHmaDyQuAzL4MJwank5O1U8,706380
 datamule/data/company_metadata.csv,sha256=yPovrCVjYwLWTU_hBUFJymp8iNO0NBYuq_QwOkRLoN8,3068599
@@ -13,7 +13,7 @@ datamule/data/sec-glossary.csv,sha256=-cN7GjiadLw5C1sv4zSeCnfeZZDYeSgJl-0ydarMAo
 datamule/data/xbrl_descriptions.csv,sha256=SQ9wUURNqG424rnTiZtopsxV2q-PvU4NMj52LqgDsvg,2621524
 datamule/dataset_builder/dataset_builder.py,sha256=NCvNbDwlEkA_eAbqbsG--YlqPBDREFTVSM1GJquR0RE,9747
 datamule/downloader/downloader.py,sha256=Cw8FAc0oi8H4okK6pVuS9hkAGoXR72fV3xAdF83J4v4,14558
-datamule/downloader/premiumdownloader.py,sha256=JH4aZ-ZwARCIACKwgzSgHAuOkKPc_GnhiUHSSu22XO4,14206
+datamule/downloader/premiumdownloader.py,sha256=WyR0lHHva9txCYzf-OrZ2o-ScPxDBtjAfnB5ebUpzjk,14251
 datamule/mulebot/__init__.py,sha256=YvZXV6xQ0iP-oGD8rloufjdwJL6D46P3NNr0CY9PQCA,29
 datamule/mulebot/helper.py,sha256=olztOwltfELZ-IERM2bRNLBavD04kfB6ueWTisJAleA,1080
 datamule/mulebot/mulebot.py,sha256=XbtgvXBSFu9OaaLW_k1KDgHVTNQGV8_0ZwNMFad-pPU,5837
@@ -39,7 +39,7 @@ datamule/parser/document_parsing/basic_13g_parser.py,sha256=sWg83-QTAzUDNs45iWtp
 datamule/parser/document_parsing/basic_8k_parser.py,sha256=inCSmlH_BkLK0Lkvt0kZ6EUJ0nijul_RkdXzccyOmRI,2466
 datamule/parser/document_parsing/form_d_parser.py,sha256=dWlGeVZRzh0kfT3gVMC8eyqeQORdVV3r8KXUwEqAW3s,2036
 datamule/parser/document_parsing/generalized_item_parser.py,sha256=67_DFb1BQbMmdHefEgoCPlEoiUT0zyxh3eBNJpjGXUk,2616
-datamule/parser/document_parsing/helper.py,sha256=QPhVxLxMSx6Qdi7sR4D4iPObGoTnVD3tXTCNWzNxStg,2533
+datamule/parser/document_parsing/helper.py,sha256=sIddTOeX3yjCXlJeW8_2-10NU9omDJrG9HyYdXisB1I,2540
 datamule/parser/document_parsing/information_table_parser_13fhr.py,sha256=R4Up1oDx3xAlzHwXzVzUkdOSsk8YPuJBPS_3I_bNQSE,1767
 datamule/parser/document_parsing/insider_trading_parser.py,sha256=OVQDeLcfaZtgmOWvWPDotftO6jxx-doFAqBYVqNgypo,7106
 datamule/parser/document_parsing/mappings.py,sha256=VKdnT3C5yPTbB4ZBa4El4jnB-6_osomm2rbJx6Ac6HE,5286
@@ -47,8 +47,8 @@ datamule/parser/document_parsing/n_port_p_parser.py,sha256=GmmQFkCZt57WikUZ5Daht
 datamule/parser/document_parsing/sec_parser.py,sha256=AS8H4h1sfUAdWP2gotULcjbylsYN_nHgTfkeVRyENPo,2716
 datamule/parser/document_parsing/sgml_parser.py,sha256=tC1cL3cdVQPWbc9QtoRUYSo2wRuYNaglFaCmP57oEfA,3317
 datamule/parser/sgml_parsing/sgml_parser_cy.c,sha256=P36bPkmVEmEwIhxumwJTmPslHqCaFcNhVSKPTQBQAH0,840575
-datamule/parser/sgml_parsing/sgml_parser_cy.cpython-310-darwin.so,sha256=xcyJC5M1WSGr8l4eZeBffUGGUTAmXxGnPvj0eqD7AaM,395800
-datamule-0.428.dist-info/METADATA,sha256=PwVnD4CKpBieBe9hsnNhv85tVRdr7MGRyfiPMI6MPDU,1006
-datamule-0.428.dist-info/WHEEL,sha256=hckVD9W218I-cSckYATMe3Ko_37-6EaiE9IaoMBKclc,114
-datamule-0.428.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-0.428.dist-info/RECORD,,
+datamule/parser/sgml_parsing/sgml_parser_cy.cpython-310-darwin.so,sha256=rW_mVXXT1n-75AG_Dfs9QHGg-kFkiQ7srk_jBWe8cwQ,395800
+datamule-0.430.dist-info/METADATA,sha256=mo_5k3kJja3arHNTKs5NzswM5Ixw1m1mxV1WarRerZo,1006
+datamule-0.430.dist-info/WHEEL,sha256=SCc8MHJxw-p1LVYhHgn-sVJQOJeWCoPWUIXCO4nz594,114
+datamule-0.430.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-0.430.dist-info/RECORD,,

{datamule-0.428.dist-info → datamule-0.430.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.7.0)
 Root-Is-Purelib: false
 Tag: cp310-cp310-macosx_10_9_universal2