datamule 0.429__tar.gz → 0.430__tar.gz

{datamule-0.429 → datamule-0.430}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 0.429
+Version: 0.430
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-0.429 → datamule-0.430}/datamule/document.py

@@ -17,6 +17,9 @@ class Document:
         self.data = None
         self.content = None
 
+    def _load_content(self):
+        self.content = load_file_content(self.path)
+
     def contains_string(self, pattern):
         """Currently only works for .htm, .html, and .txt files"""
         if self.path.suffix in ['.htm', '.html', '.txt']:

{datamule-0.429 → datamule-0.430}/datamule/downloader/premiumdownloader.py

@@ -259,7 +259,8 @@ class PremiumDownloader:
                 keepalive_timeout=60
             )
 
-            async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=3600)) as session:
+            # timeout should be max 2 hours.
+            async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=7200)) as session:
                 tasks = [self.download_and_process(session, url, semaphore, decompression_pool, output_dir, processor) for url in urls]
                 await asyncio.gather(*tasks, return_exceptions=True)
 

{datamule-0.429 → datamule-0.430}/datamule/parser/document_parsing/helper.py

@@ -62,7 +62,7 @@ def load_file_content(filename):
     elif filename.suffix in ['.html','.htm']:
         return load_html_content(filename)
     else:
-        raise ValueError(f"Unsupported file type: {filename}")
+        raise ValueError(f"Unsupported file type: {filename.suffix}")
 
 def clean_title(title: str) -> str:
     """Clean up section title by removing newlines, periods, and all whitespace, converting to lowercase."""

datamule-0.430/datamule/portfolio.py

@@ -0,0 +1,95 @@
+from pathlib import Path
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
+from .submission import Submission
+from .downloader.premiumdownloader import PremiumDownloader
+from .downloader.downloader import Downloader
+from .config import Config
+import os
+
+class Portfolio:
+    def __init__(self, path):
+        self.path = Path(path)
+        self.submissions = []
+        self.MAX_WORKERS = os.cpu_count() - 1 
+        
+        if self.path.exists():
+            self._load_submissions()
+    
+    def _load_submissions(self):
+        folders = [f for f in self.path.iterdir() if f.is_dir()]
+        print(f"Loading {len(folders)} submissions")
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            self.submissions = list(tqdm(
+                executor.map(Submission, folders),
+                total=len(folders),
+                desc="Loading submissions"
+            ))
+
+    def process_submissions(self, callback):
+        """Process all submissions using a thread pool."""
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, self.submissions),
+                total=len(self.submissions),
+                desc="Processing submissions"
+            ))
+            return results
+
+    def process_documents(self, callback):
+        """Process all documents using a thread pool."""
+        documents = [doc for sub in self.submissions for doc in sub]
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, documents),
+                total=len(documents),
+                desc="Processing documents"
+            ))
+            return results
+
+    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None):
+        if provider is None:
+            config = Config()
+            provider = config.get_default_source()
+
+        downloader = PremiumDownloader() if provider == 'datamule' else Downloader()
+        downloader.download_submissions(
+            output_dir=self.path,
+            cik=cik,
+            ticker=ticker,
+            submission_type=submission_type,
+            filing_date=filing_date
+        )
+        
+        # Reload submissions after download
+        self._load_submissions()
+
+    def __iter__(self):
+        return iter(self.submissions)
+    
+    def document_type(self, document_types):
+        """Filter documents by type(s)."""
+        if isinstance(document_types, str):
+            document_types = [document_types]
+            
+        for submission in self.submissions:
+            yield from submission.document_type(document_types)
+
+    def contains_string(self, pattern, document_types=None):
+        """Search for pattern in documents, with optional type filter."""
+        def check_document(document):
+            return document if document.contains_string(pattern) else None
+        
+        # Get documents, filtered by type if specified
+        documents = list(self.document_type(document_types)) if document_types else [
+            doc for sub in self.submissions for doc in sub
+        ]
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = executor.map(check_document, documents)
+            
+            for doc in tqdm(results, total=len(documents), desc=f"Searching for '{pattern}'"):
+                if doc is not None:
+                    yield doc

{datamule-0.429 → datamule-0.430}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 0.429
+Version: 0.430
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-0.429 → datamule-0.430}/setup.py

@@ -55,7 +55,7 @@ extras["all"] = list(all_dependencies)
 setup(
     name="datamule",
     author="John Friedman",
-    version="0.429",
+    version="0.430",
     description="Making it easier to use SEC filings.",
     packages=find_namespace_packages(include=['datamule*']),
     url="https://github.com/john-friedman/datamule-python",

datamule-0.429/datamule/portfolio.py

@@ -1,82 +0,0 @@
-from pathlib import Path
-from tqdm import tqdm
-from concurrent.futures import ProcessPoolExecutor
-from .submission import Submission
-from .downloader.premiumdownloader import PremiumDownloader
-from .downloader.downloader import Downloader
-from .config import Config
-
-class Portfolio:
-    def create(cls, path):
-        # This method handles the process pool lifecycle
-        with ProcessPoolExecutor() as executor:
-            portfolio = cls(path, executor)
-            return portfolio
-
-    def __init__(self, path, executor=None):
-        self.path = Path(path)
-        # check if path exists
-        if self.path.exists():
-            folders = [f for f in self.path.iterdir() if f.is_dir()]
-            print(f"Loading {len(folders)} submissions")
-            
-            if executor is None:
-                # Fall back to sequential loading if no executor
-                self.submissions = [Submission(f) for f in tqdm(folders, desc="Loading submissions")]
-            else:
-                # Use provided executor for parallel loading
-                self.submissions = list(tqdm(
-                    executor.map(Submission, folders),
-                    total=len(folders),
-                    desc="Loading submissions"  
-                ))
-            
-        else:
-            pass
-
-    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None):
-        if provider is None:
-            config = Config()
-            provider = config.get_default_source()
-
-        if provider == 'sec':
-            downloader = Downloader()
-        elif provider == 'datamule':
-            downloader = PremiumDownloader()
-
-        downloader.download_submissions(output_dir=self.path, cik=cik, ticker=ticker, submission_type=submission_type, filing_date=filing_date
-                                        )
-        
-        # Reload submissions after download
-        self.__init__(self.path)
-        
-    def __iter__(self):
-        return iter(self.submissions)
-    
-    def document_type(self, document_types):
-        # Convert single document type to list for consistent handling
-        if isinstance(document_types, str):
-            document_types = [document_types]
-            
-        for submission in self.submissions:
-            yield from submission.document_type(document_types)
-
-    def contains_string(self, pattern, document_types=None, executor=None):
-        def check_document(document):
-            return document if document.contains_string(pattern) else None
-        
-        documents = list(self.document_type(document_types) if document_types else (
-            doc for sub in tqdm(self.submissions, desc="Collecting documents") for doc in sub
-        ))
-        
-        if executor:
-            results = list(tqdm(
-                executor.map(check_document, documents),
-                total=len(documents),
-                desc=f"Searching for '{pattern}'"
-            ))
-            yield from (doc for doc in results if doc is not None)
-        else:
-            for document in tqdm(documents, desc=f"Searching for '{pattern}'"):
-                if document.contains_string(pattern):
-                    yield document

{datamule-0.429 → datamule-0.430}/datamule.egg-info/requires.txt

@@ -11,11 +11,11 @@ pytz
 zstandard
 
 [all]
-flask
-google-generativeai
+psutil
 pandas
+google-generativeai
 openai
-psutil
+flask
 
 [dataset_builder]
 pandas