datamule 1.6.3__py3-none-any.whl → 1.6.5__py3-none-any.whl

datamule/sec/submissions/monitor.py

@@ -65,6 +65,7 @@ def clean_efts_hits(hits):
 class Monitor():
     def __init__(self):
         self.accessions = deque(maxlen=50000)
+        self.accessions_set = set()
         self.ratelimiters = {'sec.gov': PreciseRateLimiter(rate=5)}
         self.efts_query = EFTSQuery(quiet=True)
         self.efts_query.limiter = self.ratelimiters['sec.gov']
@@ -250,7 +251,14 @@ class Monitor():
         new_items = []
         for item in items:
             accession = item['accession']
-            if accession not in self.accessions:
+            if accession not in self.accessions_set:  # O(1) lookup instead of O(n)
+                # Check if deque is at capacity and will evict an item
+                if len(self.accessions) == self.accessions.maxlen:
+                    evicted_accession = self.accessions[0]  # Get leftmost item that will be evicted
+                    self.accessions_set.discard(evicted_accession)  # Remove from set
+                
+                # Add to both structures
                 self.accessions.append(accession)
+                self.accessions_set.add(accession)
                 new_items.append(item)
         return new_items

datamule/submission.py

@@ -144,24 +144,57 @@ class Submission:
         
         # Handle regular path case
         else:
-            # Use exact filename from metadata
-            document_path = self.path / filename
-            
-            if not document_path.exists():
-                raise FileNotFoundError(f"Document file not found: {document_path}")
-            
-            with document_path.open('rb') as f:
-                content = f.read()
-            
-            # Decompress if needed based on filename extension
-            if is_compressed == 'gzip':
-                content = gzip.decompress(content)
-            elif is_compressed == 'zstd':
-                content = zstd.ZstdDecompressor().decompress(content)
+            # Check if path is a tar file (old format)
+            if self.path.suffix == '.tar':
+                with tarfile.open(self.path, 'r') as tar:
+                    # Try to extract the file, handling compression
+                    try:
+                        content = tar.extractfile(filename).read()
+                        actual_filename = filename
+                    except:
+                        try:
+                            content = tar.extractfile(filename + '.gz').read()
+                            actual_filename = filename + '.gz'
+                            is_compressed = 'gzip'
+                        except:
+                            try:
+                                content = tar.extractfile(filename + '.zst').read()
+                                actual_filename = filename + '.zst'
+                                is_compressed = 'zstd'
+                            except:
+                                raise FileNotFoundError(f"Document file not found in tar: {filename}")
+                    
+                    # Decompress if compressed
+                    if is_compressed == 'gzip':
+                        content = gzip.decompress(content)
+                    elif is_compressed == 'zstd':
+                        content = zstd.ZstdDecompressor().decompress(content)
+                    
+                    # Decode text files
+                    if extension in ['.htm', '.html', '.txt', '.xml']:
+                        content = content.decode('utf-8', errors='replace')
+                    
+                    document_path = f"{self.path}::{actual_filename}"
             
-            # Decode text files
-            if extension in ['.htm', '.html', '.txt', '.xml']:
-                content = content.decode('utf-8', errors='replace')
+            else:
+                # Regular directory case
+                document_path = self.path / filename
+                
+                if not document_path.exists():
+                    raise FileNotFoundError(f"Document file not found: {document_path}")
+                
+                with document_path.open('rb') as f:
+                    content = f.read()
+                
+                # Decompress if needed based on filename extension
+                if is_compressed == 'gzip':
+                    content = gzip.decompress(content)
+                elif is_compressed == 'zstd':
+                    content = zstd.ZstdDecompressor().decompress(content)
+                
+                # Decode text files
+                if extension in ['.htm', '.html', '.txt', '.xml']:
+                    content = content.decode('utf-8', errors='replace')
 
         return Document(
             type=doc['type'], 
@@ -171,7 +204,6 @@ class Submission:
             accession=self.accession,
             path=document_path
         )
-
     def __iter__(self):
         """Make Submission iterable by yielding all documents."""
         for idx in range(len(self.metadata.content['documents'])):

{datamule-1.6.3.dist-info → datamule-1.6.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.6.3
+Version: 1.6.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.6.3.dist-info → datamule-1.6.5.dist-info}/RECORD

@@ -6,7 +6,7 @@ datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,9
 datamule/portfolio.py,sha256=tADqQMkFaFyjanbJ0QcaOHGdJJB254rOg29FW7a13l0,11835
 datamule/portfolio_compression_utils.py,sha256=8OPYEN5zAdV1FiTxgVN3S7cTKs99Elv74bwgoIJP4QY,12654
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
-datamule/submission.py,sha256=yDPglaFJ65nXn7Lxh-JFTQGKVVmBJDHBVWTf4UEUm2M,8610
+datamule/submission.py,sha256=ooLsesZ5HkgSWyEFID4u08CobTxdo35eAUHSCB6fw2k,10332
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/datamule/sec_connector.py,sha256=T3edE7I-d4oHysqj7zYlIOxH3Fuauj9tfw39UdFWvB8,2393
@@ -52,7 +52,7 @@ datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNs
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/submissions/downloader.py,sha256=zGS0oJJI8tVF_GnVpZm20MymdYxnjrEjQioSVggw7Ck,1486
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
-datamule/sec/submissions/monitor.py,sha256=1JUMRYsTqtd31hX3UrUA_aXFUmZN6n-V7h0i1gavNOs,11395
+datamule/sec/submissions/monitor.py,sha256=6mE0NZFdPId69t4V53GwBb9sqtRN7HE54sU3WpU0bnY,11900
 datamule/sec/submissions/streamer.py,sha256=Qydj40CmWB_wsPv2dibefRohmCokegG2pR7iZ9C3xLQ,11584
 datamule/sec/submissions/textsearch.py,sha256=MKDXEz_VI_0ljl73_aw2lx4MVzJW5uDt8KxjvJBwPwM,5794
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -66,7 +66,7 @@ datamule/seclibrary/downloader.py,sha256=3jEy67oiEg8BF20KcKCx2KC0UjHzhiepdu29TOa
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=aX7ZaAp3zXHLcv4TFk_rGwjb8r7yNDQDFVg4nPf60kM,5934
-datamule-1.6.3.dist-info/METADATA,sha256=9tb_ecnMVFHYq-Jcj_O0xAYUtM6v2PEZRxdEtPnorD4,524
-datamule-1.6.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.6.3.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.6.3.dist-info/RECORD,,
+datamule-1.6.5.dist-info/METADATA,sha256=2Svo4CVhWzVDa9HFLOOwzIzgF84ItLQT-2BI6WR894s,524
+datamule-1.6.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.6.5.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.6.5.dist-info/RECORD,,