datamule 1.6.3__tar.gz → 1.6.5__tar.gz

{datamule-1.6.3 → datamule-1.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.6.3
+Version: 1.6.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.6.3 → datamule-1.6.5}/datamule/sec/submissions/monitor.py

@@ -65,6 +65,7 @@ def clean_efts_hits(hits):
 class Monitor():
     def __init__(self):
         self.accessions = deque(maxlen=50000)
+        self.accessions_set = set()
         self.ratelimiters = {'sec.gov': PreciseRateLimiter(rate=5)}
         self.efts_query = EFTSQuery(quiet=True)
         self.efts_query.limiter = self.ratelimiters['sec.gov']
@@ -250,7 +251,14 @@ class Monitor():
         new_items = []
         for item in items:
             accession = item['accession']
-            if accession not in self.accessions:
+            if accession not in self.accessions_set:  # O(1) lookup instead of O(n)
+                # Check if deque is at capacity and will evict an item
+                if len(self.accessions) == self.accessions.maxlen:
+                    evicted_accession = self.accessions[0]  # Get leftmost item that will be evicted
+                    self.accessions_set.discard(evicted_accession)  # Remove from set
+                
+                # Add to both structures
                 self.accessions.append(accession)
+                self.accessions_set.add(accession)
                 new_items.append(item)
         return new_items

{datamule-1.6.3 → datamule-1.6.5}/datamule/submission.py

@@ -144,24 +144,57 @@ class Submission:
         
         # Handle regular path case
         else:
-            # Use exact filename from metadata
-            document_path = self.path / filename
-            
-            if not document_path.exists():
-                raise FileNotFoundError(f"Document file not found: {document_path}")
-            
-            with document_path.open('rb') as f:
-                content = f.read()
-            
-            # Decompress if needed based on filename extension
-            if is_compressed == 'gzip':
-                content = gzip.decompress(content)
-            elif is_compressed == 'zstd':
-                content = zstd.ZstdDecompressor().decompress(content)
+            # Check if path is a tar file (old format)
+            if self.path.suffix == '.tar':
+                with tarfile.open(self.path, 'r') as tar:
+                    # Try to extract the file, handling compression
+                    try:
+                        content = tar.extractfile(filename).read()
+                        actual_filename = filename
+                    except:
+                        try:
+                            content = tar.extractfile(filename + '.gz').read()
+                            actual_filename = filename + '.gz'
+                            is_compressed = 'gzip'
+                        except:
+                            try:
+                                content = tar.extractfile(filename + '.zst').read()
+                                actual_filename = filename + '.zst'
+                                is_compressed = 'zstd'
+                            except:
+                                raise FileNotFoundError(f"Document file not found in tar: {filename}")
+                    
+                    # Decompress if compressed
+                    if is_compressed == 'gzip':
+                        content = gzip.decompress(content)
+                    elif is_compressed == 'zstd':
+                        content = zstd.ZstdDecompressor().decompress(content)
+                    
+                    # Decode text files
+                    if extension in ['.htm', '.html', '.txt', '.xml']:
+                        content = content.decode('utf-8', errors='replace')
+                    
+                    document_path = f"{self.path}::{actual_filename}"
             
-            # Decode text files
-            if extension in ['.htm', '.html', '.txt', '.xml']:
-                content = content.decode('utf-8', errors='replace')
+            else:
+                # Regular directory case
+                document_path = self.path / filename
+                
+                if not document_path.exists():
+                    raise FileNotFoundError(f"Document file not found: {document_path}")
+                
+                with document_path.open('rb') as f:
+                    content = f.read()
+                
+                # Decompress if needed based on filename extension
+                if is_compressed == 'gzip':
+                    content = gzip.decompress(content)
+                elif is_compressed == 'zstd':
+                    content = zstd.ZstdDecompressor().decompress(content)
+                
+                # Decode text files
+                if extension in ['.htm', '.html', '.txt', '.xml']:
+                    content = content.decode('utf-8', errors='replace')
 
         return Document(
             type=doc['type'], 
@@ -171,7 +204,6 @@ class Submission:
             accession=self.accession,
             path=document_path
         )
-
     def __iter__(self):
         """Make Submission iterable by yielding all documents."""
         for idx in range(len(self.metadata.content['documents'])):

{datamule-1.6.3 → datamule-1.6.5}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.6.3
+Version: 1.6.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.6.3 → datamule-1.6.5}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.6.3",
+    version="1.6.5",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",