datamule 2.3.7__tar.gz → 2.3.9__tar.gz

{datamule-2.3.7 → datamule-2.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.3.7
+Version: 2.3.9
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.3.7 → datamule-2.3.9}/datamule/portfolio/portfolio.py

@@ -34,7 +34,6 @@ class Portfolio:
         
         if self.path.exists():
             self._load_submissions()
-            self.submissions_loaded = True
         else:
             self.path.mkdir(parents=True, exist_ok=True)
 
@@ -81,6 +80,8 @@ class Portfolio:
         self.submissions = [s for s in (regular_submissions + batch_submissions) if s is not None]
         print(f"Successfully loaded {len(self.submissions)} submissions")
 
+        self.submissions_loaded = True
+
     def _load_batch_submissions_worker(self, batch_tar_path, pbar):
         """Worker function to load submissions from one batch tar with progress updates"""
         # Open tar handle and store it

datamule-2.3.9/datamule/providers/providers.py

@@ -0,0 +1,6 @@
+# TODO
+# make it easy for people to bring their own cloud
+
+SEC_LOOKUP_DB_ENDPOINT = ""
+SEC_FILINGS_SGML_BUCKET_ENDPOINT = "https://sec-library.datamule.xyz/" 
+SEC_FILINGS_TAR_BUCKET_ENDPOINT = "https://sec-library.datamule.xyz/tar/"

{datamule-2.3.7 → datamule-2.3.9}/datamule/sec/submissions/streamer.py

@@ -82,7 +82,7 @@ class Streamer(EFTSQuery):
             if self.accession_numbers is not None and accno_w_dash not in self.accession_numbers:
                 return None, None, None
             
-            if self.skip_accession_numbers is not None and accno_w_dash in self.skip_accession_numbers:
+            if self.skip_accession_numbers is not None and accno_no_dash in self.skip_accession_numbers:
                 return None, None, None
             
             # Construct the URL

{datamule-2.3.7 → datamule-2.3.9}/datamule/submission/submission.py

@@ -12,6 +12,7 @@ from company_fundamentals import construct_fundamentals
 from decimal import Decimal
 from ..utils.format_accession import format_accession
 from .tar_submission import tar_submission
+import zstandard as zstd
 
 # probably needs rework later
 class FundamentalsAccessor:
@@ -105,6 +106,10 @@ class Submission:
 
                 if response.getcode() == 200:
                     sgml_content=response.read()
+                    content_type = response.headers.get('Content-Type', '')
+                    if content_type == 'application/zstd':
+                        dctx = zstd.ZstdDecompressor()
+                        sgml_content = dctx.decompress(sgml_content)
                 else:
                     raise ValueError(f"URL: {url}, Error: {response.getcode()}")
 
@@ -116,6 +121,7 @@ class Submission:
             metadata = transform_metadata_string(metadata)
 
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=None)
+
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
     
             self.documents_obj_list = []

{datamule-2.3.7 → datamule-2.3.9}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.3.7
+Version: 2.3.9
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.3.7 → datamule-2.3.9}/datamule.egg-info/SOURCES.txt

@@ -29,6 +29,8 @@ datamule/mapping_dicts/xml_mapping_dicts.py
 datamule/portfolio/__init__.py
 datamule/portfolio/portfolio.py
 datamule/portfolio/portfolio_compression_utils_legacy.py
+datamule/providers/__init__.py
+datamule/providers/providers.py
 datamule/sec/__init__.py
 datamule/sec/utils.py
 datamule/sec/infrastructure/__init__.py

{datamule-2.3.7 → datamule-2.3.9}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="2.3.7",
+    version="2.3.9",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",