datamule 2.3.7__py3-none-any.whl → 2.3.9__py3-none-any.whl

datamule/portfolio/portfolio.py

@@ -34,7 +34,6 @@ class Portfolio:
         
         if self.path.exists():
             self._load_submissions()
-            self.submissions_loaded = True
         else:
             self.path.mkdir(parents=True, exist_ok=True)
 
@@ -81,6 +80,8 @@ class Portfolio:
         self.submissions = [s for s in (regular_submissions + batch_submissions) if s is not None]
         print(f"Successfully loaded {len(self.submissions)} submissions")
 
+        self.submissions_loaded = True
+
     def _load_batch_submissions_worker(self, batch_tar_path, pbar):
         """Worker function to load submissions from one batch tar with progress updates"""
         # Open tar handle and store it

datamule/providers/providers.py

@@ -0,0 +1,6 @@
+# TODO
+# make it easy for people to bring their own cloud
+
+SEC_LOOKUP_DB_ENDPOINT = ""
+SEC_FILINGS_SGML_BUCKET_ENDPOINT = "https://sec-library.datamule.xyz/" 
+SEC_FILINGS_TAR_BUCKET_ENDPOINT = "https://sec-library.datamule.xyz/tar/"

datamule/sec/submissions/streamer.py

@@ -82,7 +82,7 @@ class Streamer(EFTSQuery):
             if self.accession_numbers is not None and accno_w_dash not in self.accession_numbers:
                 return None, None, None
             
-            if self.skip_accession_numbers is not None and accno_w_dash in self.skip_accession_numbers:
+            if self.skip_accession_numbers is not None and accno_no_dash in self.skip_accession_numbers:
                 return None, None, None
             
             # Construct the URL

datamule/submission/submission.py

@@ -12,6 +12,7 @@ from company_fundamentals import construct_fundamentals
 from decimal import Decimal
 from ..utils.format_accession import format_accession
 from .tar_submission import tar_submission
+import zstandard as zstd
 
 # probably needs rework later
 class FundamentalsAccessor:
@@ -105,6 +106,10 @@ class Submission:
 
                 if response.getcode() == 200:
                     sgml_content=response.read()
+                    content_type = response.headers.get('Content-Type', '')
+                    if content_type == 'application/zstd':
+                        dctx = zstd.ZstdDecompressor()
+                        sgml_content = dctx.decompress(sgml_content)
                 else:
                     raise ValueError(f"URL: {url}, Error: {response.getcode()}")
 
@@ -116,6 +121,7 @@ class Submission:
             metadata = transform_metadata_string(metadata)
 
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=None)
+
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
     
             self.documents_obj_list = []

{datamule-2.3.7.dist-info → datamule-2.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.3.7
+Version: 2.3.9
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.3.7.dist-info → datamule-2.3.9.dist-info}/RECORD

@@ -21,8 +21,10 @@ datamule/mapping_dicts/html_mapping_dicts.py,sha256=pba3utMr2KldPeEGnMRkHyVw7D2W
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
 datamule/portfolio/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/portfolio/portfolio.py,sha256=YPIvS4KKuEtm8A1XvNqDF39f4LJHhAFWmtpJzjbGDhY,11680
+datamule/portfolio/portfolio.py,sha256=PQWROW5eDunOgu668n_RArQKBd8QFNjWCbdZabLE7AY,11677
 datamule/portfolio/portfolio_compression_utils_legacy.py,sha256=1nlbz7JfBDrI0pwTyFiBF856xqGXvQRYBulLUpk7G1A,12695
+datamule/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/providers/providers.py,sha256=mxU_YokPuUc87tdbsfLz0ReqcDc3XGkaiHnqfI_VUac,232
 datamule/sec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/utils.py,sha256=96bavyG2Kq1t8L1YA2vwYnAHKIKdRSoVXxBO5QH1HWo,2196
 datamule/sec/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -31,7 +33,7 @@ datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 datamule/sec/submissions/downloader.py,sha256=9Po1eQ6YEj3Yo9Qw_M5PjQM-OR8iocTNjPIyO3O8GMs,1513
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
 datamule/sec/submissions/monitor.py,sha256=6mE0NZFdPId69t4V53GwBb9sqtRN7HE54sU3WpU0bnY,11900
-datamule/sec/submissions/streamer.py,sha256=A6hunG_mOuBVqA9bBCXhNMcsPaZlhslA3WhopyUwdS4,11611
+datamule/sec/submissions/streamer.py,sha256=AVawZ9pzjuqS5dxmZTvGtpDtHDSKp3r6XjJaF1W19Rs,11612
 datamule/sec/submissions/textsearch.py,sha256=MKDXEz_VI_0ljl73_aw2lx4MVzJW5uDt8KxjvJBwPwM,5794
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/xbrl/downloadcompanyfacts.py,sha256=rMWRiCF9ci_gNZMJ9MC2c_PGEd-yEthawQ0CtVwWTjM,3323
@@ -43,7 +45,7 @@ datamule/seclibrary/bq.py,sha256=TOP0WA6agDKu4vE1eHd62NDpAc02LDDrOP-g1bJpxbw,180
 datamule/sheet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sheet/sheet.py,sha256=Dw979JGygS566N0Iwsvqk0h1s26GfbrIHDWiBaS2oH8,10711
 datamule/submission/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/submission/submission.py,sha256=4UtdViw-h_4Rqt09SFe8-WWdLqaD55T3vqTUVRB0CsE,17058
+datamule/submission/submission.py,sha256=I_7F658vTW1C_dsogIXdxXnV3W1Gbfj_6uzse1YHgY0,17343
 datamule/submission/tar_submission.py,sha256=lkm1neVLW2_-G26VylL6Rzx98Cavvml0Qd2wlJHD0bw,3075
 datamule/tables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/tables/tables.py,sha256=Z3Eu6bdjiaNx4pgXlTMwk2Q-DhpMpEAygF2kJdp-Pu8,5722
@@ -65,7 +67,7 @@ datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=NB_hvfxlRXPyt4Fgc-5qA8vJRItkLhBedCSTaxwW7Jg,5887
 datamule/utils/format_accession.py,sha256=60RtqoNqoT9zSKVb1DeOv1gncJxzPTFMNW4SNOVmC_g,476
 datamule/utils/pdf.py,sha256=Z9xrdVhKex2YdvjYsaPaygRE_J6P_JNiUGkwflz2Hw0,735
-datamule-2.3.7.dist-info/METADATA,sha256=1Igs40zdVpr6XPH4s2ToG5EIyAsI1lpdA1yiuGUjsx4,609
-datamule-2.3.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-2.3.7.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-2.3.7.dist-info/RECORD,,
+datamule-2.3.9.dist-info/METADATA,sha256=X9sKhS_wPqUynhpANtzNQntQUtzrVn6CIoJBmg4Rs7c,609
+datamule-2.3.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-2.3.9.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-2.3.9.dist-info/RECORD,,