datamule 2.1.3__py3-none-any.whl → 2.1.5__py3-none-any.whl

datamule/datasets.py

@@ -0,0 +1,51 @@
+# datamule/datasets.py
+from pathlib import Path
+import requests
+import gzip
+import shutil
+import csv
+
+# Dataset URLs
+DATASET_URLS = {
+    "cik_cusip_crosswalk": "https://github.com/john-friedman/datamule-data/raw/refs/heads/master/data/datasets/cik_cusip_crosswalk.csv.gz",
+    "financial_security_identifiers_crosswalk" : "https://github.com/john-friedman/datamule-data/raw/refs/heads/master/data/datasets/financial_security_identifiers_crosswalk.csv.gz"
+}
+
+def update_dataset(name):
+    """Force update a dataset by re-downloading it."""
+    return _get_dataset(name, update=True)
+
+def _get_dataset(name, update=False):
+    """Internal function to get dataset as list of dicts, downloading if necessary."""
+    if name not in DATASET_URLS:
+        raise ValueError(f"Unknown dataset: {name}")
+    
+    url = DATASET_URLS[name]
+    data_dir = Path.home() / ".datamule" / "datasets"
+    file_path = data_dir / f"{name}.csv"
+    
+    if not file_path.exists() or update:
+        print(f"Downloading {name}...")
+        data_dir.mkdir(parents=True, exist_ok=True)
+        
+        response = requests.get(url, stream=True)
+        response.raise_for_status()
+        
+        gz_path = file_path.with_suffix('.csv.gz')
+        with open(gz_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+        
+        with gzip.open(gz_path, 'rb') as f_in:
+            with open(file_path, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        
+        gz_path.unlink()
+    
+    # Read CSV and return as list of dicts
+    with open(file_path, 'r') as f:
+        return list(csv.DictReader(f))
+
+# Dataset available as list of dicts on import
+cik_cusip_crosswalk = _get_dataset("cik_cusip_crosswalk")
+financial_security_identifiers_crosswalk = _get_dataset("financial_security_identifiers_crosswalk")

datamule/document/tables/tables_proxyvotingrecord.py

@@ -13,7 +13,8 @@ proxy_voting_record_dict = {
     'vote_voteRecord_sharesVoted': 'recordSharesVoted',  # To distinguish from top-level sharesVoted
     'isin': 'isin',
     'voteSource': 'voteSource',
-    'voteSeries': 'voteSeries'
+    'voteSeries': 'voteSeries',
+    'figi': 'figi',
 }
 
 config_proxyvotingrecord = {

datamule/portfolio.py

@@ -96,12 +96,16 @@ class Portfolio:
         # Create submissions for each accession
         submissions = []
         for accession_prefix in accession_prefixes:
-            submission = Submission(
-                batch_tar_path=batch_tar_path,
-                accession_prefix=accession_prefix,
-                portfolio_ref=self
-            )
-            submissions.append(submission)
+            try:
+                submission = Submission(
+                    batch_tar_path=batch_tar_path,
+                    accession_prefix=accession_prefix,
+                    portfolio_ref=self
+                )
+                submissions.append(submission)
+            except Exception as e:
+                pass
+                #print(f"Path: {batch_tar_path}. Exception: {e}")
             pbar.update(1)  # Update progress for each successful submission
         
         return submissions

datamule/submission.py

@@ -12,6 +12,7 @@ import urllib.request
 from secxbrl import parse_inline_xbrl
 from company_fundamentals import construct_fundamentals
 from decimal import Decimal
+from .utils.format_accession import format_accession
 
 
 class Submission:
@@ -93,11 +94,10 @@ class Submission:
             # standardize metadata
             metadata = transform_metadata_string(metadata)
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=metadata_path)
-            self.accession = self.metadata.content['accession-number']
+
+            # lets just use accesion-prefix, to get around malformed metadata files (1995 has a lot!)
+            self.accession = format_accession(self.accession_prefix,'dash')
             
-            # Band-aid fix: some SGML files in the SEC are bad lol, so they have TWO header sections. Will fix post w/ my cleaned archive
-            if isinstance(self.accession,list):
-                self.accession = self.accession[0]
             #print(f"s: {self.metadata.content['accession-number']} : {batch_tar_path}")
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
 

{datamule-2.1.3.dist-info → datamule-2.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.1.3
+Version: 2.1.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.1.3.dist-info → datamule-2.1.5.dist-info}/RECORD

@@ -1,12 +1,13 @@
 datamule/__init__.py,sha256=sY9rYx9z4LADjOLmwjL3BXssIzHs8MQM6gt9IWMS85U,1192
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
+datamule/datasets.py,sha256=-2_5kTRS3mxlkKbXwBg8aiistYljLYRnZjDLZNhV8bk,1867
 datamule/helper.py,sha256=KqhAmTMdvATEh3I-O4xLcAcrHB9zXQERBuwzue7zyQw,3674
 datamule/index.py,sha256=Rrcna9FJV-Oh_K6O2IuUEIDmtay_7UZ4l4jgKCi7A7I,2079
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
-datamule/portfolio.py,sha256=YViG1JgJ9SFhg8N3tOOhBI8oc6Pmi2vwnHeHmlkC_5U,12119
+datamule/portfolio.py,sha256=0-E1ZSEjJ8hba7HxF8oCrRneNuF_KKISOY6K4dRg0Cg,12282
 datamule/portfolio_compression_utils.py,sha256=8OPYEN5zAdV1FiTxgVN3S7cTKs99Elv74bwgoIJP4QY,12654
 datamule/sheet.py,sha256=Ws_YRtpvewLVioarngVMe8cgG_sp11MP9_goGbRaiWE,23952
-datamule/submission.py,sha256=piMtTyoMZrKmLBpjyi0BBFhlkugi_CEuyox7J-jnusQ,15898
+datamule/submission.py,sha256=TdQDfFjOKXy2qAZcD6hc9kjDSxmuZLqk8WRhtMjjC-g,15822
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/datamule/datamule_lookup.py,sha256=e8djAg-ctSyHiKk7BjbtgugZ3p8roUjzsym5z3AihUg,9468
@@ -22,7 +23,7 @@ datamule/document/tables/tables_25nse.py,sha256=kpoOcIpra6i3Wx_6pUCj1fkx0wUbMhx7
 datamule/document/tables/tables_informationtable.py,sha256=3yjuxYuLoBjRd6O0BNd0jQDmS1XUDjA6xp51Csq2cH8,649
 datamule/document/tables/tables_npx.py,sha256=tZDBAonAQWLsgecVK_OwIgNcUJhuV5L2gkTSNbXAgNE,6652
 datamule/document/tables/tables_ownership.py,sha256=pRoFFRGLWp8gKAAvvUbVRxIU2xDFAQhwi9bgwddsT8A,11185
-datamule/document/tables/tables_proxyvotingrecord.py,sha256=t5h6iQtlg28Rqt1NJ1FDHCFHFjj_4_aelnFbWLtQcs0,875
+datamule/document/tables/tables_proxyvotingrecord.py,sha256=S_Th294-KWRL-QVXkexNWIksSaFePZGSVq6EU8iiK0o,896
 datamule/document/tables/tables_sbsef.py,sha256=X6VKVnAdWxn2TgRmaAd1WWlxPhcLPQ-53s0qDokkPI0,635
 datamule/document/tables/tables_sdr.py,sha256=BwHRJvtijiYvNJ2lIc_30kct6VEmLimIzX28JjZBBqo,4924
 datamule/document/tables/utils.py,sha256=2-X_1NsiWj_XsD9djxCXwTeIVlg-ip78gG11xACJiDs,738
@@ -50,7 +51,7 @@ datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,180
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=NB_hvfxlRXPyt4Fgc-5qA8vJRItkLhBedCSTaxwW7Jg,5887
 datamule/utils/format_accession.py,sha256=60RtqoNqoT9zSKVb1DeOv1gncJxzPTFMNW4SNOVmC_g,476
-datamule-2.1.3.dist-info/METADATA,sha256=jE2eNeO223VJUDysOc9ENyiOIDbVsv5DEXdCnXrhFRA,560
-datamule-2.1.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-2.1.3.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-2.1.3.dist-info/RECORD,,
+datamule-2.1.5.dist-info/METADATA,sha256=O7H7morwBBDgqmyytWiui81VS4Wt5PoyIsNGv3_j3rk,560
+datamule-2.1.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-2.1.5.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-2.1.5.dist-info/RECORD,,