datamule 2.1.3__tar.gz → 2.1.5__tar.gz

{datamule-2.1.3 → datamule-2.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.1.3
+Version: 2.1.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

datamule-2.1.5/datamule/datasets.py

@@ -0,0 +1,51 @@
+# datamule/datasets.py
+from pathlib import Path
+import requests
+import gzip
+import shutil
+import csv
+
+# Dataset URLs
+DATASET_URLS = {
+    "cik_cusip_crosswalk": "https://github.com/john-friedman/datamule-data/raw/refs/heads/master/data/datasets/cik_cusip_crosswalk.csv.gz",
+    "financial_security_identifiers_crosswalk" : "https://github.com/john-friedman/datamule-data/raw/refs/heads/master/data/datasets/financial_security_identifiers_crosswalk.csv.gz"
+}
+
+def update_dataset(name):
+    """Force update a dataset by re-downloading it."""
+    return _get_dataset(name, update=True)
+
+def _get_dataset(name, update=False):
+    """Internal function to get dataset as list of dicts, downloading if necessary."""
+    if name not in DATASET_URLS:
+        raise ValueError(f"Unknown dataset: {name}")
+    
+    url = DATASET_URLS[name]
+    data_dir = Path.home() / ".datamule" / "datasets"
+    file_path = data_dir / f"{name}.csv"
+    
+    if not file_path.exists() or update:
+        print(f"Downloading {name}...")
+        data_dir.mkdir(parents=True, exist_ok=True)
+        
+        response = requests.get(url, stream=True)
+        response.raise_for_status()
+        
+        gz_path = file_path.with_suffix('.csv.gz')
+        with open(gz_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+        
+        with gzip.open(gz_path, 'rb') as f_in:
+            with open(file_path, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        
+        gz_path.unlink()
+    
+    # Read CSV and return as list of dicts
+    with open(file_path, 'r') as f:
+        return list(csv.DictReader(f))
+
+# Dataset available as list of dicts on import
+cik_cusip_crosswalk = _get_dataset("cik_cusip_crosswalk")
+financial_security_identifiers_crosswalk = _get_dataset("financial_security_identifiers_crosswalk")

{datamule-2.1.3 → datamule-2.1.5}/datamule/document/tables/tables_proxyvotingrecord.py

@@ -13,7 +13,8 @@ proxy_voting_record_dict = {
     'vote_voteRecord_sharesVoted': 'recordSharesVoted',  # To distinguish from top-level sharesVoted
     'isin': 'isin',
     'voteSource': 'voteSource',
-    'voteSeries': 'voteSeries'
+    'voteSeries': 'voteSeries',
+    'figi': 'figi',
 }
 
 config_proxyvotingrecord = {

{datamule-2.1.3 → datamule-2.1.5}/datamule/portfolio.py

@@ -96,12 +96,16 @@ class Portfolio:
         # Create submissions for each accession
         submissions = []
         for accession_prefix in accession_prefixes:
-            submission = Submission(
-                batch_tar_path=batch_tar_path,
-                accession_prefix=accession_prefix,
-                portfolio_ref=self
-            )
-            submissions.append(submission)
+            try:
+                submission = Submission(
+                    batch_tar_path=batch_tar_path,
+                    accession_prefix=accession_prefix,
+                    portfolio_ref=self
+                )
+                submissions.append(submission)
+            except Exception as e:
+                pass
+                #print(f"Path: {batch_tar_path}. Exception: {e}")
             pbar.update(1)  # Update progress for each successful submission
         
         return submissions

{datamule-2.1.3 → datamule-2.1.5}/datamule/submission.py

@@ -12,6 +12,7 @@ import urllib.request
 from secxbrl import parse_inline_xbrl
 from company_fundamentals import construct_fundamentals
 from decimal import Decimal
+from .utils.format_accession import format_accession
 
 
 class Submission:
@@ -93,11 +94,10 @@ class Submission:
             # standardize metadata
             metadata = transform_metadata_string(metadata)
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=metadata_path)
-            self.accession = self.metadata.content['accession-number']
+
+            # lets just use accesion-prefix, to get around malformed metadata files (1995 has a lot!)
+            self.accession = format_accession(self.accession_prefix,'dash')
             
-            # Band-aid fix: some SGML files in the SEC are bad lol, so they have TWO header sections. Will fix post w/ my cleaned archive
-            if isinstance(self.accession,list):
-                self.accession = self.accession[0]
             #print(f"s: {self.metadata.content['accession-number']} : {batch_tar_path}")
             self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
 

{datamule-2.1.3 → datamule-2.1.5}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.1.3
+Version: 2.1.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.1.3 → datamule-2.1.5}/datamule.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 setup.py
 datamule/__init__.py
 datamule/config.py
+datamule/datasets.py
 datamule/helper.py
 datamule/index.py
 datamule/package_updater.py

{datamule-2.1.3 → datamule-2.1.5}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="2.1.3",
+    version="2.1.5",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",