datamule 1.2.9__py3-none-any.whl → 1.3.0__py3-none-any.whl

datamule/helper.py

@@ -1,20 +1,32 @@
 from functools import lru_cache
 import csv
 from pathlib import Path
+import os
 
 def _load_package_csv(name):
-    """Load CSV files from ~/.datamule/ directory"""
-    data_dir = Path.home() / ".datamule"
-    csv_path = data_dir / f"{name}.csv"
-    
-    data = []
-    
-    with open(csv_path, 'r') as csvfile:
-        csv_reader = csv.DictReader(csvfile)
-        for row in csv_reader:
-            data.append(row)
+    """Load CSV files from package data directory"""
+    # First try to load from the package data directory
+    try:
+        package_dir = os.path.dirname(os.path.dirname(__file__))
+        csv_path = os.path.join(package_dir, "data", f"{name}.csv")
+        
+        # Fallback to the legacy location
+        if not os.path.exists(csv_path):
+            csv_path = Path.home() / ".datamule" / f"{name}.csv"
+        
+        data = []
+        with open(csv_path, 'r') as csvfile:
+            csv_reader = csv.DictReader(csvfile)
+            for row in csv_reader:
+                data.append(row)
+        
+        return data
     
-    return data
+    except FileNotFoundError:
+        raise FileNotFoundError(
+            f"Required data file '{name}.csv' not found. "
+            f"This file should be in the datamule package directory or in ~/.datamule/"
+        )
 
 def load_package_dataset(dataset):
     if dataset =='listed_filer_metadata':
@@ -39,8 +51,6 @@ def get_cik_from_dataset(dataset_name, key, value):
     
     return result
 
-
-
 @lru_cache(maxsize=128)
 def get_ciks_from_metadata_filters(**kwargs):
     """Get CIKs from listed_filer_metadata.csv that match all provided filters."""
@@ -67,46 +77,45 @@ def get_ciks_from_metadata_filters(**kwargs):
     
     return list(result_ciks)
 
-
 def _process_cik_and_metadata_filters(cik=None, ticker=None, **kwargs):
-        """ 
-        Helper method to process CIK, ticker, and metadata filters.
-        Returns a list of CIKs after processing.
-        """
-        # Input validation
-        if cik is not None and ticker is not None:
-            raise ValueError("Only one of cik or ticker should be provided, not both.")
+    """ 
+    Helper method to process CIK, ticker, and metadata filters.
+    Returns a list of CIKs after processing.
+    """
+    # Input validation
+    if cik is not None and ticker is not None:
+        raise ValueError("Only one of cik or ticker should be provided, not both.")
+    
+    if 'tickers' in kwargs:
+        raise ValueError("Use 'ticker' instead of 'tickers'.")
 
-        # Convert ticker to CIK if provided
-        if ticker is not None:
-            if isinstance(ticker, str):
-                ticker = [ticker]
+    # Convert ticker to CIK if provided
+    if ticker is not None:
+        if isinstance(ticker, str):
+            ticker = [ticker]
+            
+        cik = []
+        for t in ticker:
+            ticker_ciks = get_cik_from_dataset('listed_filer_metadata', 'ticker', t)
+            if ticker_ciks:
+                cik.extend(ticker_ciks)
 
-            ciks_from_ticker = []
-            for t in ticker:
-                ciks = get_cik_from_dataset('listed_filer_metadata', 'ticker', t)
-                if ciks:
-                    ciks_from_ticker.extend(ciks)
+    # Normalize CIK format
+    if cik is not None:
+        if isinstance(cik, str):
+            cik = [int(cik)]
+        elif isinstance(cik, int):
+            cik = [cik]
+        elif isinstance(cik, list):
+            cik = [int(x) for x in cik]
 
-            cik = ciks
+    # Process metadata filters if provided
+    if kwargs:
+        metadata_ciks = get_ciks_from_metadata_filters(**kwargs)
 
-        # Normalize CIK format
         if cik is not None:
-            if isinstance(cik, str):
-                cik = [int(cik)]
-            elif isinstance(cik, int):
-                cik = [cik]
-            elif isinstance(cik, list):
-                cik = [int(x) for x in cik]
-
-        # Process metadata filters if provided
-        if kwargs:
-            metadata_ciks = get_ciks_from_metadata_filters(**kwargs)
-
-            if cik is not None:
-                cik = list(set(cik).intersection(metadata_ciks))
-            else:
-                cik = metadata_ciks
-                
-        return cik
-        
+            cik = list(set(cik).intersection(metadata_ciks))
+        else:
+            cik = metadata_ciks
+            
+    return cik

datamule/sec/submissions/monitor.py

@@ -48,7 +48,7 @@ async def poll_rss(limiter):
 
 def clean_efts_hits(hits):
     # clean hits
-    hits = [{'accession': int(hit['_source']['adsh'].replace('-','')), 'filing_date': hit['_source']['file_date'], 'ciks': hit['_source']['ciks']} for hit in hits]
+    hits = [{'accession': int(hit['_source']['adsh'].replace('-','')), 'filing_date': hit['_source']['file_date'], 'ciks': hit['_source']['ciks'], 'submission_type': hit['_source']['file_type']} for hit in hits]
     return hits
 
 class Monitor():

{datamule-1.2.9.dist-info → datamule-1.3.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.2.9
-Summary: Making it easier to use SEC filings.
+Version: 1.3.0
+Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
 Requires-Dist: aiohttp
@@ -16,5 +16,4 @@ Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: doc2dict
 Requires-Dist: secsgml
-Requires-Dist: lxml
 

{datamule-1.2.9.dist-info → datamule-1.3.0.dist-info}/RECORD

@@ -1,11 +1,12 @@
 datamule/__init__.py,sha256=glzwBeGJEE6-TG7mRule9GH6L59XaIRR9T7ALcdpMus,1067
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
-datamule/helper.py,sha256=g9Kb1DWbViCoON06PjOkSX5Ucu0uG7zPwhsO2LQ6C1g,3579
+datamule/helper.py,sha256=BB4v73HDd5UWqG212UvSKGTTEW_ugohFPGKAiMboQ3s,4050
 datamule/index.py,sha256=_7Ox5hyF_7RWdblVFr5rNyv_ARwBP7VY4f703pk9qQ8,2074
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
 datamule/portfolio.py,sha256=8fiK-vfZM5-NJSvOEsDR2YDb-2njjzFk6l7BiRyrzOM,7168
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
 datamule/submission.py,sha256=Yh5nG3ioumhl6z30wJdIEmKjDDNSuo0r2xycZSIaeIg,11035
+datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/document.py,sha256=menUFoeWwiY0rJnBkQiqY4NWnO0J17-qs8jFvO_1jiY,9969
 datamule/document/processing.py,sha256=jDCEzBFDSQtq7nQxRScIsbALnFcvMPOkNkMUCa7mFxg,31921
@@ -44,7 +45,7 @@ datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNs
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/submissions/downloader.py,sha256=60wX2Yml1UCuxOtU0xMxqqeyHhrypCmlDQ0jZF-StJo,2665
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
-datamule/sec/submissions/monitor.py,sha256=s6uknn1dF1EemiI3Hl4nEq3txwK7nYl6wmayuUPYpRs,7844
+datamule/sec/submissions/monitor.py,sha256=dZYuVCi_X82eYA8l_9cbnkRjiawz3K4U-FnCAyJcgk4,7892
 datamule/sec/submissions/streamer.py,sha256=EXyWNCD9N6mZmvm9lFSCFodF19zSQ8jfIbWPZNp0K5Y,11253
 datamule/sec/submissions/textsearch.py,sha256=zEr3NXdhVFL8eMh2jruVXIt7taUZTMdNy2hOAyRM2pA,5706
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -56,7 +57,7 @@ datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
 datamule/seclibrary/downloader.py,sha256=PIgz_7ASUTZOHcUZGcD1SmLaGSbq7xe7EiJT0Z7HU4M,13653
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.2.9.dist-info/METADATA,sha256=5bMwIRcARNqP6S1cdPzoIBuu1miiUJUUdWnTXvwtPNk,490
-datamule-1.2.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.2.9.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.2.9.dist-info/RECORD,,
+datamule-1.3.0.dist-info/METADATA,sha256=ANPyXCixCioGwfTqpxX0BsbnxDe7-LS3NtLWg0WledM,469
+datamule-1.3.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.3.0.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.3.0.dist-info/RECORD,,