datamule 2.1.5__tar.gz → 2.1.6__tar.gz

{datamule-2.1.5 → datamule-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.1.5
+Version: 2.1.6
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.1.5 → datamule-2.1.6}/datamule/document/document.py

@@ -119,93 +119,92 @@ class Document:
         
         if self.extension == '.txt':
             content = self.text
-            if self.type == '10-Q':
+            if self.type in ['10-Q', '10-Q/A']:
                 mapping_dict = dict_10q
-            elif self.type == '10-K':
+            elif self.type in ['10-K','10-K/A']:
                 mapping_dict = dict_10k
-            elif self.type == '8-K':
+            elif self.type in ['8-K', '8-K/A']:
                 mapping_dict = dict_8k
-            elif self.type == 'SC 13D':
+            elif self.type in ['SC 13D', 'SC 13D/A']:
                 mapping_dict = dict_13d
-            elif self.type == 'SC 13G':
+            elif self.type in ['SC 13G', 'SC 13G/A']:
                 mapping_dict = dict_13g
             
             self._data = {}
             self._data['document'] = dict2dict(txt2dict(content=content, mapping_dict=mapping_dict))
         elif self.extension in ['.htm', '.html']:
             
-            if self.type == '1-K':
+            if self.type in ['1-K', '1-K/A']:
                 mapping_dict = dict_1kpartii_html
-            elif self.type == '1-SA':
+            elif self.type in ['1-SA', '1-SA/A']:
                 mapping_dict = dict_1sa_html
-            elif self.type == '1-U':
+            elif self.type in ['1-U', '1-U/A']:
                 mapping_dict = dict_1u_html
-            elif self.type == '10-12B':
+            elif self.type in ['10-12B', '10-12B/A']:
                 mapping_dict = dict_1012b_html
-            elif self.type == '10-D':
+            elif self.type in ['10-D', '10-D/A']:
                 mapping_dict = dict_10d_html
-            elif self.type == '10-K':
+            elif self.type in ['10-K', '10-K/A']:
                 mapping_dict = dict_10k_html
-            elif self.type == '10-Q':
+            elif self.type in ['10-Q', '10-Q/A']:
                 mapping_dict = dict_10q_html
-            elif self.type == '20-F':
+            elif self.type in ['20-F', '20-F/A']:
                 mapping_dict = dict_20f_html
-            elif self.type == '8-A12B':
+            elif self.type in ['8-A12B', '8-A12B/A']:
                 mapping_dict = dict_8a12b_html
-            elif self.type == '8-A12G':
+            elif self.type in ['8-A12G', '8-A12G/A']:
                 mapping_dict = dict_8a12g_html
-            elif self.type == '8-K':
+            elif self.type in ['8-K', '8-K/A']:
                 mapping_dict = dict_8k_html
-            elif self.type == '8-K12B':
+            elif self.type in ['8-K12B', '8-K12B/A']:
                 mapping_dict = dict_8k12b_html
-            elif self.type == '8-K12G3':
+            elif self.type in ['8-K12G3', '8-K12G3/A']:
                 mapping_dict = dict_8k12g3_html
-            elif self.type == '8-K15D5':
+            elif self.type in ['8-K15D5', '8-K15D5/A']:
                 mapping_dict = dict_8k15d5_html
-            elif self.type == 'ABS-15G':
+            elif self.type in ['ABS-15G', 'ABS-15G/A']:
                 mapping_dict = dict_abs15g_html
-            elif self.type == 'ABS-EE':
+            elif self.type in ['ABS-EE', 'ABS-EE/A']:
                 mapping_dict = dict_absee_html
-            elif self.type == 'APP NTC':
-                dict_appntc_html
-            elif self.type == 'CB':
+            elif self.type in ['APP NTC', 'APP NTC/A']:
+                mapping_dict = dict_appntc_html
+            elif self.type in ['CB', 'CB/A']:
                 mapping_dict = dict_cb_html
-            elif self.type == 'DSTRBRPT':
+            elif self.type in ['DSTRBRPT', 'DSTRBRPT/A']:
                 mapping_dict = dict_dstrbrpt_html
-            elif self.type == 'N-18F1':
+            elif self.type in ['N-18F1', 'N-18F1/A']:
                 mapping_dict = dict_n18f1_html
-            elif self.type == 'N-CSRS':
+            elif self.type in ['N-CSRS', 'N-CSRS/A']:
                 mapping_dict = dict_ncsrs_html
-            elif self.type == 'NT-10K':
+            elif self.type in ['NT-10K', 'NT-10K/A']:
                 mapping_dict = dict_nt10k_html
-            elif self.type == 'NT-10Q':
+            elif self.type in ['NT-10Q', 'NT-10Q/A']:
                 mapping_dict = dict_nt10q_html
-            elif self.type == 'NT 20-F':
+            elif self.type in ['NT 20-F', 'NT 20-F/A']:
                 mapping_dict = dict_nt20f_html
-            elif self.type == 'NT-NCEN':
+            elif self.type in ['NT-NCEN', 'NT-NCEN/A']:
                 mapping_dict = dict_ntncen_html
-            elif self.type == 'NT-NCSR':
+            elif self.type in ['NT-NCSR', 'NT-NCSR/A']:
                 mapping_dict = dict_ntncsr_html
-            elif self.type == 'NTFNCEN':
+            elif self.type in ['NTFNCEN', 'NTFNCEN/A']:
                 mapping_dict = dict_ntfcen_html
-            elif self.type == 'NTFNCSR':
+            elif self.type in ['NTFNCSR', 'NTFNCSR/A']:
                 mapping_dict = dict_ntfncsr_html
-            elif self.type == 'EX-99.CERT':
+            elif self.type in ['EX-99.CERT', 'EX-99.CERT/A']:
                 mapping_dict = dict_ex99cert_html
-            elif self.type == 'SC 13E3':
+            elif self.type in ['SC 13E3', 'SC 13E3/A']:
                 mapping_dict = dict_sc13e3_html
-            elif self.type == 'SC 14D9':
+            elif self.type in ['SC 14D9', 'SC 14D9/A']:
                 mapping_dict = dict_sc14d9_html
-            elif self.type == 'SP 15D2':
+            elif self.type in ['SP 15D2', 'SP 15D2/A']:
                 mapping_dict = dict_sp15d2_html
-
-            elif self.type == 'SD':
+            elif self.type in ['SD', 'SD/A']:
                 mapping_dict = dict_sd_html
-            elif self.type == 'S-1':
+            elif self.type in ['S-1', 'S-1/A']:
                 mapping_dict = dict_s1_html
-            elif self.type == 'T-3':
+            elif self.type in ['T-3', 'T-3/A']:
                 mapping_dict = dict_t3_html
-            elif self.type in ['NT 10-K', 'NT 10-Q','NT 20-F']:
+            elif self.type in ['NT 10-K', 'NT 10-K/A', 'NT 10-Q', 'NT 10-Q/A', 'NT 20-F', 'NT 20-F/A']:
                 mapping_dict = dict_nt10k_html
             
             dct = html2dict(content=self.content, mapping_dict=mapping_dict)
@@ -233,7 +232,7 @@ class Document:
                 self._preprocess_html_content()
             elif self.extension == '.txt':
                 self._preprocess_txt_content()
-            return self._text
+        return self._text
     
     def write_json(self, output_filename=None):
         if not self.data:

datamule-2.1.6/datamule/sheet.py

@@ -0,0 +1,306 @@
+from pathlib import Path
+import csv
+import os
+from .helper import _process_cik_and_metadata_filters, load_package_dataset
+from .sec.xbrl.downloadcompanyfacts import download_company_facts
+from .datamule.datamule_lookup import datamule_lookup
+from .datamule.datamule_mysql_rds import query_mysql_rds
+from company_fundamentals.utils import get_fundamental_mappings
+from company_fundamentals import construct_fundamentals
+class Sheet:
+    def __init__(self, path):
+        self.path = Path(path)
+
+    # Keep
+    def get_submissions(self,cik=None, accession_number=None, submission_type=None, filing_date=None, 
+                   columns=None, distinct=False, page_size=25000, quiet=False, api_key=None):
+        
+        return datamule_lookup(cik, accession_number, submission_type, filing_date, 
+                   columns, distinct, page_size, quiet, api_key)
+    
+    def get_table(self,table,cik=None,ticker=None,**kwargs):
+        cik = _process_cik_and_metadata_filters(cik, ticker)
+
+        if table == 'fundamentals':
+            fundamentals = kwargs.pop('fundamentals', None)
+            if fundamentals is None:
+                raise ValueError("fundamentals parameter required for fundamentals table")
+            
+            categories = kwargs.pop('categories',None)
+            
+            mappings = get_fundamental_mappings(fundamentals=fundamentals)
+            #print(mappings)
+            taxonomies = [item[0] for item in mappings]
+            names = [item[1] for item in mappings]
+            xbrl = query_mysql_rds(table='simple_xbrl',cik=cik,taxonomy=taxonomies,name=names,**kwargs)
+            #print(xbrl)
+
+            return construct_fundamentals(xbrl, 'taxonomy', 'name', 'period_start_date', 'period_end_date', categories=categories,fundamentals=fundamentals)
+            
+        else:
+            return query_mysql_rds(table=table,cik=cik,**kwargs)
+
+    def download_xbrl(
+        self, 
+        cik=None, 
+        ticker=None, 
+        **kwargs
+    ):
+        # If no CIK or ticker specified, get all companies with tickers
+        if cik is None and ticker is None:
+            cik = [row['cik'] for row in load_package_dataset('company_tickers')]
+            
+        # Normalize cik to list format
+        if isinstance(cik, (str, int)):
+            cik = [cik]
+            
+        # Process CIK and metadata filters
+        cik_list = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
+        
+        # Download facts for all CIKs in parallel
+        download_company_facts(cik=cik_list, output_dir=self.path)
+
+    def get_information_table(
+        self,
+        # Optional filtering parameters
+        columns=None,
+        name_of_issuer=None,
+        title_of_class=None,
+        cusip=None,
+        value=None,
+        ssh_prnamt=None,
+        ssh_prnamt_type=None,
+        investment_discretion=None,
+        voting_authority_sole=None,
+        voting_authority_shared=None,
+        voting_authority_none=None,
+        reporting_owner_cik=None,
+        put_call=None,
+        other_manager=None,
+        figi=None,
+        accession=None,
+        filing_date=None,
+        
+        # API key handling
+        api_key=None,
+        
+        # Additional options
+        print_cost=True,
+        verbose=False
+    ):
+        """
+        Query the SEC BigQuery API for 13F-HR information table data.
+        
+        Parameters:
+        -----------
+        columns : List[str], optional
+            Specific columns to return. If None, all columns are returned.
+        
+        # Filter parameters
+        name_of_issuer, title_of_class, etc. : Various filters that can be:
+            - str: Exact match
+            - List[str]: Match any in list
+            - tuple: (min, max) range for numeric/date fields
+        
+        api_key : str, optional
+            SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+        print_cost : bool
+            Whether to print the query cost information
+        verbose : bool
+            Whether to print additional information about the query
+            
+        Returns:
+        --------
+        List[Dict]
+            A list of dictionaries containing the query results
+            
+        Raises:
+        -------
+        ValueError
+            If API key is missing or invalid
+        Exception
+            For API errors or other issues
+        """
+
+        return get_information_table(
+            columns=columns,
+            name_of_issuer=name_of_issuer,
+            title_of_class=title_of_class,
+            cusip=cusip,
+            value=value,
+            ssh_prnamt=ssh_prnamt,
+            ssh_prnamt_type=ssh_prnamt_type,
+            investment_discretion=investment_discretion,
+            voting_authority_sole=voting_authority_sole,
+            voting_authority_shared=voting_authority_shared,
+            voting_authority_none=voting_authority_none,
+            reporting_owner_cik=reporting_owner_cik,
+            put_call=put_call,
+            other_manager=other_manager,
+            figi=figi,
+            accession=accession,
+            filing_date=filing_date,
+            
+            # API key handling
+            api_key=api_key,
+            
+            # Additional options
+            print_cost=print_cost,
+            verbose=verbose
+        )
+
+    def get_345(
+        self,
+        # Optional filtering parameters
+        columns=None,
+        is_derivative=None,
+        is_non_derivative=None,
+        security_title=None,
+        transaction_date=None,
+        document_type=None,
+        transaction_code=None,
+        equity_swap_involved=None,
+        transaction_timeliness=None,
+        transaction_shares=None,
+        transaction_price_per_share=None,
+        shares_owned_following_transaction=None,
+        ownership_type=None,
+        deemed_execution_date=None,
+        conversion_or_exercise_price=None,
+        exercise_date=None,
+        expiration_date=None,
+        underlying_security_title=None,
+        underlying_security_shares=None,
+        underlying_security_value=None,
+        accession=None,
+        reporting_owner_cik=None,
+        issuer_cik=None,
+        filing_date=None,
+        
+        # API key handling
+        api_key=None,
+        
+        # Additional options
+        print_cost=True,
+        verbose=False
+    ):
+        """
+        Query the SEC BigQuery API for Form 345 insider transaction data.
+        
+        Parameters:
+        -----------
+        columns : List[str], optional
+            Specific columns to return. If None, all columns are returned.
+        
+        # Filter parameters
+        is_derivative, security_title, etc. : Various filters that can be:
+            - str/bool: Exact match
+            - List[str]: Match any in list
+            - tuple: (min, max) range for numeric/date fields
+            
+        reporting_owner_cik : str or List[str]
+            CIK(s) of the reporting insider(s). This is matched against an array in BigQuery.
+            Any match within the array will return the record.
+            
+        issuer_cik : str or List[str]
+            CIK(s) of the company/companies
+        
+        api_key : str, optional
+            SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+        print_cost : bool
+            Whether to print the query cost information
+        verbose : bool
+            Whether to print additional information about the query
+            
+        Returns:
+        --------
+        List[Dict]
+            A list of dictionaries containing the query results
+            
+        Raises:
+        -------
+        ValueError
+            If API key is missing or invalid
+        Exception
+            For API errors or other issues
+        """
+
+        return get_345(
+            columns=columns,
+            is_derivative=is_derivative,
+            is_non_derivative=is_non_derivative,
+            security_title=security_title,
+            transaction_date=transaction_date,
+            document_type=document_type,
+            transaction_code=transaction_code,
+            equity_swap_involved=equity_swap_involved,
+            transaction_timeliness=transaction_timeliness,
+            transaction_shares=transaction_shares,
+            transaction_price_per_share=transaction_price_per_share,
+            shares_owned_following_transaction=shares_owned_following_transaction,
+            ownership_type=ownership_type,
+            deemed_execution_date=deemed_execution_date,
+            conversion_or_exercise_price=conversion_or_exercise_price,
+            exercise_date=exercise_date,
+            expiration_date=expiration_date,
+            underlying_security_title=underlying_security_title,
+            underlying_security_shares=underlying_security_shares,
+            underlying_security_value=underlying_security_value,
+            accession=accession,
+            reporting_owner_cik=reporting_owner_cik,
+            issuer_cik=issuer_cik,
+            filing_date=filing_date,
+            
+            # API key handling
+            api_key=api_key,
+            
+            # Additional options
+            print_cost=print_cost,
+            verbose=verbose
+        )
+
+    def _download_to_csv(self, data, filepath, verbose=False):
+        """
+        Helper method to download data to a CSV file.
+        
+        Parameters:
+        -----------
+        data : List[Dict]
+            The data to save
+        filepath : str or Path
+            Path where to save the CSV file. If relative, it will be relative to the Sheet's path.
+        verbose : bool
+            Whether to print additional information
+            
+        Returns:
+        --------
+        List[Dict]
+            The input data (for method chaining)
+        """
+        # If no data returned, nothing to save
+        if not data:
+            if verbose:
+                print("No data returned from API. No file was created.")
+            return data
+        
+        # Resolve filepath - if it's not absolute, make it relative to self.path
+        filepath_obj = Path(filepath)
+        if not filepath_obj.is_absolute():
+            filepath_obj = self.path / filepath_obj
+        
+        # Create directory if it doesn't exist
+        os.makedirs(filepath_obj.parent, exist_ok=True)
+        
+        # Get fieldnames from the first record
+        fieldnames = data[0].keys()
+        
+        # Write to CSV
+        with open(filepath_obj, 'w', newline='') as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(data)
+            
+        if verbose:
+            print(f"Saved {len(data)} records to {filepath_obj}")
+            
+    

{datamule-2.1.5 → datamule-2.1.6}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.1.5
+Version: 2.1.6
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.1.5 → datamule-2.1.6}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="2.1.5",
+    version="2.1.6",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",