datamule 1.7.0__py3-none-any.whl → 1.8.0__py3-none-any.whl

datamule/__init__.py

@@ -7,7 +7,7 @@ from .sheet import Sheet
 from .index import Index
 from .package_updater import PackageUpdater
 from .utils.format_accession import format_accession
-
+from .utils.construct_submissions_data import construct_submissions_data
 
 
 # Keep the notebook environment setup

datamule/seclibrary/datamule_lookup.py

@@ -0,0 +1,236 @@
+import os
+import asyncio
+import aiohttp
+import urllib.parse
+import ssl
+import json
+import time
+from tqdm import tqdm
+
+class DatamuleLookup:
+    def __init__(self, api_key=None):
+        self.API_BASE_URL = "https://datamule-lookup.jgfriedman99.workers.dev/"
+        self._api_key = api_key
+        self.total_cost = 0
+        self.remaining_balance = None
+        self.start_time = None
+
+    @property
+    def api_key(self):
+        return getattr(self, '_api_key', None) or os.getenv('DATAMULE_API_KEY')
+
+    @api_key.setter
+    def api_key(self, value):
+        if not value:
+            raise ValueError("API key cannot be empty")
+        self._api_key = value
+
+    async def _fetch_page(self, session, cik=None, accession_number=None, submission_type=None, 
+                         filing_date=None, columns=None, distinct=False, page=1, page_size=25000):
+        params = {
+            'api_key': self.api_key,
+            'page': page,
+            'pageSize': page_size
+        }
+        
+        # Handle CIK parameter
+        if cik:
+            if isinstance(cik, list):
+                params['cik'] = ','.join(str(x) for x in cik)
+            else:
+                params['cik'] = str(cik)
+        
+        # Handle accession number parameter
+        if accession_number:
+            if isinstance(accession_number, list):
+                params['accessionNumber'] = ','.join(str(x) for x in accession_number)
+            else:
+                params['accessionNumber'] = str(accession_number)
+        
+        # Handle submission_type parameter
+        if submission_type:
+            if isinstance(submission_type, list):
+                params['submissionType'] = ','.join(str(x) for x in submission_type)
+            else:
+                params['submissionType'] = str(submission_type)
+        
+        # Handle filing_date parameter
+        if filing_date:
+            if isinstance(filing_date, tuple):
+                params['startDate'] = str(filing_date[0])
+                params['endDate'] = str(filing_date[1])
+            else:
+                if isinstance(filing_date, list):
+                    params['filingDate'] = ','.join(str(x) for x in filing_date)
+                else:
+                    params['filingDate'] = str(filing_date)
+        
+        # Handle columns parameter
+        if columns:
+            if isinstance(columns, list):
+                params['columns'] = ','.join(columns)
+            else:
+                params['columns'] = str(columns)
+        
+        # Handle distinct parameter
+        if distinct:
+            params['distinct'] = 'true'
+
+        url = f"{self.API_BASE_URL}?{urllib.parse.urlencode(params)}"
+        
+        async with session.get(url) as response:
+            data = await response.json()
+            if not data.get('success'):
+                raise ValueError(f"API request failed: {data.get('error')}")
+            
+            # Track costs and balance
+            billing = data['metadata']['billing']
+            page_cost = billing['total_charge']
+            self.total_cost += page_cost
+            self.remaining_balance = billing['remaining_balance']
+            
+            return data['data'], data['metadata']['pagination'], page_cost
+
+    async def execute_query(self, cik=None, accession_number=None, submission_type=None, 
+                          filing_date=None, columns=None, distinct=False, page_size=25000, quiet=False):
+        if self.api_key is None:
+            raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key in constructor")
+        
+        self.start_time = time.time()
+        total_items = 0
+        pages_processed = 0
+        
+        # Display query parameters
+        query_desc = []
+        if cik:
+            query_desc.append(f"CIK={cik}")
+        if accession_number:
+            query_desc.append(f"Accession={accession_number}")
+        if submission_type:
+            query_desc.append(f"Type={submission_type}")
+        if filing_date:
+            if isinstance(filing_date, tuple):
+                query_desc.append(f"Date={filing_date[0]} to {filing_date[1]}")
+            else:
+                query_desc.append(f"Date={filing_date}")
+        if columns:
+            query_desc.append(f"Columns={columns}")
+        if distinct:
+            query_desc.append("DISTINCT=True")
+        
+        if query_desc and not quiet:
+            print(f"QUERY: {', '.join(query_desc)}")
+        
+        connector = aiohttp.TCPConnector(ssl=ssl.create_default_context())
+        async with aiohttp.ClientSession(connector=connector) as session:
+            # Initialize progress bar only if not quiet
+            if not quiet:
+                pbar = tqdm(unit="page", bar_format="{desc}: {n_fmt} {unit} [{elapsed}<{remaining}, {rate_fmt}{postfix}]")
+                pbar.set_description("Fetching data")
+            
+            current_page = 1
+            has_more = True
+            results = []
+            
+            while has_more:
+                # Fetch page
+                page_results, pagination, page_cost = await self._fetch_page(
+                    session, 
+                    cik=cik,
+                    accession_number=accession_number,
+                    submission_type=submission_type, 
+                    filing_date=filing_date,
+                    columns=columns,
+                    distinct=distinct,
+                    page=current_page,
+                    page_size=page_size
+                )
+                
+                # Accumulate results
+                results.extend(page_results)
+                
+                pages_processed += 1
+                total_items += len(page_results)
+                
+                # Update progress bar only if not quiet
+                if not quiet:
+                    pbar.set_description(f"Fetching data (page {current_page})")
+                    pbar.set_postfix_str(f"cost=${self.total_cost:.4f} | balance=${self.remaining_balance:.2f}")
+                    pbar.update(1)
+                
+                # Check if we need to fetch more pages
+                has_more = pagination.get('hasMore', False)
+                current_page += 1
+                
+                # For the first page, display record info only if not quiet
+                if pages_processed == 1 and not quiet:
+                    records_per_page = pagination.get('currentPageRecords', len(page_results))
+                    if records_per_page > 0:
+                        pbar.write(f"Retrieved {records_per_page} records (page 1) - Fetching additional pages...")
+                    else:
+                        pbar.write("No records found matching criteria")
+                        break
+            
+            if not quiet:
+                pbar.close()
+            
+            # Final summary only if not quiet
+            if not quiet:
+                elapsed_time = time.time() - self.start_time
+                print("\nQuery complete:")
+                print(f"- Retrieved {total_items} records across {pages_processed} pages")
+                print(f"- Total cost: ${self.total_cost:.4f}")
+                print(f"- Remaining balance: ${self.remaining_balance:.2f}")
+                print(f"- Time: {elapsed_time:.1f} seconds")
+            
+            return results
+
+
+def datamule_lookup(cik=None, accession_number=None, submission_type=None, filing_date=None, 
+                   columns=None, distinct=False, page_size=25000, quiet=False, api_key=None):
+    """
+    Query SEC filing data from Datamule with optional filtering
+    
+    Parameters:
+    - cik: Company CIK number(s), can be string, int, or list
+    - accession_number: Accession number(s), can be string or list
+    - submission_type: Filing type(s), can be string or list (e.g., '10-K', ['10-K', '10-Q'])
+    - filing_date: Filing date(s), can be string, list, or tuple of (start_date, end_date)
+    - columns: Column(s) to return, can be string or list. Options: 'accessionNumber', 'cik', 'filingDate', 'submissionType'
+    - distinct: Boolean, whether to return distinct results only
+    - page_size: Number of records per page (max 25000)
+    - quiet: Boolean, whether to suppress progress output and summary
+    - api_key: Optional API key (can also use DATAMULE_API_KEY environment variable)
+    
+    Returns:
+    - List of dictionaries containing the requested data (ready for pandas DataFrame)
+    """
+    # Create a DatamuleLookup instance for this request
+    dl = DatamuleLookup(api_key=api_key)
+    
+    # Format dates by removing dashes if present
+    if isinstance(filing_date, tuple):
+        filing_date = (filing_date[0].replace('-', ''), filing_date[1].replace('-', ''))
+    elif isinstance(filing_date, str):
+        filing_date = filing_date.replace('-', '')
+    elif isinstance(filing_date, list):
+        filing_date = [x.replace('-', '') for x in filing_date]
+    
+    # Set default columns if none specified
+    if columns is None:
+        columns = ['accessionNumber', 'cik', 'filingDate', 'submissionType']
+    
+    # Validate page_size
+    page_size = min(max(1, page_size), 25000)
+    
+    # Run the query and return results
+    return asyncio.run(dl.execute_query(
+        cik=cik,
+        accession_number=accession_number,
+        submission_type=submission_type,
+        filing_date=filing_date,
+        columns=columns,
+        distinct=distinct,
+        page_size=page_size,
+        quiet=quiet
+    ))

datamule/seclibrary/downloader.py

@@ -9,20 +9,24 @@ import zstandard as zstd
 import io
 import json
 import tarfile
+import logging
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from queue import Queue, Empty
+from queue import Queue
 from threading import Thread, Lock
-from .query import query
 from os import cpu_count
 from secsgml import parse_sgml_content_into_memory
 from secsgml.utils import bytes_to_str
+from .datamule_lookup import datamule_lookup
 
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
 
 
 class Downloader:
     def __init__(self, api_key=None):
-        self.BASE_URL = "https://library.datamule.xyz/original/nc/"
+        self.BASE_URL = "https://sec-library.datamule.xyz/"
         self.CHUNK_SIZE = 2 * 1024 * 1024
         self.MAX_CONCURRENT_DOWNLOADS = 100
         self.MAX_DECOMPRESSION_WORKERS = cpu_count()
@@ -66,7 +70,7 @@ class Downloader:
             with open(error_file, 'w') as f:
                 json.dump(errors, f, indent=2)
         except Exception as e:
-            print(f"Failed to log error to {error_file}: {str(e)}")
+            logger.error(f"Failed to log error to {error_file}: {str(e)}")
 
     class TarManager:
         def __init__(self, output_dir, num_tar_files, max_batch_size=1024*1024*1024):
@@ -81,7 +85,7 @@ class Downloader:
             
             for i in range(num_tar_files):
                 tar_path = os.path.join(output_dir, f'batch_{i:03d}_001.tar')
-                self.tar_files[i] = tarfile.open(tar_path, 'w')
+                self.tar_files[i] = tarfile.open(tar_path, 'a')
                 self.tar_locks[i] = Lock()
                 self.file_counters[i] = 0
                 self.tar_sizes[i] = 0
@@ -105,7 +109,7 @@ class Downloader:
 
                     self.tar_sequences[tar_index] += 1
                     new_tar_path = os.path.join(self.output_dir, f'batch_{tar_index:03d}_{self.tar_sequences[tar_index]:03d}.tar')
-                    self.tar_files[tar_index] = tarfile.open(new_tar_path, 'w')
+                    self.tar_files[tar_index] = tarfile.open(new_tar_path, 'a')
                     self.file_counters[tar_index] = 0
                     self.tar_sizes[tar_index] = 0
                 
@@ -127,7 +131,7 @@ class Downloader:
                     return True
                     
                 except Exception as e:
-                    print(f"Error writing {filename} to tar {tar_index}: {str(e)}")
+                    logger.error(f"Error writing {filename} to tar {tar_index}: {str(e)}")
                     return False
         
         def _get_document_name(self, metadata, file_num, standardize_metadata):
@@ -153,7 +157,7 @@ class Downloader:
                 try:
                     tar.close()
                 except Exception as e:
-                    print(f"Error closing tar {i}: {str(e)}")
+                    logger.error(f"Error closing tar {i}: {str(e)}")
 
     def decompress_and_parse_and_write(self, compressed_chunks, filename, keep_document_types, keep_filtered_metadata, standardize_metadata, tar_manager, output_dir):
         dctx = zstd.ZstdDecompressor()
@@ -221,17 +225,21 @@ class Downloader:
                 }
                 
                 async with session.get(url, headers=headers) as response:
+                    content_type = response.headers.get('Content-Type', '')
+                    
                     if response.status == 200:
                         async for chunk in response.content.iter_chunked(self.CHUNK_SIZE):
                             chunks.append(chunk)
 
                         loop = asyncio.get_running_loop()
-                        if filename.endswith('.zst'):
+                        if content_type == 'application/zstd':
+                            logger.debug(f"Processing {filename} as compressed (zstd)")
                             success = await loop.run_in_executor(
                                 decompression_pool,
                                 partial(self.decompress_and_parse_and_write, chunks, filename, keep_document_types, keep_filtered_metadata, standardize_metadata, tar_manager, output_dir)
                             )
                         else:
+                            logger.debug(f"Processing {filename} as uncompressed")
                             success = await loop.run_in_executor(
                                 decompression_pool,
                                 partial(self.parse_and_write_regular_file, chunks, filename, keep_document_types, keep_filtered_metadata, standardize_metadata, tar_manager, output_dir)
@@ -293,32 +301,27 @@ class Downloader:
         if self.api_key is None:
             raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key in constructor")
 
-        print("Querying SEC filings...")
-        filings = query(
-            submission_type=submission_type,
-            cik=cik,
-            filing_date=filing_date,
-            api_key=self.api_key
-        )
+        logger.debug("Querying SEC filings...")
+
+        filings = datamule_lookup(cik=cik, submission_type=submission_type, filing_date=filing_date, 
+                   columns=['accessionNumber'], distinct=True, page_size=25000, quiet=False)
 
         if accession_numbers:
             accession_numbers = [str(int(item.replace('-',''))) for item in accession_numbers]
-            filings = [filing for filing in filings if filing['accession_number'] in accession_numbers]
+            filings = [filing for filing in filings if filing['accessionNumber'] in accession_numbers]
         
         if skip_accession_numbers:
             skip_accession_numbers = [int(item.replace('-','')) for item in skip_accession_numbers]
-            filings = [filing for filing in filings if filing['accession_number'] not in skip_accession_numbers]
+            filings = [filing for filing in filings if filing['accessionNumber'] not in skip_accession_numbers]
 
-        print(f"Generating URLs for {len(filings)} filings...")
+        logger.debug(f"Generating URLs for {len(filings)} filings...")
         urls = []
         for item in filings:
-            url = f"{self.BASE_URL}{str(item['accession_number']).zfill(18)}.sgml"
-            if item['compressed'] == True or item['compressed'] == 'true' or item['compressed'] == 'True':
-                url += '.zst'
+            url = f"{self.BASE_URL}{str(item['accessionNumber']).zfill(18)}.sgml"
             urls.append(url)
         
         if not urls:
-            print("No submissions found matching the criteria")
+            logger.warning("No submissions found matching the criteria")
             return
 
         urls = list(set(urls))
@@ -328,8 +331,8 @@ class Downloader:
         asyncio.run(self.process_batch(urls, output_dir, keep_document_types=keep_document_types, keep_filtered_metadata=keep_filtered_metadata, standardize_metadata=standardize_metadata, max_batch_size=max_batch_size))
         
         elapsed_time = time.time() - start_time
-        print(f"\nProcessing completed in {elapsed_time:.2f} seconds")
-        print(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
+        logger.debug(f"Processing completed in {elapsed_time:.2f} seconds")
+        logger.debug(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
     
     def __del__(self):
         if hasattr(self, 'loop') and self.loop.is_running():
@@ -348,10 +351,10 @@ class Downloader:
         for filename in filenames:
             if not isinstance(filename, str):
                 raise ValueError(f"Invalid filename type: {type(filename)}. Expected string.")
-            if not (filename.endswith('.sgml') or filename.endswith('.sgml.zst')):
-                raise ValueError(f"Invalid filename format: {filename}. Expected .sgml or .sgml.zst extension.")
+            if not filename.endswith('.sgml'):
+                raise ValueError(f"Invalid filename format: {filename}. Expected .sgml extension.")
         
-        print(f"Generating URLs for {len(filenames)} files...")
+        logger.debug(f"Generating URLs for {len(filenames)} files...")
         urls = []
         for filename in filenames:
             url = f"{self.BASE_URL}{filename}"
@@ -360,7 +363,7 @@ class Downloader:
         seen = set()
         urls = [url for url in urls if not (url in seen or seen.add(url))]
         
-        print(f"Downloading {len(urls)} files...")
+        logger.debug(f"Downloading {len(urls)} files...")
         
         start_time = time.time()
         
@@ -374,12 +377,13 @@ class Downloader:
         ))
         
         elapsed_time = time.time() - start_time
-        print(f"\nProcessing completed in {elapsed_time:.2f} seconds")
-        print(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
+        logger.debug(f"Processing completed in {elapsed_time:.2f} seconds")
+        logger.debug(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
 
 
 def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
              skip_accession_numbers=[], max_batch_size=1024*1024*1024):
+    
     if accession_numbers:
         accession_numbers = [int(str(x).replace('-', '')) for x in accession_numbers]
     elif accession_numbers == []:

datamule/sheet.py

@@ -3,12 +3,21 @@ import csv
 import os
 from .helper import _process_cik_and_metadata_filters, load_package_dataset
 from .sec.xbrl.downloadcompanyfacts import download_company_facts
+from .seclibrary.datamule_lookup import datamule_lookup
+
+# slated for deprecation?
 from .seclibrary.bq import get_information_table, get_345, get_proxy_voting_record
 
 class Sheet:
     def __init__(self, path):
         self.path = Path(path)
 
+    def get_submissions(self,cik=None, accession_number=None, submission_type=None, filing_date=None, 
+                   columns=None, distinct=False, page_size=25000, quiet=False, api_key=None):
+        
+        return datamule_lookup(cik, accession_number, submission_type, filing_date, 
+                   columns, distinct, page_size, quiet, api_key)
+
     def download_xbrl(
         self, 
         cik=None, 

{datamule-1.7.0.dist-info → datamule-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.7.0
+Version: 1.8.0
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.7.0.dist-info → datamule-1.8.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
-datamule/__init__.py,sha256=PB4eZtwWbS3PXykgJ8n6zHBKxUvJmx6ea7-XBrONbDE,1120
+datamule/__init__.py,sha256=sY9rYx9z4LADjOLmwjL3BXssIzHs8MQM6gt9IWMS85U,1192
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
 datamule/helper.py,sha256=KqhAmTMdvATEh3I-O4xLcAcrHB9zXQERBuwzue7zyQw,3674
 datamule/index.py,sha256=Rrcna9FJV-Oh_K6O2IuUEIDmtay_7UZ4l4jgKCi7A7I,2079
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
 datamule/portfolio.py,sha256=tADqQMkFaFyjanbJ0QcaOHGdJJB254rOg29FW7a13l0,11835
 datamule/portfolio_compression_utils.py,sha256=8OPYEN5zAdV1FiTxgVN3S7cTKs99Elv74bwgoIJP4QY,12654
-datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
+datamule/sheet.py,sha256=V5iR9_LkuwTFxfHCfzgadO6qgB6qOhzWiCAED-y8ZJQ,22744
 datamule/submission.py,sha256=ooLsesZ5HkgSWyEFID4u08CobTxdo35eAUHSCB6fw2k,10332
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -38,9 +38,6 @@ datamule/document/mappings/ta.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 datamule/document/mappings/thirteenfhr.py,sha256=XpYRIMPZnGLfEE4TqBI0BPXbyuq0xf3hut1fePOF6kU,4250
 datamule/document/mappings/twentyfivense.py,sha256=lKyj0ZBhkHX9gQJMTUPrQlxYFg3k-aBnWqtoS5bujZM,905
 datamule/document/mappings/twentyfourf2nt.py,sha256=Q7RPT3JgJHjYdjMuaSyAxclt6QPT_LgCQloxp-ByDuI,4118
-datamule/document/mappings_new/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/mappings_new/mappings.py,sha256=sP94GK3-klMCTD6XFajAP9KxJ7Wq5YMMaXcHx1rQEKA,281
-datamule/document/mappings_new/ownership.py,sha256=GVtyROefvEC_X5l6kayvZv57-kHxj8bHckAru8JtFOQ,10656
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/mapping_dicts/html_mapping_dicts.py,sha256=G2PWB__FNg4VH9iFJFkflM0u-qOEtk67IWtGoqesb0k,5388
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
@@ -62,12 +59,13 @@ datamule/sec/xbrl/streamcompanyfacts.py,sha256=Qq88PqW5_j1k3Aqrl0KRmKeF54D6Wbb6H
 datamule/sec/xbrl/xbrlmonitor.py,sha256=TKFVfSyyUUfUgFQw4WxEVs4g8Nh-2C0tygNIRmTqW3Y,5848
 datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
-datamule/seclibrary/downloader.py,sha256=3jEy67oiEg8BF20KcKCx2KC0UjHzhiepdu29TOaHWXs,17564
+datamule/seclibrary/datamule_lookup.py,sha256=_opEh-DRY3ZBXFbuE2Ua_aRwoc1IsV-cPSWK0c61ofY,9465
+datamule/seclibrary/downloader.py,sha256=6cPPddjXekOwlzsyratUqzpCSbvdaNyRCGjQXUtVoJU,17930
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=aX7ZaAp3zXHLcv4TFk_rGwjb8r7yNDQDFVg4nPf60kM,5934
 datamule/utils/format_accession.py,sha256=60RtqoNqoT9zSKVb1DeOv1gncJxzPTFMNW4SNOVmC_g,476
-datamule-1.7.0.dist-info/METADATA,sha256=kxtkvpOh-1yak0DS5j2AzWe5-d_vn9K6X2hCleut3aE,524
-datamule-1.7.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.7.0.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.7.0.dist-info/RECORD,,
+datamule-1.8.0.dist-info/METADATA,sha256=QGhncvN1wEhQ3m9mrqZ2-jrb7LZ9BwJAzhCp72CEUWc,524
+datamule-1.8.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.8.0.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.8.0.dist-info/RECORD,,

datamule/document/mappings_new/mappings.py

@@ -1,13 +0,0 @@
-import ownership
-
-
-# key is document type
-# note: this assumes XML format.
-table_mappings = {
-    '3' : ownership.mappings,
-    '3/A' : ownership.mappings,
-    '4' : ownership.mappings,
-    '4/A' : ownership.mappings,
-    '5' : ownership.mappings,
-    '5/A' : ownership.mappings,
-}

datamule/document/mappings_new/ownership.py

@@ -1,174 +0,0 @@
-
-
-
-# Non-derivative transaction ownership mapping
-ownership_non_derivative_transactions_dict = {
-    'securityTitle_value': 'securityTitle',
-    'securityTitle_footnote': 'securityTitleFootnote',
-    'transactionDate_value': 'transactionDate',
-    'transactionDate_footnote': 'transactionDateFootnote',
-    'deemedExecutionDate_value': 'deemedExecutionDate',
-    'deemedExecutionDate_footnote': 'deemedExecutionDateFootnote',
-    'transactionCoding_transactionFormType': 'transactionFormType',
-    'transactionCoding_transactionCode': 'transactionCode',
-    'transactionCoding_equitySwapInvolved': 'equitySwapInvolved',
-    'transactionCoding_footnote': 'transactionCodingFootnote',
-    'transactionAmounts_transactionShares_value': 'transactionShares',
-    'transactionAmounts_transactionShares_footnote': 'transactionSharesFootnote',
-    'transactionAmounts_transactionPricePerShare_value': 'transactionPricePerShare',
-    'transactionAmounts_transactionPricePerShare_footnote': 'transactionPricePerShareFootnote',
-    'transactionAmounts_transactionAcquiredDisposedCode_value': 'transactionAcquiredDisposedCode',
-    'transactionAmounts_transactionAcquiredDisposedCode_footnote': 'transactionAcquiredDisposedCodeFootnote',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
-    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
-    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
-    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
-    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
-    'transactionTimeliness_value': 'transactionTimeliness',
-    'transactionTimeliness_footnote': 'transactionTimelinessFootnote',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote'
-}
-
-# Derivative transaction ownership mapping
-derivative_transaction_ownership_dict = {
-    'securityTitle_value': 'securityTitle',
-    'securityTitle_footnote': 'securityTitleFootnote',
-    'conversionOrExercisePrice_value': 'conversionOrExercisePrice',
-    'conversionOrExercisePrice_footnote': 'conversionOrExercisePriceFootnote',
-    'transactionDate_value': 'transactionDate',
-    'transactionDate_footnote': 'transactionDateFootnote',
-    'deemedExecutionDate_value': 'deemedExecutionDate',
-    'deemedExecutionDate_footnote': 'deemedExecutionDateFootnote',
-    'transactionCoding_transactionFormType': 'transactionFormType',
-    'transactionCoding_transactionCode': 'transactionCode',
-    'transactionCoding_equitySwapInvolved': 'equitySwapInvolved',
-    'transactionCoding_footnote': 'transactionCodingFootnote',
-    'transactionAmounts_transactionShares_value': 'transactionShares',
-    'transactionAmounts_transactionShares_footnote': 'transactionSharesFootnote',
-    'transactionAmounts_transactionPricePerShare_value': 'transactionPricePerShare',
-    'transactionAmounts_transactionPricePerShare_footnote': 'transactionPricePerShareFootnote',
-    'transactionAmounts_transactionAcquiredDisposedCode_value': 'transactionAcquiredDisposedCode',
-    'transactionAmounts_transactionTotalValue_value': 'transactionTotalValue',
-    'transactionAmounts_transactionTotalValue_footnote': 'transactionTotalValueFootnote',
-    'exerciseDate_value': 'exerciseDate',
-    'exerciseDate_footnote': 'exerciseDateFootnote',
-    'expirationDate_value': 'expirationDate',
-    'expirationDate_footnote': 'expirationDateFootnote',
-    'underlyingSecurity_underlyingSecurityTitle_value': 'underlyingSecurityTitle',
-    'underlyingSecurity_underlyingSecurityTitle_footnote': 'underlyingSecurityTitleFootnote',
-    'underlyingSecurity_underlyingSecurityShares_value': 'underlyingSecurityShares',
-    'underlyingSecurity_underlyingSecurityShares_footnote': 'underlyingSecuritySharesFootnote',
-    'underlyingSecurity_underlyingSecurityValue_value': 'underlyingSecurityValue',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
-    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
-    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
-    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
-    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
-    'transactionTimeliness_value': 'transactionTimeliness',
-    'transactionTimeliness_footnote': 'transactionTimelinessFootnote',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote',
-    'transactionAmounts_transactionAcquiredDisposedCode_footnote': 'transactionAcquiredDisposedCodeFootnote',
-    'underlyingSecurity_underlyingSecurityValue_footnote': 'underlyingSecurityValueFootnote'
-}
-
-# Non-derivative holding ownership mapping
-non_derivative_holding_ownership_dict = {
-    'securityTitle_value': 'securityTitle',
-    'securityTitle_footnote': 'securityTitleFootnote',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
-    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
-    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
-    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
-    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
-    'transactionCoding_footnote': 'transactionCodingFootnote',
-    'transactionCoding_transactionFormType': 'transactionFormType',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote'
-}
-
-# Derivative holding ownership mapping
-derivative_holding_ownership_dict = {
-    'securityTitle_value': 'securityTitle',
-    'securityTitle_footnote': 'securityTitleFootnote',
-    'conversionOrExercisePrice_value': 'conversionOrExercisePrice',
-    'conversionOrExercisePrice_footnote': 'conversionOrExercisePriceFootnote',
-    'exerciseDate_value': 'exerciseDate',
-    'exerciseDate_footnote': 'exerciseDateFootnote',
-    'expirationDate_value': 'expirationDate',
-    'expirationDate_footnote': 'expirationDateFootnote',
-    'underlyingSecurity_underlyingSecurityTitle_value': 'underlyingSecurityTitle',
-    'underlyingSecurity_underlyingSecurityTitle_footnote': 'underlyingSecurityTitleFootnote',
-    'underlyingSecurity_underlyingSecurityShares_value': 'underlyingSecurityShares',
-    'underlyingSecurity_underlyingSecurityShares_footnote': 'underlyingSecuritySharesFootnote',
-    'underlyingSecurity_underlyingSecurityValue_value': 'underlyingSecurityValue',
-    'underlyingSecurity_underlyingSecurityValue_footnote': 'underlyingSecurityValueFootnote',
-    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
-    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
-    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
-    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
-    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
-    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote',
-    'transactionCoding_transactionFormType': 'transactionFormType',
-    'transactionCoding_footnote': 'transactionCodingFootnote'
-}
-
-# Reporting owner ownership mapping
-reporting_owner_ownership_dict = {
-    'reportingOwnerAddress_rptOwnerCity': 'rptOwnerCity',
-    'reportingOwnerAddress_rptOwnerState': 'rptOwnerState',
-    'reportingOwnerAddress_rptOwnerStateDescription': 'rptOwnerStateDescription',
-    'reportingOwnerAddress_rptOwnerStreet1': 'rptOwnerStreet1',
-    'reportingOwnerAddress_rptOwnerStreet2': 'rptOwnerStreet2',
-    'reportingOwnerAddress_rptOwnerZipCode': 'rptOwnerZipCode',
-    'reportingOwnerId_rptOwnerCik': 'rptOwnerCik',
-    'reportingOwnerId_rptOwnerName': 'rptOwnerName',
-    'reportingOwnerRelationship_isDirector': 'rptOwnerIsDirector',
-    'reportingOwnerRelationship_isOfficer': 'rptOwnerIsOfficer',
-    'reportingOwnerRelationship_isTenPercentOwner': 'rptOwnerIsTenPercentOwner',
-    'reportingOwnerRelationship_isOther': 'rptOwnerIsOther',
-    'reportingOwnerRelationship_officerTitle': 'rptOwnerOfficerTitle',
-    'reportingOwnerRelationship_otherText': 'rptOwnerOtherText'
-}
-
-# Metadata ownership mapping
-metadata_ownership_dict = {
-    'periodOfReport': 'periodOfReport',
-    'issuer_issuerCik': 'issuerCik',
-    'issuer_issuerName': 'issuerName',
-    'issuer_issuerTradingSymbol': 'issuerTradingSymbol',
-    'documentType': 'documentType',
-    'remarks': 'remarks',
-    'documentDescription': 'documentDescription',
-    'footnotes': 'footnotes',
-    'notSubjectToSection16': 'notSubjectToSection16',
-    'form3HoldingsReported': 'form3HoldingsReported',
-    'form4TransactionsReported': 'form4TransactionsReported',
-    'noSecuritiesOwned': 'noSecuritiesOwned',
-    'aff10b5One': 'aff10b5One',
-    'dateOfOriginalSubmission': 'dateOfOriginalSubmission',
-    'schemaVersion': 'schemaVersion'
-}
-
-# Owner signature ownership mapping
-owner_signature_ownership_dict = {
-    'signatureName': 'signatureName',
-    'signatureDate': 'signatureDate'
-}
-
-
-mappings = {
-    'ownership_non_derivative_transactions' : ownership_non_derivative_transactions_dict,
-    'ownership_derivative_transactions' : derivative_transaction_ownership_dict,
-    'ownership_non_derivative_holdings' : non_derivative_holding_ownership_dict,
-    'ownership_derivative_holdings' : derivative_holding_ownership_dict,
-    'ownership_reporting_owner' : reporting_owner_ownership_dict,
-    'ownership_metadata' : metadata_ownership_dict,
-    'ownership_owner_signature' : owner_signature_ownership_dict
-}