datamule-hub 0.2.3__py3-none-win_amd64.whl

datamule_hub-0.2.3.dist-info/METADATA

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: datamule-hub
+Version: 0.2.3
+Summary: Access Datamule cloud
+Home-page: https://github.com/john-friedman/datamule-hub
+Author: John Friedman
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: tqdm
+Requires-Dist: aiohttp
+Requires-Dist: aioboto3
+Requires-Dist: gcloud-aio-storage
+Requires-Dist: google-auth
+Requires-Dist: google-cloud-storage
+Requires-Dist: pyarrow
+Requires-Dist: websocket-client
+Requires-Dist: zstandard
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

datamule_hub-0.2.3.dist-info/RECORD

@@ -0,0 +1,35 @@
+datamule_hub-0.2.3.dist-info/licenses/LICENSE,sha256=e4vO5__LjkwRec1EOexxayzJEymAkwH9gtWEpkF2oRY,1066
+datamulehub/__init__.py,sha256=1San9XeBEKF6Fsp8tBr2zs0VU-1W-w9mERjH8YmJHwc,368
+datamulehub/api_key.py,sha256=ku5HQhtnORtomdka6uPdij3H5INiQZHj2XdNDfYZeEE,270
+datamulehub/bin/datamule-archive-downloader.exe,sha256=qifj77g4rOtq5AS71Bv8MRV7Y7d44q6FcNUYrROGWDk,4059648
+datamulehub/object_transfer/__init__.py,sha256=wy4EgEwMDvDZWv0T4tlTfNvKxfleDsNLs2n5pWEY8nE,432
+datamulehub/object_transfer/utils.py,sha256=2nRtl9LSfkPnWHNmRWaLddEycz6S-zTFtj55z2Gc9bo,2442
+datamulehub/object_transfer/gcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamulehub/object_transfer/gcs/bucket_transfer.py,sha256=OxgCL2ccmh-CKvDf5jDefnYyiu3PcLKmv5MItIDIDVc,4003
+datamulehub/object_transfer/gcs/datasets_transfer.py,sha256=8ZpXRMrW4JC49xVCvi7po1zCez75PNAzuNxaSSFuA70,3656
+datamulehub/object_transfer/gcs/utils.py,sha256=yOQZJoOpYphoSy3YKQXtNkW-nkPoJQzM3VUzi66SGo0,664
+datamulehub/object_transfer/s3/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamulehub/object_transfer/s3/bucket_transfer.py,sha256=t79qKzew2yHGIzq1LeemOl620LBTFSSDQo2QUy-YUUs,4251
+datamulehub/object_transfer/s3/datasets_transfer.py,sha256=r6a54lyzACuqH69SPKOSJNS8FOpZme2BjkzH-HJCTAU,6737
+datamulehub/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamulehub/utils/format_accession.py,sha256=Es_Y48lOq_Z_tByDbWTifcO3-1GA4izNAA2_nFIgAy8,697
+datamulehub/v3/__init__.py,sha256=9kZ5EV1j8xnW1yW8jvh9ho3xU7vu8dt8vCt66d6SMtk,301
+datamulehub/v3/databases/__init__.py,sha256=4rIDESjUEJpUWDMRFpYyKDo2D66zJOQtYKRlq_CE0aQ,76
+datamulehub/v3/databases/athena.py,sha256=fSuzORl4pon8BCfG2IKxy40fEF8tu4rwZ91ZP2VXvWE,7247
+datamulehub/v3/databases/fastest.py,sha256=JsfGwcdTeMQgGck8mfL15-GvC3xvp5Z5Q_XB-4LV4S0,10113
+datamulehub/v3/datasets/__init__.py,sha256=Z2TaPOCpIjWYY6hv7Abi-uplUk27fMBxDcXnjg9bk_U,110
+datamulehub/v3/datasets/datasets.py,sha256=f0GylDgsT9yoKXAoIiNpes2vP4TRccVXCZrSKl2S_io,5564
+datamulehub/v3/sec_filings_archive/__init__.py,sha256=VxEAv15gqJ3LfUbeaALE7We0lU8Vwc-SuSaBSi0HTKM,111
+datamulehub/v3/sec_filings_archive/_rust.py,sha256=Wvk3abMqRnzdQxuVijyOVB4KfSNsMBkPseRxKdzMrWY,3257
+datamulehub/v3/sec_filings_archive/sgml.py,sha256=fVNgoRK6nr8xXcvXovgXx4HAxTaS4wIuLuRVJdjzGbo,9649
+datamulehub/v3/sec_filings_archive/tar.py,sha256=wkENjqweIcbjnbjRYAFv0PHTW9Cy2Om_82GQlCS3oXs,26742
+datamulehub/v3/sec_filings_archive/utils.py,sha256=6SYP-O5Kok-4BAASEjOCZwLgHAuL4m_g5VCE1lDUgMQ,4427
+datamulehub/v3/sec_filings_lookup/__init__.py,sha256=fjAgAQwr8Jjx1VV9zqx6fW0uCLlg-2wJzFYV-1qhY-Y,154
+datamulehub/v3/sec_filings_lookup/sec_filings_lookup.py,sha256=abxWSfquspOMqb0PzLajrpeGsb2CjjphdV-aGbXkrwY,9665
+datamulehub/v3/sec_filings_notifications/__init__.py,sha256=M2a6PEnwGqgr6I3j6levqC44dWKkFpLXQQtcbhspJgs,272
+datamulehub/v3/sec_filings_notifications/webhooks.py,sha256=MLhMuYc88YyQNDhMTsBI_q7njaKVn6D5Bqp0dqU9JTQ,2991
+datamulehub/v3/sec_filings_notifications/websocket.py,sha256=vHrsvWQqodc7ApVVpkzVHWfaNBZXSMdXQhF-V9AX0Ac,12176
+datamule_hub-0.2.3.dist-info/METADATA,sha256=0ppVioqPa7e7ZjUcppCsWv74SngoeIxDsFmsq-3X844,600
+datamule_hub-0.2.3.dist-info/WHEEL,sha256=Zx98gwb_dQKckJK3HEOVKnxxD52EfU_PXDG_usMJ2ng,98
+datamule_hub-0.2.3.dist-info/top_level.txt,sha256=aef5Z5BB_fOwDXmIWr9yJ1zncoNVce6MWgNwtaxrUSk,12
+datamule_hub-0.2.3.dist-info/RECORD,,

datamule_hub-0.2.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: false
+Tag: py3-none-win_amd64
+

datamule_hub-0.2.3.dist-info/licenses/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2026 John Friedman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

datamule_hub-0.2.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+datamulehub

datamulehub/__init__.py

@@ -0,0 +1,16 @@
+from . import object_transfer
+from .v3 import databases
+from .v3 import datasets
+from .v3 import sec_filings_archive
+from .v3 import sec_filings_lookup
+from .v3 import sec_filings_notifications
+
+
+__all__ = [
+    "databases",
+    "datasets",
+    "object_transfer",
+    "sec_filings_archive",
+    "sec_filings_lookup",
+    "sec_filings_notifications",
+]

datamulehub/api_key.py

@@ -0,0 +1,10 @@
+import os
+
+def get_api_key(api_key=None):
+    key = api_key or os.environ.get('DATAMULE_API_KEY')
+    if not key:
+        raise EnvironmentError("DATAMULE_API_KEY environment variable is not set.")
+    return key
+
+
+api_key = os.environ.get('DATAMULE_API_KEY')

datamulehub/object_transfer/__init__.py

@@ -0,0 +1,11 @@
+from .gcs.bucket_transfer import bucket_transfer as gcs_archive_transfer
+from .gcs.datasets_transfer import datasets_transfer as gcs_dataset_transfer
+from .s3.bucket_transfer import bucket_transfer as s3_archive_transfer
+from .s3.datasets_transfer import datasets_transfer as s3_dataset_transfer
+
+__all__ = [
+    "gcs_archive_transfer",
+    "gcs_dataset_transfer",
+    "s3_archive_transfer",
+    "s3_dataset_transfer",
+]

datamulehub/object_transfer/gcs/bucket_transfer.py

@@ -0,0 +1,80 @@
+import asyncio
+import aiohttp
+import ssl
+import json
+from urllib.parse import urlparse
+from tqdm import tqdm
+from datetime import datetime, timezone
+
+from ..utils import _generate_dates, _get_urls
+from .utils import _get_storage
+
+
+async def _transfer_file(session, storage, semaphore, url, bucket, prefix=None, retry_errors=3):
+    async with semaphore:
+        key = urlparse(url).path.split('/')[-1]
+        if prefix:
+            key = f"{prefix.rstrip('/')}/{key}"
+        last_error = None
+        for attempt in range(retry_errors + 1):
+            try:
+                async with session.get(url) as response:
+                    if response.status != 200:
+                        raise aiohttp.ClientResponseError(response.request_info, response.history, status=response.status)
+                    content = await response.read()
+                    await storage.upload(bucket, key, content,
+                        content_type=response.headers.get('Content-Type', 'application/octet-stream'),
+                        metadata={'source-url': url, 'transfer-date': datetime.now(timezone.utc).isoformat()}
+                    )
+                    return {'success': True, 'url': url, 'size_bytes': len(content)}
+            except Exception as e:
+                if attempt < retry_errors:
+                    await asyncio.sleep(2 ** attempt)
+                last_error = e
+        return {'success': False, 'url': url, 'error': str(last_error)}
+
+
+async def _transfer_urls(urls, gcs_credentials, max_workers, retry_errors, prefix=None):
+    connector = aiohttp.TCPConnector(limit=max_workers, ssl=ssl.create_default_context(), ttl_dns_cache=300)
+    async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=7200)) as session:
+        async with _get_storage(gcs_credentials, session) as storage:
+            semaphore = asyncio.Semaphore(max_workers)
+            tasks = [_transfer_file(session, storage, semaphore, url, gcs_credentials['bucket_name'], prefix, retry_errors) for url in urls]
+
+            failed, total_bytes = [], 0
+            with tqdm(total=len(urls), desc="Transferring files", unit="file") as pbar:
+                for coro in asyncio.as_completed(tasks):
+                    result = await coro
+                    if result['success']:
+                        total_bytes += result['size_bytes']
+                    else:
+                        failed.append(result)
+                    pbar.set_postfix({'Total': f"{total_bytes / (1024**3):.2f} GB"})
+                    pbar.update(1)
+
+            return failed
+
+
+def bucket_transfer(datamule_bucket, gcs_credentials, max_workers=4, errors_json_filename='errors_bucket_transfer.json',
+                retry_errors=3, force_daily=True, cik=None, submission_type=None, filing_date=None, accession_number=None, prefix=None):
+
+    if datamule_bucket not in ['filings_sgml_r2', 'sec_filings_sgml_r2']:
+        raise ValueError('Datamule S3 bucket not found.')
+
+    if accession_number is not None and any(p is not None for p in [cik, submission_type, filing_date]):
+        raise ValueError('If accession_number is provided, cik, submission_type, and filing_date must be None.')
+
+    dates = _generate_dates(filing_date) if force_daily and filing_date else [filing_date]
+
+    for date in dates:
+        if len(dates) > 1:
+            print(f"Transferring {date}")
+        urls = _get_urls(submission_type=submission_type, cik=cik, filing_date=date, accession_number=accession_number)
+        failed = asyncio.run(_transfer_urls(urls, gcs_credentials, max_workers, retry_errors, prefix=prefix))
+
+        if failed and errors_json_filename:
+            with open(errors_json_filename, 'w') as f:
+                json.dump(failed, f, indent=2)
+            print(f"Saved {len(failed)} errors to {errors_json_filename}")
+
+        print(f"Transfer complete: {len(urls) - len(failed)}/{len(urls)} files successful")

datamulehub/object_transfer/gcs/datasets_transfer.py

@@ -0,0 +1,87 @@
+import json
+import urllib
+import urllib.request
+from tqdm import tqdm
+from datetime import datetime, timezone
+from google.cloud import storage as gcs
+
+from ...api_key import get_api_key
+from ...v3.datasets import resolve_path
+
+
+def _get_dataset_url(dataset, api_key=None):
+    object_key = resolve_path(dataset)
+    body = json.dumps({"path": object_key}).encode("utf-8")
+    request = urllib.request.Request(
+        "https://api.datamule.xyz/v3/get-s3-link",
+        data=body,
+        headers={
+            "Authorization": f"Bearer {get_api_key(api_key)}",
+            "Content-Type": "application/json",
+            "User-Agent": "datamule-hub",
+        },
+        method="POST",
+    )
+    with urllib.request.urlopen(request) as response:
+        data = json.loads(response.read().decode('utf-8'))
+    if not data.get('success'):
+        raise Exception(f"API error: {data.get('error', 'Unknown error')}")
+    billing = data.get('metadata', {}).get('billing', {})
+    return data['data']['download_url'], data['data']['size_gb'], billing, data['data'].get('object_key', object_key)
+
+
+def _get_gcs_client(gcs_credentials):
+    if 'service_file' in gcs_credentials:
+        return gcs.Client.from_service_account_json(gcs_credentials['service_file'])
+    return gcs.Client()
+
+
+def _transfer_dataset(client, bucket, dataset, prefix=None, retry_errors=3):
+    download_url, size_gb, billing, object_key = _get_dataset_url(dataset)
+
+    key = object_key.strip("/") or f"{dataset}.download"
+    if prefix:
+        key = f"{prefix.rstrip('/')}/{key}"
+
+    last_error = None
+    for attempt in range(retry_errors + 1):
+        try:
+            request = urllib.request.Request(download_url, headers={'User-Agent': 'datamule-python'})
+            with urllib.request.urlopen(request) as response:
+                content_type = response.headers.get('Content-Type', 'application/octet-stream')
+                blob = bucket.blob(key)
+                blob.metadata = {
+                    'source-url': download_url,
+                    'transfer-date': datetime.now(timezone.utc).isoformat(),
+                }
+                blob.upload_from_file(response, content_type=content_type)
+                return {'success': True, 'dataset': dataset, 'size_bytes': blob.size, 'billing': billing}
+        except Exception as e:
+            if attempt < retry_errors:
+                download_url, _, _, _ = _get_dataset_url(dataset)
+            last_error = e
+
+    return {'success': False, 'dataset': dataset, 'error': str(last_error)}
+
+
+def datasets_transfer(datasets, gcs_credentials, errors_json_filename='errors_datasets_transfer.json', retry_errors=3, prefix=None):
+    client = _get_gcs_client(gcs_credentials)
+    bucket = client.bucket(gcs_credentials['bucket_name'])
+
+    failed, total_bytes = [], 0
+    with tqdm(total=len(datasets), desc="Transferring datasets", unit="dataset") as pbar:
+        for dataset in datasets:
+            result = _transfer_dataset(client, bucket, dataset, prefix, retry_errors)
+            if result['success']:
+                total_bytes += result.get('size_bytes', 0)
+            else:
+                failed.append(result)
+            pbar.set_postfix({'Total': f"{total_bytes / (1024**3):.2f} GB"})
+            pbar.update(1)
+
+    if failed and errors_json_filename:
+        with open(errors_json_filename, 'w') as f:
+            json.dump(failed, f, indent=2)
+        print(f"Saved {len(failed)} errors to {errors_json_filename}")
+
+    print(f"Transfer complete: {len(datasets) - len(failed)}/{len(datasets)} datasets successful")

datamulehub/object_transfer/gcs/utils.py

@@ -0,0 +1,18 @@
+import os
+import platform
+
+def set_adc_credentials():
+    if platform.system() == 'Windows':
+        path = os.path.join(os.environ['APPDATA'], 'gcloud', 'application_default_credentials.json')
+    else:
+        path = os.path.expanduser('~/.config/gcloud/application_default_credentials.json')
+    os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = path
+
+
+def _get_storage(gcs_credentials, session):
+    from gcloud.aio.storage import Storage
+    if 'service_file' in gcs_credentials:
+        return Storage(service_file=gcs_credentials['service_file'], session=session)
+    else:
+        set_adc_credentials()
+        return Storage(session=session)

datamulehub/object_transfer/s3/bucket_transfer.py

@@ -0,0 +1,85 @@
+import asyncio
+import aiohttp
+import aioboto3
+import ssl
+import json
+from urllib.parse import urlparse
+from tqdm import tqdm
+from datetime import datetime, timezone
+
+from ..utils import _generate_dates, _get_urls
+
+async def _transfer_file(session, s3_client, semaphore, url, bucket, prefix=None, retry_errors=3):
+    async with semaphore:
+        key = urlparse(url).path.split('/')[-1]
+        if prefix:
+            key = f"{prefix.rstrip('/')}/{key}"
+        last_error = None
+        for attempt in range(retry_errors + 1):
+            try:
+                async with session.get(url) as response:
+                    if response.status != 200:
+                        raise aiohttp.ClientResponseError(response.request_info, response.history, status=response.status)
+                    content = await response.read()
+                    await s3_client.put_object(
+                        Bucket=bucket, Key=key, Body=content,
+                        ContentType=response.headers.get('Content-Type', 'application/octet-stream'),
+                        Metadata={'source-url': url, 'transfer-date': datetime.now(timezone.utc).isoformat()}
+                    )
+                    return {'success': True, 'url': url, 'size_bytes': len(content)}
+            except Exception as e:
+                if attempt < retry_errors:
+                    await asyncio.sleep(2 ** attempt)
+                last_error = e
+        return {'success': False, 'url': url, 'error': str(last_error)}
+
+
+async def _transfer_urls(urls, s3_credentials, max_workers, retry_errors, prefix=None):
+    connector = aiohttp.TCPConnector(limit=max_workers, ssl=ssl.create_default_context(), ttl_dns_cache=300)
+    async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=7200)) as session:
+        async with aioboto3.Session().client(
+            's3',
+            aws_access_key_id=s3_credentials['aws_access_key_id'],
+            aws_secret_access_key=s3_credentials['aws_secret_access_key'],
+            region_name=s3_credentials['region_name']
+        ) as s3_client:
+            semaphore = asyncio.Semaphore(max_workers)
+            tasks = [_transfer_file(session, s3_client, semaphore, url, s3_credentials['bucket_name'], prefix, retry_errors) for url in urls]
+
+            failed, total_bytes = [], 0
+            with tqdm(total=len(urls), desc="Transferring files", unit="file") as pbar:
+                for coro in asyncio.as_completed(tasks):
+                    result = await coro
+                    if result['success']:
+                        total_bytes += result['size_bytes']
+                    else:
+                        failed.append(result)
+                    pbar.set_postfix({'Total': f"{total_bytes / (1024**3):.2f} GB"})
+                    pbar.update(1)
+
+            return failed
+
+
+def bucket_transfer(datamule_bucket, s3_credentials, max_workers=4, errors_json_filename='errors_bucket_transfer.json',
+                retry_errors=3, force_daily=True, cik=None, submission_type=None, filing_date=None, accession_number=None, prefix=None):
+
+    if datamule_bucket not in ['filings_sgml_r2', 'sec_filings_sgml_r2']:
+        raise ValueError('Datamule S3 bucket not found.')
+
+    if accession_number is not None and any(p is not None for p in [cik, submission_type, filing_date]):
+        raise ValueError('If accession_number is provided, cik, submission_type, and filing_date must be None.')
+
+    dates = _generate_dates(filing_date) if force_daily and filing_date else [filing_date]
+
+    for date in dates:
+        if len(dates) > 1:
+            print(f"Transferring {date}")
+        urls = _get_urls(submission_type=submission_type, cik=cik, filing_date=date, accession_number=accession_number)
+        failed = asyncio.run(_transfer_urls(urls, s3_credentials, max_workers, retry_errors, prefix=prefix))
+
+        if failed and errors_json_filename:
+            with open(errors_json_filename, 'w') as f:
+                json.dump(failed, f, indent=2)
+            print(f"Saved {len(failed)} errors to {errors_json_filename}")
+
+        print(f"Transfer complete: {len(urls) - len(failed)}/{len(urls)} files successful")

datamulehub/object_transfer/s3/datasets_transfer.py

@@ -0,0 +1,142 @@
+import asyncio
+import aiohttp
+import aioboto3
+import ssl
+import json
+import urllib
+from tqdm import tqdm
+from datetime import datetime, timezone
+
+from ...api_key import get_api_key
+from ...v3.datasets import resolve_path
+
+
+async def _get_dataset_url(session, dataset, api_key=None):
+    object_key = resolve_path(dataset)
+    async with session.post(
+        "https://api.datamule.xyz/v3/get-s3-link",
+        json={"path": object_key},
+        headers={
+            "Authorization": f"Bearer {get_api_key(api_key)}",
+            "User-Agent": "datamule-hub",
+        },
+    ) as response:
+        data = await response.json()
+    if not data.get('success'):
+        raise Exception(f"API error: {data.get('error', 'Unknown error')}")
+    billing = data.get('metadata', {}).get('billing', {})
+    return data['data']['download_url'], data['data']['size_gb'], billing, data['data'].get('object_key', object_key)
+
+
+async def _transfer_dataset(session, s3_client, semaphore, dataset, bucket, prefix=None, retry_errors=3, multipart_threshold_mb=100, chunk_size_mb=8):
+    download_url, size_gb, billing, object_key = await _get_dataset_url(session, dataset)
+
+    key = object_key.strip("/") or f"{dataset}.download"
+    if prefix:
+        key = f"{prefix.rstrip('/')}/{key}"
+
+    use_multipart = size_gb * 1024 > multipart_threshold_mb
+    chunk_size = chunk_size_mb * 1024 * 1024
+
+    async with semaphore:
+        last_error = None
+        for attempt in range(retry_errors + 1):
+            mpu_id = None
+            try:
+                async with session.get(download_url) as response:
+                    if response.status != 200:
+                        raise aiohttp.ClientResponseError(response.request_info, response.history, status=response.status)
+
+                    if use_multipart:
+                        mpu = await s3_client.create_multipart_upload(
+                            Bucket=bucket, Key=key,
+                            ContentType=response.headers.get('Content-Type', 'application/octet-stream'),
+                            Metadata={'source-url': download_url, 'transfer-date': datetime.now(timezone.utc).isoformat()}
+                        )
+                        mpu_id = mpu['UploadId']
+                        parts = []
+                        part_number = 1
+                        buffer = b""
+
+                        async for chunk in response.content.iter_chunked(chunk_size):
+                            buffer += chunk
+                            if len(buffer) >= chunk_size:
+                                part = await s3_client.upload_part(
+                                    Bucket=bucket, Key=key,
+                                    PartNumber=part_number, UploadId=mpu_id, Body=buffer
+                                )
+                                parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
+                                part_number += 1
+                                buffer = b""
+
+                        if buffer:
+                            part = await s3_client.upload_part(
+                                Bucket=bucket, Key=key,
+                                PartNumber=part_number, UploadId=mpu_id, Body=buffer
+                            )
+                            parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
+
+                        await s3_client.complete_multipart_upload(
+                            Bucket=bucket, Key=key,
+                            UploadId=mpu_id, MultipartUpload={'Parts': parts}
+                        )
+
+                    else:
+                        content = await response.read()
+                        await s3_client.put_object(
+                            Bucket=bucket, Key=key, Body=content,
+                            ContentType=response.headers.get('Content-Type', 'application/octet-stream'),
+                            Metadata={'source-url': download_url, 'transfer-date': datetime.now(timezone.utc).isoformat()}
+                        )
+
+                    return {'success': True, 'dataset': dataset, 'size_bytes': int(size_gb * 1024**3), 'billing': billing}
+
+            except Exception as e:
+                if mpu_id:
+                    try:
+                        await s3_client.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=mpu_id)
+                    except Exception:
+                        pass
+                if attempt < retry_errors:
+                    download_url, _, _, _ = await _get_dataset_url(session, dataset)
+                    await asyncio.sleep(2 ** attempt)
+                last_error = e
+
+        return {'success': False, 'dataset': dataset, 'error': str(last_error)}
+
+
+async def _transfer_datasets(datasets, s3_credentials, max_workers, retry_errors, prefix=None):
+    connector = aiohttp.TCPConnector(limit=max_workers, ssl=ssl.create_default_context())
+    async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=7200)) as session:
+        async with aioboto3.Session().client(
+            's3',
+            aws_access_key_id=s3_credentials['aws_access_key_id'],
+            aws_secret_access_key=s3_credentials['aws_secret_access_key'],
+            region_name=s3_credentials['region_name']
+        ) as s3_client:
+            semaphore = asyncio.Semaphore(max_workers)
+            tasks = [_transfer_dataset(session, s3_client, semaphore, d, s3_credentials['bucket_name'], prefix, retry_errors) for d in datasets]
+
+            failed, total_bytes = [], 0
+            with tqdm(total=len(datasets), desc="Transferring datasets", unit="dataset") as pbar:
+                for coro in asyncio.as_completed(tasks):
+                    result = await coro
+                    if result['success']:
+                        total_bytes += result['size_bytes']
+                    else:
+                        failed.append(result)
+                    pbar.set_postfix({'Total': f"{total_bytes / (1024**3):.2f} GB"})
+                    pbar.update(1)
+
+            return failed
+
+
+def datasets_transfer(datasets, s3_credentials, max_workers=4, errors_json_filename='errors_datasets_transfer.json', retry_errors=3, prefix=None):
+    failed = asyncio.run(_transfer_datasets(datasets, s3_credentials, max_workers, retry_errors, prefix=prefix))
+
+    if failed and errors_json_filename:
+        with open(errors_json_filename, 'w') as f:
+            json.dump(failed, f, indent=2)
+        print(f"Saved {len(failed)} errors to {errors_json_filename}")
+
+    print(f"Transfer complete: {len(datasets) - len(failed)}/{len(datasets)} datasets successful")

datamulehub/object_transfer/utils.py

@@ -0,0 +1,63 @@
+from datetime import datetime, timedelta
+from ..v3.databases import read_query
+from ..utils.format_accession import format_accession
+
+def _generate_dates(filing_date):
+    if isinstance(filing_date, str):
+        return [filing_date]
+    elif isinstance(filing_date, list):
+        return filing_date
+    elif isinstance(filing_date, tuple):
+        start = datetime.strptime(filing_date[0], '%Y-%m-%d')
+        end = datetime.strptime(filing_date[1], '%Y-%m-%d')
+        dates = []
+        current = start
+        while current <= end:
+            dates.append(current.strftime('%Y-%m-%d'))
+            current += timedelta(days=1)
+        return dates
+    raise ValueError('filing_date must be a string, list, or (start, end) tuple')
+
+def _sql_literal(value):
+    return "'" + str(value).replace("'", "''") + "'"
+
+def _normalize_accession(value):
+    return str(value).replace("-", "")
+
+def _sql_value(value, quote=True):
+    return _sql_literal(value) if quote else str(value)
+
+def _sql_filter(column, value, transform=None, quote=True):
+    if value is None:
+        return None
+
+    if isinstance(value, tuple):
+        start = transform(value[0]) if transform else value[0]
+        end = transform(value[1]) if transform else value[1]
+        return f"{column} BETWEEN {_sql_value(start, quote)} AND {_sql_value(end, quote)}"
+
+    if isinstance(value, (list, set)):
+        values = [transform(item) if transform else item for item in value]
+        return f"{column} IN ({', '.join(_sql_value(item, quote) for item in values)})"
+
+    value = transform(value) if transform else value
+    return f"{column} = {_sql_value(value, quote)}"
+
+def _get_urls(submission_type=None, cik=None, filing_date=None, accession_number=None):
+    filters = [
+        _sql_filter("cik", cik, quote=False),
+        _sql_filter("form", submission_type),
+        _sql_filter("filingdate", filing_date),
+        _sql_filter("accessionnumber", accession_number, _normalize_accession, quote=False),
+    ]
+    where = " AND ".join(item for item in filters if item)
+    if where:
+        where = f"WHERE {where}"
+
+    rows = read_query(f"""
+        SELECT accessionnumber
+        FROM submissions_metadata
+        {where}
+        ORDER BY filingdate, accessionnumber
+    """)
+    return [f"https://sec-library.datamule.xyz/{format_accession(row['accessionnumber'], 'no-dash')}.sgml" for row in rows]

datamulehub/utils/format_accession.py

@@ -0,0 +1,20 @@
+def format_accession(accession, format):
+    if format == 'int':
+        accession = int(str(accession).replace('-',''))
+    elif format == 'dash':
+        accession = str(int(str(accession).replace('-',''))).zfill(18)
+        accession = f"{accession[:10]}-{accession[10:12]}-{accession[12:]}"
+    elif format == 'no-dash':
+        accession = str(int(str(accession).replace('-',''))).zfill(18)
+    else:
+        raise ValueError("unrecognized format")
+    return accession
+
+def detect_accession_type(accession):
+    accession = str(accession)
+    if '-' in accession:
+        return 'dash'
+    elif len(accession) == 18:
+        return 'no-dash'
+    else:
+        return 'int'

datamulehub/v3/__init__.py

@@ -0,0 +1,13 @@
+from . import databases
+from . import datasets
+from . import sec_filings_archive
+from . import sec_filings_lookup
+from . import sec_filings_notifications
+
+__all__ = [
+    "databases",
+    "datasets",
+    "sec_filings_archive",
+    "sec_filings_lookup",
+    "sec_filings_notifications",
+]

datamulehub/v3/databases/__init__.py

@@ -0,0 +1,3 @@
+from .athena import query, read_query
+
+__all__ = ["query", "read_query"]