rclone-api 1.3.28__py2.py3-none-any.whl → 1.4.1__py2.py3-none-any.whl

rclone_api/profile/mount_copy_bytes.py

@@ -1,311 +0,0 @@
-"""
-Unit test file.
-"""
-
-import argparse
-import os
-import shutil
-import time
-from concurrent.futures import Future
-from dataclasses import dataclass
-from pathlib import Path
-
-import psutil
-from dotenv import load_dotenv
-
-from rclone_api import Config, Rclone, SizeSuffix
-from rclone_api.mount_read_chunker import MultiMountFileChunker
-from rclone_api.types import FilePart
-
-os.environ["RCLONE_API_VERBOSE"] = "1"
-
-
-@dataclass
-class Args:
-    direct_io: bool
-    num: int
-    size: SizeSuffix | None
-
-
-@dataclass
-class Credentials:
-    BUCKET_KEY_SECRET: str
-    BUCKET_KEY_PUBLIC: str
-    BUCKET_NAME: str
-    SRC_SFTP_HOST: str
-    SRC_SFTP_USER: str
-    SRC_SFTP_PORT: str
-    SRC_SFTP_PASS: str
-    BUCKET_URL: str
-
-
-def _generate_rclone_config() -> tuple[Config, Credentials]:
-
-    cwd = Path.cwd()
-    env_path = cwd / ".env"
-    assert (
-        env_path.exists()
-    ), "this test requires that the secret .env file exists with the credentials"
-    load_dotenv(env_path, verbose=True)
-    print(f"Current working directory: {cwd}")
-
-    # assert that .env exists for this test
-    assert os.path.exists(
-        ".env"
-    ), "this test requires that the secret .env file exists with the credentials"
-
-    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
-
-    # Load additional environment variables
-    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
-    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
-    BUCKET_NAME = os.getenv("BUCKET_NAME")
-    SRC_SFTP_HOST = os.getenv("SRC_SFTP_HOST")
-    SRC_SFTP_USER = os.getenv("SRC_SFTP_USER")
-    SRC_SFTP_PORT = os.getenv("SRC_SFTP_PORT")
-    SRC_SFTP_PASS = os.getenv("SRC_SFTP_PASS")
-    # BUCKET_URL = os.getenv("BUCKET_URL")
-    BUCKET_URL = "sfo3.digitaloceanspaces.com"
-
-    config_text = f"""
-[dst]
-type = s3
-provider = DigitalOcean
-access_key_id = {BUCKET_KEY_PUBLIC}
-secret_access_key = {BUCKET_KEY_SECRET}
-endpoint = {BUCKET_URL}
-bucket = {BUCKET_NAME}
-
-[src]
-type = sftp
-host = {SRC_SFTP_HOST}
-user = {SRC_SFTP_USER}
-port = {SRC_SFTP_PORT}
-pass = {SRC_SFTP_PASS}
-
-"""
-    # print("Config text:")
-    # print(config_text)
-    # _CONFIG_PATH.write_text(config_text, encoding="utf-8")
-    # print(f"Config file written to: {_CONFIG_PATH}")
-
-    creds = Credentials(
-        BUCKET_KEY_SECRET=str(BUCKET_KEY_SECRET),
-        BUCKET_KEY_PUBLIC=str(BUCKET_KEY_PUBLIC),
-        BUCKET_NAME=str(BUCKET_NAME),
-        SRC_SFTP_HOST=str(SRC_SFTP_HOST),
-        SRC_SFTP_USER=str(SRC_SFTP_USER),
-        SRC_SFTP_PORT=str(SRC_SFTP_PORT),
-        SRC_SFTP_PASS=str(SRC_SFTP_PASS),
-        BUCKET_URL=str(BUCKET_URL),
-    )
-
-    return Config(config_text), creds
-
-
-def _run_profile(
-    rclone: Rclone,
-    src_file: str,
-    transfers: int,
-    offset: SizeSuffix,
-    size: SizeSuffix,
-    log_dir: Path,
-    num: int,
-    direct_io: bool,
-) -> None:
-
-    mount_log = log_dir / f"mount_{SizeSuffix(size)}_threads_{transfers}.log"
-    print("\n\n")
-    print("#" * 80)
-    print(f"# Started test download of {SizeSuffix(size)} with {transfers} transfers")
-    print("#" * 80)
-
-    chunk_size = size
-
-    filechunker: MultiMountFileChunker = rclone.get_multi_mount_file_chunker(
-        src=src_file,
-        chunk_size=chunk_size,
-        threads=transfers,
-        direct_io=direct_io,
-        mount_log=mount_log,
-    )
-    bytes_count = 0
-
-    futures: list[Future[FilePart]] = []
-    for i in range(num):
-        offset = SizeSuffix(i * chunk_size.as_int()) + offset
-        future = filechunker.fetch(offset.as_int(), size.as_int(), "TEST OBJECT")
-        futures.append(future)
-
-    # dry run to warm up the mounts, then read a different byte range.
-    for future in futures:
-        filepart_or_err = future.result()
-        if isinstance(filepart_or_err, Exception):
-            assert False, f"Error: {filepart_or_err}"
-        filepart_or_err.dispose()
-    futures.clear()
-
-    start = time.time()
-    net_io_start = psutil.net_io_counters()
-
-    offset = SizeSuffix("1G")
-
-    for i in range(num):
-        offset = SizeSuffix(i * chunk_size.as_int()) + offset
-        future = filechunker.fetch(offset.as_int(), size.as_int(), "TEST OBJECT")
-        futures.append(future)
-
-    for future in futures:
-        bytes_or_err = future.result()
-        if isinstance(bytes_or_err, Exception):
-            assert False, f"Error: {bytes_or_err}"
-        bytes_count += bytes_or_err.n_bytes()
-
-    diff = (time.time() - start) / num
-    net_io_end = psutil.net_io_counters()
-    # self.assertEqual(len(bytes_or_err), size)
-    # assert (
-    #     bytes_count == SizeSuffix(size * num).as_int()
-    # ), f"Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}"
-
-    if bytes_count != SizeSuffix(size * num).as_int():
-        print("\n######################## ERROR ########################")
-        print(f"Error: Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}")
-        print(f"  Bytes count: {bytes_count}")
-        print(f"  Size: {SizeSuffix(size * num)}")
-        print(f"  num: {num}")
-        print("########################################################\n")
-
-    # print io stats
-    bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num
-    bytes_recv = (net_io_end.bytes_recv - net_io_start.bytes_recv) // num
-    packets_sent = (net_io_end.packets_sent - net_io_start.packets_sent) // num
-    efficiency = size.as_int() / (bytes_recv)
-    efficiency_100 = efficiency * 100
-    efficiency_str = f"{efficiency_100:.2f}"
-
-    bytes_send_suffix = SizeSuffix(bytes_sent)
-    bytes_recv_suffix = SizeSuffix(bytes_recv)
-    range_size = SizeSuffix(size)
-
-    print(f"\nFinished downloading {range_size} with {transfers} transfers")
-    print("Net IO stats:")
-    print(f"Bytes sent: {bytes_send_suffix}")
-    print(f"Bytes received: {bytes_recv_suffix}")
-    print(f"Packets sent: {packets_sent}")
-    print(f"Efficiency: {efficiency_str}%")
-    print(f"Time: {diff:.1f} seconds")
-
-
-def test_profile_copy_bytes(
-    args: Args,
-    rclone: Rclone,
-    offset: SizeSuffix,
-    transfer_list: list[int] | None,
-    mount_root_path: Path,
-    size: SizeSuffix | None,
-    num: int,
-) -> None:
-
-    if size:
-        sizes = [size.as_int()]
-    else:
-        sizes = [
-            1024 * 1024 * 1,
-            # 1024 * 1024 * 2,
-            1024 * 1024 * 4,
-            # 1024 * 1024 * 8,
-            1024 * 1024 * 16,
-            # 1024 * 1024 * 32,
-            1024 * 1024 * 64,
-            # 1024 * 1024 * 128,
-            1024 * 1024 * 256,
-        ]
-    # transfer_list = [1, 2, 4, 8, 16]
-    transfer_list = transfer_list or [1, 2, 4]
-
-    # src_file = "dst:rclone-api-unit-test/zachs_video/internaly_ai_alignment.mp4"
-    # sftp mount
-    src_file = "src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
-
-    for sz in sizes:
-        for transfers in transfer_list:
-            _run_profile(
-                rclone=rclone,
-                src_file=src_file,
-                transfers=transfers,
-                offset=offset,
-                size=SizeSuffix(sz),
-                direct_io=args.direct_io,
-                log_dir=mount_root_path,
-                num=num,
-            )
-    print("done")
-
-
-def _parse_args() -> Args:
-    parser = argparse.ArgumentParser(description="Profile copy_bytes")
-    parser.add_argument("--direct-io", help="Use direct IO", action="store_true")
-    parser.add_argument("-n", "--num", help="Number of workers", type=int, default=1)
-    parser.add_argument(
-        "--size", help="Size of the file to download", type=SizeSuffix, default=None
-    )
-    args = parser.parse_args()
-    return Args(direct_io=args.direct_io, num=args.num, size=args.size)
-
-
-_SHOW_CREDS = False
-
-
-def main() -> None:
-    """Main entry point."""
-    print("Running test_profile_copy_bytes")
-    config, creds = _generate_rclone_config()
-    if _SHOW_CREDS:
-        print("Config:")
-        print(config)
-        print("Credentials:")
-        print(creds)
-    rclone = Rclone(config)
-
-    mount_root_path = Path("rclone_logs") / "mount"
-    if mount_root_path.exists():
-        shutil.rmtree(mount_root_path)
-
-    args = _parse_args()
-    transfer_list = None
-    # parallel_workers = args.num
-
-    def task(
-        offset: SizeSuffix,
-        args=args,
-        rclone=rclone,
-        transfer_list=transfer_list,
-        mount_root_path=mount_root_path,
-    ):
-        return test_profile_copy_bytes(
-            args=args,
-            rclone=rclone,
-            offset=offset,
-            mount_root_path=mount_root_path,
-            transfer_list=transfer_list,
-            size=args.size,
-            num=args.num,
-        )
-
-    task(offset=SizeSuffix(0))
-
-    # with ThreadPoolExecutor(max_workers=parallel_workers) as _:
-    #     tasks: list[Future] = []
-    #     for i in range(parallel_workers):
-    #         offset = SizeSuffix(i * 1024 * 1024 * 256)
-    #         future = ThreadPoolExecutor().submit(lambda: task(offset=offset))
-    #         tasks.append(future)
-
-
-if __name__ == "__main__":
-    import sys
-
-    sys.argv.append("--size")
-    sys.argv.append("16MB")
-    main()

rclone_api-1.3.28.dist-info/RECORD

@@ -1,51 +0,0 @@
-rclone_api/__init__.py,sha256=gOQJgOs0_oaV_pOwlY00LXRYAHk1_MDwN59od1VpoC0,1334
-rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
-rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
-rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
-rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
-rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
-rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
-rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
-rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
-rclone_api/exec.py,sha256=Bq0gkyZ10mEY0FRyzNZgdN4FaWP9vpeCk1kjpg-gN_8,1083
-rclone_api/file.py,sha256=cz-7_nJArkVdJ9z2QaC_XZYpihXe3IPBC90Z5_3g2aw,5419
-rclone_api/file_item.py,sha256=cH-AQYsxedhNPp4c8NHY1ad4Z7St4yf_VGbmiGD59no,1770
-rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
-rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/http_server.py,sha256=SmeUDDKaMpJGDqRNkHoImHTRNkvHEtGFzu_8jYfoeZU,8113
-rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
-rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
-rclone_api/mount_read_chunker.py,sha256=7jaF1Rsjr-kXIZW--Ol1QuG7WArBgdIcpQ0AJMYn7bI,4764
-rclone_api/process.py,sha256=BGXJTZVT__jeaDyjN8_kRycliOhkBErMPdHO1hKRvJE,5271
-rclone_api/rclone.py,sha256=yRhQoCBJI-tfhxySR17a-vSEhWw5cMbk8_WbYs5WqRc,54117
-rclone_api/remote.py,sha256=jq3dPbAGvYZFW5cTviqxT2w6_jG2LLfS1RIcYSmMsQQ,503
-rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
-rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=ous70ea5oEs0gz0KLCWfimVyF0_fgrXRA8e0_HLk8nk,12301
-rclone_api/util.py,sha256=F9Q3zbWRsgPF4NG6OWB63cZ7GVq82lsraP47gmmDohU,5416
-rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
-rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
-rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
-rclone_api/cmd/copy_large_s3.py,sha256=etOMCyN2FECev7sF_Z3Ftp-2UHRE09Yhfg6AgflDHek,3573
-rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
-rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
-rclone_api/db/db.py,sha256=ZpYfeCUe8MKg_fdJucRSe6-fwGY_rWqUn7WkHCNFH_4,10074
-rclone_api/db/models.py,sha256=v7qaXUehvsDvU51uk69JI23fSIs9JFGcOa-Tv1c_wVs,1600
-rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
-rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=M1vZn-Mrga14Ik7MHGZHbnwYli41Ep6Tyll7hQc7Wmo,9071
-rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
-rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_task.py,sha256=AanVCygDoUjmMOUdEIYl-hEpPEGSJTIU_MSFGQ0tI0Q,7421
-rclone_api/s3/chunk_types.py,sha256=oSWv8No9V3BeM7IcGnowyR2a7YrszdAXzEJlxaeZcp0,8852
-rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
-rclone_api/s3/s3_multipart_uploader.py,sha256=JxUjf-Dpeq_JnMr8RBGuNCFcljGr_GM11529W53n3AM,766
-rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
-rclone_api/s3/upload_file_multipart.py,sha256=55TXp9q1It5nKkwBo49nCZNgvdMsqvIGByDMBFzo_ik,12576
-rclone_api-1.3.28.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.3.28.dist-info/METADATA,sha256=JkcK7CZWmK6d5PEF3vYvlWIOGnb7ZIXNraPEH3HQUHM,4628
-rclone_api-1.3.28.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.3.28.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.3.28.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.3.28.dist-info/RECORD,,