s3duct 0.3.0__tar.gz

s3duct-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest tests/ -v --tb=short

s3duct-0.3.0/.github/workflows/integration.yml

@@ -0,0 +1,373 @@
+name: Cloud Integration Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      simple:
+        description: "Basic S3 roundtrip (upload, download, verify)"
+        type: boolean
+        default: false
+      backends:
+        description: "All S3-compatible backends (R2, B2, MinIO)"
+        type: boolean
+        default: false
+      huge_file:
+        description: "Large file stress test (multi-GB)"
+        type: boolean
+        default: false
+      glacier:
+        description: "Glacier lifecycle (upload, thaw, restore)"
+        type: boolean
+        default: false
+      compat:
+        description: "Backward compatibility (restore old manifests)"
+        type: boolean
+        default: false
+
+env:
+  S3DUCT_TEST_PREFIX: "ci-${{ github.run_id }}"
+
+jobs:
+  # --------------------------------------------------------------------------
+  # Simple: basic upload/download/verify on AWS S3
+  # --------------------------------------------------------------------------
+  simple:
+    if: ${{ inputs.simple }}
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      S3DUCT_TEST_BUCKET: ${{ secrets.S3DUCT_TEST_BUCKET }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Run roundtrip
+        run: bash tests/integration/roundtrip.sh
+
+      - name: Cleanup
+        if: always()
+        run: |
+          aws s3 rm "s3://${S3DUCT_TEST_BUCKET}/${S3DUCT_TEST_PREFIX}-" --recursive 2>/dev/null || true
+
+  # --------------------------------------------------------------------------
+  # Backends: test all S3-compatible services
+  # --------------------------------------------------------------------------
+  backends-minio:
+    if: ${{ inputs.backends }}
+    runs-on: ubuntu-latest
+    services:
+      minio:
+        image: minio/minio
+        ports:
+          - 9000:9000
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+        options: >-
+          --health-cmd "mc ready local || exit 1"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: |
+          pip install -e ".[dev]" awscli
+          for i in $(seq 1 30); do
+            aws --endpoint-url http://localhost:9000 s3 ls 2>/dev/null && break
+            sleep 2
+          done
+          aws --endpoint-url http://localhost:9000 s3 mb s3://s3duct-test
+        env:
+          AWS_ACCESS_KEY_ID: minioadmin
+          AWS_SECRET_ACCESS_KEY: minioadmin
+      - name: Run roundtrip
+        env:
+          AWS_ACCESS_KEY_ID: minioadmin
+          AWS_SECRET_ACCESS_KEY: minioadmin
+          S3DUCT_TEST_BUCKET: s3duct-test
+          S3DUCT_ENDPOINT_URL: http://localhost:9000
+        run: bash tests/integration/roundtrip.sh
+
+  backends-r2:
+    if: ${{ inputs.backends }}
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      S3DUCT_TEST_BUCKET: ${{ secrets.R2_TEST_BUCKET }}
+      S3DUCT_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: pip install -e ".[dev]"
+      - name: Run roundtrip
+        if: env.AWS_ACCESS_KEY_ID != ''
+        run: bash tests/integration/roundtrip.sh
+      - name: Skip (no secrets)
+        if: env.AWS_ACCESS_KEY_ID == ''
+        run: echo "R2 secrets not configured, skipping"
+      - name: Cleanup
+        if: always() && env.AWS_ACCESS_KEY_ID != ''
+        run: |
+          pip install awscli
+          aws --endpoint-url "${S3DUCT_ENDPOINT_URL}" s3 rm "s3://${S3DUCT_TEST_BUCKET}/${S3DUCT_TEST_PREFIX}-" --recursive 2>/dev/null || true
+
+  backends-b2:
+    if: ${{ inputs.backends }}
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.B2_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.B2_SECRET_ACCESS_KEY }}
+      S3DUCT_TEST_BUCKET: ${{ secrets.B2_TEST_BUCKET }}
+      S3DUCT_ENDPOINT_URL: ${{ secrets.B2_ENDPOINT_URL }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: pip install -e ".[dev]"
+      - name: Run roundtrip
+        if: env.AWS_ACCESS_KEY_ID != ''
+        run: bash tests/integration/roundtrip.sh
+      - name: Skip (no secrets)
+        if: env.AWS_ACCESS_KEY_ID == ''
+        run: echo "B2 secrets not configured, skipping"
+      - name: Cleanup
+        if: always() && env.AWS_ACCESS_KEY_ID != ''
+        run: |
+          pip install awscli
+          aws --endpoint-url "${S3DUCT_ENDPOINT_URL}" s3 rm "s3://${S3DUCT_TEST_BUCKET}/${S3DUCT_TEST_PREFIX}-" --recursive 2>/dev/null || true
+
+  # --------------------------------------------------------------------------
+  # Huge file: multi-GB stress test
+  # --------------------------------------------------------------------------
+  huge-file:
+    if: ${{ inputs.huge_file }}
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      S3DUCT_TEST_BUCKET: ${{ secrets.S3DUCT_TEST_BUCKET }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Generate 2GB test file
+        run: dd if=/dev/urandom of=/tmp/huge-test.bin bs=1M count=2048
+
+      - name: Compute expected hash
+        run: sha256sum /tmp/huge-test.bin | cut -d' ' -f1 > /tmp/expected.sha256
+
+      - name: Upload with backpressure
+        run: |
+          cat /tmp/huge-test.bin | s3duct put \
+            --bucket "${S3DUCT_TEST_BUCKET}" \
+            --name "${S3DUCT_TEST_PREFIX}-huge" \
+            --chunk-size 256M \
+            --diskspace-limit 1G \
+            --tag test=huge-file \
+            --no-encrypt
+
+      - name: Download and verify
+        run: |
+          s3duct get \
+            --bucket "${S3DUCT_TEST_BUCKET}" \
+            --name "${S3DUCT_TEST_PREFIX}-huge" \
+            > /tmp/restored.bin
+          ACTUAL=$(sha256sum /tmp/restored.bin | cut -d' ' -f1)
+          EXPECTED=$(cat /tmp/expected.sha256)
+          if [ "$ACTUAL" != "$EXPECTED" ]; then
+            echo "HASH MISMATCH: expected=$EXPECTED actual=$ACTUAL"
+            exit 1
+          fi
+          echo "Huge file roundtrip OK: $ACTUAL"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          aws s3 rm "s3://${S3DUCT_TEST_BUCKET}/${S3DUCT_TEST_PREFIX}-huge/" --recursive 2>/dev/null || true
+          rm -f /tmp/huge-test.bin /tmp/restored.bin /tmp/expected.sha256
+
+  # --------------------------------------------------------------------------
+  # Glacier: upload to GLACIER, thaw, restore
+  # --------------------------------------------------------------------------
+  glacier:
+    if: ${{ inputs.glacier }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 360  # thaw can take hours
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      S3DUCT_TEST_BUCKET: ${{ secrets.S3DUCT_TEST_BUCKET }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Upload to GLACIER
+        run: |
+          dd if=/dev/urandom bs=1K count=64 | s3duct put \
+            --bucket "${S3DUCT_TEST_BUCKET}" \
+            --name "${S3DUCT_TEST_PREFIX}-glacier" \
+            --chunk-size 32K \
+            --storage-class GLACIER \
+            --tag test=glacier \
+            --no-encrypt
+
+      - name: Initiate restore (Expedited tier)
+        run: |
+          # Expedited tier: 1-5 minutes for GLACIER (not available for DEEP_ARCHIVE)
+          # This uses aws cli directly since s3duct thaw isn't implemented yet
+          MANIFEST_KEY="${S3DUCT_TEST_PREFIX}-glacier/.manifest.json"
+          aws s3 cp "s3://${S3DUCT_TEST_BUCKET}/${MANIFEST_KEY}" /tmp/manifest.json
+          # Extract chunk keys from manifest and request restore for each
+          python3 -c "
+          import json, subprocess, sys, os
+          m = json.load(open('/tmp/manifest.json'))
+          bucket = os.environ['S3DUCT_TEST_BUCKET']
+          for c in m['chunks']:
+              key = c['s3_key']
+              print(f'Restoring {key}...')
+              subprocess.run([
+                  'aws', 's3api', 'restore-object',
+                  '--bucket', bucket,
+                  '--key', key,
+                  '--restore-request', '{\"Days\":1,\"GlacierJobParameters\":{\"Tier\":\"Expedited\"}}'
+              ], check=True)
+          "
+
+      - name: Wait for thaw (poll every 60s, up to 30 min)
+        run: |
+          python3 -c "
+          import json, subprocess, time, os
+          m = json.load(open('/tmp/manifest.json'))
+          bucket = os.environ['S3DUCT_TEST_BUCKET']
+          keys = [c['s3_key'] for c in m['chunks']]
+          for attempt in range(30):
+              all_ready = True
+              for key in keys:
+                  result = subprocess.run(
+                      ['aws', 's3api', 'head-object', '--bucket', bucket, '--key', key],
+                      capture_output=True, text=True
+                  )
+                  output = json.loads(result.stdout) if result.stdout else {}
+                  restore = output.get('Restore', '')
+                  if 'ongoing-request=\"true\"' in restore:
+                      all_ready = False
+                      break
+              if all_ready:
+                  print(f'All chunks thawed after {attempt + 1} poll(s)')
+                  break
+              print(f'Poll {attempt + 1}/30: still thawing...')
+              time.sleep(60)
+          else:
+              print('Timed out waiting for thaw')
+              exit(1)
+          "
+
+      - name: Download and verify
+        run: |
+          s3duct verify \
+            --bucket "${S3DUCT_TEST_BUCKET}" \
+            --name "${S3DUCT_TEST_PREFIX}-glacier"
+          echo "Glacier roundtrip verified"
+
+      - name: Cleanup
+        if: always()
+        run: |
+          aws s3 rm "s3://${S3DUCT_TEST_BUCKET}/${S3DUCT_TEST_PREFIX}-glacier/" --recursive 2>/dev/null || true
+
+  # --------------------------------------------------------------------------
+  # Compat: backward compatibility with older manifest versions
+  # --------------------------------------------------------------------------
+  compat:
+    if: ${{ inputs.compat }}
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      S3DUCT_TEST_BUCKET: ${{ secrets.S3DUCT_TEST_BUCKET }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Test backward compat fixtures
+        run: |
+          for VERSION_DIR in tests/fixtures/compat/v*/; do
+            VERSION=$(basename "$VERSION_DIR")
+            echo "Testing compat with $VERSION..."
+
+            STREAM_NAME="${S3DUCT_TEST_PREFIX}-compat-${VERSION}"
+            MANIFEST="${VERSION_DIR}/manifest.json"
+
+            if [ ! -f "$MANIFEST" ]; then
+              echo "  No manifest.json in $VERSION_DIR, skipping"
+              continue
+            fi
+
+            # Rewrite manifest s3_keys to use our CI stream name, then upload
+            python3 -c "
+          import json, sys
+          stream = '${STREAM_NAME}'
+          m = json.load(open('${MANIFEST}'))
+          m['name'] = stream
+          for c in m['chunks']:
+              fname = c['s3_key'].rsplit('/', 1)[-1]
+              c['s3_key'] = f'{stream}/{fname}'
+          json.dump(m, sys.stdout)
+          " > /tmp/compat-manifest.json
+            aws s3 cp /tmp/compat-manifest.json \
+              "s3://${S3DUCT_TEST_BUCKET}/${STREAM_NAME}/.manifest.json"
+
+            # Upload chunks
+            for CHUNK in "${VERSION_DIR}"/chunks/*; do
+              [ -f "$CHUNK" ] || continue
+              CHUNK_NAME=$(basename "$CHUNK")
+              aws s3 cp "$CHUNK" \
+                "s3://${S3DUCT_TEST_BUCKET}/${STREAM_NAME}/${CHUNK_NAME}"
+            done
+
+            # Download with current tool and verify
+            EXPECTED_HASH=$(cat "${VERSION_DIR}/expected_sha256")
+            s3duct get \
+              --bucket "${S3DUCT_TEST_BUCKET}" \
+              --name "${STREAM_NAME}" \
+              > /tmp/compat-restored.bin
+
+            ACTUAL_HASH=$(sha256sum /tmp/compat-restored.bin | cut -d' ' -f1)
+            if [ "$ACTUAL_HASH" != "$EXPECTED_HASH" ]; then
+              echo "  COMPAT FAIL ($VERSION): expected=$EXPECTED_HASH actual=$ACTUAL_HASH"
+              exit 1
+            fi
+            echo "  $VERSION OK"
+            rm -f /tmp/compat-restored.bin /tmp/compat-manifest.json
+          done
+
+      - name: Cleanup
+        if: always()
+        run: |
+          aws s3 rm "s3://${S3DUCT_TEST_BUCKET}/${S3DUCT_TEST_PREFIX}-compat-" --recursive 2>/dev/null || true

s3duct-0.3.0/.github/workflows/minio.yml

@@ -0,0 +1,66 @@
+name: MinIO Integration
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  minio-roundtrip:
+    runs-on: ubuntu-latest
+    services:
+      minio:
+        image: minio/minio
+        ports:
+          - 9000:9000
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+        options: >-
+          --health-cmd "mc ready local || exit 1"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+        # minio server needs to be started via command
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install s3duct
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Wait for MinIO and create test bucket
+        run: |
+          pip install awscli
+          export AWS_ACCESS_KEY_ID=minioadmin
+          export AWS_SECRET_ACCESS_KEY=minioadmin
+          # Wait for MinIO to be ready
+          for i in $(seq 1 30); do
+            aws --endpoint-url http://localhost:9000 s3 ls 2>/dev/null && break
+            echo "Waiting for MinIO... ($i)"
+            sleep 2
+          done
+          aws --endpoint-url http://localhost:9000 s3 mb s3://s3duct-test
+
+      - name: Run roundtrip test
+        env:
+          AWS_ACCESS_KEY_ID: minioadmin
+          AWS_SECRET_ACCESS_KEY: minioadmin
+          S3DUCT_TEST_BUCKET: s3duct-test
+          S3DUCT_ENDPOINT_URL: http://localhost:9000
+        run: bash tests/integration/roundtrip.sh
+
+      - name: Cleanup
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: minioadmin
+          AWS_SECRET_ACCESS_KEY: minioadmin
+        run: |
+          aws --endpoint-url http://localhost:9000 s3 rb s3://s3duct-test --force 2>/dev/null || true

s3duct-0.3.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+*.egg-info/
+__pycache__/
+*.pyc
+.pytest_cache/
+dist/
+build/

s3duct-0.3.0/CLA.md

@@ -0,0 +1,64 @@
+# Contributor License Agreement
+
+By submitting a contribution (including but not limited to code, documentation,
+or other materials) to this project, you agree to the following terms:
+
+## 1. Grant of Copyright License
+
+You grant to the project maintainers and to recipients of software distributed
+by the project maintainers a perpetual, worldwide, non-exclusive, no-charge,
+royalty-free, irrevocable copyright license to reproduce, prepare derivative
+works of, publicly display, publicly perform, sublicense, and distribute your
+contributions and such derivative works.
+
+## 2. Grant of Patent License
+
+You grant to the project maintainers and to recipients of software distributed
+by the project maintainers a perpetual, worldwide, non-exclusive, no-charge,
+royalty-free, irrevocable patent license to make, have made, use, offer to sell,
+sell, import, and otherwise transfer the work, where such license applies only
+to those patent claims licensable by you that are necessarily infringed by your
+contribution alone or by combination of your contribution with the project to
+which it was submitted.
+
+## 3. Right to Grant
+
+You represent that you are legally entitled to grant the above licenses. If your
+employer has rights to intellectual property that you create, you represent that
+you have received permission to make contributions on behalf of that employer,
+or that your employer has waived such rights for your contributions to this
+project.
+
+## 4. Original Work
+
+You represent that each of your contributions is your original creation. You
+represent that your contribution submissions include complete details of any
+third-party license or other restriction (including, but not limited to, related
+patents and trademarks) of which you are aware and which are associated with any
+part of your contributions.
+
+## 5. Re-licensing
+
+You acknowledge and agree that the project maintainers may re-license the
+project (including your contributions) under any license terms they see fit,
+including proprietary licenses. This right is granted to ensure the long-term
+viability and flexibility of the project.
+
+## 6. No Obligation
+
+You understand that the decision to include your contribution in any project or
+source repository is entirely at the discretion of the project maintainers, and
+this agreement does not obligate the project maintainers to use or include your
+contribution.
+
+## 7. Support
+
+You are not expected to provide support for your contributions, except to the
+extent you desire to provide support. You may provide support for free, for a
+fee, or not at all.
+
+## How to Sign
+
+By opening a pull request or otherwise submitting a contribution to this
+project, you indicate that you have read this Contributor License Agreement and
+agree to its terms.

s3duct-0.3.0/LICENSE

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor's trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.