twat-fs 1.7.5__tar.gz

twat_fs-1.7.5/.github/workflows/push.yml

@@ -0,0 +1,112 @@
+name: Build & Test
+
+on:
+  push:
+    branches: [main]
+    tags-ignore: ["v*"]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  quality:
+    name: Code Quality
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run Ruff lint
+        uses: astral-sh/ruff-action@v3
+        with:
+          version: "latest"
+          args: "check --output-format=github"
+
+      - name: Run Ruff Format
+        uses: astral-sh/ruff-action@v3
+        with:
+          version: "latest"
+          args: "format --check --respect-gitignore"
+
+  test:
+    name: Run Tests
+    needs: quality
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-latest]
+      fail-fast: true
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install UV
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          cache-suffix: ${{ matrix.os }}-${{ matrix.python-version }}
+
+      - name: Install test dependencies
+        run: |
+          uv pip install --system --upgrade pip
+          uv pip install --system ".[test]"
+
+      - name: Run tests with Pytest
+        run: uv run pytest -n auto --maxfail=1 --disable-warnings --cov-report=xml --cov-config=pyproject.toml --cov=src/twat_fs --cov=tests tests/
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.python-version }}-${{ matrix.os }}
+          path: coverage.xml
+
+  build:
+    name: Build Distribution
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install UV
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+          python-version: "3.12"
+          enable-cache: true
+
+      - name: Install build tools
+        run: uv pip install build hatchling hatch-vcs
+
+      - name: Build distributions
+        run: uv run python -m build --outdir dist
+
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-files
+          path: dist/
+          retention-days: 5 

twat_fs-1.7.5/.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  release:
+    name: Release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/twat-fs
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install UV
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+          python-version: "3.12"
+          enable-cache: true
+
+      - name: Install build tools
+        run: uv pip install build hatchling hatch-vcs
+
+      - name: Build distributions
+        run: uv run python -m build --outdir dist
+
+      - name: Verify distribution files
+        run: |
+          ls -la dist/
+          test -n "$(find dist -name '*.whl')" || (echo "Wheel file missing" && exit 1)
+          test -n "$(find dist -name '*.tar.gz')" || (echo "Source distribution missing" && exit 1)
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: dist/*
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 

twat_fs-1.7.5/.gitignore

@@ -0,0 +1,315 @@
+*_autogen/
+.DS_Store
+__version__.py
+__pycache__/
+_Chutzpah*
+_deps
+_NCrunch_*
+_pkginfo.txt
+_Pvt_Extensions
+_ReSharper*/
+_TeamCity*
+_UpgradeReport_Files/
+!?*.[Cc]ache/
+!.axoCover/settings.json
+!.vscode/extensions.json
+!.vscode/launch.json
+!.vscode/settings.json
+!.vscode/tasks.json
+!**/[Pp]ackages/build/
+!Directory.Build.rsp
+.*crunch*.local.xml
+.axoCover/*
+.builds
+.cr/personal
+.fake/
+.history/
+.ionide/
+.localhistory/
+.mfractor/
+.ntvs_analysis.dat
+.paket/paket.exe
+.sass-cache/
+.vs/
+.vscode
+.vscode/*
+.vshistory/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+[Bb]in/
+[Bb]uild[Ll]og.*
+[Dd]ebug/
+[Dd]ebugPS/
+[Dd]ebugPublic/
+[Ee]xpress/
+[Ll]og/
+[Ll]ogs/
+[Oo]bj/
+[Rr]elease/
+[Rr]eleasePS/
+[Rr]eleases/
+[Tt]est[Rr]esult*/
+[Ww][Ii][Nn]32/
+*_h.h
+*_i.c
+*_p.c
+*_wpftmp.csproj
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+*- [Bb]ackup.rdl
+*.[Cc]ache
+*.[Pp]ublish.xml
+*.[Rr]e[Ss]harper
+*.a
+*.app
+*.appx
+*.appxbundle
+*.appxupload
+*.aps
+*.azurePubxml
+*.bim_*.settings
+*.bim.layout
+*.binlog
+*.btm.cs
+*.btp.cs
+*.build.csdef
+*.cab
+*.cachefile
+*.code-workspace
+*.coverage
+*.coveragexml
+*.d
+*.dbmdl
+*.dbproj.schemaview
+*.dll
+*.dotCover
+*.DotSettings.user
+*.dsp
+*.dsw
+*.dylib
+*.e2e
+*.exe
+*.gch
+*.GhostDoc.xml
+*.gpState
+*.ilk
+*.iobj
+*.ipdb
+*.jfm
+*.jmconfig
+*.la
+*.lai
+*.ldf
+*.lib
+*.lo
+*.log
+*.mdf
+*.meta
+*.mm.*
+*.mod
+*.msi
+*.msix
+*.msm
+*.msp
+*.ncb
+*.ndf
+*.nuget.props
+*.nuget.targets
+*.nupkg
+*.nvuser
+*.o
+*.obj
+*.odx.cs
+*.opendb
+*.opensdf
+*.opt
+*.out
+*.pch
+*.pdb
+*.pfx
+*.pgc
+*.pgd
+*.pidb
+*.plg
+*.psess
+*.publishproj
+*.publishsettings
+*.pubxml
+*.pyc
+*.rdl.data
+*.rptproj.bak
+*.rptproj.rsuser
+*.rsp
+*.rsuser
+*.sap
+*.sbr
+*.scc
+*.sdf
+*.sln.docstates
+*.sln.iml
+*.slo
+*.smod
+*.snupkg
+*.so
+*.suo
+*.svclog
+*.tlb
+*.tlh
+*.tli
+*.tlog
+*.tmp
+*.tmp_proj
+*.tss
+*.user
+*.userosscache
+*.userprefs
+*.vbp
+*.vbw
+*.VC.db
+*.VC.VC.opendb
+*.VisualState.xml
+*.vsp
+*.vspscc
+*.vspx
+*.vssscc
+*.xsd.cs
+**/[Pp]ackages/*
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.HTMLClient/GeneratedArtifacts
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+*~
+~$*
+$tf/
+AppPackages/
+artifacts/
+ASALocalRun/
+AutoTest.Net/
+Backup*/
+BenchmarkDotNet.Artifacts/
+bld/
+BundleArtifacts/
+ClientBin/
+cmake_install.cmake
+CMakeCache.txt
+CMakeFiles
+CMakeLists.txt.user
+CMakeScripts
+CMakeUserPresets.json
+compile_commands.json
+coverage*.info
+coverage*.json
+coverage*.xml
+csx/
+CTestTestfile.cmake
+dlldata.c
+DocProject/buildhelp/
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/*.HxC
+DocProject/Help/*.HxT
+DocProject/Help/html
+DocProject/Help/Html2
+ecf/
+FakesAssemblies/
+FodyWeavers.xsd
+Generated_Code/
+Generated\ Files/
+healthchecksdb
+install_manifest.txt
+ipch/
+Makefile
+MigrationBackup/
+mono_crash.*
+nCrunchTemp_*
+node_modules/
+nunit-*.xml
+OpenCover/
+orleans.codegen.cs
+Package.StoreAssociation.xml
+paket-files/
+project.fragment.lock.json
+project.lock.json
+publish/
+PublishScripts/
+rcf/
+ScaffoldingReadMe.txt
+ServiceFabricBackup/
+StyleCopReport.xml
+Testing
+TestResult.xml
+UpgradeLog*.htm
+UpgradeLog*.XML
+x64/
+x86/
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Distribution / packaging
+!dist/.gitkeep
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+.ruff_cache/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Project specific
+__version__.py
+_private

twat_fs-1.7.5/.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+        args: [--respect-gitignore]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+      - id: debug-statements
+      - id: check-case-conflict
+      - id: mixed-line-ending
+        args: [--fix=lf] 

twat_fs-1.7.5/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Adam Twardoch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

twat_fs-1.7.5/LOG.md

@@ -0,0 +1,160 @@
+# LOG
+
+## Completed Work
+
+1. Created file upload functionality with multiple providers:
+   - Created `src/twat_fs/upload_providers/` directory
+   - Implemented `fal.py` provider with simple file upload functionality
+   - Adapted private Dropbox implementation into `dropbox.py` provider
+   - Created main `upload.py` module with provider selection logic
+   - Added S3 provider with AWS S3 upload functionality
+
+2. Implemented CLI interface:
+   - Created `__main__.py` with Fire CLI integration
+   - Added `upload_file` command with provider selection
+   - Updated `__init__.py` to expose main functionality
+
+3. Updated documentation:
+   - Added new functionality to README.md
+   - Documented provider configuration
+   - Added usage examples for both Python API and CLI
+
+4. Enhanced provider functionality:
+   - Added `provider_auth` functions to check authentication
+   - Added loguru warnings for missing credentials
+   - Implemented provider fallback mechanism
+   - Added support for provider lists and preferences
+   - Added S3 as the default preferred provider
+
+5. Created test framework:
+   - Added comprehensive test skeleton in `tests/test_upload.py`
+   - Created test fixtures and mocks
+   - Added test data directory
+   - Implemented tests for auth, upload, and fallback functionality
+   - Added S3-specific tests for authentication and upload scenarios
+   - Added integration tests with real providers in `test_integration.py`
+   - Added edge cases and error condition tests
+   - Added performance tests with different file sizes
+   - Added tests for S3 multipart uploads
+   - Added tests for different S3 endpoint configurations
+   - Added tests for various AWS credential providers
+
+6. Refactored provider authentication and initialization:
+   - Split provider initialization into two steps:
+     1. `get_credentials`: Simple function to check and return required env variables
+     2. `get_provider`: Function to import dependencies and initialize provider client
+   - Updated all providers (S3, FAL, Dropbox) to use the new pattern
+   - Added proper typing and error handling
+   - Improved logging and error messages
+   - Made provider initialization more efficient by checking credentials before importing dependencies
+   - Updated main upload module to use the new provider pattern
+   - Added better provider client management
+
+7. Added provider setup functionality:
+   - Implemented `setup_provider` function to check individual provider status
+   - Added `setup_providers` function to check all available providers
+   - Added detailed help messages for each provider's setup requirements
+   - Added dependency installation instructions
+   - Updated all tests to use the new provider pattern:
+     - Updated unit tests in `test_upload.py`
+     - Updated integration tests in `test_integration.py`
+     - Updated S3 advanced tests in `test_s3_advanced.py`
+   - Added new test cases for provider setup
+   - Improved error messages and setup instructions
+   - Added comprehensive credential configuration guides
+   - Added CLI commands for checking provider setup:
+     - `setup provider <name>`: Check setup status for a specific provider
+     - `setup all`: Check setup status for all available providers
+
+8. Improved provider interface and type system:
+   - Created formal provider interfaces using Python's Protocol system:
+     - `Provider` protocol defining what each provider module must implement
+     - `ProviderClient` protocol defining what each provider's client must implement
+   - Added runtime protocol checking with `@runtime_checkable`
+   - Moved provider-specific help messages to their respective provider modules
+   - Created a centralized provider registry in `upload_providers/__init__.py`:
+     - Added `ProviderType` that automatically stays in sync with available providers
+     - Added lazy loading of provider modules with caching
+     - Added proper type hints and documentation for all interfaces
+   - Improved type safety throughout the codebase:
+     - Added proper return type hints for all functions
+     - Added TypedDict for provider help messages
+     - Added better error handling for protocol violations
+   - Made the provider system more maintainable:
+     - Single source of truth for provider list
+     - Self-documenting interfaces
+     - Better IDE support through type hints
+     - Clearer separation between module and client interfaces
+
+## TODO4:
+
+1. ✅ Provider API Improvements (Completed in update 8):
+   - Created formal provider interface using Protocol system
+   - Added provider discovery with lazy loading
+   - Implemented runtime protocol checking
+   - Added comprehensive type hints
+   - Moved provider-specific help messages to modules
+   - Created centralized provider registry
+
+2. Improve S3 provider API:
+   - Add specific error types for S3-related failures:
+     - Create custom exception hierarchy
+     - Map AWS error codes to specific exceptions
+     - Add detailed error messages and recovery hints
+   - Add support for advanced AWS authentication:
+     - IAM roles and instance profiles
+     - AWS SSO credentials
+     - Role assumption and chaining
+   - Enhance S3 functionality:
+     - Add support for bucket policies and ACLs
+     - Implement S3 transfer acceleration
+     - Add credential caching mechanism
+     - Add support for S3-compatible services
+   - Improve multipart uploads:
+     - Add progress tracking
+     - Add resume capability
+     - Add parallel upload support
+     - Add automatic retries
+
+3. Enhance error handling and logging:
+   - Replace broad exception catches (BLE001)
+   - Add structured error logging
+   - Improve error recovery mechanisms
+   - Add detailed troubleshooting guides
+
+4. Improve code quality:
+   - Fix linter warnings about complexity
+   - Refactor boolean arguments to use enums/options
+   - Add more comprehensive integration tests
+   - Add performance benchmarks
+
+# Development Log
+
+## 2025-02-14
+
+### Changes Made
+- Updated the main upload_file function to add proper file validation checks (existence, type, permissions)
+- Modified _try_provider helper to properly handle remote_path parameter
+- Revised FAL provider to handle missing set_key method gracefully
+- Fixed S3 provider implementation:
+  - Added proper credentials handling
+  - Improved multipart upload with abort on failure
+  - Fixed config handling for path-style endpoints
+  - Added better error handling and logging
+- Implemented formal provider interface using Python's Protocol system
+- Added runtime protocol checking for better error detection
+- Moved provider-specific help messages to their respective modules
+- Created centralized provider registry with lazy loading
+- Added comprehensive type hints and documentation
+- Fixed S3 multipart upload tests:
+  - Corrected mocking strategy for boto3.client
+  - Fixed test function calls to use correct module path
+  - Improved test assertions and error handling
+  - Added better test documentation
+
+### Next Steps
+- Refactor broad exception catches (BLE001)
+- Fix linter warnings about complexity and boolean args
+- Review and update Dropbox provider if needed
+- Add more comprehensive integration tests
+- Improve error messages and logging