hctef 0.1.0__tar.gz

hctef-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: "Continuous integration"
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: false
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  ci:
+    name: Continuous integration
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+          - "3.13"
+          - "3.14"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Sync
+        run: |
+          uv sync \
+            --locked \
+            --all-extras \
+            --no-editable
+      - name: Pre-Commit Hooks
+        run: uv run pre-commit run --all-files
+      - name: Test
+        run: uv run pytest
+      - name: "Upload coverage to Codecov"
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          fail_ci_if_error: false
+          verbose: true

hctef-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,47 @@
+name: Build and release
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  release:
+    types:
+      - published
+
+jobs:
+  build-package:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - name: Build
+        run: uv build
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        if: startsWith(github.ref, 'refs/tags')
+        with:
+          name: dist-{github.ref}
+          path: dist/
+          overwrite: true
+          if-no-files-found: error
+
+  release-package:
+    if: startsWith(github.ref, 'refs/tags')
+    needs: build-package
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/hctef
+    permissions:
+      id-token: write
+    steps:
+      - name: Download a single artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: dist-{github.ref}
+          path: dist/
+      - name: Upload release
+        uses: pypa/gh-action-pypi-publish@release/v1

hctef-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+.coverage
+__version__.py

hctef-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,51 @@
+exclude: "^tests/fixtures/"
+repos:
+  - repo: local
+    hooks:
+      - id: ruff_check
+        name: ruff check
+        entry: ruff check --force-exclude
+        language: python
+        'types_or': [python, pyi]
+        args: [--fix, --exit-non-zero-on-fix]
+        require_serial: true
+      - id: ruff_format
+        name: ruff format
+        entry: ruff format --force-exclude
+        language: python
+        'types_or': [python, pyi]
+        args: []
+        require_serial: true
+      - id: check-added-large-files
+        name: Check for added large files
+        entry: check-added-large-files
+        language: system
+      - id: check-toml
+        name: Check Toml
+        entry: check-toml
+        language: system
+        types: [toml]
+      - id: check-yaml
+        name: Check Yaml
+        entry: check-yaml
+        language: system
+        types: [yaml]
+      - id: end-of-file-fixer
+        name: Fix End of Files
+        entry: end-of-file-fixer
+        language: system
+        types: [text]
+        stages: [pre-commit, pre-push, manual]
+      - id: trailing-whitespace
+        name: Trim Trailing Whitespace
+        entry: trailing-whitespace-fixer
+        language: system
+        types: [text]
+        stages: [pre-commit, pre-push, manual]
+      - id: mypy
+        name: mypy
+        entry: mypy
+        language: python
+        'types_or': [python, pyi]
+        args: []
+        require_serial: true

hctef-0.1.0/PKG-INFO

@@ -0,0 +1,236 @@
+Metadata-Version: 2.4
+Name: hctef
+Version: 0.1.0
+Summary: Helper classes to read files over HTTP using Range requests, with caching
+Project-URL: Repository, https://github.com/jkeifer/hctef
+Author-email: Jarrett Keifer <jkeifer0@gmail.com>
+License: Apache License 2.0
+Requires-Python: >=3.12
+Provides-Extra: async
+Requires-Dist: aiohttp>=3.13.0; extra == 'async'
+Description-Content-Type: text/markdown
+
+[![Tests](https://github.com/jkeifer/hctef/actions/workflows/ci.yml/badge.svg)](https://github.com/jkeifer/hctef/actions/workflows/ci.yml)
+[![PyPI version](https://badge.fury.io/py/hctef.svg)](https://badge.fury.io/py/hctef)
+
+# hctef
+
+Python library with helper classes to read files over HTTP using Range
+requests, with caching.
+
+## Overview
+
+`hctef` provides a file-like interface for reading files over HTTP/HTTPS, using
+HTTP Range requests to fetch only the data you need. It includes intelligent
+caching to minimize network requests and supports both synchronous and
+asynchronous operations.
+
+## Features
+
+- **File-like API**: Works like a regular Python file object with `read()`,
+  `seek()`, and `tell()` methods
+- **Efficient Range Requests**: Fetches only the data you need using HTTP Range
+  headers
+- **Intelligent Caching**: Uses an interval tree to track cached byte ranges
+  and minimize redundant requests
+- **Prefetching**: Optionally prefetch data from the start or end of the file
+- **Sync and Async**: Both synchronous and asynchronous implementations
+  available
+- **Context Manager Support**: Use with `with` statements for automatic cleanup
+
+## Installation
+
+```bash
+pip install hctef
+```
+
+To include async support:
+
+```bash
+pip install hctef[async]
+```
+
+## Quick Start
+
+### Synchronous Usage
+
+```python
+from hctef import HttpFile
+
+url = "https://example.com/large-file.bin"
+
+with HttpFile(url) as f:
+    # Read first 100 bytes
+    data = f.read(100)
+
+    # Seek to a specific position
+    f.seek(1000)
+
+    # Read from current position
+    more_data = f.read(50)
+
+    # Get current position
+    position = f.tell()
+
+    # Seek relative to end of file
+    f.seek(-100, 2)
+```
+
+### Asynchronous Usage
+
+The async implementation supports independent cursors for concurrent reads:
+
+```python
+import asyncio
+from hctef.aio import AsyncHttpFile
+
+url = "https://example.com/large-file.bin"
+
+async with AsyncHttpFile(url) as f:
+    # Read first 100 bytes
+    data = await f.read(100)
+
+    # Seek to a specific position (synchronous - no I/O)
+    f.seek(1000)
+
+    # Read from current position
+    more_data = await f.read(50)
+```
+
+#### Parallel Reads with Multiple Cursors
+
+Create independent cursors to read from different positions concurrently:
+
+```python
+import asyncio
+from hctef.aio import AsyncHttpFile
+
+url = "https://example.com/large-file.bin"
+
+async with AsyncHttpFile(url) as f:
+    # Create independent cursors for parallel reading
+    cursor1 = f.clone()
+    cursor2 = f.clone()
+
+    # Position each cursor at different locations
+    f.seek(0)
+    cursor1.seek(1000)
+    cursor2.seek(2000)
+
+    # Read from all three positions in parallel
+    # All cursors share the same cache and HTTP session
+    results = await asyncio.gather(
+        f.read(100),        # Read bytes 0-100
+        cursor1.read(100),  # Read bytes 1000-1100
+        cursor2.read(100),  # Read bytes 2000-2100
+    )
+
+    # Each cursor maintains independent position
+    print(f.tell())        # 100
+    print(cursor1.tell())  # 1100
+    print(cursor2.tell())  # 2100
+```
+
+Cursors are lightweight and share:
+
+- HTTP session (connection pooling)
+- Byte range cache (deduplication of overlapping requests)
+- File metadata
+
+## Configuration Options
+
+Both `HttpFile` and `AsyncHttpFile` accept the following parameters:
+
+```python
+HttpFile(
+    url,
+    minimum_range_request_bytes=8192,  # Minimum bytes per request (default: 8KB)
+    prefetch_bytes=1048576,             # Bytes to prefetch on open (default: 1MB)
+    prefetch_direction='END'            # 'START' or 'END' (default: 'END')
+)
+```
+
+- **`minimum_range_request_bytes`**: The minimum number of bytes to request in
+  a single HTTP Range request (except when filling small cache gaps)
+- **`prefetch_bytes`**: How many bytes to fetch immediately when opening the
+  file. Set to 0 to disable prefetching
+- **`prefetch_direction`**: Whether to prefetch from the start (`'START'`) or
+  end (`'END'`) of the file
+
+## Requirements
+
+- Python 3.12 or higher
+- HTTP server must support Range requests
+- For async: `aiohttp>=3.13.0`
+
+## How It Works
+
+When you open an HTTP file, `hctef`:
+
+1. Sends an initial Range request to determine the file size and verify Range
+   support
+1. Optionally prefetches data from the start or end of the file
+1. Maintains an in-memory cache of fetched byte ranges (not suitable for
+   downloading complete large files)
+1. On `read()`, checks the cache first and only fetches missing data from the
+   server
+1. Combines multiple small requests into larger ones based on
+   `minimum_range_request_bytes`
+
+This approach minimizes HTTP requests while providing efficient random access
+to remote files.
+
+## Error Handling
+
+`hctef` defines custom exceptions:
+
+- `HctefError`: Base exception class
+- `HctefNetworkError`: Raised for network-related errors (inherits from
+  `IOError`)
+- `HctefUrlError`: Raised for invalid URLs (inherits from `ValueError`)
+
+```python
+from hctef import HttpFile
+from hctef.exceptions import HctefNetworkError, HctefUrlError
+
+try:
+    with HttpFile("https://example.com/file.bin") as f:
+        data = f.read(100)
+except HctefNetworkError as e:
+    print(f"Network error: {e}")
+except HctefUrlError as e:
+    print(f"Invalid URL: {e}")
+```
+
+## Development
+
+To set up for development:
+
+```bash
+# Clone the repository
+git clone https://github.com/jkeifer/hctef
+cd hctef
+
+# Install dependencies
+uv sync --all-extras --dev
+
+# Setup pre-commit
+pre-commit install
+
+# Run tests
+pytest
+
+# Run all checks with pre-commit
+pre-commit run --all-files
+```
+
+## Future Ideas
+
+- Consoldiate sync/async implementations
+- Allow uncached "cursor" for reading a large file segement
+- Cursors with separate caches (to allow clearing memory when done)
+  - would allow cursor-based access with non-async implementation
+
+## License
+
+Apache License 2.0

hctef-0.1.0/README.md

@@ -0,0 +1,224 @@
+[![Tests](https://github.com/jkeifer/hctef/actions/workflows/ci.yml/badge.svg)](https://github.com/jkeifer/hctef/actions/workflows/ci.yml)
+[![PyPI version](https://badge.fury.io/py/hctef.svg)](https://badge.fury.io/py/hctef)
+
+# hctef
+
+Python library with helper classes to read files over HTTP using Range
+requests, with caching.
+
+## Overview
+
+`hctef` provides a file-like interface for reading files over HTTP/HTTPS, using
+HTTP Range requests to fetch only the data you need. It includes intelligent
+caching to minimize network requests and supports both synchronous and
+asynchronous operations.
+
+## Features
+
+- **File-like API**: Works like a regular Python file object with `read()`,
+  `seek()`, and `tell()` methods
+- **Efficient Range Requests**: Fetches only the data you need using HTTP Range
+  headers
+- **Intelligent Caching**: Uses an interval tree to track cached byte ranges
+  and minimize redundant requests
+- **Prefetching**: Optionally prefetch data from the start or end of the file
+- **Sync and Async**: Both synchronous and asynchronous implementations
+  available
+- **Context Manager Support**: Use with `with` statements for automatic cleanup
+
+## Installation
+
+```bash
+pip install hctef
+```
+
+To include async support:
+
+```bash
+pip install hctef[async]
+```
+
+## Quick Start
+
+### Synchronous Usage
+
+```python
+from hctef import HttpFile
+
+url = "https://example.com/large-file.bin"
+
+with HttpFile(url) as f:
+    # Read first 100 bytes
+    data = f.read(100)
+
+    # Seek to a specific position
+    f.seek(1000)
+
+    # Read from current position
+    more_data = f.read(50)
+
+    # Get current position
+    position = f.tell()
+
+    # Seek relative to end of file
+    f.seek(-100, 2)
+```
+
+### Asynchronous Usage
+
+The async implementation supports independent cursors for concurrent reads:
+
+```python
+import asyncio
+from hctef.aio import AsyncHttpFile
+
+url = "https://example.com/large-file.bin"
+
+async with AsyncHttpFile(url) as f:
+    # Read first 100 bytes
+    data = await f.read(100)
+
+    # Seek to a specific position (synchronous - no I/O)
+    f.seek(1000)
+
+    # Read from current position
+    more_data = await f.read(50)
+```
+
+#### Parallel Reads with Multiple Cursors
+
+Create independent cursors to read from different positions concurrently:
+
+```python
+import asyncio
+from hctef.aio import AsyncHttpFile
+
+url = "https://example.com/large-file.bin"
+
+async with AsyncHttpFile(url) as f:
+    # Create independent cursors for parallel reading
+    cursor1 = f.clone()
+    cursor2 = f.clone()
+
+    # Position each cursor at different locations
+    f.seek(0)
+    cursor1.seek(1000)
+    cursor2.seek(2000)
+
+    # Read from all three positions in parallel
+    # All cursors share the same cache and HTTP session
+    results = await asyncio.gather(
+        f.read(100),        # Read bytes 0-100
+        cursor1.read(100),  # Read bytes 1000-1100
+        cursor2.read(100),  # Read bytes 2000-2100
+    )
+
+    # Each cursor maintains independent position
+    print(f.tell())        # 100
+    print(cursor1.tell())  # 1100
+    print(cursor2.tell())  # 2100
+```
+
+Cursors are lightweight and share:
+
+- HTTP session (connection pooling)
+- Byte range cache (deduplication of overlapping requests)
+- File metadata
+
+## Configuration Options
+
+Both `HttpFile` and `AsyncHttpFile` accept the following parameters:
+
+```python
+HttpFile(
+    url,
+    minimum_range_request_bytes=8192,  # Minimum bytes per request (default: 8KB)
+    prefetch_bytes=1048576,             # Bytes to prefetch on open (default: 1MB)
+    prefetch_direction='END'            # 'START' or 'END' (default: 'END')
+)
+```
+
+- **`minimum_range_request_bytes`**: The minimum number of bytes to request in
+  a single HTTP Range request (except when filling small cache gaps)
+- **`prefetch_bytes`**: How many bytes to fetch immediately when opening the
+  file. Set to 0 to disable prefetching
+- **`prefetch_direction`**: Whether to prefetch from the start (`'START'`) or
+  end (`'END'`) of the file
+
+## Requirements
+
+- Python 3.12 or higher
+- HTTP server must support Range requests
+- For async: `aiohttp>=3.13.0`
+
+## How It Works
+
+When you open an HTTP file, `hctef`:
+
+1. Sends an initial Range request to determine the file size and verify Range
+   support
+1. Optionally prefetches data from the start or end of the file
+1. Maintains an in-memory cache of fetched byte ranges (not suitable for
+   downloading complete large files)
+1. On `read()`, checks the cache first and only fetches missing data from the
+   server
+1. Combines multiple small requests into larger ones based on
+   `minimum_range_request_bytes`
+
+This approach minimizes HTTP requests while providing efficient random access
+to remote files.
+
+## Error Handling
+
+`hctef` defines custom exceptions:
+
+- `HctefError`: Base exception class
+- `HctefNetworkError`: Raised for network-related errors (inherits from
+  `IOError`)
+- `HctefUrlError`: Raised for invalid URLs (inherits from `ValueError`)
+
+```python
+from hctef import HttpFile
+from hctef.exceptions import HctefNetworkError, HctefUrlError
+
+try:
+    with HttpFile("https://example.com/file.bin") as f:
+        data = f.read(100)
+except HctefNetworkError as e:
+    print(f"Network error: {e}")
+except HctefUrlError as e:
+    print(f"Invalid URL: {e}")
+```
+
+## Development
+
+To set up for development:
+
+```bash
+# Clone the repository
+git clone https://github.com/jkeifer/hctef
+cd hctef
+
+# Install dependencies
+uv sync --all-extras --dev
+
+# Setup pre-commit
+pre-commit install
+
+# Run tests
+pytest
+
+# Run all checks with pre-commit
+pre-commit run --all-files
+```
+
+## Future Ideas
+
+- Consoldiate sync/async implementations
+- Allow uncached "cursor" for reading a large file segement
+- Cursors with separate caches (to allow clearing memory when done)
+  - would allow cursor-based access with non-async implementation
+
+## License
+
+Apache License 2.0

hctef-0.1.0/pyproject.toml

@@ -0,0 +1,99 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hctef"
+description = "Helper classes to read files over HTTP using Range requests, with caching"
+readme = "README.md"
+authors = [
+    { name = "Jarrett Keifer", email = "jkeifer0@gmail.com" }
+]
+requires-python = ">=3.12"
+license = {text = "Apache License 2.0"}
+dependencies = []
+dynamic = ["version"]
+
+[project.urls]
+Repository = 'https://github.com/jkeifer/hctef'
+
+[project.optional-dependencies]
+async = [
+    "aiohttp>=3.13.0",
+]
+
+[dependency-groups]
+dev = [
+    "mypy>=1.15.0",
+    "pre-commit>=4.2.0",
+    "pre-commit-hooks>=5.0.0",
+    "pytest>=8.3.5",
+    "pytest-asyncio>=1.2.0",
+    "pytest-cov>=6.1.1",
+    "ruff>=0.11.6",
+]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/hctef/__version__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hctef"]
+
+[tool.ruff.format]
+quote-style = 'single'
+
+[tool.ruff.lint]
+exclude = [
+    'tests/fixtures/',
+]
+# https://docs.astral.sh/ruff/rules/
+select = [
+    'B',
+    'BLE',
+    'C4',
+    'C90',
+    'COM',
+    'DTZ',
+    'E',
+    'ERA',
+    'F',
+    'FA',
+    'G',
+    'I',
+    'INP',
+    'N',
+    'NPY',
+    'PT',
+    'PTH',
+    'RET',
+    'RUF',
+    'S',
+    'SIM',
+    'T20',
+    'UP',
+    'W',
+
+]
+
+[tool.ruff.lint.per-file-ignores]
+'__init__.py' = ['E402']
+'tests/**/*' = ['T201', 'S101', 'S603']
+
+[tool.ruff.lint.isort]
+lines-between-types = 1
+
+[tool.mypy]
+ignore_missing_imports = true
+scripts_are_modules = true
+disable_error_code = 'prop-decorator'
+
+[tool.pytest.ini_options]
+addopts="--cov=hctef"
+
+[tool.coverage.report]
+show_missing = true
+skip_empty = true
+sort = "Cover"