django-fsspec 0.1.0__tar.gz

django_fsspec-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,366 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+  pull_request:
+    branches: [main]
+
+jobs:
+  # ---------------------------------------------------------------
+  # Stage 1: Unit tests (SQLite, fast, matrix)
+  # ---------------------------------------------------------------
+  unit-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+        django-version: ["3.2", "4.2", "5.2"]
+        exclude:
+          - python-version: "3.12"
+            django-version: "3.2"
+          - python-version: "3.13"
+            django-version: "3.2"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install -e ".[dev]"
+          pip install "django~=${{ matrix.django-version }}.0"
+
+      - name: Run unit tests
+        run: |
+          python -m pytest django_fsspec/tests/ -v --cov=django_fsspec --cov-report=term-missing
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Check migrations
+        run: |
+          DJANGO_SETTINGS_MODULE=tests.settings python -m django makemigrations --check --dry-run
+
+  # ---------------------------------------------------------------
+  # Stage 2: E2E + Benchmark (real databases, after unit tests pass)
+  # ---------------------------------------------------------------
+  e2e-sqlite:
+    needs: unit-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: E2E tests
+        run: DJANGO_FSSPEC_BENCH_DB=sqlite python benchmarks/e2e_test.py
+
+      - name: Benchmark
+        run: DJANGO_FSSPEC_BENCH_DB=sqlite python benchmarks/run.py --db sqlite --json benchmarks/results-sqlite.json
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-sqlite
+          path: benchmarks/results-sqlite.json
+
+  e2e-mysql:
+    needs: unit-test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - mysql-version: "5.7"
+            django-version: "3.2"
+          - mysql-version: "8.0"
+            django-version: "5.2"
+    services:
+      mysql:
+        image: mysql:${{ matrix.mysql-version }}
+        env:
+          MYSQL_ROOT_PASSWORD: fsspec_test
+          MYSQL_DATABASE: fsspec_test
+          MYSQL_USER: fsspec
+          MYSQL_PASSWORD: fsspec_test
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h localhost -u root -pfsspec_test"
+          --health-interval=5s
+          --health-timeout=3s
+          --health-retries=10
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Configure MySQL charset
+        run: |
+          for i in 1 2 3 4 5; do
+            mysql -h 127.0.0.1 -P 3306 -u root -pfsspec_test -e "
+              ALTER DATABASE fsspec_test CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+            " && break || sleep 3
+          done
+
+      - name: Install dependencies
+        run: |
+          pip install -e ".[dev]"
+          pip install mysqlclient
+          pip install "django~=${{ matrix.django-version }}.0"
+
+      - name: E2E tests
+        env:
+          DJANGO_FSSPEC_BENCH_DB: mysql
+          MYSQL_PORT: "3306"
+        run: python benchmarks/e2e_test.py
+
+      - name: Benchmark
+        env:
+          DJANGO_FSSPEC_BENCH_DB: mysql
+          MYSQL_PORT: "3306"
+        run: python benchmarks/run.py --db mysql --json benchmarks/results-mysql-${{ matrix.mysql-version }}.json
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-mysql-${{ matrix.mysql-version }}
+          path: benchmarks/results-mysql-${{ matrix.mysql-version }}.json
+
+  e2e-postgres:
+    needs: unit-test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - pg-version: "9.6"
+            django-version: "3.2"
+          - pg-version: "16"
+            django-version: "5.2"
+    services:
+      postgres:
+        image: postgres:${{ matrix.pg-version }}
+        env:
+          POSTGRES_DB: fsspec_test
+          POSTGRES_USER: fsspec
+          POSTGRES_PASSWORD: fsspec_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U fsspec -d fsspec_test"
+          --health-interval=5s
+          --health-timeout=3s
+          --health-retries=10
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          pip install -e ".[dev]"
+          pip install psycopg2-binary
+          pip install "django~=${{ matrix.django-version }}.0"
+
+      - name: E2E tests
+        env:
+          DJANGO_FSSPEC_BENCH_DB: postgres
+          POSTGRES_PORT: "5432"
+        run: python benchmarks/e2e_test.py
+
+      - name: Benchmark
+        env:
+          DJANGO_FSSPEC_BENCH_DB: postgres
+          POSTGRES_PORT: "5432"
+        run: python benchmarks/run.py --db postgres --json benchmarks/results-postgres-${{ matrix.pg-version }}.json
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-postgres-${{ matrix.pg-version }}
+          path: benchmarks/results-postgres-${{ matrix.pg-version }}.json
+
+  e2e-oracle:
+    needs: unit-test
+    runs-on: ubuntu-latest
+    services:
+      oracle:
+        image: gvenzl/oracle-free:23-slim
+        env:
+          ORACLE_PASSWORD: fsspec_test
+          APP_USER: fsspec
+          APP_USER_PASSWORD: fsspec_test
+        ports:
+          - 1521:1521
+        options: >-
+          --health-cmd="healthcheck.sh"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=30
+          --health-start-period=60s
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          pip install -e ".[dev]"
+          pip install "oracledb<4"
+
+      - name: E2E tests
+        env:
+          DJANGO_FSSPEC_BENCH_DB: oracle
+        run: python benchmarks/e2e_test.py
+
+      - name: Benchmark
+        env:
+          DJANGO_FSSPEC_BENCH_DB: oracle
+        run: python benchmarks/run.py --db oracle --json benchmarks/results-oracle.json
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-oracle
+          path: benchmarks/results-oracle.json
+
+  # ---------------------------------------------------------------
+  # Stage 3: Collect baseline
+  # ---------------------------------------------------------------
+  baseline:
+    needs: [e2e-sqlite, e2e-mysql, e2e-postgres, e2e-oracle]
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download all benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          path: benchmarks/results/
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Generate baseline report
+        run: |
+          python -c "
+          import json, glob
+
+          all_results = []
+          for f in sorted(glob.glob('benchmarks/results/**/*.json', recursive=True)):
+              with open(f) as fh:
+                  all_results.extend(json.load(fh))
+
+          ops = {}
+          for r in all_results:
+              op = r.get('op', '?')
+              if op not in ops:
+                  ops[op] = {}
+              ops[op][r.get('db', '?')] = r
+
+          print('## Performance Baseline')
+          print()
+          print('| Operation | Database | Avg (ms) | P95 (ms) | Ops/s |')
+          print('|-----------|----------|----------|----------|-------|')
+          for op, dbs in sorted(ops.items()):
+              for db, r in sorted(dbs.items()):
+                  if 'error' in r:
+                      print(f'| {op} | {db} | ERROR | | |')
+                  else:
+                      print(f'| {op} | {db} | {r[\"avg_ms\"]:.2f} | {r[\"p95_ms\"]:.2f} | {r[\"ops_per_sec\"]:.0f} |')
+
+          with open('benchmarks/baseline.json', 'w') as f:
+              json.dump(all_results, f, indent=2, default=str)
+          "
+
+      - name: Upload baseline
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-baseline
+          path: benchmarks/baseline.json
+          retention-days: 90
+
+  # ---------------------------------------------------------------
+  # Stage 4: Publish to PyPI (tag push only)
+  # ---------------------------------------------------------------
+  publish:
+    needs: [unit-test, lint, e2e-sqlite, e2e-mysql, e2e-postgres, e2e-oracle]
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tools
+        run: pip install build twine
+
+      - name: Build package
+        run: python -m build
+
+      - name: Verify tag matches version
+        run: |
+          TAG=${GITHUB_REF#refs/tags/v}
+          VERSION=$(python -c "
+          import zipfile, glob
+          whl = glob.glob('dist/*.whl')[0]
+          # Extract version from wheel filename: name-version-...
+          version = whl.split('/')[-1].split('-')[1]
+          print(version)
+          ")
+          echo "Tag: $TAG, Package version: $VERSION"
+          if [ "$TAG" != "$VERSION" ]; then
+            echo "ERROR: Tag v$TAG does not match package version $VERSION"
+            exit 1
+          fi
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: twine upload dist/*

django_fsspec-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+dist/
+build/
+.coverage
+.pytest_cache/
+*.sqlite3

django_fsspec-0.1.0/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+## [0.1.0] - 2026-06-27
+
+### Added
+- Initial release
+- FileNode, StorageBlock, FileBlock models with configurable block size
+- fsspec integration via `DjangoFileSystem` (protocol: `django`)
+- File modes: `rb`, `wb`, `ab`, `xb`
+- Optimistic locking with version field for concurrent write detection
+- Batch block allocation with free block pool reuse
+- Path validation (blacklist + Unicode NFC normalization)
+- Implicit directory support with database-side pushdown queries
+- `RechunkOperation` for block size migration
+- Management commands: `fsspec_gc`, `fsspec_fsck`, `fsspec_stats`
+- Django Admin integration (FileNode read-only)
+- Multi-database support (MySQL, PostgreSQL, Oracle, domestic databases)
+- Namespace-based multi-tenancy

django_fsspec-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 MrLYC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

django_fsspec-0.1.0/PKG-INFO

@@ -0,0 +1,109 @@
+Metadata-Version: 2.4
+Name: django-fsspec
+Version: 0.1.0
+Summary: A Django app that provides a file system interface via fsspec, backed by Django ORM
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: django>=3.2
+Requires-Dist: fsspec
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest-django; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# django-fsspec
+
+A Django app that provides a file system interface via [fsspec](https://filesystem-spec.readthedocs.io/), backed by Django ORM.
+
+## Features
+
+- **fsspec compatible** — use standard `fsspec.filesystem("django")` API
+- **Multi-database** — MySQL, PostgreSQL, Oracle, SQLite, and domestic databases
+- **Configurable block size** — tune storage granularity per deployment
+- **Optimistic locking** — safe concurrent writes with conflict detection
+- **Block pool reuse** — efficient storage with free block recycling
+- **Namespace isolation** — multi-tenant support via integer namespace
+- **Path validation** — blacklist rules + Unicode NFC normalization
+- **Implicit directories** — no directory records, derived from file paths
+- **Management commands** — `fsspec_gc`, `fsspec_fsck`, `fsspec_stats`
+
+## Quick Start
+
+```bash
+pip install django-fsspec
+```
+
+Add to `INSTALLED_APPS`:
+
+```python
+INSTALLED_APPS = [
+    ...
+    "django_fsspec",
+]
+```
+
+Run migrations:
+
+```bash
+python manage.py migrate
+```
+
+Use it:
+
+```python
+import fsspec
+
+fs = fsspec.filesystem("django", namespace=0)
+
+# Write
+with fs.open("/hello.txt", "wb") as f:
+    f.write(b"Hello World")
+
+# Read
+data = fs.cat("/hello.txt")  # b"Hello World"
+
+# List
+fs.ls("/")  # ["/hello.txt"]
+
+# Delete
+fs.rm("/hello.txt")
+```
+
+## Configuration
+
+Add to your Django `settings.py`:
+
+```python
+# Block size in bytes (default: 256KB)
+DJANGO_FSSPEC_BLOCK_SIZE = 64 * 1024
+
+# Maximum file size in bytes (default: 2MB)
+DJANGO_FSSPEC_MAX_FILE_SIZE = 2 * 1024 * 1024
+```
+
+## Supported File Modes
+
+| Mode | Description |
+|------|-------------|
+| `rb` | Read (file must exist) |
+| `wb` | Write (create or overwrite) |
+| `ab` | Append (create or append) |
+| `xb` | Exclusive create (file must not exist) |
+
+## Documentation
+
+- [Getting Started](docs/en/getting-started.md)
+- [Configuration](docs/en/configuration.md)
+- [Usage Guide](docs/en/usage.md)
+- [Architecture](docs/en/architecture.md)
+- [Management Commands](docs/en/management-commands.md)
+- [Block Size Migration](docs/en/migration-guide.md)
+- [Exceptions](docs/en/exceptions.md)
+
+[中文文档](README_zh.md) | [Chinese Documentation](README_zh.md)
+
+## License
+
+MIT — see [LICENSE](LICENSE).

django_fsspec-0.1.0/README.md

@@ -0,0 +1,94 @@
+# django-fsspec
+
+A Django app that provides a file system interface via [fsspec](https://filesystem-spec.readthedocs.io/), backed by Django ORM.
+
+## Features
+
+- **fsspec compatible** — use standard `fsspec.filesystem("django")` API
+- **Multi-database** — MySQL, PostgreSQL, Oracle, SQLite, and domestic databases
+- **Configurable block size** — tune storage granularity per deployment
+- **Optimistic locking** — safe concurrent writes with conflict detection
+- **Block pool reuse** — efficient storage with free block recycling
+- **Namespace isolation** — multi-tenant support via integer namespace
+- **Path validation** — blacklist rules + Unicode NFC normalization
+- **Implicit directories** — no directory records, derived from file paths
+- **Management commands** — `fsspec_gc`, `fsspec_fsck`, `fsspec_stats`
+
+## Quick Start
+
+```bash
+pip install django-fsspec
+```
+
+Add to `INSTALLED_APPS`:
+
+```python
+INSTALLED_APPS = [
+    ...
+    "django_fsspec",
+]
+```
+
+Run migrations:
+
+```bash
+python manage.py migrate
+```
+
+Use it:
+
+```python
+import fsspec
+
+fs = fsspec.filesystem("django", namespace=0)
+
+# Write
+with fs.open("/hello.txt", "wb") as f:
+    f.write(b"Hello World")
+
+# Read
+data = fs.cat("/hello.txt")  # b"Hello World"
+
+# List
+fs.ls("/")  # ["/hello.txt"]
+
+# Delete
+fs.rm("/hello.txt")
+```
+
+## Configuration
+
+Add to your Django `settings.py`:
+
+```python
+# Block size in bytes (default: 256KB)
+DJANGO_FSSPEC_BLOCK_SIZE = 64 * 1024
+
+# Maximum file size in bytes (default: 2MB)
+DJANGO_FSSPEC_MAX_FILE_SIZE = 2 * 1024 * 1024
+```
+
+## Supported File Modes
+
+| Mode | Description |
+|------|-------------|
+| `rb` | Read (file must exist) |
+| `wb` | Write (create or overwrite) |
+| `ab` | Append (create or append) |
+| `xb` | Exclusive create (file must not exist) |
+
+## Documentation
+
+- [Getting Started](docs/en/getting-started.md)
+- [Configuration](docs/en/configuration.md)
+- [Usage Guide](docs/en/usage.md)
+- [Architecture](docs/en/architecture.md)
+- [Management Commands](docs/en/management-commands.md)
+- [Block Size Migration](docs/en/migration-guide.md)
+- [Exceptions](docs/en/exceptions.md)
+
+[中文文档](README_zh.md) | [Chinese Documentation](README_zh.md)
+
+## License
+
+MIT — see [LICENSE](LICENSE).

django_fsspec-0.1.0/README_zh.md

@@ -0,0 +1,94 @@
+# django-fsspec
+
+基于 Django ORM 的文件系统，通过 [fsspec](https://filesystem-spec.readthedocs.io/) 提供标准接口。
+
+## 特性
+
+- **fsspec 兼容** — 使用标准 `fsspec.filesystem("django")` API
+- **多数据库支持** — MySQL、PostgreSQL、Oracle、SQLite、信创数据库
+- **可配置块大小** — 按部署需求调整存储粒度
+- **乐观锁** — 安全的并发写入与冲突检测
+- **块池复用** — 空闲块回收，高效存储
+- **命名空间隔离** — 整数命名空间实现多租户
+- **路径校验** — 黑名单规则 + Unicode NFC 归一化
+- **隐式目录** — 无目录记录，从文件路径推导
+- **管理命令** — `fsspec_gc`、`fsspec_fsck`、`fsspec_stats`
+
+## 快速开始
+
+```bash
+pip install django-fsspec
+```
+
+添加到 `INSTALLED_APPS`：
+
+```python
+INSTALLED_APPS = [
+    ...
+    "django_fsspec",
+]
+```
+
+运行迁移：
+
+```bash
+python manage.py migrate
+```
+
+使用：
+
+```python
+import fsspec
+
+fs = fsspec.filesystem("django", namespace=0)
+
+# 写入
+with fs.open("/hello.txt", "wb") as f:
+    f.write(b"Hello World")
+
+# 读取
+data = fs.cat("/hello.txt")  # b"Hello World"
+
+# 列目录
+fs.ls("/")  # ["/hello.txt"]
+
+# 删除
+fs.rm("/hello.txt")
+```
+
+## 配置
+
+在 Django `settings.py` 中添加：
+
+```python
+# 块大小（字节），默认 256KB
+DJANGO_FSSPEC_BLOCK_SIZE = 64 * 1024
+
+# 文件大小上限（字节），默认 2MB
+DJANGO_FSSPEC_MAX_FILE_SIZE = 2 * 1024 * 1024
+```
+
+## 支持的文件模式
+
+| 模式 | 说明 |
+|------|------|
+| `rb` | 只读（文件必须存在） |
+| `wb` | 写入（创建或覆盖） |
+| `ab` | 追加（创建或追加） |
+| `xb` | 排他创建（文件必须不存在） |
+
+## 文档
+
+- [快速入门](docs/zh/getting-started.md)
+- [配置说明](docs/zh/configuration.md)
+- [使用指南](docs/zh/usage.md)
+- [架构设计](docs/zh/architecture.md)
+- [管理命令](docs/zh/management-commands.md)
+- [块大小迁移](docs/zh/migration-guide.md)
+- [异常体系](docs/zh/exceptions.md)
+
+[English](README.md) | [英文文档](README.md)
+
+## 许可证
+
+MIT — 见 [LICENSE](LICENSE)。