tabctx 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tabctx-0.9.0/.github/workflows/code_checks.yml +51 -0
- tabctx-0.9.0/.github/workflows/integration_tests.yml +57 -0
- tabctx-0.9.0/.github/workflows/publish.yml +64 -0
- tabctx-0.9.0/.github/workflows/unit_tests.yml +47 -0
- tabctx-0.9.0/.gitignore +7 -0
- tabctx-0.9.0/.pre-commit-config.yaml +29 -0
- tabctx-0.9.0/.python-version +1 -0
- tabctx-0.9.0/CHANGELOG.md +79 -0
- tabctx-0.9.0/LICENSE.md +201 -0
- tabctx-0.9.0/PKG-INFO +344 -0
- tabctx-0.9.0/README.md +308 -0
- tabctx-0.9.0/ROADMAP.md +35 -0
- tabctx-0.9.0/benchmarks/README.md +50 -0
- tabctx-0.9.0/benchmarks/baselines/v0.5.0-feature-sweep.json +87 -0
- tabctx-0.9.0/benchmarks/baselines/v0.5.0.json +102 -0
- tabctx-0.9.0/benchmarks/baselines/v0.7.0-2replica.json +103 -0
- tabctx-0.9.0/benchmarks/baselines/v0.7.0-vs-hosted-api-cpu.json +55 -0
- tabctx-0.9.0/benchmarks/bench_concurrency.py +442 -0
- tabctx-0.9.0/benchmarks/bench_vs_hosted_api.py +269 -0
- tabctx-0.9.0/benchmarks/calibrate_memory.py +199 -0
- tabctx-0.9.0/benchmarks/gen_calibration_data.py +89 -0
- tabctx-0.9.0/examples/local_fit_predict.py +37 -0
- tabctx-0.9.0/pyproject.toml +78 -0
- tabctx-0.9.0/src/tabctx/__init__.py +41 -0
- tabctx-0.9.0/src/tabctx/backends/__init__.py +3 -0
- tabctx-0.9.0/src/tabctx/backends/base.py +58 -0
- tabctx-0.9.0/src/tabctx/backends/fake.py +99 -0
- tabctx-0.9.0/src/tabctx/backends/tabicl.py +263 -0
- tabctx-0.9.0/src/tabctx/backends/tabpfn.py +147 -0
- tabctx-0.9.0/src/tabctx/batching.py +206 -0
- tabctx-0.9.0/src/tabctx/cache/__init__.py +9 -0
- tabctx-0.9.0/src/tabctx/cache/manager.py +158 -0
- tabctx-0.9.0/src/tabctx/cache/policies.py +29 -0
- tabctx-0.9.0/src/tabctx/cache/spill.py +173 -0
- tabctx-0.9.0/src/tabctx/chunking.py +54 -0
- tabctx-0.9.0/src/tabctx/client.py +285 -0
- tabctx-0.9.0/src/tabctx/engine.py +321 -0
- tabctx-0.9.0/src/tabctx/errors.py +56 -0
- tabctx-0.9.0/src/tabctx/memory/__init__.py +16 -0
- tabctx-0.9.0/src/tabctx/memory/adaptive.py +164 -0
- tabctx-0.9.0/src/tabctx/memory/calibration_data.py +47 -0
- tabctx-0.9.0/src/tabctx/memory/calibration_tabicl_a100.py +121 -0
- tabctx-0.9.0/src/tabctx/memory/estimator.py +198 -0
- tabctx-0.9.0/src/tabctx/serve/__init__.py +0 -0
- tabctx-0.9.0/src/tabctx/serve/affinity.py +91 -0
- tabctx-0.9.0/src/tabctx/serve/app.py +588 -0
- tabctx-0.9.0/src/tabctx/serve/csv_io.py +117 -0
- tabctx-0.9.0/src/tabctx/serve/factory.py +291 -0
- tabctx-0.9.0/src/tabctx/serve/tenancy.py +148 -0
- tabctx-0.9.0/src/tabctx/serve/uploads.py +203 -0
- tabctx-0.9.0/src/tabctx/types.py +35 -0
- tabctx-0.9.0/tests/integration/conftest.py +37 -0
- tabctx-0.9.0/tests/integration/test_client.py +103 -0
- tabctx-0.9.0/tests/integration/test_multi_replica_affinity.py +190 -0
- tabctx-0.9.0/tests/integration/test_tabpfn_backend.py +116 -0
- tabctx-0.9.0/tests/integration/test_upload_flow.py +122 -0
- tabctx-0.9.0/tests/unit/test_adaptive_estimator.py +96 -0
- tabctx-0.9.0/tests/unit/test_batching.py +163 -0
- tabctx-0.9.0/tests/unit/test_cache_manager.py +87 -0
- tabctx-0.9.0/tests/unit/test_chunking.py +37 -0
- tabctx-0.9.0/tests/unit/test_csv_io.py +81 -0
- tabctx-0.9.0/tests/unit/test_engine.py +171 -0
- tabctx-0.9.0/tests/unit/test_memory_estimator.py +69 -0
- tabctx-0.9.0/tests/unit/test_multi_backend_engine.py +103 -0
- tabctx-0.9.0/tests/unit/test_peak_admission.py +156 -0
- tabctx-0.9.0/tests/unit/test_serve_affinity.py +72 -0
- tabctx-0.9.0/tests/unit/test_serve_factory.py +86 -0
- tabctx-0.9.0/tests/unit/test_serve_tenancy.py +91 -0
- tabctx-0.9.0/tests/unit/test_spill.py +111 -0
- tabctx-0.9.0/tests/unit/test_uploads.py +89 -0
- tabctx-0.9.0/uv.lock +3413 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: code checks
|
|
2
|
+
permissions:
|
|
3
|
+
contents: read
|
|
4
|
+
pull-requests: write
|
|
5
|
+
|
|
6
|
+
concurrency:
|
|
7
|
+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
8
|
+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
branches: [main]
|
|
13
|
+
paths:
|
|
14
|
+
- .pre-commit-config.yaml
|
|
15
|
+
- .github/workflows/code_checks.yml
|
|
16
|
+
- '**.py'
|
|
17
|
+
- uv.lock
|
|
18
|
+
- pyproject.toml
|
|
19
|
+
pull_request:
|
|
20
|
+
branches: [main]
|
|
21
|
+
paths:
|
|
22
|
+
- .pre-commit-config.yaml
|
|
23
|
+
- .github/workflows/code_checks.yml
|
|
24
|
+
- '**.py'
|
|
25
|
+
- uv.lock
|
|
26
|
+
- pyproject.toml
|
|
27
|
+
|
|
28
|
+
jobs:
|
|
29
|
+
run-code-check:
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
timeout-minutes: 30
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v4
|
|
34
|
+
|
|
35
|
+
- name: Install uv
|
|
36
|
+
uses: astral-sh/setup-uv@v4
|
|
37
|
+
with:
|
|
38
|
+
enable-cache: true
|
|
39
|
+
|
|
40
|
+
- name: Set up Python
|
|
41
|
+
uses: actions/setup-python@v5
|
|
42
|
+
with:
|
|
43
|
+
python-version-file: ".python-version"
|
|
44
|
+
|
|
45
|
+
- name: Install the project
|
|
46
|
+
run: uv sync --dev
|
|
47
|
+
|
|
48
|
+
- name: Run pre-commit checks
|
|
49
|
+
run: |
|
|
50
|
+
source .venv/bin/activate
|
|
51
|
+
pre-commit run --all-files
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
name: integration tests
|
|
2
|
+
permissions:
|
|
3
|
+
contents: read
|
|
4
|
+
pull-requests: write
|
|
5
|
+
|
|
6
|
+
concurrency:
|
|
7
|
+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
8
|
+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
branches: [main]
|
|
13
|
+
paths:
|
|
14
|
+
- .github/workflows/integration_tests.yml
|
|
15
|
+
- '**.py'
|
|
16
|
+
- uv.lock
|
|
17
|
+
- pyproject.toml
|
|
18
|
+
pull_request:
|
|
19
|
+
branches: [main]
|
|
20
|
+
paths:
|
|
21
|
+
- .github/workflows/integration_tests.yml
|
|
22
|
+
- '**.py'
|
|
23
|
+
- uv.lock
|
|
24
|
+
- pyproject.toml
|
|
25
|
+
|
|
26
|
+
jobs:
|
|
27
|
+
# Boots a real local 2-replica Ray Serve cluster (fake backend, no GPU)
|
|
28
|
+
# and proves the serving contracts end to end: session affinity, tenant
|
|
29
|
+
# scoping, upload/fit-by-reference, the client SDK. Heavier than unit
|
|
30
|
+
# tests (installs ray[serve]) -- kept as its own job on purpose.
|
|
31
|
+
integration-tests:
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
timeout-minutes: 30
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v4
|
|
36
|
+
|
|
37
|
+
- name: Install uv
|
|
38
|
+
uses: astral-sh/setup-uv@v4
|
|
39
|
+
with:
|
|
40
|
+
enable-cache: true
|
|
41
|
+
|
|
42
|
+
- name: Set up Python
|
|
43
|
+
uses: actions/setup-python@v5
|
|
44
|
+
with:
|
|
45
|
+
python-version-file: ".python-version"
|
|
46
|
+
|
|
47
|
+
- name: Install the project (serve extra)
|
|
48
|
+
run: |
|
|
49
|
+
uv venv && uv pip install -e ".[serve,dev]"
|
|
50
|
+
|
|
51
|
+
- name: Run integration tests
|
|
52
|
+
# Invoked via the venv's python, NOT `uv run`: under `uv run`,
|
|
53
|
+
# Ray launches its worker processes through uv, which re-syncs
|
|
54
|
+
# the project env from the lockfile (dev group only -- no
|
|
55
|
+
# ray[serve]) and every replica dies with "No module named
|
|
56
|
+
# 'ray'". Observed on CI run 33260341506.
|
|
57
|
+
run: .venv/bin/python -m pytest tests/integration -q
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Publish tabctx to PyPI + create a GitHub Release on a version tag
|
|
2
|
+
# (e.g. `git tag v0.9.0 && git push --tags`). Modeled on motive's
|
|
3
|
+
# publish workflow (uv build, PyPI Trusted Publishing via OIDC -- no API
|
|
4
|
+
# token stored in GitHub, `pypi` environment as the release gate).
|
|
5
|
+
#
|
|
6
|
+
# ONE-TIME SETUP before the first release:
|
|
7
|
+
# 1. PyPI -> project (or pending publisher) -> Publishing -> add a
|
|
8
|
+
# trusted publisher: owner=VectorInstitute, repo=tabctx,
|
|
9
|
+
# workflow=publish.yml, environment=pypi.
|
|
10
|
+
# 2. GitHub repo -> Settings -> Environments -> create "pypi"
|
|
11
|
+
# (optionally with required reviewers as a release gate).
|
|
12
|
+
name: publish package
|
|
13
|
+
|
|
14
|
+
on:
|
|
15
|
+
push:
|
|
16
|
+
tags: ["v*"]
|
|
17
|
+
|
|
18
|
+
permissions:
|
|
19
|
+
contents: read
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
publish:
|
|
23
|
+
runs-on: ubuntu-latest
|
|
24
|
+
environment: pypi
|
|
25
|
+
permissions:
|
|
26
|
+
id-token: write # OIDC for PyPI Trusted Publishing
|
|
27
|
+
contents: write # GitHub Release creation
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v4
|
|
30
|
+
|
|
31
|
+
- name: Install uv
|
|
32
|
+
uses: astral-sh/setup-uv@v4
|
|
33
|
+
|
|
34
|
+
- name: Set up Python
|
|
35
|
+
uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version-file: ".python-version"
|
|
38
|
+
|
|
39
|
+
- name: Verify tag matches pyproject version
|
|
40
|
+
run: |
|
|
41
|
+
PYPROJECT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
42
|
+
TAG_VERSION="${GITHUB_REF_NAME#v}"
|
|
43
|
+
if [ "$PYPROJECT_VERSION" != "$TAG_VERSION" ]; then
|
|
44
|
+
echo "tag v$TAG_VERSION != pyproject version $PYPROJECT_VERSION" >&2
|
|
45
|
+
exit 1
|
|
46
|
+
fi
|
|
47
|
+
|
|
48
|
+
- name: Run unit tests
|
|
49
|
+
run: |
|
|
50
|
+
uv venv && uv pip install -e ".[dev]"
|
|
51
|
+
uv run pytest tests/unit/ -q
|
|
52
|
+
|
|
53
|
+
- name: Build package
|
|
54
|
+
run: uv build
|
|
55
|
+
|
|
56
|
+
- name: Publish to PyPI
|
|
57
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
58
|
+
|
|
59
|
+
- name: Create GitHub Release
|
|
60
|
+
uses: ncipollo/release-action@v1
|
|
61
|
+
with:
|
|
62
|
+
artifacts: "dist/*"
|
|
63
|
+
generateReleaseNotes: true
|
|
64
|
+
allowUpdates: true
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: unit tests
|
|
2
|
+
permissions:
|
|
3
|
+
contents: read
|
|
4
|
+
pull-requests: write
|
|
5
|
+
|
|
6
|
+
concurrency:
|
|
7
|
+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
8
|
+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
branches: [main]
|
|
13
|
+
paths:
|
|
14
|
+
- .github/workflows/unit_tests.yml
|
|
15
|
+
- '**.py'
|
|
16
|
+
- uv.lock
|
|
17
|
+
- pyproject.toml
|
|
18
|
+
pull_request:
|
|
19
|
+
branches: [main]
|
|
20
|
+
paths:
|
|
21
|
+
- .github/workflows/unit_tests.yml
|
|
22
|
+
- '**.py'
|
|
23
|
+
- uv.lock
|
|
24
|
+
- pyproject.toml
|
|
25
|
+
|
|
26
|
+
jobs:
|
|
27
|
+
unit-tests:
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
timeout-minutes: 30
|
|
30
|
+
steps:
|
|
31
|
+
- uses: actions/checkout@v4
|
|
32
|
+
|
|
33
|
+
- name: Install uv
|
|
34
|
+
uses: astral-sh/setup-uv@v4
|
|
35
|
+
with:
|
|
36
|
+
enable-cache: true
|
|
37
|
+
|
|
38
|
+
- name: Set up Python
|
|
39
|
+
uses: actions/setup-python@v5
|
|
40
|
+
with:
|
|
41
|
+
python-version-file: ".python-version"
|
|
42
|
+
|
|
43
|
+
- name: Install the project
|
|
44
|
+
run: uv sync --dev
|
|
45
|
+
|
|
46
|
+
- name: Run unit tests with coverage
|
|
47
|
+
run: uv run pytest tests/unit --cov tabctx --cov-report=term-missing -q
|
tabctx-0.9.0/.gitignore
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
+
rev: v6.0.0
|
|
4
|
+
hooks:
|
|
5
|
+
- id: trailing-whitespace
|
|
6
|
+
- id: check-ast
|
|
7
|
+
- id: check-builtin-literals
|
|
8
|
+
- id: check-docstring-first
|
|
9
|
+
- id: debug-statements
|
|
10
|
+
- id: end-of-file-fixer
|
|
11
|
+
- id: mixed-line-ending
|
|
12
|
+
args: [--fix=lf]
|
|
13
|
+
- id: check-merge-conflict
|
|
14
|
+
- id: detect-private-key
|
|
15
|
+
- id: check-yaml
|
|
16
|
+
args: [--unsafe]
|
|
17
|
+
- id: check-toml
|
|
18
|
+
|
|
19
|
+
- repo: https://github.com/astral-sh/uv-pre-commit
|
|
20
|
+
rev: 0.12.5
|
|
21
|
+
hooks:
|
|
22
|
+
- id: uv-lock
|
|
23
|
+
|
|
24
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
25
|
+
rev: v0.14.14
|
|
26
|
+
hooks:
|
|
27
|
+
- id: ruff-check
|
|
28
|
+
args: [--fix, --exit-non-zero-on-fix]
|
|
29
|
+
- id: ruff-format
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## [0.9.0] - 2026-08-29
|
|
6
|
+
### Changed
|
|
7
|
+
- **Standard multi-model endpoint (chat-completions style).** One deployment now serves several models over ONE shared context cache and GPU budget: `TABCTX_BACKEND=tabicl,tabpfn` (first listed = default), requests select a model via `"model": "<exact id>"` (`tabicl-v2`, `tabpfn-3` -- named for the checkpoints actually loaded), `GET /v1/models` lists what's served. Contexts remember which model fit them, so predict dispatches automatically; a single shared budget is the point -- two separate deployments on one GPU would each be blind to the other's memory use. Single-model deployments are unchanged. Requires Python >= 3.12.
|
|
8
|
+
- **Admission control now bounds the right quantity, measured on real hardware** (the former Priority-1 rewrite):
|
|
9
|
+
- Backends report TWO measurements per fit: the transient PEAK allocation (what actually OOMs -- can exceed the resident context by an order of magnitude) and the post-fit RESIDENT size. Admission learns from peaks; cache accounting charges residents. (v0.4.0 conflated these -- admission was learning from residents.)
|
|
10
|
+
- Admission is usage-aware: a fit's estimated peak must fit within 85% of device capacity MINUS what the cache already holds resident, so the same shape honestly stops being admissible as the replica fills and recovers after eviction. `/v1/tabctx/limits` reports the live boundary.
|
|
11
|
+
- `AdaptiveMemoryEstimator` ships with a factory-preloaded calibration grid (`memory/calibration_tabicl_a100.py`, generated by `benchmarks/calibrate_memory.py` + `gen_calibration_data.py` from an A100-40GB sweep of TabICL with the kv cache on): measured peaks for shapes up to 100,000 rows x 200 features in "kv" mode and 500,000 rows in "repr" mode (whose contexts are ~10x smaller resident: 1.6GB vs 14.8GB at 100k x 100 -- the high-tenant-density option), with the real boundaries recorded: ~200k rows GPU-OOMs in kv mode, and ~10M-cell tables (200k x 100, 100k x 200) exhaust ~85GB of HOST RAM in the model's CPU-side pipeline -- pod memory limits are the backstop for that. Calibration points carry a 1.1x margin (direct measurements) vs 1.5x for runtime observations. Net effect, validated end to end on an A100: a 100,000 x 10 fit -- rejected by every prior version -- admits and completes in 17s, and fits beyond the measured boundary still 413 cleanly.
|
|
12
|
+
- **Disk spillover tier** (`TABCTX_SPILL_DIR`, off by default): capacity-evicted contexts serialize to local disk and reload transparently on the next request -- no training data needed from the caller; bounded by its own LRU (`TABCTX_SPILL_CAPACITY_BYTES`), best-effort by contract, warm-restart capable. TabICL spills exclude the shared pretrained backbone. Validated under real pressure on an A100 (3 contexts / 39GB spilled, evicted context reloaded with correct predictions). Measured economics are in the module docstring: on slow disks a reload can lose to a re-fit -- enable it for fast NVMe, expensive fits, or callers who can't re-send data.
|
|
13
|
+
- **Evict-ahead-of-fit** (found by the OOM it prevents, on a real A100): when a fit's estimated transient peak exceeds current headroom, cold contexts are evicted/spilled until it fits, instead of rejecting a fit that is safe once they step aside. Admission now runs under the cache lock (no two fits can pass on one snapshot), and the transient budget is capped at 85% of device capacity -- a fit admitted with ~400MB nominal slack at 90% still OOMed to allocator fragmentation.
|
|
14
|
+
- **Fixed a multi-GB eviction leak**: tabicl's `forward_with_cache` leaves a pointer to the active context's kv cache on the (shared) backbone module, so the most recently used context stayed pinned on the GPU after tabctx evicted it -- ~15GB the accounting said was freed but never left the device, eventually OOMing a fit the books called safe. The backend now clears the module pointer after every fit/predict.
|
|
15
|
+
- Cross-context batching: investigated, mapped, and deliberately deferred with the full feasibility map preserved in `batching.py`'s docstring (same-shape wrapper-level batching is exact but narrow; cross-shape needs upstream model changes and has a silent-accuracy trap in SSMax). Same-context coalescing remains the batching layer; real-user traffic will decide whether the narrow case is worth building.
|
|
16
|
+
- Tooling: Python >= 3.12; ruff + pre-commit + uv lockfile; CI split into code checks / unit tests (with coverage) / integration tests; `publish.yml` (PyPI Trusted Publishing, `pypi` environment). README's "Why" section rewritten for first-time readers.
|
|
17
|
+
|
|
18
|
+
## [0.8.0] - 2026-08-29
|
|
19
|
+
### Added
|
|
20
|
+
- **Large-table ingestion: upload, then fit/predict by reference** (ROADMAP Priority 3, inspired by comparing API shapes with PriorLabs' hosted service). New `POST /v1/tabctx/upload` streams a raw CSV to replica-local disk (never buffered in memory, size cap enforced mid-stream via `TABCTX_MAX_UPLOAD_BYTES`, TTL expiry via `TABCTX_UPLOAD_TTL_S`); `fit` accepts `train_upload_id`/`target_column` and `predict` accepts `test_upload_id` as alternatives to inline JSON. Uploads are single-use, tenant-scoped like dataset ids, and replica-local -- correct at `num_replicas >= 2` because the same `x-session-id` affinity contract routes upload, fit, and predicts to one replica. Test-CSV headers are checked against the training schema (reordered columns 422 rather than silently mispredicting). CSV parsing (`serve/csv_io.py`, numpy C parser, float32) and the store (`serve/uploads.py`) are transport-agnostic modules with their own unit tests; the engine is unchanged apart from an O(1) rectangularity fast path for 2D arrays (replacing a per-row Python loop that would cost real time at 10^6 rows). `TabctxClient` gains `upload_csv`/`upload_csv_file`/`fit_uploaded` and `predict(test_upload_id=...)`. New errors: `UploadNotFoundError` (404), `UploadTooLargeError` (413). Transport and admission remain separate layers: a huge CSV uploads fine and the memory-admission gate then rules on the table itself.
|
|
21
|
+
- **TabPFN backend** (`backends/tabpfn.py`, `TABCTX_BACKEND=tabpfn`, `pip install tabctx[tabpfn]`): the second real backend behind the `TabularICLBackend` protocol -- engine, cache, estimator, and serving stack all unchanged, which was the design claim. The TabICL kv-cache lesson applies verbatim: TabPFN's default `fit_mode="fit_preprocessors"` still re-runs the training-context forward pass per predict, so tabctx maps `TABCTX_KV_CACHE` onto it (`kv`->`fit_with_cache`, `repr`->`fit_preprocessors`, `off`->`low_memory`). TabPFN's pretraining-limit rejections surface as 422s, CUDA OOM as `BackendComputeError`. Validated on CPU with the real model (license portal: ux.priorlabs.ai + TABPFN_TOKEN): engine-stack roundtrip, cache-mode prediction equivalence (labels identical; probabilities drift up to ~1e-2 between fit_with_cache and fit_preprocessors due to TabPFN's reduced kv_cache_precision -- documented in the test), regression, and clean 422s past the ~2,000-feature checkpoint limit. GPU validation and TabPFN-specific memory calibration still pending (estimator calibration is TabICL data).
|
|
22
|
+
- `/v1/tabctx/limits` + `TabctxClient.limits()`: capability discovery (inspired by PriorLabs' `get_model_limits`) reporting the live, adaptive admission boundary -- max admissible train rows at representative feature counts -- so clients validate shapes before uploading tables.
|
|
23
|
+
- `benchmarks/bench_vs_hosted_api.py` + `baselines/v0.7.0-vs-hosted-api-cpu.json`: serving-architecture comparison against PriorLabs' hosted TabPFN API on the fit-once-predict-many workload. tabctx warm predicts are 6-14x faster at p50 even with the local side on a laptop CPU (each hosted predict pays upload + network round trips); the hosted side wins cold-fit on larger tables absent a local GPU. Explicitly NOT a model-speed comparison -- the baseline file and script both say so.
|
|
24
|
+
|
|
25
|
+
## [0.7.0] - 2026-08-29
|
|
26
|
+
### Added
|
|
27
|
+
- `tabctx.client.TabctxClient`: pure-stdlib HTTP client that makes the serving contracts impossible to get wrong -- it sends `x-session-id` (== dataset_id, required for correct multi-replica routing) and `x-tabctx-tenant-id` automatically, maps HTTP errors back to the same tabctx exception types the in-process engine raises, and retries 503 backpressure with exponential backoff (safe by construction under strict affinity). End-to-end tested against the local 2-replica deployment.
|
|
28
|
+
|
|
29
|
+
### Performance
|
|
30
|
+
- **TabICL's kv-cache is now enabled by default** (`TabICLBackend(kv_cache="kv")`, env-tunable via `TABCTX_KV_CACHE=kv|repr|off`). tabicl defaults it OFF, and without it every `predict()` silently re-encodes the ENTIRE training set through all three transformer stages -- i.e. the exact repeat cost tabctx exists to eliminate. Found by tracing tabicl's `predict_proba` internals (`transform(mode="both")` re-concatenates train+test per call; with the cache, test rows only attend to fit-time-cached K/V). Verified on CPU that kv-cached predictions are identical to the uncached path (max probability diff ~1e-6) and warm predict is 2.9x faster even for a tiny 40-row context; the win grows with training-set size since the avoided re-encode is O(n_train^2). The cache's GPU memory cost lands inside `fit()`'s measured allocation delta, so cache accounting and the adaptive admission gate price it automatically.
|
|
31
|
+
- **Pretrained backbone weights are loaded once per process and shared across fits** instead of `torch.load` + H2D-transferring hundreds of MB per fitted context (5.8x faster re-fit on CPU). Uses tabicl's own sanctioned sharing pattern (`_unsupervised`/`_finetune` do exactly this). Safe because the engine serializes every backend call behind its cache lock, so the shared module's per-call mutable state is never touched concurrently.
|
|
32
|
+
- **Same-context predict coalescing** (`batching.py`, `CoalescingPredictor`): concurrent requests against one cached context are packed into a single backend call within a small window (`TABCTX_BATCH_WINDOW_MS`, default 5ms; 0 disables) and results are split back per caller. One malformed member is retried individually so it fails alone (422) without poisoning batchmates. Cannot exceed the memory budget: the engine still runs exactly one GPU call at a time and chunks oversized merged batches. `/readyz` reports coalescing counters.
|
|
33
|
+
- `max_ongoing_requests` raised 2 -> 8 (and `max_queued_requests` 8 -> 16): safe since GPU work is serialized by the engine lock regardless; more in-flight requests just means more coalescing opportunity.
|
|
34
|
+
|
|
35
|
+
## [0.6.0] - 2026-08-28
|
|
36
|
+
### Fixed
|
|
37
|
+
- **Multi-replica deployments are now correct** (previously the architecture was only safe at exactly one replica -- ROADMAP.md's former Priority 1). The Ray Serve deployment now ships with Ray's consistent-hash request router (`ray.serve.experimental.consistent_hash_router.ConsistentHashRouter`, requires ray >= 2.58) configured for strict session affinity (`num_fallback_replicas=0`): requests carrying the `x-session-id` header always land on the same replica. The contract, enforced in the new `serve/affinity.py`, is that the session id IS the dataset_id -- clients send `x-session-id: <dataset_id>` on every `/v1/tabctx/fit` and `/v1/tabctx/predict` call (fit adopts the header value as the dataset_id when the body omits one; a mismatched header/body pair is rejected 422 loudly rather than silently mis-routing). Requests without the header behave as before, which only routes correctly on single-replica deployments.
|
|
38
|
+
|
|
39
|
+
### Security
|
|
40
|
+
- **Tenant isolation** (former ROADMAP.md Priority 2): the serving layer now scopes every dataset_id by the caller's `x-tabctx-tenant-id` header before it touches the cache (`serve/tenancy.py`), so a guessed/shared dataset_id no longer reaches another tenant's cached model. `TABCTX_REQUIRE_TENANT=true` makes the header mandatory (401 without it), eliminating the unscoped namespace entirely; the default keeps the header optional for dev/backward compatibility. Tenant identity is caller-supplied and unverified by design (pair with an authenticating proxy for real security -- trust model documented in the module). Malformed tenant ids are rejected 422 rather than silently unscoped; responses never leak the internal scoping prefix.
|
|
41
|
+
|
|
42
|
+
### Added
|
|
43
|
+
- `serve/factory.py`: environment-driven engine construction, shared by any host. `TABCTX_BACKEND=fake` runs the full serve app against the deterministic fake backend (no GPU/torch), which is what makes multi-replica routing testable on a laptop and in CI. `TABCTX_GPU_MEMORY_FRACTION` scales the estimator's admission ceiling and the cache's capacity budget for replicas sharing one physical GPU (e.g. two replicas at `num_gpus: 0.5` on one A100).
|
|
44
|
+
- `served_by` (replica tag) in fit/predict responses and `replica` in `/readyz`, so clients and probes can verify affinity actually pinned a tenant's traffic to one replica.
|
|
45
|
+
- `tests/integration/test_multi_replica_affinity.py`: boots a real local 2-replica Ray Serve cluster (fake backend) and asserts 16 datasets x 6 predicts see zero spurious 404s, every predict lands on its fit replica, and fits spread across both replicas. This is the regression test that would have caught the routing gap immediately; also run in CI (new `integration` job).
|
|
46
|
+
- `benchmarks/bench_concurrency.py` now sends the session header on all tabctx-native calls, so its numbers stay valid at `num_replicas >= 2`.
|
|
47
|
+
|
|
48
|
+
## [0.5.0] - 2026-08-28
|
|
49
|
+
### Fixed
|
|
50
|
+
- Malformed input (mismatched `train_X`/`train_y` lengths, empty tables, ragged rows, zero-feature rows, a `predict()` feature count mismatched against the cached context) now raises `InvalidInputError` -> HTTP 422, instead of reaching the backend as raw numpy/sklearn arrays and surfacing as an unhandled, untranslated exception (a bare 500 with no useful detail). Found via `probe_scale.py`'s malformed-input test against a real deployment.
|
|
51
|
+
|
|
52
|
+
### Context
|
|
53
|
+
Found during a dedicated race-condition/throughput/input-validation test pass (`tests/gke-tabicl-test/probe_scale.py` in inference-platform), run to validate the service is robust to one careless client's bad request -- a real multi-tenant requirement, not just a nice-to-have. That same pass confirmed no data corruption under concurrent `fit()`/`predict()` races on the same `dataset_id`, and measured real sustained throughput (~1.2 fit+predict cycles/sec across 8 concurrent users, p95 latency ~7.7s) -- concrete evidence of the coarse per-replica lock's real-world cost under load, which is already a documented gap.
|
|
54
|
+
|
|
55
|
+
## [0.4.0] - 2026-08-28
|
|
56
|
+
### Added
|
|
57
|
+
- `AdaptiveMemoryEstimator` (`memory/adaptive.py`): wraps the static `PowerLawMemoryEstimator` as a fallback, but the pre-fit admission gate now uses real per-fit measurements (fed back via `engine.fit()` -> `record_observation()`) whenever a past fit at least as large in both rows and features has been observed -- safe by construction (memory cost is assumed monotonic in table size, so a real measurement on a larger-or-equal shape is a valid upper bound for a smaller query), with a configurable safety margin on top. Falls back to the static formula for genuinely novel or larger-than-anything-seen shapes, and always for predict()-time chunking queries (a different, unmeasured quantity). Deployed as the default estimator in `serve/app.py`.
|
|
58
|
+
- `MemoryEstimator` protocol gained `record_observation()`; `PowerLawMemoryEstimator` implements it as a no-op.
|
|
59
|
+
|
|
60
|
+
### Context
|
|
61
|
+
Direct extension of the v0.3.0 cache-accounting fix: now that fit() calls report real measured GPU cost, this closes the loop by feeding that data back into the PRE-FIT admission gate too, so admission decisions get progressively less conservative for shapes the service has actually served -- compounding the multi-tenant capacity improvement from v0.3.0 over the life of a running replica.
|
|
62
|
+
|
|
63
|
+
## [0.3.0] - 2026-08-28
|
|
64
|
+
### Fixed
|
|
65
|
+
- Cache-accounting size for a fitted context now comes from the backend *after* `fit()` runs, not before. `TabICLBackend` now reports the real `torch.cuda.memory_allocated()` delta measured across `fit()` instead of the pre-fit formula-based estimate. Found via extensive multi-tenant load testing on a real A100-40GB: the formula-based estimate was ~14x higher than real GPU memory for a realistic shape (21.85GB accounted vs 1.56GB actually resident), needlessly throttling effective multi-tenant cache capacity to a fraction of what the hardware supports.
|
|
66
|
+
- The pre-fit admission-control gate (which must stay conservative, since nothing has run yet to measure) is unchanged -- this fix only affects how much of the cache's capacity budget a *successfully* fitted context is charged.
|
|
67
|
+
|
|
68
|
+
### Context
|
|
69
|
+
Found and fixed during an extensive multi-tenant test suite (concurrent users, cache eviction under pressure, leak detection) run against a real GKE A100-40GB deployment, in direct response to a requirement that this service support several concurrent users at scale.
|
|
70
|
+
|
|
71
|
+
## [0.2.0] - 2026-08-28
|
|
72
|
+
### Added
|
|
73
|
+
- `/readyz` now reports real GPU memory (`torch.cuda.memory_allocated`/`memory_reserved`) alongside the cache's own byte-accounting, so the two can be compared to catch the estimator drifting from reality or evicted contexts not actually releasing GPU memory.
|
|
74
|
+
|
|
75
|
+
### Context
|
|
76
|
+
Added while running an extensive multi-tenant test suite (concurrent users, cache eviction under pressure, regression task, dataset_id reuse, repeated fit/evict cycles) against a real GKE A100-40GB deployment, per the requirement that this service support several users at scale and be tested accordingly.
|
|
77
|
+
|
|
78
|
+
## [0.1.0] - 2026-08-28
|
|
79
|
+
Initial release. Multi-tenant context caching engine for tabular ICL models (TabICL backend), calibrated memory estimator, estimator-driven test-row chunking, and a Ray Serve deployment. See README.md for full v1 scope and known gaps.
|
tabctx-0.9.0/LICENSE.md
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2026, Vector Institute
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|