sparklint 0.1.0__py3-none-win_amd64.whl

sparklint/__init__.py

@@ -0,0 +1,38 @@
+"""sparklint: a fast, Spark-aware linter for PySpark code (Rust binary)."""
+
+from __future__ import annotations
+
+import os
+import sys
+import sysconfig
+
+__all__ = ["find_sparklint_bin"]
+
+
+def find_sparklint_bin() -> str:
+    """Return the path to the bundled ``sparklint`` binary."""
+    exe = "sparklint" + (sysconfig.get_config_var("EXE") or "")
+
+    candidates = [
+        os.path.join(sysconfig.get_path("scripts"), exe),
+    ]
+    if sys.version_info >= (3, 10):
+        user_scheme = sysconfig.get_preferred_scheme("user")
+    elif os.name == "nt":
+        user_scheme = "nt_user"
+    elif sys.platform == "darwin" and sys._framework:  # type: ignore[attr-defined]
+        user_scheme = "osx_framework_user"
+    else:
+        user_scheme = "posix_user"
+    candidates.append(os.path.join(sysconfig.get_path("scripts", scheme=user_scheme), exe))
+    # `pip install --target` / unusual layouts: <prefix>/bin next to the package.
+    pkg_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    candidates.append(os.path.join(pkg_root, "bin", exe))
+    candidates.append(os.path.join(os.path.dirname(pkg_root), "bin", exe))
+
+    for path in candidates:
+        if os.path.isfile(path):
+            return path
+    raise FileNotFoundError(
+        "sparklint binary not found; searched: " + ", ".join(candidates)
+    )

sparklint/__main__.py

@@ -0,0 +1,21 @@
+"""``python -m sparklint`` — run the bundled binary."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+from sparklint import find_sparklint_bin
+
+
+def _run() -> None:
+    sparklint = find_sparklint_bin()
+    if sys.platform == "win32":
+        completed = subprocess.run([sparklint, *sys.argv[1:]])
+        sys.exit(completed.returncode)
+    os.execvp(sparklint, [sparklint, *sys.argv[1:]])
+
+
+if __name__ == "__main__":
+    _run()

sparklint-0.1.0.dist-info/METADATA

@@ -0,0 +1,249 @@
+Metadata-Version: 2.4
+Name: sparklint
+Version: 0.1.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+License-File: LICENSE
+License-File: NOTICE
+Summary: A fast, Spark-aware linter for PySpark code and Databricks notebooks, written in Rust.
+Keywords: pyspark,spark,databricks,linter,static-analysis,performance,cost
+Home-Page: https://github.com/clusterbite/sparklint
+Author-email: Marthinus Bosman <hello@clusterbite.com>
+Maintainer-email: ClusterBite <hello@clusterbite.com>
+License: Apache-2.0
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Changelog, https://github.com/clusterbite/sparklint/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://github.com/clusterbite/sparklint#readme
+Project-URL: Homepage, https://github.com/clusterbite/sparklint
+Project-URL: Issues, https://github.com/clusterbite/sparklint/issues
+Project-URL: Repository, https://github.com/clusterbite/sparklint
+Project-URL: Rules, https://github.com/clusterbite/sparklint/blob/main/docs/RULES_CATALOG.md
+
+# sparklint
+
+A fast linter for PySpark code and Databricks notebooks, written in Rust and installed
+with pip.
+
+sparklint looks for Spark code habits that make jobs slow and expensive: unbounded
+`collect()`s, `count()` used as an emptiness check or only for a log line, Python UDFs
+where a built-in function exists, actions and writes inside loops, caches used once,
+cartesian joins, `coalesce(1)` before a write, risky Spark configs, and so on. On
+Databricks these patterns turn directly into extra cluster hours.
+
+It tracks Spark types (SparkSession, DataFrame, Column, GroupedData, Window, RDD, readers,
+writers, streams), so rules fire only on real Spark objects. A `count()` on a Python list
+or a pandas DataFrame is not reported, and DataFrames passed into functions or returned
+from helpers in other modules are still recognised.
+
+## What sparklint does not tell you
+
+sparklint reads code. It cannot tell you which finding costs real money, or how much:
+that depends on how big the data is, how often the job runs and what the cluster costs.
+And many Databricks savings are not in the code at all: cluster and warehouse sizing,
+idle time, job compute versus all-purpose compute, spot instances, tables nobody reads.
+
+[ClusterBite](https://clusterbite.com/?utm_source=github&utm_medium=readme&utm_campaign=sparklint) finds those from your workspace's own usage data and measures what each change saved.
+
+**Status: early (0.x).** Rules, codes and output may still change between minor versions.
+Reports of false positives are especially welcome.
+
+## Install
+
+```bash
+pip install sparklint      # or: uv tool install sparklint / pipx install sparklint
+sparklint --version
+```
+
+Wheels are published for Linux (x86_64, aarch64), macOS (x86_64, arm64) and Windows
+(x86_64). They contain a native binary, so Python is only needed to install it. Other
+platforms build from the source distribution, which needs a Rust toolchain (1.88+).
+
+## Quickstart
+
+```bash
+sparklint check                    # lint the current directory
+sparklint check src/ notebooks/    # .py files, Databricks source notebooks, .ipynb
+sparklint check --fix              # apply safe fixes
+sparklint check --unsafe-fixes     # also apply fixes that can change behaviour
+sparklint check --statistics       # counts per rule
+sparklint rules                    # list the rules
+sparklint rule ACT002              # explain one rule (code or name, e.g. collect-unbounded)
+```
+
+Example: save this as `etl.py` and run `sparklint check etl.py`.
+
+```python
+from pyspark.sql import SparkSession
+
+spark = SparkSession.builder.getOrCreate()
+orders = spark.read.parquet("/data/orders")
+print(f"rows: {orders.count()}")
+if orders.count() > 0:
+    orders.write.mode("overwrite").saveAsTable("analytics.big_orders")
+```
+
+```text
+etl.py:5:23: ACT007 orders.count() is computed only to be logged — a full job for a log line.
+  |
+4 | orders = spark.read.parquet("/data/orders")
+5 | print(f"rows: {orders.count()}")
+  |                       ^^^^^^^ ACT007
+6 | if orders.count() > 0:
+  |
+  = help: Drop it, read numOutputRows from the Delta commit, or use df.observe() to compute the metric in the same job.
+
+etl.py:6:4: ACT006 [*] orders.count() used as an emptiness check scans every row — use orders.isEmpty().
+  |
+5 | print(f"rows: {orders.count()}")
+6 | if orders.count() > 0:
+  |    ^^^^^^^^^^^^^^^^^^ ACT006
+7 |     orders.write.mode("overwrite").saveAsTable("analytics.big_orders")
+  |
+  = fix: Replace with `not orders.isEmpty()`
+
+Found 2 findings in 1 file checked (0.00s).
+[*] 1 fixable with the `--fix` option.
+```
+
+Exit codes: `0` no findings, `1` findings, `2` error (bad config, unknown rule selector,
+missing path). `--exit-zero` exits 0 even when there are findings.
+
+### Output formats
+
+`--output-format text|json|github|sarif`
+
+- `text` (default): ruff-style, with a code frame and the suggested fix.
+- `json`: one object per finding (code, name, severity, location, fix edits).
+- `github`: GitHub Actions workflow annotations.
+- `sarif`: SARIF 2.1.0, for GitHub code scanning and other SARIF viewers.
+
+### Selecting rules
+
+Rule codes are a three-letter family plus three digits: `ACT` actions and driver, `PRF`
+plan and shuffle performance, `UDF` user-defined functions, `IOD` I/O and Delta, `CFG`
+Spark config, `STM` Structured Streaming, `COR` correctness, `DBX` Databricks platform,
+`PDS` pandas-on-Spark, `STY` style.
+
+```bash
+sparklint check --select ACT,PRF          # families or single codes
+sparklint check --extend-select ACT001    # add to the configured selection
+sparklint check --ignore STY              # turn rules off
+sparklint check --select tag:cost         # by tag
+sparklint check --select review           # presets: default, review, strict, cylint-compat, serverless
+```
+
+### Configuration
+
+Configuration goes in `[tool.sparklint]` in `pyproject.toml`, or in `sparklint.toml`
+(the closest file wins):
+
+```toml
+[tool.sparklint]
+extend-select = ["ACT001"]
+ignore = ["STY"]
+extend-exclude = ["generated/**"]
+spark-version = "3.5"        # fixes that need a newer Spark are not offered
+```
+
+Test code usually calls actions on purpose: an assert like `df.count() > 0` or a
+`collect()` on a small fixture is fine there. Turn the action rules off for tests:
+
+```toml
+[tool.sparklint.per-file-ignores]
+"tests/**" = ["ACT"]
+```
+
+All options are listed in [docs/DEVELOPMENT.md](https://github.com/clusterbite/sparklint/blob/main/docs/DEVELOPMENT.md#cli).
+
+### Suppressions
+
+Suppress a finding with `# noqa: ACT002` or `# sparklint: ignore[ACT002]` on the line, or
+`# sparklint: file-ignore[ACT002]` for a whole file. cylint's `# cy:ignore CY001` comments
+are honoured too.
+
+## Rules
+
+Every rule, with its rationale, detection logic and examples, is in
+[docs/RULES_CATALOG.md](https://github.com/clusterbite/sparklint/blob/main/docs/RULES_CATALOG.md).
+`sparklint rules` lists the rules of your installed version, and `sparklint rule <CODE>`
+prints the details of one.
+
+## CI and editors
+
+### GitHub Actions
+
+```yaml
+# .github/workflows/sparklint.yml
+name: sparklint
+on: [push, pull_request]
+jobs:
+  sparklint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - run: pip install sparklint
+      - run: sparklint check --output-format github .
+```
+
+For GitHub code scanning, write SARIF and upload it:
+
+```yaml
+      - run: sparklint check --output-format sarif --exit-zero . > sparklint.sarif
+      - uses: github/codeql-action/upload-sarif@v4
+        with:
+          sarif_file: sparklint.sarif
+```
+
+### pre-commit
+
+With the published wheel (fast, no Rust toolchain needed):
+
+```yaml
+# .pre-commit-config.yaml
+repos:
+  - repo: local
+    hooks:
+      - id: sparklint
+        name: sparklint
+        entry: sparklint check
+        language: python
+        additional_dependencies: ["sparklint==0.1.0"]
+        types_or: [python, jupyter]
+```
+
+This repository also has a `.pre-commit-hooks.yaml`, so
+`repo: https://github.com/clusterbite/sparklint` with `rev: v0.1.0` and `id: sparklint`
+works as well, but pre-commit then builds sparklint from source, which needs a Rust
+toolchain and takes a few minutes the first time.
+
+### Editors
+
+There is no language server yet. Run `sparklint check` from your editor's task runner,
+or feed the `json` / `sarif` output to a plugin that reads them.
+
+## Contributing
+
+Issues and pull requests are welcome; see [CONTRIBUTING.md](https://github.com/clusterbite/sparklint/blob/main/CONTRIBUTING.md). False
+positives are bugs: please include a minimal snippet. For security issues see
+[SECURITY.md](https://github.com/clusterbite/sparklint/blob/main/SECURITY.md). Design notes are in [docs/DESIGN.md](https://github.com/clusterbite/sparklint/blob/main/docs/DESIGN.md).
+
+## License
+
+Apache-2.0. See [LICENSE](https://github.com/clusterbite/sparklint/blob/main/LICENSE) and [NOTICE](https://github.com/clusterbite/sparklint/blob/main/NOTICE).
+
+Built by [ClusterBite](https://clusterbite.com/?utm_source=github&utm_medium=readme&utm_campaign=sparklint),
+which works on Databricks cost optimization.
+

sparklint-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+sparklint/__init__.py,sha256=BONaFUS8D5CWQOA8MsRKBRY4WjdKY77pTGbg8hzKrUo,1372
+sparklint/__main__.py,sha256=Q9yIGlETyQYZjyUF9P95onD_tey7WaxkYEGs-_U6Tio,483
+sparklint-0.1.0.data/scripts/sparklint.exe,sha256=AGYzmAlxwhkFdNSqJf0MMpupkPgIUqEvYRXhVr3PXrA,5891072
+sparklint-0.1.0.dist-info/METADATA,sha256=6D9RALfUeFEwruJNH8Xp-Q9n2SCMXMlMs8CtYofsnck,9879
+sparklint-0.1.0.dist-info/WHEEL,sha256=Uvw7WSmy7Kj2FBk8Va4L_dq4MDW8X5Ynby-pOX3ZNS0,94
+sparklint-0.1.0.dist-info/licenses/LICENSE,sha256=Pd-b5cKP4n2tFDpdx27qJSIq0d1ok0oEcGTlbtL6QMU,11560
+sparklint-0.1.0.dist-info/licenses/NOTICE,sha256=KDHOvPfoqHbzK6aYfMq1dabZAtuC1A23sYb2JJJ3QJM,529
+sparklint-0.1.0.dist-info/sboms/sparklint.cyclonedx.json,sha256=XR6ew3f4428LkacVgkozGy6b6ivcSEk_OidmSpEoMYU,129319
+sparklint-0.1.0.dist-info/RECORD,,

sparklint-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.16.0)
+Root-Is-Purelib: false
+Tag: py3-none-win_amd64

sparklint-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

sparklint-0.1.0.dist-info/licenses/NOTICE

@@ -0,0 +1,14 @@
+sparklint
+Copyright 2026 Marthinus Bosman / ClusterBite
+
+This product is licensed under the Apache License, Version 2.0 (see LICENSE).
+
+Third-party material
+--------------------
+
+Compiled dependencies:
+
+The sparklint binary statically links Rust crates published on crates.io, including the
+Python parser crates of Ruff (https://github.com/astral-sh/ruff, Copyright (c) 2022
+Charles Marsh, MIT License). The licenses of all linked crates are listed in their
+crates.io packages; Cargo.lock lists the exact versions.