finelint 0.2.0__tar.gz

finelint-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 FineLint contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

finelint-0.2.0/PKG-INFO

@@ -0,0 +1,227 @@
+Metadata-Version: 2.4
+Name: finelint
+Version: 0.2.0
+Summary: Facts for fine-tuning data. No models, no verdicts.
+Author: FineLint contributors
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/denizaslan36/FineLint
+Project-URL: Repository, https://github.com/denizaslan36/FineLint
+Project-URL: Issues, https://github.com/denizaslan36/FineLint/issues
+Keywords: fine-tuning,dataset,data-quality,llm,cli
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+<div align="center">
+
+# FineLint
+
+**Facts for fine-tuning data. No models, no verdicts.**
+
+[![Test](https://github.com/denizaslan36/FineLint/actions/workflows/test.yml/badge.svg)](https://github.com/denizaslan36/FineLint/actions/workflows/test.yml)
+[![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/)
+[![AI free](https://img.shields.io/badge/AI-free-2ea44f)](#what-finelint-reports)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+Terminal only · Deterministic · Local · Vendor neutral
+
+</div>
+
+FineLint inspects text datasets before fine-tuning. It identifies malformed chat
+records, duplicate examples, suspicious conversation flows, conflicting targets,
+and leakage between train, validation, and test splits.
+
+FineLint does **not** call a model, create embeddings, use an API, change your source
+files, or label a dataset as good or bad. Every result points back to concrete rows,
+lines, turns, and deterministic evidence.
+
+## Install
+
+```bash
+pipx install finelint
+```
+
+Or install the current source version:
+
+```bash
+git clone https://github.com/denizaslan36/FineLint.git
+cd FineLint
+pipx install .
+```
+
+FineLint has no runtime dependencies and supports Python 3.10 through 3.14.
+
+## Inspect one dataset
+
+```bash
+finelint inspect conversations.jsonl
+```
+
+FineLint detects these chat schemas automatically:
+
+| Schema | Recognized shape |
+| --- | --- |
+| OpenAI chat | `messages` with `role`, `content`, and optional tool calls |
+| ShareGPT | `conversations` with `from` and `value` |
+| Alpaca | `instruction`, optional `input`, and `output` |
+| Generic text | Top-level string fields in CSV or JSONL |
+
+Choose a schema explicitly when needed:
+
+```bash
+finelint inspect conversations.jsonl --schema openai
+finelint inspect sharegpt.jsonl --schema sharegpt
+finelint inspect alpaca.jsonl --schema alpaca
+finelint inspect custom.csv --schema generic --fields prompt --fields response
+```
+
+If a file contains conflicting chat shapes, auto-detection stops and asks for an
+explicit schema instead of guessing.
+
+## Compare dataset splits
+
+```bash
+finelint compare \
+  --split train=train.jsonl \
+  --split validation=validation.jsonl \
+  --split test=test.jsonl
+```
+
+`compare` finds cross-split overlap at two levels:
+
+- whole examples: exact, normalized, and highly similar conversations;
+- training contexts: repeated prompts or histories, including contexts paired with
+  different assistant targets.
+
+Each split is adapted independently, so an OpenAI-formatted train file can be
+compared with a ShareGPT validation file. Override one split when auto-detection is
+not enough:
+
+```bash
+finelint compare \
+  --split train=train.jsonl \
+  --split validation=validation.jsonl \
+  --schema train=openai \
+  --schema validation=sharegpt
+```
+
+## What FineLint reports
+
+| Area | Examples |
+| --- | --- |
+| File structure | Malformed JSONL, non-object rows, empty lines, malformed CSV, wrong row widths |
+| Chat schema | Missing messages, invalid roles or content, empty turns, missing assistant targets |
+| Tool calling | Invalid tool definitions, malformed arguments JSON, undefined tools, orphaned results |
+| Conversation flow | Consecutive roles, late system messages, repeated turns |
+| Duplication | Exact copies, Unicode/case/whitespace-hidden copies, near duplicates |
+| Split leakage | Repeated examples or contexts across named splits, conflicting targets |
+
+Definite schema violations are `error` findings. Patterns that may be intentional are
+`warning` findings. Neither severity is an overall dataset score or verdict.
+
+Near-duplicate candidates come from a bounded deterministic MinHash/LSH index and
+are verified with character 5-gram Jaccard similarity. Adjust the conservative
+default threshold when needed:
+
+```bash
+finelint inspect data.jsonl --similarity-threshold 0.92 --min-text-length 60
+```
+
+## Outputs
+
+Every successful command writes two JSON files:
+
+```text
+conversations.finelint-report.json
+conversations.finelint-affected.json
+```
+
+The report contains metadata, the effective configuration, counts, and complete
+findings. Every finding has a deterministic ID such as `FL-a62db0194f2e9c11`.
+
+The affected-records file is a compact index of every referenced record and its
+reason codes. It deliberately contains no `keep`, `delete`, or `fix` decision.
+
+```json
+{
+  "report_schema_version": 2,
+  "records": [
+    {
+      "split": "validation",
+      "dataset": "/data/validation.jsonl",
+      "row": 204,
+      "line": 204,
+      "turn": 1,
+      "reasons": [
+        {
+          "finding_id": "FL-a62db0194f2e9c11",
+          "code": "CROSS_SPLIT_EXACT_CONTEXT",
+          "category": "split_leakage",
+          "severity": "warning"
+        }
+      ]
+    }
+  ]
+}
+```
+
+Use custom output paths with `--report` and `--affected-records`. FineLint refuses
+to overwrite an input dataset.
+
+A completed scan exits with code `0`, even when it reports findings. Invalid input,
+configuration problems, and fatal processing errors return a nonzero code.
+
+## Generic datasets
+
+The original CSV/JSONL field inspector remains available:
+
+```bash
+finelint inspect data.jsonl \
+  --schema generic \
+  --fields instruction \
+  --fields response \
+  --input-fields instruction \
+  --output-fields response \
+  --required-fields instruction \
+  --id-field id
+```
+
+Field options are repeatable. `--input-fields` and `--output-fields` must be supplied
+together.
+
+## Development
+
+```bash
+git clone https://github.com/denizaslan36/FineLint.git
+cd FineLint
+
+PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src \
+  python3 -m unittest discover -s tests -v
+
+PYTHONPATH=src python3 -m finelint --help
+```
+
+The runtime uses only the Python standard library. Pull requests should preserve
+deterministic output, source-file safety, and the no-model/no-verdict boundary.
+
+## Roadmap
+
+- dataset facts: length, role, source, Unicode script, and template distributions;
+- reproducible project rules through `.finelint.toml` and report diffs;
+- DPO, evaluation, and deterministic PII/secret checks;
+- a stable adapter and rule extension interface.
+
+## License
+
+[MIT](LICENSE)

finelint-0.2.0/README.md

@@ -0,0 +1,202 @@
+<div align="center">
+
+# FineLint
+
+**Facts for fine-tuning data. No models, no verdicts.**
+
+[![Test](https://github.com/denizaslan36/FineLint/actions/workflows/test.yml/badge.svg)](https://github.com/denizaslan36/FineLint/actions/workflows/test.yml)
+[![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/)
+[![AI free](https://img.shields.io/badge/AI-free-2ea44f)](#what-finelint-reports)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+Terminal only · Deterministic · Local · Vendor neutral
+
+</div>
+
+FineLint inspects text datasets before fine-tuning. It identifies malformed chat
+records, duplicate examples, suspicious conversation flows, conflicting targets,
+and leakage between train, validation, and test splits.
+
+FineLint does **not** call a model, create embeddings, use an API, change your source
+files, or label a dataset as good or bad. Every result points back to concrete rows,
+lines, turns, and deterministic evidence.
+
+## Install
+
+```bash
+pipx install finelint
+```
+
+Or install the current source version:
+
+```bash
+git clone https://github.com/denizaslan36/FineLint.git
+cd FineLint
+pipx install .
+```
+
+FineLint has no runtime dependencies and supports Python 3.10 through 3.14.
+
+## Inspect one dataset
+
+```bash
+finelint inspect conversations.jsonl
+```
+
+FineLint detects these chat schemas automatically:
+
+| Schema | Recognized shape |
+| --- | --- |
+| OpenAI chat | `messages` with `role`, `content`, and optional tool calls |
+| ShareGPT | `conversations` with `from` and `value` |
+| Alpaca | `instruction`, optional `input`, and `output` |
+| Generic text | Top-level string fields in CSV or JSONL |
+
+Choose a schema explicitly when needed:
+
+```bash
+finelint inspect conversations.jsonl --schema openai
+finelint inspect sharegpt.jsonl --schema sharegpt
+finelint inspect alpaca.jsonl --schema alpaca
+finelint inspect custom.csv --schema generic --fields prompt --fields response
+```
+
+If a file contains conflicting chat shapes, auto-detection stops and asks for an
+explicit schema instead of guessing.
+
+## Compare dataset splits
+
+```bash
+finelint compare \
+  --split train=train.jsonl \
+  --split validation=validation.jsonl \
+  --split test=test.jsonl
+```
+
+`compare` finds cross-split overlap at two levels:
+
+- whole examples: exact, normalized, and highly similar conversations;
+- training contexts: repeated prompts or histories, including contexts paired with
+  different assistant targets.
+
+Each split is adapted independently, so an OpenAI-formatted train file can be
+compared with a ShareGPT validation file. Override one split when auto-detection is
+not enough:
+
+```bash
+finelint compare \
+  --split train=train.jsonl \
+  --split validation=validation.jsonl \
+  --schema train=openai \
+  --schema validation=sharegpt
+```
+
+## What FineLint reports
+
+| Area | Examples |
+| --- | --- |
+| File structure | Malformed JSONL, non-object rows, empty lines, malformed CSV, wrong row widths |
+| Chat schema | Missing messages, invalid roles or content, empty turns, missing assistant targets |
+| Tool calling | Invalid tool definitions, malformed arguments JSON, undefined tools, orphaned results |
+| Conversation flow | Consecutive roles, late system messages, repeated turns |
+| Duplication | Exact copies, Unicode/case/whitespace-hidden copies, near duplicates |
+| Split leakage | Repeated examples or contexts across named splits, conflicting targets |
+
+Definite schema violations are `error` findings. Patterns that may be intentional are
+`warning` findings. Neither severity is an overall dataset score or verdict.
+
+Near-duplicate candidates come from a bounded deterministic MinHash/LSH index and
+are verified with character 5-gram Jaccard similarity. Adjust the conservative
+default threshold when needed:
+
+```bash
+finelint inspect data.jsonl --similarity-threshold 0.92 --min-text-length 60
+```
+
+## Outputs
+
+Every successful command writes two JSON files:
+
+```text
+conversations.finelint-report.json
+conversations.finelint-affected.json
+```
+
+The report contains metadata, the effective configuration, counts, and complete
+findings. Every finding has a deterministic ID such as `FL-a62db0194f2e9c11`.
+
+The affected-records file is a compact index of every referenced record and its
+reason codes. It deliberately contains no `keep`, `delete`, or `fix` decision.
+
+```json
+{
+  "report_schema_version": 2,
+  "records": [
+    {
+      "split": "validation",
+      "dataset": "/data/validation.jsonl",
+      "row": 204,
+      "line": 204,
+      "turn": 1,
+      "reasons": [
+        {
+          "finding_id": "FL-a62db0194f2e9c11",
+          "code": "CROSS_SPLIT_EXACT_CONTEXT",
+          "category": "split_leakage",
+          "severity": "warning"
+        }
+      ]
+    }
+  ]
+}
+```
+
+Use custom output paths with `--report` and `--affected-records`. FineLint refuses
+to overwrite an input dataset.
+
+A completed scan exits with code `0`, even when it reports findings. Invalid input,
+configuration problems, and fatal processing errors return a nonzero code.
+
+## Generic datasets
+
+The original CSV/JSONL field inspector remains available:
+
+```bash
+finelint inspect data.jsonl \
+  --schema generic \
+  --fields instruction \
+  --fields response \
+  --input-fields instruction \
+  --output-fields response \
+  --required-fields instruction \
+  --id-field id
+```
+
+Field options are repeatable. `--input-fields` and `--output-fields` must be supplied
+together.
+
+## Development
+
+```bash
+git clone https://github.com/denizaslan36/FineLint.git
+cd FineLint
+
+PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src \
+  python3 -m unittest discover -s tests -v
+
+PYTHONPATH=src python3 -m finelint --help
+```
+
+The runtime uses only the Python standard library. Pull requests should preserve
+deterministic output, source-file safety, and the no-model/no-verdict boundary.
+
+## Roadmap
+
+- dataset facts: length, role, source, Unicode script, and template distributions;
+- reproducible project rules through `.finelint.toml` and report diffs;
+- DPO, evaluation, and deterministic PII/secret checks;
+- a stable adapter and rule extension interface.
+
+## License
+
+[MIT](LICENSE)

finelint-0.2.0/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=69"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "finelint"
+version = "0.2.0"
+description = "Facts for fine-tuning data. No models, no verdicts."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "FineLint contributors" }]
+dependencies = []
+keywords = ["fine-tuning", "dataset", "data-quality", "llm", "cli"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Software Development :: Quality Assurance",
+]
+
+[project.urls]
+Homepage = "https://github.com/denizaslan36/FineLint"
+Repository = "https://github.com/denizaslan36/FineLint"
+Issues = "https://github.com/denizaslan36/FineLint/issues"
+
+[project.scripts]
+finelint = "finelint.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]

finelint-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

finelint-0.2.0/src/finelint/__init__.py

@@ -0,0 +1,3 @@
+"""FineLint: deterministic facts for fine-tuning datasets."""
+
+__version__ = "0.2.0"

finelint-0.2.0/src/finelint/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()