samyak 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. samyak-0.1.0/.gitignore +49 -0
  2. samyak-0.1.0/LICENSE +21 -0
  3. samyak-0.1.0/PKG-INFO +325 -0
  4. samyak-0.1.0/README.md +296 -0
  5. samyak-0.1.0/examples/sample-corpus/duplicate_policy_copy.txt +8 -0
  6. samyak-0.1.0/examples/sample-corpus/empty_doc.txt +0 -0
  7. samyak-0.1.0/examples/sample-corpus/noisy_repeated.txt +10 -0
  8. samyak-0.1.0/examples/sample-corpus/normal_overview.md +21 -0
  9. samyak-0.1.0/examples/sample-corpus/normal_policy.txt +8 -0
  10. samyak-0.1.0/examples/sample-corpus/notes.pdf +1 -0
  11. samyak-0.1.0/examples/sample-corpus/tiny_note.txt +1 -0
  12. samyak-0.1.0/pyproject.toml +79 -0
  13. samyak-0.1.0/src/samyak/__init__.py +21 -0
  14. samyak-0.1.0/src/samyak/__main__.py +6 -0
  15. samyak-0.1.0/src/samyak/__version__.py +3 -0
  16. samyak-0.1.0/src/samyak/cli.py +157 -0
  17. samyak-0.1.0/src/samyak/corpus/__init__.py +15 -0
  18. samyak-0.1.0/src/samyak/corpus/analyzers/__init__.py +4 -0
  19. samyak-0.1.0/src/samyak/corpus/analyzers/_helpers.py +18 -0
  20. samyak-0.1.0/src/samyak/corpus/analyzers/accumulator.py +64 -0
  21. samyak-0.1.0/src/samyak/corpus/analyzers/document_pass.py +231 -0
  22. samyak-0.1.0/src/samyak/corpus/analyzers/findings_builder.py +633 -0
  23. samyak-0.1.0/src/samyak/corpus/analyzers/hashing.py +51 -0
  24. samyak-0.1.0/src/samyak/corpus/analyzers/text_stats.py +305 -0
  25. samyak-0.1.0/src/samyak/corpus/config.py +127 -0
  26. samyak-0.1.0/src/samyak/corpus/discovery.py +163 -0
  27. samyak-0.1.0/src/samyak/corpus/html_report.py +323 -0
  28. samyak-0.1.0/src/samyak/corpus/loaders/__init__.py +36 -0
  29. samyak-0.1.0/src/samyak/corpus/loaders/errors.py +5 -0
  30. samyak-0.1.0/src/samyak/corpus/loaders/html.py +165 -0
  31. samyak-0.1.0/src/samyak/corpus/loaders/pdf.py +161 -0
  32. samyak-0.1.0/src/samyak/corpus/loaders/text.py +47 -0
  33. samyak-0.1.0/src/samyak/corpus/models.py +155 -0
  34. samyak-0.1.0/src/samyak/corpus/pipeline.py +228 -0
  35. samyak-0.1.0/src/samyak/corpus/report.py +134 -0
  36. samyak-0.1.0/src/samyak/py.typed +0 -0
  37. samyak-0.1.0/tests/conftest.py +1 -0
  38. samyak-0.1.0/tests/helpers.py +19 -0
  39. samyak-0.1.0/tests/test_analyze.py +464 -0
  40. samyak-0.1.0/tests/test_config.py +85 -0
  41. samyak-0.1.0/tests/test_discovery.py +212 -0
  42. samyak-0.1.0/tests/test_html_report.py +272 -0
  43. samyak-0.1.0/tests/test_inputs.py +121 -0
  44. samyak-0.1.0/tests/test_packaging.py +40 -0
  45. samyak-0.1.0/tests/test_pdf_html.py +291 -0
  46. samyak-0.1.0/tests/test_product.py +222 -0
  47. samyak-0.1.0/tests/test_streaming.py +119 -0
  48. samyak-0.1.0/tests/test_text_stats.py +132 -0
@@ -0,0 +1,49 @@
1
+ # OS / editor
2
+ .DS_Store
3
+ Thumbs.db
4
+ *.swp
5
+ *~
6
+ .idea/
7
+ .vscode/
8
+ *.code-workspace
9
+
10
+ # Private data (must never be committed)
11
+ .private/
12
+
13
+ # Local analysis reports generated in the repo root (must never be committed)
14
+ /report.html
15
+ /report.json
16
+ /report.txt
17
+ /corpus-report.json
18
+
19
+ # Python
20
+ __pycache__/
21
+ *.py[cod]
22
+ *$py.class
23
+ *.egg-info/
24
+ .eggs/
25
+ dist/
26
+ build/
27
+ .venv/
28
+ venv/
29
+ .env
30
+ .env.*
31
+
32
+ # Node / JS (keep ignores ready; no JS tooling yet)
33
+ node_modules/
34
+ npm-debug.log*
35
+ yarn-debug.log*
36
+ yarn-error.log*
37
+ .pnpm-store/
38
+
39
+ # Test / coverage / type checkers
40
+ .pytest_cache/
41
+ .mypy_cache/
42
+ .ruff_cache/
43
+ .coverage
44
+ htmlcov/
45
+ coverage/
46
+
47
+ # Local env / secrets
48
+ *.pem
49
+ secrets/
samyak-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 DataAIHub
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
samyak-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,325 @@
1
+ Metadata-Version: 2.5
2
+ Name: samyak
3
+ Version: 0.1.0
4
+ Summary: AI engineering tooling for building reliable AI applications.
5
+ Project-URL: Homepage, https://www.dataaihub.co
6
+ Project-URL: Repository, https://github.com/tapansharma04/dataaihub-ai-engineering-toolkit
7
+ Project-URL: Issues, https://github.com/tapansharma04/dataaihub-ai-engineering-toolkit/issues
8
+ Author: DataAIHub
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: ai-engineering,corpus,data-quality,rag,retrieval-augmented-generation,samyak
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Software Development :: Quality Assurance
21
+ Classifier: Typing :: Typed
22
+ Requires-Python: >=3.12
23
+ Requires-Dist: beautifulsoup4>=4.12
24
+ Requires-Dist: pypdf>=5.0
25
+ Provides-Extra: dev
26
+ Requires-Dist: pytest>=8.0; extra == 'dev'
27
+ Requires-Dist: ruff>=0.6; extra == 'dev'
28
+ Description-Content-Type: text/markdown
29
+
30
+ # Samyak
31
+
32
+ AI engineering tooling for building reliable AI applications.
33
+
34
+ **Samyak is an open-source project by [DataAIHub](https://www.dataaihub.co).**
35
+
36
+ The current release provides **Corpus Intelligence** for analyzing document collections before they are used in AI and RAG systems.
37
+
38
+ ## Corpus Intelligence
39
+
40
+ Analyze your document corpus before it becomes a RAG problem.
41
+
42
+ Many AI/RAG issues start in the source corpus — empty files, duplicates, tiny stubs, oversized dumps, noisy extraction — before retrieval, embeddings, or generation are involved.
43
+
44
+ Corpus Intelligence inspects a local document directory, collects evidence, explains why findings matter, and recommends practical next actions. It identifies characteristics that **may** affect downstream systems. It does **not** guarantee or predict retrieval or answer quality.
45
+
46
+ ## Who it is for
47
+
48
+ - AI / ML engineers preparing document corpora for RAG or other retrieval workflows
49
+ - Data engineers validating knowledge-base handoffs
50
+ - Platform and QA teams adding corpus checks to local workflows or CI
51
+
52
+ ## Features (v0.1)
53
+
54
+ - Corpus inventory (supported / unsupported / analyzed, by format)
55
+ - Empty and whitespace-only document detection
56
+ - Very small / low-information document signals
57
+ - Very large document heuristics
58
+ - Exact duplicate detection (normalized content hashing)
59
+ - Lightweight text-quality / noise indicators
60
+ - Simple chunkability indicators (not a chunking engine)
61
+ - **PDF** text-layer extraction with page-level stats (no OCR)
62
+ - PDF-specific checks (OCR-likely pages, headers/footers, complexity, table-like text)
63
+ - **HTML** local parsing (scripts/styles removed; no network fetches)
64
+ - HTML-specific checks (boilerplate, low content ratio, code-heavy pages)
65
+ - Human-readable CLI report, JSON output, and self-contained HTML reports
66
+ - Incremental corpus processing (documents are not all held in memory at once)
67
+ - Progress on stderr for large corpora (`--no-progress` to disable)
68
+ - Fully local analysis — no network, no API keys, no telemetry
69
+
70
+ ## Privacy / local processing
71
+
72
+ Samyak:
73
+
74
+ - processes documents on your machine
75
+ - makes **no** network or API calls
76
+ - sends **no** document content externally
77
+ - includes **no** telemetry
78
+ - requires **no** account and **no** API key
79
+
80
+ Treat corpus files as untrusted input: contents are never executed or interpreted as instructions.
81
+
82
+ ## Supported formats (v0.1)
83
+
84
+ | Extension | Support |
85
+ | --- | --- |
86
+ | `.txt` | Yes |
87
+ | `.md` | Yes |
88
+ | `.pdf` | Yes (text layer via `pypdf`; no OCR) |
89
+ | `.html` / `.htm` | Yes (local files via BeautifulSoup; no crawling) |
90
+ | Other formats | Discovered and reported as unsupported (analysis continues) |
91
+
92
+ Point Samyak at a **document corpus directory**, not a source-code repository. Discovery is recursive and will include unsupported files it finds (for example under `.git` or `node_modules`) as inventory, not as a project linter.
93
+
94
+ ## Installation
95
+
96
+ Install from PyPI to use Samyak:
97
+
98
+ ```bash
99
+ pip install samyak
100
+ ```
101
+
102
+ Requires Python 3.12+.
103
+
104
+ The CLI command is:
105
+
106
+ ```bash
107
+ samyak
108
+ ```
109
+
110
+ ## Quick start
111
+
112
+ Run Corpus Intelligence on a local document directory:
113
+
114
+ ```bash
115
+ samyak corpus ./documents
116
+ ```
117
+
118
+ JSON:
119
+
120
+ ```bash
121
+ samyak corpus ./documents --format json
122
+ ```
123
+
124
+ HTML (self-contained file; open locally, no network required):
125
+
126
+ ```bash
127
+ samyak corpus ./documents --format html --output report.html
128
+ ```
129
+
130
+ Write a JSON report file:
131
+
132
+ ```bash
133
+ samyak corpus ./documents --format json --output report.json
134
+ ```
135
+
136
+ Help and version:
137
+
138
+ ```bash
139
+ samyak --help
140
+ samyak corpus --help
141
+ samyak --version
142
+ ```
143
+
144
+ The `examples/sample-corpus` directory is in this Git repository (and the source distribution). It is **not** included when you `pip install samyak`. After installing Samyak, clone the repository and analyze the example directory:
145
+
146
+ ```bash
147
+ pip install samyak
148
+ git clone https://github.com/tapansharma04/dataaihub-ai-engineering-toolkit.git
149
+ samyak corpus dataaihub-ai-engineering-toolkit/examples/sample-corpus
150
+ ```
151
+
152
+ ## Checks implemented
153
+
154
+ | Check | What it looks for |
155
+ | --- | --- |
156
+ | Inventory | Discovered, supported, unsupported, analyzed counts and size stats |
157
+ | Empty documents | No meaningful text after stripping whitespace |
158
+ | Very small documents | Meaningful text below a documented character threshold (default: 100) |
159
+ | Very large documents | Meaningful text at/above a documented threshold (default: 100,000) |
160
+ | Exact duplicates | Identical normalized textual content (SHA-256 of normalized text) |
161
+ | Text quality | Conservative heuristics: high symbol ratio, excessive whitespace, repeated lines |
162
+ | Chunkability | Long uninterrupted blocks; documents dominated by very short lines |
163
+ | PDF (format-specific) | OCR-likely/text-poor pages, uneven page text, repeated headers/footers, extraction anomalies, large/complex PDFs, table-like text (heuristic) |
164
+ | HTML (format-specific) | Boilerplate domination, low main-content ratio, large pages, code-heavy pages, repeated navigation-like lines |
165
+ | Load failures | Supported files that could not be parsed (analysis continues) |
166
+ | Discovery access failures | Paths that could not be listed or inspected (analysis continues) |
167
+
168
+ Default thresholds are documented on `AnalysisConfig` and included in JSON output under `config`. Override size thresholds via `--small-chars` / `--large-chars`.
169
+
170
+ ## Example output (text)
171
+
172
+ ```text
173
+ Samyak Corpus Intelligence Report
174
+ =================================
175
+
176
+ Product: samyak
177
+ Capability: corpus
178
+ Version: 0.1.0
179
+
180
+ Corpus
181
+ ------
182
+ Documents analyzed: ...
183
+ Unsupported files: ...
184
+
185
+ Findings
186
+ --------
187
+
188
+ HIGH Exact duplicate documents
189
+
190
+ 2 documents belong to 1 exact-duplicate group(s).
191
+
192
+ Why it matters:
193
+ Duplicate content can produce redundant retrieval results...
194
+
195
+ Recommendation:
196
+ Review and remove duplicate documents before indexing...
197
+
198
+ Summary
199
+ -------
200
+ High: ...
201
+ Medium: ...
202
+ Low: ...
203
+ Info: ...
204
+ ```
205
+
206
+ ## JSON output
207
+
208
+ JSON includes:
209
+
210
+ - `product` / `capability` / `version` (product identity)
211
+ - `config` (thresholds used)
212
+ - `summary` (inventory and size statistics)
213
+ - `findings` (code, category, severity, message, why_it_matters, recommendation, evidence, affected_documents)
214
+ - `severity_counts`
215
+
216
+ Example identity fields:
217
+
218
+ ```json
219
+ {
220
+ "product": "samyak",
221
+ "capability": "corpus",
222
+ "version": "0.1.0"
223
+ }
224
+ ```
225
+
226
+ Paths in reports are **relative to the corpus root** whenever possible.
227
+
228
+ ### HTML output
229
+
230
+ `--format html` renders the same `AnalysisReport` as a standalone HTML page. Open the file in a browser; it requires no network access, JavaScript framework, or external assets.
231
+
232
+ ```bash
233
+ samyak corpus ./documents --format html --output report.html
234
+ ```
235
+
236
+ Omit `--output` to print HTML to stdout (same contract as text and JSON).
237
+
238
+ Exit codes (v0.1):
239
+
240
+ - `0` — analysis completed (findings do not change the exit code)
241
+ - `2` — invalid path, invalid configuration, missing subcommand, or I/O failure writing `--output`
242
+
243
+ Findings (including `HIGH`) do **not** fail the process by themselves. v0.1 reports findings but does not fail CI based on finding severity.
244
+
245
+ ### CI usage
246
+
247
+ ```bash
248
+ samyak corpus ./documents --format json --output corpus-report.json
249
+ ```
250
+
251
+ Keep stdout JSON-only for piping; progress messages go to stderr. Use `--no-progress` when you want silent stderr.
252
+
253
+ ## Library usage
254
+
255
+ ```python
256
+ from samyak import AnalysisConfig, analyze_corpus
257
+
258
+ report = analyze_corpus("./documents")
259
+ print(report.product, report.capability, report.version)
260
+ print(report.severity_counts())
261
+ for finding in report.findings:
262
+ print(finding.severity, finding.title, finding.recommendation)
263
+
264
+ report = analyze_corpus(
265
+ "./documents",
266
+ config=AnalysisConfig(small_document_chars=50),
267
+ )
268
+ ```
269
+
270
+ ## Validation & scale characteristics
271
+
272
+ Corpus Intelligence is validated beyond unit tests using public corpora and controlled defect injection. This repository does **not** redistribute third-party dataset files.
273
+
274
+ Documents are processed **incrementally**: peak memory for multi-document corpora is driven primarily by the largest active document plus compact corpus metadata (hashes, counters, bounded finding samples), not by total corpus text size.
275
+
276
+ Measured on Apple M1, 16 GB RAM, macOS arm64, Python 3.12 for v0.1.0:
277
+
278
+ | Scenario | Approximate result |
279
+ | --- | --- |
280
+ | Multi-GB text corpora (≈250 MB → ≈5 GB) | Completes with low hundreds of MB peak RSS (not proportional to full corpus size) |
281
+ | Many small files (up to 100,000 docs) | Completes; runtime scales primarily with file count |
282
+ | Very large individual files (10–250 MB) | Peak RSS scales with the active file (typically several times the file size) |
283
+
284
+ Controlled detection validation covers empty documents, exact duplicates, size outliers, fragmentation/noise signals, PDF OCR-likely/text-poor pages, and HTML code-heavy pages. Performance varies by hardware, storage, and document formats. PDF analysis is substantially more expensive than plain text. These figures are measured characteristics, not guarantees.
285
+
286
+ ## Limitations
287
+
288
+ - No OCR — scanned/image-only PDFs are *detected* but not converted to text
289
+ - PDF table detection is a conservative text heuristic, not a table extractor
290
+ - HTML main-content / boilerplate detection is heuristic; site structure varies widely
291
+ - No DOCX or other office formats yet
292
+ - No near-duplicate / semantic duplicate detection
293
+ - No embeddings, vector databases, or LLM judges
294
+ - Findings identify characteristics that **may** cause ingestion/chunking/retrieval problems; they do **not** predict downstream answer quality
295
+ - Thresholds may need adjustment for specialized document collections
296
+ - No single readiness score — findings are evidence-based, not oracles
297
+ - Very large *individual* documents still require memory proportional to that file (PDF and HTML included)
298
+ - Discovery materializes the list of discovered files; duplicate tracking is proportional to file count
299
+ - Finding path lists in reports are bounded samples; full counts remain in evidence
300
+ - Multi-gigabyte *corpora* are processed incrementally and do not require holding all document text in RAM
301
+
302
+ ## Samyak by DataAIHub
303
+
304
+ | Project | Role |
305
+ | --- | --- |
306
+ | [DataAIHub](https://www.dataaihub.co) | Knowledge, ecosystem, research, guides, comparisons, and discovery |
307
+ | [DataAIHub Cookbook](https://github.com/tapansharma04/dataaihub-cookbook) | Practical, runnable examples for learning AI engineering patterns |
308
+ | **[Samyak](https://github.com/tapansharma04/dataaihub-ai-engineering-toolkit)** (this repository) | Open-source AI engineering product; current capability: Corpus Intelligence |
309
+
310
+ ## Development
311
+
312
+ Requirements: Python 3.12+
313
+
314
+ ```bash
315
+ python -m pip install -e ".[dev]"
316
+ pytest
317
+ ruff check src tests
318
+ ruff format --check src tests
319
+ ```
320
+
321
+ Tests require the package to be installed (editable is fine) so distribution metadata can be inspected.
322
+
323
+ ## License
324
+
325
+ This project is licensed under the MIT License. See [LICENSE](LICENSE).
samyak-0.1.0/README.md ADDED
@@ -0,0 +1,296 @@
1
+ # Samyak
2
+
3
+ AI engineering tooling for building reliable AI applications.
4
+
5
+ **Samyak is an open-source project by [DataAIHub](https://www.dataaihub.co).**
6
+
7
+ The current release provides **Corpus Intelligence** for analyzing document collections before they are used in AI and RAG systems.
8
+
9
+ ## Corpus Intelligence
10
+
11
+ Analyze your document corpus before it becomes a RAG problem.
12
+
13
+ Many AI/RAG issues start in the source corpus — empty files, duplicates, tiny stubs, oversized dumps, noisy extraction — before retrieval, embeddings, or generation are involved.
14
+
15
+ Corpus Intelligence inspects a local document directory, collects evidence, explains why findings matter, and recommends practical next actions. It identifies characteristics that **may** affect downstream systems. It does **not** guarantee or predict retrieval or answer quality.
16
+
17
+ ## Who it is for
18
+
19
+ - AI / ML engineers preparing document corpora for RAG or other retrieval workflows
20
+ - Data engineers validating knowledge-base handoffs
21
+ - Platform and QA teams adding corpus checks to local workflows or CI
22
+
23
+ ## Features (v0.1)
24
+
25
+ - Corpus inventory (supported / unsupported / analyzed, by format)
26
+ - Empty and whitespace-only document detection
27
+ - Very small / low-information document signals
28
+ - Very large document heuristics
29
+ - Exact duplicate detection (normalized content hashing)
30
+ - Lightweight text-quality / noise indicators
31
+ - Simple chunkability indicators (not a chunking engine)
32
+ - **PDF** text-layer extraction with page-level stats (no OCR)
33
+ - PDF-specific checks (OCR-likely pages, headers/footers, complexity, table-like text)
34
+ - **HTML** local parsing (scripts/styles removed; no network fetches)
35
+ - HTML-specific checks (boilerplate, low content ratio, code-heavy pages)
36
+ - Human-readable CLI report, JSON output, and self-contained HTML reports
37
+ - Incremental corpus processing (documents are not all held in memory at once)
38
+ - Progress on stderr for large corpora (`--no-progress` to disable)
39
+ - Fully local analysis — no network, no API keys, no telemetry
40
+
41
+ ## Privacy / local processing
42
+
43
+ Samyak:
44
+
45
+ - processes documents on your machine
46
+ - makes **no** network or API calls
47
+ - sends **no** document content externally
48
+ - includes **no** telemetry
49
+ - requires **no** account and **no** API key
50
+
51
+ Treat corpus files as untrusted input: contents are never executed or interpreted as instructions.
52
+
53
+ ## Supported formats (v0.1)
54
+
55
+ | Extension | Support |
56
+ | --- | --- |
57
+ | `.txt` | Yes |
58
+ | `.md` | Yes |
59
+ | `.pdf` | Yes (text layer via `pypdf`; no OCR) |
60
+ | `.html` / `.htm` | Yes (local files via BeautifulSoup; no crawling) |
61
+ | Other formats | Discovered and reported as unsupported (analysis continues) |
62
+
63
+ Point Samyak at a **document corpus directory**, not a source-code repository. Discovery is recursive and will include unsupported files it finds (for example under `.git` or `node_modules`) as inventory, not as a project linter.
64
+
65
+ ## Installation
66
+
67
+ Install from PyPI to use Samyak:
68
+
69
+ ```bash
70
+ pip install samyak
71
+ ```
72
+
73
+ Requires Python 3.12+.
74
+
75
+ The CLI command is:
76
+
77
+ ```bash
78
+ samyak
79
+ ```
80
+
81
+ ## Quick start
82
+
83
+ Run Corpus Intelligence on a local document directory:
84
+
85
+ ```bash
86
+ samyak corpus ./documents
87
+ ```
88
+
89
+ JSON:
90
+
91
+ ```bash
92
+ samyak corpus ./documents --format json
93
+ ```
94
+
95
+ HTML (self-contained file; open locally, no network required):
96
+
97
+ ```bash
98
+ samyak corpus ./documents --format html --output report.html
99
+ ```
100
+
101
+ Write a JSON report file:
102
+
103
+ ```bash
104
+ samyak corpus ./documents --format json --output report.json
105
+ ```
106
+
107
+ Help and version:
108
+
109
+ ```bash
110
+ samyak --help
111
+ samyak corpus --help
112
+ samyak --version
113
+ ```
114
+
115
+ The `examples/sample-corpus` directory is in this Git repository (and the source distribution). It is **not** included when you `pip install samyak`. After installing Samyak, clone the repository and analyze the example directory:
116
+
117
+ ```bash
118
+ pip install samyak
119
+ git clone https://github.com/tapansharma04/dataaihub-ai-engineering-toolkit.git
120
+ samyak corpus dataaihub-ai-engineering-toolkit/examples/sample-corpus
121
+ ```
122
+
123
+ ## Checks implemented
124
+
125
+ | Check | What it looks for |
126
+ | --- | --- |
127
+ | Inventory | Discovered, supported, unsupported, analyzed counts and size stats |
128
+ | Empty documents | No meaningful text after stripping whitespace |
129
+ | Very small documents | Meaningful text below a documented character threshold (default: 100) |
130
+ | Very large documents | Meaningful text at/above a documented threshold (default: 100,000) |
131
+ | Exact duplicates | Identical normalized textual content (SHA-256 of normalized text) |
132
+ | Text quality | Conservative heuristics: high symbol ratio, excessive whitespace, repeated lines |
133
+ | Chunkability | Long uninterrupted blocks; documents dominated by very short lines |
134
+ | PDF (format-specific) | OCR-likely/text-poor pages, uneven page text, repeated headers/footers, extraction anomalies, large/complex PDFs, table-like text (heuristic) |
135
+ | HTML (format-specific) | Boilerplate domination, low main-content ratio, large pages, code-heavy pages, repeated navigation-like lines |
136
+ | Load failures | Supported files that could not be parsed (analysis continues) |
137
+ | Discovery access failures | Paths that could not be listed or inspected (analysis continues) |
138
+
139
+ Default thresholds are documented on `AnalysisConfig` and included in JSON output under `config`. Override size thresholds via `--small-chars` / `--large-chars`.
140
+
141
+ ## Example output (text)
142
+
143
+ ```text
144
+ Samyak Corpus Intelligence Report
145
+ =================================
146
+
147
+ Product: samyak
148
+ Capability: corpus
149
+ Version: 0.1.0
150
+
151
+ Corpus
152
+ ------
153
+ Documents analyzed: ...
154
+ Unsupported files: ...
155
+
156
+ Findings
157
+ --------
158
+
159
+ HIGH Exact duplicate documents
160
+
161
+ 2 documents belong to 1 exact-duplicate group(s).
162
+
163
+ Why it matters:
164
+ Duplicate content can produce redundant retrieval results...
165
+
166
+ Recommendation:
167
+ Review and remove duplicate documents before indexing...
168
+
169
+ Summary
170
+ -------
171
+ High: ...
172
+ Medium: ...
173
+ Low: ...
174
+ Info: ...
175
+ ```
176
+
177
+ ## JSON output
178
+
179
+ JSON includes:
180
+
181
+ - `product` / `capability` / `version` (product identity)
182
+ - `config` (thresholds used)
183
+ - `summary` (inventory and size statistics)
184
+ - `findings` (code, category, severity, message, why_it_matters, recommendation, evidence, affected_documents)
185
+ - `severity_counts`
186
+
187
+ Example identity fields:
188
+
189
+ ```json
190
+ {
191
+ "product": "samyak",
192
+ "capability": "corpus",
193
+ "version": "0.1.0"
194
+ }
195
+ ```
196
+
197
+ Paths in reports are **relative to the corpus root** whenever possible.
198
+
199
+ ### HTML output
200
+
201
+ `--format html` renders the same `AnalysisReport` as a standalone HTML page. Open the file in a browser; it requires no network access, JavaScript framework, or external assets.
202
+
203
+ ```bash
204
+ samyak corpus ./documents --format html --output report.html
205
+ ```
206
+
207
+ Omit `--output` to print HTML to stdout (same contract as text and JSON).
208
+
209
+ Exit codes (v0.1):
210
+
211
+ - `0` — analysis completed (findings do not change the exit code)
212
+ - `2` — invalid path, invalid configuration, missing subcommand, or I/O failure writing `--output`
213
+
214
+ Findings (including `HIGH`) do **not** fail the process by themselves. v0.1 reports findings but does not fail CI based on finding severity.
215
+
216
+ ### CI usage
217
+
218
+ ```bash
219
+ samyak corpus ./documents --format json --output corpus-report.json
220
+ ```
221
+
222
+ Keep stdout JSON-only for piping; progress messages go to stderr. Use `--no-progress` when you want silent stderr.
223
+
224
+ ## Library usage
225
+
226
+ ```python
227
+ from samyak import AnalysisConfig, analyze_corpus
228
+
229
+ report = analyze_corpus("./documents")
230
+ print(report.product, report.capability, report.version)
231
+ print(report.severity_counts())
232
+ for finding in report.findings:
233
+ print(finding.severity, finding.title, finding.recommendation)
234
+
235
+ report = analyze_corpus(
236
+ "./documents",
237
+ config=AnalysisConfig(small_document_chars=50),
238
+ )
239
+ ```
240
+
241
+ ## Validation & scale characteristics
242
+
243
+ Corpus Intelligence is validated beyond unit tests using public corpora and controlled defect injection. This repository does **not** redistribute third-party dataset files.
244
+
245
+ Documents are processed **incrementally**: peak memory for multi-document corpora is driven primarily by the largest active document plus compact corpus metadata (hashes, counters, bounded finding samples), not by total corpus text size.
246
+
247
+ Measured on Apple M1, 16 GB RAM, macOS arm64, Python 3.12 for v0.1.0:
248
+
249
+ | Scenario | Approximate result |
250
+ | --- | --- |
251
+ | Multi-GB text corpora (≈250 MB → ≈5 GB) | Completes with low hundreds of MB peak RSS (not proportional to full corpus size) |
252
+ | Many small files (up to 100,000 docs) | Completes; runtime scales primarily with file count |
253
+ | Very large individual files (10–250 MB) | Peak RSS scales with the active file (typically several times the file size) |
254
+
255
+ Controlled detection validation covers empty documents, exact duplicates, size outliers, fragmentation/noise signals, PDF OCR-likely/text-poor pages, and HTML code-heavy pages. Performance varies by hardware, storage, and document formats. PDF analysis is substantially more expensive than plain text. These figures are measured characteristics, not guarantees.
256
+
257
+ ## Limitations
258
+
259
+ - No OCR — scanned/image-only PDFs are *detected* but not converted to text
260
+ - PDF table detection is a conservative text heuristic, not a table extractor
261
+ - HTML main-content / boilerplate detection is heuristic; site structure varies widely
262
+ - No DOCX or other office formats yet
263
+ - No near-duplicate / semantic duplicate detection
264
+ - No embeddings, vector databases, or LLM judges
265
+ - Findings identify characteristics that **may** cause ingestion/chunking/retrieval problems; they do **not** predict downstream answer quality
266
+ - Thresholds may need adjustment for specialized document collections
267
+ - No single readiness score — findings are evidence-based, not oracles
268
+ - Very large *individual* documents still require memory proportional to that file (PDF and HTML included)
269
+ - Discovery materializes the list of discovered files; duplicate tracking is proportional to file count
270
+ - Finding path lists in reports are bounded samples; full counts remain in evidence
271
+ - Multi-gigabyte *corpora* are processed incrementally and do not require holding all document text in RAM
272
+
273
+ ## Samyak by DataAIHub
274
+
275
+ | Project | Role |
276
+ | --- | --- |
277
+ | [DataAIHub](https://www.dataaihub.co) | Knowledge, ecosystem, research, guides, comparisons, and discovery |
278
+ | [DataAIHub Cookbook](https://github.com/tapansharma04/dataaihub-cookbook) | Practical, runnable examples for learning AI engineering patterns |
279
+ | **[Samyak](https://github.com/tapansharma04/dataaihub-ai-engineering-toolkit)** (this repository) | Open-source AI engineering product; current capability: Corpus Intelligence |
280
+
281
+ ## Development
282
+
283
+ Requirements: Python 3.12+
284
+
285
+ ```bash
286
+ python -m pip install -e ".[dev]"
287
+ pytest
288
+ ruff check src tests
289
+ ruff format --check src tests
290
+ ```
291
+
292
+ Tests require the package to be installed (editable is fine) so distribution metadata can be inspected.
293
+
294
+ ## License
295
+
296
+ This project is licensed under the MIT License. See [LICENSE](LICENSE).
@@ -0,0 +1,8 @@
1
+ Customer support policy for returns.
2
+
3
+ Customers may return unused items within 30 days of purchase with a receipt.
4
+ Refunds are issued to the original payment method within five business days.
5
+ Damaged items must be reported within 48 hours of delivery.
6
+ International orders may have different timelines depending on local regulations.
7
+
8
+ This policy applies to online purchases only.
File without changes