cureta 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. cureta-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md +43 -0
  2. cureta-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md +29 -0
  3. cureta-0.1.0/.github/ISSUE_TEMPLATE/question.md +21 -0
  4. cureta-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +30 -0
  5. cureta-0.1.0/.github/workflows/ci.yml +86 -0
  6. cureta-0.1.0/.github/workflows/docs.yml +55 -0
  7. cureta-0.1.0/.gitignore +219 -0
  8. cureta-0.1.0/CHANGELOG.md +419 -0
  9. cureta-0.1.0/CONTRIBUTING.md +59 -0
  10. cureta-0.1.0/PKG-INFO +176 -0
  11. cureta-0.1.0/README.md +102 -0
  12. cureta-0.1.0/clusters/aclust.yaml +9 -0
  13. cureta-0.1.0/clusters/example.yaml +28 -0
  14. cureta-0.1.0/clusters/jclust.yaml +12 -0
  15. cureta-0.1.0/clusters/nysa-b.yaml +10 -0
  16. cureta-0.1.0/cureta/__init__.py +136 -0
  17. cureta-0.1.0/cureta/base_tagger.py +270 -0
  18. cureta-0.1.0/cureta/cli.py +816 -0
  19. cureta-0.1.0/cureta/config.py +632 -0
  20. cureta-0.1.0/cureta/data_sources.py +417 -0
  21. cureta-0.1.0/cureta/document.py +135 -0
  22. cureta-0.1.0/cureta/external_taggers/__init__.py +22 -0
  23. cureta-0.1.0/cureta/external_taggers/base.py +88 -0
  24. cureta-0.1.0/cureta/external_taggers/datatrove.py +333 -0
  25. cureta-0.1.0/cureta/external_taggers/datatrove_fused.py +143 -0
  26. cureta-0.1.0/cureta/external_taggers/nemo.py +218 -0
  27. cureta-0.1.0/cureta/heuristic_taggers/c4_quality_tagger.py +85 -0
  28. cureta-0.1.0/cureta/heuristic_taggers/compression_ratio_tagger.py +37 -0
  29. cureta-0.1.0/cureta/heuristic_taggers/content_type_tagger.py +74 -0
  30. cureta-0.1.0/cureta/heuristic_taggers/date_mention_tagger.py +64 -0
  31. cureta-0.1.0/cureta/heuristic_taggers/exact_norm_hash_tagger.py +48 -0
  32. cureta-0.1.0/cureta/heuristic_taggers/fineweb_quality_tagger.py +73 -0
  33. cureta-0.1.0/cureta/heuristic_taggers/gopher_quality_tagger.py +95 -0
  34. cureta-0.1.0/cureta/heuristic_taggers/heuristic_taggers.py +285 -0
  35. cureta-0.1.0/cureta/heuristic_taggers/html_density_tagger.py +47 -0
  36. cureta-0.1.0/cureta/heuristic_taggers/markdown_structure_tagger.py +57 -0
  37. cureta-0.1.0/cureta/heuristic_taggers/minhash_tagger.py +58 -0
  38. cureta-0.1.0/cureta/heuristic_taggers/paragraph_stats_tagger.py +39 -0
  39. cureta-0.1.0/cureta/heuristic_taggers/pii_regex_tagger.py +57 -0
  40. cureta-0.1.0/cureta/heuristic_taggers/readability_tagger.py +57 -0
  41. cureta-0.1.0/cureta/heuristic_taggers/setu/constants.py +282 -0
  42. cureta-0.1.0/cureta/heuristic_taggers/setu/setu_filters.py +631 -0
  43. cureta-0.1.0/cureta/heuristic_taggers/simhash_tagger.py +54 -0
  44. cureta-0.1.0/cureta/heuristic_taggers/type_token_ratio_tagger.py +58 -0
  45. cureta-0.1.0/cureta/heuristic_taggers/url_blocklist_tagger.py +93 -0
  46. cureta-0.1.0/cureta/heuristic_taggers/url_parse_tagger.py +53 -0
  47. cureta-0.1.0/cureta/heuristic_taggers/utils.py +15 -0
  48. cureta-0.1.0/cureta/heuristic_taggers/word_length_stats_tagger.py +38 -0
  49. cureta-0.1.0/cureta/infra/__init__.py +15 -0
  50. cureta-0.1.0/cureta/infra/distributed_runner.py +183 -0
  51. cureta-0.1.0/cureta/infra/engine.py +329 -0
  52. cureta-0.1.0/cureta/infra/skypilot_cluster.py +291 -0
  53. cureta-0.1.0/cureta/manifest.py +196 -0
  54. cureta-0.1.0/cureta/pipeline.py +353 -0
  55. cureta-0.1.0/cureta/registry.py +584 -0
  56. cureta-0.1.0/cureta/results.py +83 -0
  57. cureta-0.1.0/cureta/runner.py +763 -0
  58. cureta-0.1.0/cureta/tag_ops.py +96 -0
  59. cureta-0.1.0/cureta/taggers/__init__.py +31 -0
  60. cureta-0.1.0/cureta/taggers/detoxify_tagger.py +67 -0
  61. cureta-0.1.0/cureta/taggers/domain_classifier_tagger.py +91 -0
  62. cureta-0.1.0/cureta/taggers/fasttext_quality_tagger.py +77 -0
  63. cureta-0.1.0/cureta/taggers/fineweb_edu_tagger.py +103 -0
  64. cureta-0.1.0/cureta/taggers/glotlid_tagger.py +85 -0
  65. cureta-0.1.0/cureta/taggers/indic_lid_tagger.py +200 -0
  66. cureta-0.1.0/cureta/taggers/llm_judge_tagger.py +128 -0
  67. cureta-0.1.0/cureta/taggers/mmbert_quality_tagger.py +263 -0
  68. cureta-0.1.0/cureta/taggers/paragraph_langid_tagger.py +68 -0
  69. cureta-0.1.0/cureta/taggers/perplexity_tagger.py +586 -0
  70. cureta-0.1.0/cureta/taggers/presidio_pii_tagger.py +73 -0
  71. cureta-0.1.0/cureta/taggers/programming_language_tagger.py +90 -0
  72. cureta-0.1.0/cureta/taggers/sentence_embedding_tagger.py +76 -0
  73. cureta-0.1.0/docs/concepts/data_lineage.md +170 -0
  74. cureta-0.1.0/docs/concepts/execution_model.md +175 -0
  75. cureta-0.1.0/docs/concepts/memory_model.md +150 -0
  76. cureta-0.1.0/docs/concepts/plugin_system.md +161 -0
  77. cureta-0.1.0/docs/concepts/tagger_architecture.md +142 -0
  78. cureta-0.1.0/docs/contributing/adr/0001_two_tier_tagger_api.md +39 -0
  79. cureta-0.1.0/docs/contributing/adr/0002_sdmi_default_for_cpu.md +35 -0
  80. cureta-0.1.0/docs/contributing/adr/0003_pipeline_config_plugin_discovery.md +38 -0
  81. cureta-0.1.0/docs/contributing/adr/0004_document_as_authoring_api.md +54 -0
  82. cureta-0.1.0/docs/contributing/adr/0005_streaming_chunk_extraction.md +39 -0
  83. cureta-0.1.0/docs/contributing/adr/0006_output_manifests.md +42 -0
  84. cureta-0.1.0/docs/contributing/adr/0007_datasource_abstraction.md +45 -0
  85. cureta-0.1.0/docs/contributing/architecture_decisions.md +48 -0
  86. cureta-0.1.0/docs/contributing/setup.md +198 -0
  87. cureta-0.1.0/docs/contributing/style_guide.md +172 -0
  88. cureta-0.1.0/docs/contributing/test_suite.md +158 -0
  89. cureta-0.1.0/docs/how_to/choose_a_tagger.md +115 -0
  90. cureta-0.1.0/docs/how_to/configure_skypilot.md +186 -0
  91. cureta-0.1.0/docs/how_to/deploy_to_kubernetes.md +173 -0
  92. cureta-0.1.0/docs/how_to/load_huggingface_data.md +124 -0
  93. cureta-0.1.0/docs/how_to/load_local_data.md +128 -0
  94. cureta-0.1.0/docs/how_to/resume_a_failed_run.md +122 -0
  95. cureta-0.1.0/docs/how_to/tune_chunk_size.md +108 -0
  96. cureta-0.1.0/docs/how_to/use_external_taggers.md +246 -0
  97. cureta-0.1.0/docs/index.md +107 -0
  98. cureta-0.1.0/docs/reference/cli.md +316 -0
  99. cureta-0.1.0/docs/reference/cluster_config.md +142 -0
  100. cureta-0.1.0/docs/reference/data_sources.md +182 -0
  101. cureta-0.1.0/docs/reference/external_taggers.md +199 -0
  102. cureta-0.1.0/docs/reference/output_manifest.md +225 -0
  103. cureta-0.1.0/docs/reference/pipeline_config.md +184 -0
  104. cureta-0.1.0/docs/reference/python_api.md +266 -0
  105. cureta-0.1.0/docs/reference/taggers.md +1345 -0
  106. cureta-0.1.0/docs/tutorials/data_quality_pipeline.md +188 -0
  107. cureta-0.1.0/docs/tutorials/deploying_on_cloud.md +277 -0
  108. cureta-0.1.0/docs/tutorials/first_pipeline.md +198 -0
  109. cureta-0.1.0/docs/tutorials/writing_a_custom_tagger.md +336 -0
  110. cureta-0.1.0/docs/writing_taggers.md +447 -0
  111. cureta-0.1.0/examples/01_quickstart/README.md +36 -0
  112. cureta-0.1.0/examples/01_quickstart/pipeline_config.yaml +5 -0
  113. cureta-0.1.0/examples/01_quickstart/run.py +68 -0
  114. cureta-0.1.0/examples/02_custom_tagger/README.md +38 -0
  115. cureta-0.1.0/examples/02_custom_tagger/my_tagger.py +94 -0
  116. cureta-0.1.0/examples/02_custom_tagger/pipeline_config.yaml +5 -0
  117. cureta-0.1.0/examples/02_custom_tagger/run.py +61 -0
  118. cureta-0.1.0/examples/03_huggingface/README.md +81 -0
  119. cureta-0.1.0/examples/03_huggingface/pipeline_config.yaml +23 -0
  120. cureta-0.1.0/examples/03_huggingface/run.py +112 -0
  121. cureta-0.1.0/examples/04_programmatic_api/README.md +45 -0
  122. cureta-0.1.0/examples/04_programmatic_api/pipeline_config.yaml +3 -0
  123. cureta-0.1.0/examples/04_programmatic_api/run.py +72 -0
  124. cureta-0.1.0/examples/05_streaming/README.md +44 -0
  125. cureta-0.1.0/examples/05_streaming/pipeline_config.yaml +2 -0
  126. cureta-0.1.0/examples/05_streaming/run.py +124 -0
  127. cureta-0.1.0/examples/README.md +13 -0
  128. cureta-0.1.0/examples/pipeline_config.yaml +29 -0
  129. cureta-0.1.0/mkdocs.yml +97 -0
  130. cureta-0.1.0/pipelines/aclust_test/pipeline_config.yaml +41 -0
  131. cureta-0.1.0/pipelines/example/pipeline_config.yaml +24 -0
  132. cureta-0.1.0/pipelines/global/global_defaults.yaml +55 -0
  133. cureta-0.1.0/pipelines/jclust_test/pipeline_config.yaml +41 -0
  134. cureta-0.1.0/pipelines/sangraha_hin_verified/pipeline_config.yaml +45 -0
  135. cureta-0.1.0/pipelines/sangraha_hin_verified/smoke_test.yaml +21 -0
  136. cureta-0.1.0/pyproject.toml +114 -0
  137. cureta-0.1.0/scripts/check_dataset_size.py +0 -0
  138. cureta-0.1.0/scripts/download_sentiment_dataset.py +89 -0
  139. cureta-0.1.0/scripts/execute_ray_pipeline.py +0 -0
  140. cureta-0.1.0/scripts/regenerate_cli_docs.py +72 -0
  141. cureta-0.1.0/scripts/submit_pipeline.py +0 -0
  142. cureta-0.1.0/tests/__init__.py +12 -0
  143. cureta-0.1.0/tests/conftest.py +112 -0
  144. cureta-0.1.0/tests/integration/__init__.py +1 -0
  145. cureta-0.1.0/tests/integration/conftest.py +110 -0
  146. cureta-0.1.0/tests/integration/sample_taggers.py +37 -0
  147. cureta-0.1.0/tests/integration/sky_test_task.yaml +25 -0
  148. cureta-0.1.0/tests/integration/test_distributed_ray.py +160 -0
  149. cureta-0.1.0/tests/integration/test_engine_equivalence.py +300 -0
  150. cureta-0.1.0/tests/integration/test_external_taggers_integration.py +244 -0
  151. cureta-0.1.0/tests/integration/test_manifest_provenance.py +301 -0
  152. cureta-0.1.0/tests/integration/test_notebook_usage.py +245 -0
  153. cureta-0.1.0/tests/integration/test_pipeline_workflow.py +132 -0
  154. cureta-0.1.0/tests/integration/test_skypilot_integration.py +320 -0
  155. cureta-0.1.0/tests/integration/test_streaming_chunks.py +72 -0
  156. cureta-0.1.0/tests/integration/test_two_tier_pipeline.py +138 -0
  157. cureta-0.1.0/tests/nysa_test/__init__.py +1 -0
  158. cureta-0.1.0/tests/nysa_test/configs/global_defaults.yaml +96 -0
  159. cureta-0.1.0/tests/nysa_test/configs/heuristic_pipeline_config.yaml +27 -0
  160. cureta-0.1.0/tests/nysa_test/configs/perf_sdmi.yaml +21 -0
  161. cureta-0.1.0/tests/nysa_test/configs/perf_simd.yaml +21 -0
  162. cureta-0.1.0/tests/nysa_test/configs/pipeline_config.yaml +21 -0
  163. cureta-0.1.0/tests/nysa_test/configs/pipeline_config_indic_lid.yaml +6 -0
  164. cureta-0.1.0/tests/nysa_test/configs/pipeline_config_mmbert.yaml +7 -0
  165. cureta-0.1.0/tests/nysa_test/configs/pipeline_config_perplexity.yaml +7 -0
  166. cureta-0.1.0/tests/nysa_test/configs/pipeline_config_sdmi.yaml +25 -0
  167. cureta-0.1.0/tests/nysa_test/fixtures/sample_docs.jsonl +5 -0
  168. cureta-0.1.0/tests/nysa_test/run_heuristic_pipeline.py +197 -0
  169. cureta-0.1.0/tests/nysa_test/run_indic_lid_pipeline.py +146 -0
  170. cureta-0.1.0/tests/nysa_test/run_mmbert_quality_pipeline.py +193 -0
  171. cureta-0.1.0/tests/nysa_test/run_perf_pipeline.py +106 -0
  172. cureta-0.1.0/tests/nysa_test/run_perplexity_pipeline.py +191 -0
  173. cureta-0.1.0/tests/nysa_test/run_pipeline.py +170 -0
  174. cureta-0.1.0/tests/nysa_test/taggers/__init__.py +1 -0
  175. cureta-0.1.0/tests/nysa_test/taggers/sentence_splitter.py +22 -0
  176. cureta-0.1.0/tests/nysa_test/taggers/sentiment_analyzer.py +61 -0
  177. cureta-0.1.0/tests/nysa_test/taggers/slow_tagger.py +34 -0
  178. cureta-0.1.0/tests/nysa_test/taggers/word_counter.py +16 -0
  179. cureta-0.1.0/tests/nysa_test/test_nysa_integration.py +544 -0
  180. cureta-0.1.0/tests/test_indic_lid_local.py +59 -0
  181. cureta-0.1.0/tests/unit/__init__.py +1 -0
  182. cureta-0.1.0/tests/unit/test_base_tagger.py +248 -0
  183. cureta-0.1.0/tests/unit/test_c4_quality_tagger.py +97 -0
  184. cureta-0.1.0/tests/unit/test_chunk_extraction.py +83 -0
  185. cureta-0.1.0/tests/unit/test_cli.py +127 -0
  186. cureta-0.1.0/tests/unit/test_compression_ratio_tagger.py +64 -0
  187. cureta-0.1.0/tests/unit/test_config.py +444 -0
  188. cureta-0.1.0/tests/unit/test_content_type_tagger.py +65 -0
  189. cureta-0.1.0/tests/unit/test_data_sources.py +308 -0
  190. cureta-0.1.0/tests/unit/test_date_mention_tagger.py +68 -0
  191. cureta-0.1.0/tests/unit/test_detoxify_tagger.py +111 -0
  192. cureta-0.1.0/tests/unit/test_distributed_runner.py +270 -0
  193. cureta-0.1.0/tests/unit/test_document.py +82 -0
  194. cureta-0.1.0/tests/unit/test_domain_classifier_tagger.py +110 -0
  195. cureta-0.1.0/tests/unit/test_engine.py +424 -0
  196. cureta-0.1.0/tests/unit/test_exact_norm_hash_tagger.py +66 -0
  197. cureta-0.1.0/tests/unit/test_execution_modes.py +115 -0
  198. cureta-0.1.0/tests/unit/test_external_taggers.py +803 -0
  199. cureta-0.1.0/tests/unit/test_fasttext_quality_tagger.py +93 -0
  200. cureta-0.1.0/tests/unit/test_fineweb_edu_tagger.py +122 -0
  201. cureta-0.1.0/tests/unit/test_fineweb_quality_tagger.py +80 -0
  202. cureta-0.1.0/tests/unit/test_glotlid_tagger.py +92 -0
  203. cureta-0.1.0/tests/unit/test_gopher_quality_tagger.py +90 -0
  204. cureta-0.1.0/tests/unit/test_html_density_tagger.py +57 -0
  205. cureta-0.1.0/tests/unit/test_init.py +92 -0
  206. cureta-0.1.0/tests/unit/test_llm_judge_tagger.py +111 -0
  207. cureta-0.1.0/tests/unit/test_manifest.py +414 -0
  208. cureta-0.1.0/tests/unit/test_markdown_structure_tagger.py +68 -0
  209. cureta-0.1.0/tests/unit/test_minhash_tagger.py +76 -0
  210. cureta-0.1.0/tests/unit/test_paragraph_langid_tagger.py +108 -0
  211. cureta-0.1.0/tests/unit/test_paragraph_stats_tagger.py +66 -0
  212. cureta-0.1.0/tests/unit/test_perplexity_tagger.py +298 -0
  213. cureta-0.1.0/tests/unit/test_pii_regex_tagger.py +89 -0
  214. cureta-0.1.0/tests/unit/test_pipeline.py +355 -0
  215. cureta-0.1.0/tests/unit/test_presidio_pii_tagger.py +96 -0
  216. cureta-0.1.0/tests/unit/test_programming_language_tagger.py +77 -0
  217. cureta-0.1.0/tests/unit/test_quick_tag.py +176 -0
  218. cureta-0.1.0/tests/unit/test_readability_tagger.py +60 -0
  219. cureta-0.1.0/tests/unit/test_rename_identity.py +64 -0
  220. cureta-0.1.0/tests/unit/test_sentence_embedding_tagger.py +92 -0
  221. cureta-0.1.0/tests/unit/test_simhash_tagger.py +60 -0
  222. cureta-0.1.0/tests/unit/test_skypilot_cluster.py +122 -0
  223. cureta-0.1.0/tests/unit/test_tag_ops.py +115 -0
  224. cureta-0.1.0/tests/unit/test_type_token_ratio_tagger.py +68 -0
  225. cureta-0.1.0/tests/unit/test_url_blocklist_tagger.py +86 -0
  226. cureta-0.1.0/tests/unit/test_url_parse_tagger.py +74 -0
  227. cureta-0.1.0/tests/unit/test_word_length_stats_tagger.py +63 -0
@@ -0,0 +1,43 @@
1
+ ---
2
+ name: Bug report
3
+ about: Something is broken or behaving unexpectedly
4
+ title: 'bug: '
5
+ labels: bug
6
+ assignees: ''
7
+ ---
8
+
9
+ ## Description
10
+
11
+ A clear, one-sentence description of the bug.
12
+
13
+ ## Steps to reproduce
14
+
15
+ ```python
16
+ # Minimal code that reproduces the issue
17
+ ```
18
+
19
+ Or the CLI command:
20
+
21
+ ```bash
22
+ cureta run ...
23
+ ```
24
+
25
+ ## Expected behaviour
26
+
27
+ What you expected to happen.
28
+
29
+ ## Actual behaviour
30
+
31
+ What actually happened. Include the full error message and traceback if applicable.
32
+
33
+ ## Environment
34
+
35
+ - **Cureta version**: (run `cureta --version`)
36
+ - **Python version**: (run `python --version`)
37
+ - **Ray version**: (run `python -c "import ray; print(ray.__version__)"`)
38
+ - **OS**: (e.g. Ubuntu 22.04, macOS 14)
39
+ - **GPU**: (if relevant — model and VRAM)
40
+
41
+ ## Additional context
42
+
43
+ Any other relevant information (pipeline config, dataset characteristics, cluster setup).
@@ -0,0 +1,29 @@
1
+ ---
2
+ name: Feature request
3
+ about: Suggest a new feature or improvement
4
+ title: 'feat: '
5
+ labels: enhancement
6
+ assignees: ''
7
+ ---
8
+
9
+ ## Problem
10
+
11
+ A clear description of the problem this feature would solve. Example: "I need to load data from Elasticsearch but there is no built-in DataSource for it."
12
+
13
+ ## Proposed solution
14
+
15
+ A description of the feature you would like, with enough detail to evaluate feasibility. Include a code sketch if helpful:
16
+
17
+ ```python
18
+ # Example of what the feature would look like to use
19
+ ```
20
+
21
+ ## Alternatives considered
22
+
23
+ Other approaches you have thought about and why you prefer this solution.
24
+
25
+ ## Willingness to contribute
26
+
27
+ - [ ] I am willing to open a PR for this feature
28
+ - [ ] I would appreciate guidance but could attempt a PR
29
+ - [ ] I am requesting this for others to implement
@@ -0,0 +1,21 @@
1
+ ---
2
+ name: Question
3
+ about: Ask a question about using or understanding Cureta
4
+ title: 'question: '
5
+ labels: question
6
+ assignees: ''
7
+ ---
8
+
9
+ ## Question
10
+
11
+ Your question, as clearly as possible.
12
+
13
+ ## What I have tried
14
+
15
+ Steps you have already taken to answer the question yourself:
16
+ - Docs I read:
17
+ - Code I looked at:
18
+
19
+ ## Relevant context
20
+
21
+ Any code, configs, or error messages relevant to the question.
@@ -0,0 +1,30 @@
1
+ ## Summary
2
+
3
+ Brief description of what this PR does and why.
4
+
5
+ ## Type of change
6
+
7
+ - [ ] Bug fix (fixes a broken behaviour without changing the API)
8
+ - [ ] New feature (adds functionality without breaking existing behaviour)
9
+ - [ ] Breaking change (changes existing API or output format)
10
+ - [ ] Documentation only (no code changes)
11
+ - [ ] Refactor (no functionality change)
12
+
13
+ ## Testing
14
+
15
+ - [ ] Unit tests pass: `pytest tests/unit/ -v`
16
+ - [ ] Integration tests pass: `pytest tests/integration/ -v`
17
+ - [ ] Type check passes: `pyright .`
18
+ - [ ] Linter passes: `ruff check .`
19
+
20
+ ## Documentation
21
+
22
+ - [ ] Docstrings updated for any changed public APIs
23
+ - [ ] Relevant docs in `docs/` updated
24
+ - [ ] `CHANGELOG.md` updated (for user-visible changes)
25
+
26
+ ## Checklist
27
+
28
+ - [ ] PR title clearly describes the change (will be used in the changelog)
29
+ - [ ] Tests added or updated for new/changed functionality
30
+ - [ ] No secrets, credentials, or large binary files added
@@ -0,0 +1,86 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ test:
11
+ name: Tests (Python ${{ matrix.python-version }})
12
+ runs-on: ubuntu-latest
13
+ strategy:
14
+ fail-fast: false
15
+ matrix:
16
+ python-version: ["3.10", "3.11", "3.12"]
17
+
18
+ steps:
19
+ - uses: actions/checkout@v4
20
+
21
+ - name: Set up Python ${{ matrix.python-version }}
22
+ uses: actions/setup-python@v5
23
+ with:
24
+ python-version: ${{ matrix.python-version }}
25
+
26
+ - name: Cache pip
27
+ uses: actions/cache@v4
28
+ with:
29
+ path: ~/.cache/pip
30
+ key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
31
+ restore-keys: |
32
+ ${{ runner.os }}-pip-${{ matrix.python-version }}-
33
+
34
+ - name: Install dependencies
35
+ run: |
36
+ pip install -e ".[ray,dev]"
37
+
38
+ - name: Run unit tests
39
+ run: |
40
+ pytest tests/unit/ -v --tb=short
41
+
42
+ - name: Run integration tests
43
+ run: |
44
+ pytest tests/integration/ -v --tb=short \
45
+ --ignore=tests/integration/test_skypilot_integration.py
46
+
47
+ typecheck:
48
+ name: Type check
49
+ runs-on: ubuntu-latest
50
+
51
+ steps:
52
+ - uses: actions/checkout@v4
53
+
54
+ - name: Set up Python 3.11
55
+ uses: actions/setup-python@v5
56
+ with:
57
+ python-version: "3.11"
58
+
59
+ - name: Install dependencies
60
+ run: |
61
+ pip install -e ".[ray,dev]"
62
+
63
+ - name: Run pyright
64
+ run: |
65
+ pyright .
66
+
67
+ lint:
68
+ name: Lint
69
+ runs-on: ubuntu-latest
70
+
71
+ steps:
72
+ - uses: actions/checkout@v4
73
+
74
+ - name: Set up Python 3.11
75
+ uses: actions/setup-python@v5
76
+ with:
77
+ python-version: "3.11"
78
+
79
+ - name: Install ruff
80
+ run: pip install ruff
81
+
82
+ - name: Run ruff check
83
+ run: ruff check .
84
+
85
+ - name: Run ruff format check
86
+ run: ruff format --check .
@@ -0,0 +1,55 @@
1
+ name: Docs
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ permissions:
10
+ contents: write
11
+
12
+ jobs:
13
+ build:
14
+ name: Build docs
15
+ runs-on: ubuntu-latest
16
+
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+
20
+ - name: Set up Python 3.11
21
+ uses: actions/setup-python@v5
22
+ with:
23
+ python-version: "3.11"
24
+
25
+ - name: Install docs dependencies
26
+ run: |
27
+ pip install -e ".[docs]"
28
+
29
+ - name: Build docs (strict — fails on broken links)
30
+ run: |
31
+ mkdocs build --strict
32
+
33
+ deploy:
34
+ name: Deploy to GitHub Pages
35
+ runs-on: ubuntu-latest
36
+ needs: build
37
+ if: github.ref == 'refs/heads/main' && github.event_name == 'push'
38
+
39
+ steps:
40
+ - uses: actions/checkout@v4
41
+
42
+ - name: Set up Python 3.11
43
+ uses: actions/setup-python@v5
44
+ with:
45
+ python-version: "3.11"
46
+
47
+ - name: Install docs dependencies
48
+ run: |
49
+ pip install -e ".[docs]"
50
+
51
+ - name: Deploy to GitHub Pages
52
+ run: |
53
+ git config user.name "github-actions[bot]"
54
+ git config user.email "github-actions[bot]@users.noreply.github.com"
55
+ mkdocs gh-deploy --force
@@ -0,0 +1,219 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ #poetry.lock
109
+ #poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ #pdm.lock
116
+ #pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ #pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # SageMath parsed files
135
+ *.sage.py
136
+
137
+ # Environments
138
+ .env
139
+ .envrc
140
+ .venv
141
+ .venv*
142
+ env/
143
+ venv/
144
+ ENV/
145
+ env.bak/
146
+ venv.bak/
147
+
148
+ # Spyder project settings
149
+ .spyderproject
150
+ .spyproject
151
+
152
+ # Rope project settings
153
+ .ropeproject
154
+
155
+ # mkdocs documentation
156
+ /site
157
+
158
+ # mypy
159
+ .mypy_cache/
160
+ .dmypy.json
161
+ dmypy.json
162
+
163
+ # Pyre type checker
164
+ .pyre/
165
+
166
+ # pytype static type analyzer
167
+ .pytype/
168
+
169
+ # Cython debug symbols
170
+ cython_debug/
171
+
172
+ # PyCharm
173
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
174
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
175
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
176
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
177
+ #.idea/
178
+
179
+ # Abstra
180
+ # Abstra is an AI-powered process automation framework.
181
+ # Ignore directories containing user credentials, local state, and settings.
182
+ # Learn more at https://abstra.io/docs
183
+ .abstra/
184
+
185
+ # Visual Studio Code
186
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
187
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
188
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
189
+ # you could uncomment the following to ignore the entire vscode folder
190
+ # .vscode/
191
+
192
+ # Ruff stuff:
193
+ .ruff_cache/
194
+
195
+ # PyPI configuration file
196
+ .pypirc
197
+
198
+ # Cursor
199
+ # Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
200
+ # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
201
+ # refer to https://docs.cursor.com/context/ignore-files
202
+ .cursorignore
203
+ .cursorindexingignore
204
+
205
+ # Marimo
206
+ marimo/_static/
207
+ marimo/_lsp/
208
+ __marimo__/
209
+
210
+ # Dev Resources
211
+ *.txt
212
+ dev_scripts/
213
+
214
+ # Data
215
+ data/
216
+
217
+ # Benchmarking run outputs (regenerated by run_all.sh; not committed)
218
+ .benchmarking/results/full_pipeline/
219
+ .benchmarking/results/external_tagger_comparison