cureta 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cureta-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md +43 -0
- cureta-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md +29 -0
- cureta-0.1.0/.github/ISSUE_TEMPLATE/question.md +21 -0
- cureta-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +30 -0
- cureta-0.1.0/.github/workflows/ci.yml +86 -0
- cureta-0.1.0/.github/workflows/docs.yml +55 -0
- cureta-0.1.0/.gitignore +219 -0
- cureta-0.1.0/CHANGELOG.md +419 -0
- cureta-0.1.0/CONTRIBUTING.md +59 -0
- cureta-0.1.0/PKG-INFO +176 -0
- cureta-0.1.0/README.md +102 -0
- cureta-0.1.0/clusters/aclust.yaml +9 -0
- cureta-0.1.0/clusters/example.yaml +28 -0
- cureta-0.1.0/clusters/jclust.yaml +12 -0
- cureta-0.1.0/clusters/nysa-b.yaml +10 -0
- cureta-0.1.0/cureta/__init__.py +136 -0
- cureta-0.1.0/cureta/base_tagger.py +270 -0
- cureta-0.1.0/cureta/cli.py +816 -0
- cureta-0.1.0/cureta/config.py +632 -0
- cureta-0.1.0/cureta/data_sources.py +417 -0
- cureta-0.1.0/cureta/document.py +135 -0
- cureta-0.1.0/cureta/external_taggers/__init__.py +22 -0
- cureta-0.1.0/cureta/external_taggers/base.py +88 -0
- cureta-0.1.0/cureta/external_taggers/datatrove.py +333 -0
- cureta-0.1.0/cureta/external_taggers/datatrove_fused.py +143 -0
- cureta-0.1.0/cureta/external_taggers/nemo.py +218 -0
- cureta-0.1.0/cureta/heuristic_taggers/c4_quality_tagger.py +85 -0
- cureta-0.1.0/cureta/heuristic_taggers/compression_ratio_tagger.py +37 -0
- cureta-0.1.0/cureta/heuristic_taggers/content_type_tagger.py +74 -0
- cureta-0.1.0/cureta/heuristic_taggers/date_mention_tagger.py +64 -0
- cureta-0.1.0/cureta/heuristic_taggers/exact_norm_hash_tagger.py +48 -0
- cureta-0.1.0/cureta/heuristic_taggers/fineweb_quality_tagger.py +73 -0
- cureta-0.1.0/cureta/heuristic_taggers/gopher_quality_tagger.py +95 -0
- cureta-0.1.0/cureta/heuristic_taggers/heuristic_taggers.py +285 -0
- cureta-0.1.0/cureta/heuristic_taggers/html_density_tagger.py +47 -0
- cureta-0.1.0/cureta/heuristic_taggers/markdown_structure_tagger.py +57 -0
- cureta-0.1.0/cureta/heuristic_taggers/minhash_tagger.py +58 -0
- cureta-0.1.0/cureta/heuristic_taggers/paragraph_stats_tagger.py +39 -0
- cureta-0.1.0/cureta/heuristic_taggers/pii_regex_tagger.py +57 -0
- cureta-0.1.0/cureta/heuristic_taggers/readability_tagger.py +57 -0
- cureta-0.1.0/cureta/heuristic_taggers/setu/constants.py +282 -0
- cureta-0.1.0/cureta/heuristic_taggers/setu/setu_filters.py +631 -0
- cureta-0.1.0/cureta/heuristic_taggers/simhash_tagger.py +54 -0
- cureta-0.1.0/cureta/heuristic_taggers/type_token_ratio_tagger.py +58 -0
- cureta-0.1.0/cureta/heuristic_taggers/url_blocklist_tagger.py +93 -0
- cureta-0.1.0/cureta/heuristic_taggers/url_parse_tagger.py +53 -0
- cureta-0.1.0/cureta/heuristic_taggers/utils.py +15 -0
- cureta-0.1.0/cureta/heuristic_taggers/word_length_stats_tagger.py +38 -0
- cureta-0.1.0/cureta/infra/__init__.py +15 -0
- cureta-0.1.0/cureta/infra/distributed_runner.py +183 -0
- cureta-0.1.0/cureta/infra/engine.py +329 -0
- cureta-0.1.0/cureta/infra/skypilot_cluster.py +291 -0
- cureta-0.1.0/cureta/manifest.py +196 -0
- cureta-0.1.0/cureta/pipeline.py +353 -0
- cureta-0.1.0/cureta/registry.py +584 -0
- cureta-0.1.0/cureta/results.py +83 -0
- cureta-0.1.0/cureta/runner.py +763 -0
- cureta-0.1.0/cureta/tag_ops.py +96 -0
- cureta-0.1.0/cureta/taggers/__init__.py +31 -0
- cureta-0.1.0/cureta/taggers/detoxify_tagger.py +67 -0
- cureta-0.1.0/cureta/taggers/domain_classifier_tagger.py +91 -0
- cureta-0.1.0/cureta/taggers/fasttext_quality_tagger.py +77 -0
- cureta-0.1.0/cureta/taggers/fineweb_edu_tagger.py +103 -0
- cureta-0.1.0/cureta/taggers/glotlid_tagger.py +85 -0
- cureta-0.1.0/cureta/taggers/indic_lid_tagger.py +200 -0
- cureta-0.1.0/cureta/taggers/llm_judge_tagger.py +128 -0
- cureta-0.1.0/cureta/taggers/mmbert_quality_tagger.py +263 -0
- cureta-0.1.0/cureta/taggers/paragraph_langid_tagger.py +68 -0
- cureta-0.1.0/cureta/taggers/perplexity_tagger.py +586 -0
- cureta-0.1.0/cureta/taggers/presidio_pii_tagger.py +73 -0
- cureta-0.1.0/cureta/taggers/programming_language_tagger.py +90 -0
- cureta-0.1.0/cureta/taggers/sentence_embedding_tagger.py +76 -0
- cureta-0.1.0/docs/concepts/data_lineage.md +170 -0
- cureta-0.1.0/docs/concepts/execution_model.md +175 -0
- cureta-0.1.0/docs/concepts/memory_model.md +150 -0
- cureta-0.1.0/docs/concepts/plugin_system.md +161 -0
- cureta-0.1.0/docs/concepts/tagger_architecture.md +142 -0
- cureta-0.1.0/docs/contributing/adr/0001_two_tier_tagger_api.md +39 -0
- cureta-0.1.0/docs/contributing/adr/0002_sdmi_default_for_cpu.md +35 -0
- cureta-0.1.0/docs/contributing/adr/0003_pipeline_config_plugin_discovery.md +38 -0
- cureta-0.1.0/docs/contributing/adr/0004_document_as_authoring_api.md +54 -0
- cureta-0.1.0/docs/contributing/adr/0005_streaming_chunk_extraction.md +39 -0
- cureta-0.1.0/docs/contributing/adr/0006_output_manifests.md +42 -0
- cureta-0.1.0/docs/contributing/adr/0007_datasource_abstraction.md +45 -0
- cureta-0.1.0/docs/contributing/architecture_decisions.md +48 -0
- cureta-0.1.0/docs/contributing/setup.md +198 -0
- cureta-0.1.0/docs/contributing/style_guide.md +172 -0
- cureta-0.1.0/docs/contributing/test_suite.md +158 -0
- cureta-0.1.0/docs/how_to/choose_a_tagger.md +115 -0
- cureta-0.1.0/docs/how_to/configure_skypilot.md +186 -0
- cureta-0.1.0/docs/how_to/deploy_to_kubernetes.md +173 -0
- cureta-0.1.0/docs/how_to/load_huggingface_data.md +124 -0
- cureta-0.1.0/docs/how_to/load_local_data.md +128 -0
- cureta-0.1.0/docs/how_to/resume_a_failed_run.md +122 -0
- cureta-0.1.0/docs/how_to/tune_chunk_size.md +108 -0
- cureta-0.1.0/docs/how_to/use_external_taggers.md +246 -0
- cureta-0.1.0/docs/index.md +107 -0
- cureta-0.1.0/docs/reference/cli.md +316 -0
- cureta-0.1.0/docs/reference/cluster_config.md +142 -0
- cureta-0.1.0/docs/reference/data_sources.md +182 -0
- cureta-0.1.0/docs/reference/external_taggers.md +199 -0
- cureta-0.1.0/docs/reference/output_manifest.md +225 -0
- cureta-0.1.0/docs/reference/pipeline_config.md +184 -0
- cureta-0.1.0/docs/reference/python_api.md +266 -0
- cureta-0.1.0/docs/reference/taggers.md +1345 -0
- cureta-0.1.0/docs/tutorials/data_quality_pipeline.md +188 -0
- cureta-0.1.0/docs/tutorials/deploying_on_cloud.md +277 -0
- cureta-0.1.0/docs/tutorials/first_pipeline.md +198 -0
- cureta-0.1.0/docs/tutorials/writing_a_custom_tagger.md +336 -0
- cureta-0.1.0/docs/writing_taggers.md +447 -0
- cureta-0.1.0/examples/01_quickstart/README.md +36 -0
- cureta-0.1.0/examples/01_quickstart/pipeline_config.yaml +5 -0
- cureta-0.1.0/examples/01_quickstart/run.py +68 -0
- cureta-0.1.0/examples/02_custom_tagger/README.md +38 -0
- cureta-0.1.0/examples/02_custom_tagger/my_tagger.py +94 -0
- cureta-0.1.0/examples/02_custom_tagger/pipeline_config.yaml +5 -0
- cureta-0.1.0/examples/02_custom_tagger/run.py +61 -0
- cureta-0.1.0/examples/03_huggingface/README.md +81 -0
- cureta-0.1.0/examples/03_huggingface/pipeline_config.yaml +23 -0
- cureta-0.1.0/examples/03_huggingface/run.py +112 -0
- cureta-0.1.0/examples/04_programmatic_api/README.md +45 -0
- cureta-0.1.0/examples/04_programmatic_api/pipeline_config.yaml +3 -0
- cureta-0.1.0/examples/04_programmatic_api/run.py +72 -0
- cureta-0.1.0/examples/05_streaming/README.md +44 -0
- cureta-0.1.0/examples/05_streaming/pipeline_config.yaml +2 -0
- cureta-0.1.0/examples/05_streaming/run.py +124 -0
- cureta-0.1.0/examples/README.md +13 -0
- cureta-0.1.0/examples/pipeline_config.yaml +29 -0
- cureta-0.1.0/mkdocs.yml +97 -0
- cureta-0.1.0/pipelines/aclust_test/pipeline_config.yaml +41 -0
- cureta-0.1.0/pipelines/example/pipeline_config.yaml +24 -0
- cureta-0.1.0/pipelines/global/global_defaults.yaml +55 -0
- cureta-0.1.0/pipelines/jclust_test/pipeline_config.yaml +41 -0
- cureta-0.1.0/pipelines/sangraha_hin_verified/pipeline_config.yaml +45 -0
- cureta-0.1.0/pipelines/sangraha_hin_verified/smoke_test.yaml +21 -0
- cureta-0.1.0/pyproject.toml +114 -0
- cureta-0.1.0/scripts/check_dataset_size.py +0 -0
- cureta-0.1.0/scripts/download_sentiment_dataset.py +89 -0
- cureta-0.1.0/scripts/execute_ray_pipeline.py +0 -0
- cureta-0.1.0/scripts/regenerate_cli_docs.py +72 -0
- cureta-0.1.0/scripts/submit_pipeline.py +0 -0
- cureta-0.1.0/tests/__init__.py +12 -0
- cureta-0.1.0/tests/conftest.py +112 -0
- cureta-0.1.0/tests/integration/__init__.py +1 -0
- cureta-0.1.0/tests/integration/conftest.py +110 -0
- cureta-0.1.0/tests/integration/sample_taggers.py +37 -0
- cureta-0.1.0/tests/integration/sky_test_task.yaml +25 -0
- cureta-0.1.0/tests/integration/test_distributed_ray.py +160 -0
- cureta-0.1.0/tests/integration/test_engine_equivalence.py +300 -0
- cureta-0.1.0/tests/integration/test_external_taggers_integration.py +244 -0
- cureta-0.1.0/tests/integration/test_manifest_provenance.py +301 -0
- cureta-0.1.0/tests/integration/test_notebook_usage.py +245 -0
- cureta-0.1.0/tests/integration/test_pipeline_workflow.py +132 -0
- cureta-0.1.0/tests/integration/test_skypilot_integration.py +320 -0
- cureta-0.1.0/tests/integration/test_streaming_chunks.py +72 -0
- cureta-0.1.0/tests/integration/test_two_tier_pipeline.py +138 -0
- cureta-0.1.0/tests/nysa_test/__init__.py +1 -0
- cureta-0.1.0/tests/nysa_test/configs/global_defaults.yaml +96 -0
- cureta-0.1.0/tests/nysa_test/configs/heuristic_pipeline_config.yaml +27 -0
- cureta-0.1.0/tests/nysa_test/configs/perf_sdmi.yaml +21 -0
- cureta-0.1.0/tests/nysa_test/configs/perf_simd.yaml +21 -0
- cureta-0.1.0/tests/nysa_test/configs/pipeline_config.yaml +21 -0
- cureta-0.1.0/tests/nysa_test/configs/pipeline_config_indic_lid.yaml +6 -0
- cureta-0.1.0/tests/nysa_test/configs/pipeline_config_mmbert.yaml +7 -0
- cureta-0.1.0/tests/nysa_test/configs/pipeline_config_perplexity.yaml +7 -0
- cureta-0.1.0/tests/nysa_test/configs/pipeline_config_sdmi.yaml +25 -0
- cureta-0.1.0/tests/nysa_test/fixtures/sample_docs.jsonl +5 -0
- cureta-0.1.0/tests/nysa_test/run_heuristic_pipeline.py +197 -0
- cureta-0.1.0/tests/nysa_test/run_indic_lid_pipeline.py +146 -0
- cureta-0.1.0/tests/nysa_test/run_mmbert_quality_pipeline.py +193 -0
- cureta-0.1.0/tests/nysa_test/run_perf_pipeline.py +106 -0
- cureta-0.1.0/tests/nysa_test/run_perplexity_pipeline.py +191 -0
- cureta-0.1.0/tests/nysa_test/run_pipeline.py +170 -0
- cureta-0.1.0/tests/nysa_test/taggers/__init__.py +1 -0
- cureta-0.1.0/tests/nysa_test/taggers/sentence_splitter.py +22 -0
- cureta-0.1.0/tests/nysa_test/taggers/sentiment_analyzer.py +61 -0
- cureta-0.1.0/tests/nysa_test/taggers/slow_tagger.py +34 -0
- cureta-0.1.0/tests/nysa_test/taggers/word_counter.py +16 -0
- cureta-0.1.0/tests/nysa_test/test_nysa_integration.py +544 -0
- cureta-0.1.0/tests/test_indic_lid_local.py +59 -0
- cureta-0.1.0/tests/unit/__init__.py +1 -0
- cureta-0.1.0/tests/unit/test_base_tagger.py +248 -0
- cureta-0.1.0/tests/unit/test_c4_quality_tagger.py +97 -0
- cureta-0.1.0/tests/unit/test_chunk_extraction.py +83 -0
- cureta-0.1.0/tests/unit/test_cli.py +127 -0
- cureta-0.1.0/tests/unit/test_compression_ratio_tagger.py +64 -0
- cureta-0.1.0/tests/unit/test_config.py +444 -0
- cureta-0.1.0/tests/unit/test_content_type_tagger.py +65 -0
- cureta-0.1.0/tests/unit/test_data_sources.py +308 -0
- cureta-0.1.0/tests/unit/test_date_mention_tagger.py +68 -0
- cureta-0.1.0/tests/unit/test_detoxify_tagger.py +111 -0
- cureta-0.1.0/tests/unit/test_distributed_runner.py +270 -0
- cureta-0.1.0/tests/unit/test_document.py +82 -0
- cureta-0.1.0/tests/unit/test_domain_classifier_tagger.py +110 -0
- cureta-0.1.0/tests/unit/test_engine.py +424 -0
- cureta-0.1.0/tests/unit/test_exact_norm_hash_tagger.py +66 -0
- cureta-0.1.0/tests/unit/test_execution_modes.py +115 -0
- cureta-0.1.0/tests/unit/test_external_taggers.py +803 -0
- cureta-0.1.0/tests/unit/test_fasttext_quality_tagger.py +93 -0
- cureta-0.1.0/tests/unit/test_fineweb_edu_tagger.py +122 -0
- cureta-0.1.0/tests/unit/test_fineweb_quality_tagger.py +80 -0
- cureta-0.1.0/tests/unit/test_glotlid_tagger.py +92 -0
- cureta-0.1.0/tests/unit/test_gopher_quality_tagger.py +90 -0
- cureta-0.1.0/tests/unit/test_html_density_tagger.py +57 -0
- cureta-0.1.0/tests/unit/test_init.py +92 -0
- cureta-0.1.0/tests/unit/test_llm_judge_tagger.py +111 -0
- cureta-0.1.0/tests/unit/test_manifest.py +414 -0
- cureta-0.1.0/tests/unit/test_markdown_structure_tagger.py +68 -0
- cureta-0.1.0/tests/unit/test_minhash_tagger.py +76 -0
- cureta-0.1.0/tests/unit/test_paragraph_langid_tagger.py +108 -0
- cureta-0.1.0/tests/unit/test_paragraph_stats_tagger.py +66 -0
- cureta-0.1.0/tests/unit/test_perplexity_tagger.py +298 -0
- cureta-0.1.0/tests/unit/test_pii_regex_tagger.py +89 -0
- cureta-0.1.0/tests/unit/test_pipeline.py +355 -0
- cureta-0.1.0/tests/unit/test_presidio_pii_tagger.py +96 -0
- cureta-0.1.0/tests/unit/test_programming_language_tagger.py +77 -0
- cureta-0.1.0/tests/unit/test_quick_tag.py +176 -0
- cureta-0.1.0/tests/unit/test_readability_tagger.py +60 -0
- cureta-0.1.0/tests/unit/test_rename_identity.py +64 -0
- cureta-0.1.0/tests/unit/test_sentence_embedding_tagger.py +92 -0
- cureta-0.1.0/tests/unit/test_simhash_tagger.py +60 -0
- cureta-0.1.0/tests/unit/test_skypilot_cluster.py +122 -0
- cureta-0.1.0/tests/unit/test_tag_ops.py +115 -0
- cureta-0.1.0/tests/unit/test_type_token_ratio_tagger.py +68 -0
- cureta-0.1.0/tests/unit/test_url_blocklist_tagger.py +86 -0
- cureta-0.1.0/tests/unit/test_url_parse_tagger.py +74 -0
- cureta-0.1.0/tests/unit/test_word_length_stats_tagger.py +63 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Bug report
|
|
3
|
+
about: Something is broken or behaving unexpectedly
|
|
4
|
+
title: 'bug: '
|
|
5
|
+
labels: bug
|
|
6
|
+
assignees: ''
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Description
|
|
10
|
+
|
|
11
|
+
A clear, one-sentence description of the bug.
|
|
12
|
+
|
|
13
|
+
## Steps to reproduce
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
# Minimal code that reproduces the issue
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Or the CLI command:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
cureta run ...
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Expected behaviour
|
|
26
|
+
|
|
27
|
+
What you expected to happen.
|
|
28
|
+
|
|
29
|
+
## Actual behaviour
|
|
30
|
+
|
|
31
|
+
What actually happened. Include the full error message and traceback if applicable.
|
|
32
|
+
|
|
33
|
+
## Environment
|
|
34
|
+
|
|
35
|
+
- **Cureta version**: (run `cureta --version`)
|
|
36
|
+
- **Python version**: (run `python --version`)
|
|
37
|
+
- **Ray version**: (run `python -c "import ray; print(ray.__version__)"`)
|
|
38
|
+
- **OS**: (e.g. Ubuntu 22.04, macOS 14)
|
|
39
|
+
- **GPU**: (if relevant — model and VRAM)
|
|
40
|
+
|
|
41
|
+
## Additional context
|
|
42
|
+
|
|
43
|
+
Any other relevant information (pipeline config, dataset characteristics, cluster setup).
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Feature request
|
|
3
|
+
about: Suggest a new feature or improvement
|
|
4
|
+
title: 'feat: '
|
|
5
|
+
labels: enhancement
|
|
6
|
+
assignees: ''
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Problem
|
|
10
|
+
|
|
11
|
+
A clear description of the problem this feature would solve. Example: "I need to load data from Elasticsearch but there is no built-in DataSource for it."
|
|
12
|
+
|
|
13
|
+
## Proposed solution
|
|
14
|
+
|
|
15
|
+
A description of the feature you would like, with enough detail to evaluate feasibility. Include a code sketch if helpful:
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
# Example of what the feature would look like to use
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Alternatives considered
|
|
22
|
+
|
|
23
|
+
Other approaches you have thought about and why you prefer this solution.
|
|
24
|
+
|
|
25
|
+
## Willingness to contribute
|
|
26
|
+
|
|
27
|
+
- [ ] I am willing to open a PR for this feature
|
|
28
|
+
- [ ] I would appreciate guidance but could attempt a PR
|
|
29
|
+
- [ ] I am requesting this for others to implement
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Question
|
|
3
|
+
about: Ask a question about using or understanding Cureta
|
|
4
|
+
title: 'question: '
|
|
5
|
+
labels: question
|
|
6
|
+
assignees: ''
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Question
|
|
10
|
+
|
|
11
|
+
Your question, as clearly as possible.
|
|
12
|
+
|
|
13
|
+
## What I have tried
|
|
14
|
+
|
|
15
|
+
Steps you have already taken to answer the question yourself:
|
|
16
|
+
- Docs I read:
|
|
17
|
+
- Code I looked at:
|
|
18
|
+
|
|
19
|
+
## Relevant context
|
|
20
|
+
|
|
21
|
+
Any code, configs, or error messages relevant to the question.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
## Summary
|
|
2
|
+
|
|
3
|
+
Brief description of what this PR does and why.
|
|
4
|
+
|
|
5
|
+
## Type of change
|
|
6
|
+
|
|
7
|
+
- [ ] Bug fix (fixes a broken behaviour without changing the API)
|
|
8
|
+
- [ ] New feature (adds functionality without breaking existing behaviour)
|
|
9
|
+
- [ ] Breaking change (changes existing API or output format)
|
|
10
|
+
- [ ] Documentation only (no code changes)
|
|
11
|
+
- [ ] Refactor (no functionality change)
|
|
12
|
+
|
|
13
|
+
## Testing
|
|
14
|
+
|
|
15
|
+
- [ ] Unit tests pass: `pytest tests/unit/ -v`
|
|
16
|
+
- [ ] Integration tests pass: `pytest tests/integration/ -v`
|
|
17
|
+
- [ ] Type check passes: `pyright .`
|
|
18
|
+
- [ ] Linter passes: `ruff check .`
|
|
19
|
+
|
|
20
|
+
## Documentation
|
|
21
|
+
|
|
22
|
+
- [ ] Docstrings updated for any changed public APIs
|
|
23
|
+
- [ ] Relevant docs in `docs/` updated
|
|
24
|
+
- [ ] `CHANGELOG.md` updated (for user-visible changes)
|
|
25
|
+
|
|
26
|
+
## Checklist
|
|
27
|
+
|
|
28
|
+
- [ ] PR title clearly describes the change (will be used in the changelog)
|
|
29
|
+
- [ ] Tests added or updated for new/changed functionality
|
|
30
|
+
- [ ] No secrets, credentials, or large binary files added
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
name: Tests (Python ${{ matrix.python-version }})
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
22
|
+
uses: actions/setup-python@v5
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python-version }}
|
|
25
|
+
|
|
26
|
+
- name: Cache pip
|
|
27
|
+
uses: actions/cache@v4
|
|
28
|
+
with:
|
|
29
|
+
path: ~/.cache/pip
|
|
30
|
+
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
|
|
31
|
+
restore-keys: |
|
|
32
|
+
${{ runner.os }}-pip-${{ matrix.python-version }}-
|
|
33
|
+
|
|
34
|
+
- name: Install dependencies
|
|
35
|
+
run: |
|
|
36
|
+
pip install -e ".[ray,dev]"
|
|
37
|
+
|
|
38
|
+
- name: Run unit tests
|
|
39
|
+
run: |
|
|
40
|
+
pytest tests/unit/ -v --tb=short
|
|
41
|
+
|
|
42
|
+
- name: Run integration tests
|
|
43
|
+
run: |
|
|
44
|
+
pytest tests/integration/ -v --tb=short \
|
|
45
|
+
--ignore=tests/integration/test_skypilot_integration.py
|
|
46
|
+
|
|
47
|
+
typecheck:
|
|
48
|
+
name: Type check
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v4
|
|
53
|
+
|
|
54
|
+
- name: Set up Python 3.11
|
|
55
|
+
uses: actions/setup-python@v5
|
|
56
|
+
with:
|
|
57
|
+
python-version: "3.11"
|
|
58
|
+
|
|
59
|
+
- name: Install dependencies
|
|
60
|
+
run: |
|
|
61
|
+
pip install -e ".[ray,dev]"
|
|
62
|
+
|
|
63
|
+
- name: Run pyright
|
|
64
|
+
run: |
|
|
65
|
+
pyright .
|
|
66
|
+
|
|
67
|
+
lint:
|
|
68
|
+
name: Lint
|
|
69
|
+
runs-on: ubuntu-latest
|
|
70
|
+
|
|
71
|
+
steps:
|
|
72
|
+
- uses: actions/checkout@v4
|
|
73
|
+
|
|
74
|
+
- name: Set up Python 3.11
|
|
75
|
+
uses: actions/setup-python@v5
|
|
76
|
+
with:
|
|
77
|
+
python-version: "3.11"
|
|
78
|
+
|
|
79
|
+
- name: Install ruff
|
|
80
|
+
run: pip install ruff
|
|
81
|
+
|
|
82
|
+
- name: Run ruff check
|
|
83
|
+
run: ruff check .
|
|
84
|
+
|
|
85
|
+
- name: Run ruff format check
|
|
86
|
+
run: ruff format --check .
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
name: Docs
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build:
|
|
14
|
+
name: Build docs
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- name: Set up Python 3.11
|
|
21
|
+
uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.11"
|
|
24
|
+
|
|
25
|
+
- name: Install docs dependencies
|
|
26
|
+
run: |
|
|
27
|
+
pip install -e ".[docs]"
|
|
28
|
+
|
|
29
|
+
- name: Build docs (strict — fails on broken links)
|
|
30
|
+
run: |
|
|
31
|
+
mkdocs build --strict
|
|
32
|
+
|
|
33
|
+
deploy:
|
|
34
|
+
name: Deploy to GitHub Pages
|
|
35
|
+
runs-on: ubuntu-latest
|
|
36
|
+
needs: build
|
|
37
|
+
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
|
38
|
+
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/checkout@v4
|
|
41
|
+
|
|
42
|
+
- name: Set up Python 3.11
|
|
43
|
+
uses: actions/setup-python@v5
|
|
44
|
+
with:
|
|
45
|
+
python-version: "3.11"
|
|
46
|
+
|
|
47
|
+
- name: Install docs dependencies
|
|
48
|
+
run: |
|
|
49
|
+
pip install -e ".[docs]"
|
|
50
|
+
|
|
51
|
+
- name: Deploy to GitHub Pages
|
|
52
|
+
run: |
|
|
53
|
+
git config user.name "github-actions[bot]"
|
|
54
|
+
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
55
|
+
mkdocs gh-deploy --force
|
cureta-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
#poetry.lock
|
|
109
|
+
#poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
#pdm.lock
|
|
116
|
+
#pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
#pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# SageMath parsed files
|
|
135
|
+
*.sage.py
|
|
136
|
+
|
|
137
|
+
# Environments
|
|
138
|
+
.env
|
|
139
|
+
.envrc
|
|
140
|
+
.venv
|
|
141
|
+
.venv*
|
|
142
|
+
env/
|
|
143
|
+
venv/
|
|
144
|
+
ENV/
|
|
145
|
+
env.bak/
|
|
146
|
+
venv.bak/
|
|
147
|
+
|
|
148
|
+
# Spyder project settings
|
|
149
|
+
.spyderproject
|
|
150
|
+
.spyproject
|
|
151
|
+
|
|
152
|
+
# Rope project settings
|
|
153
|
+
.ropeproject
|
|
154
|
+
|
|
155
|
+
# mkdocs documentation
|
|
156
|
+
/site
|
|
157
|
+
|
|
158
|
+
# mypy
|
|
159
|
+
.mypy_cache/
|
|
160
|
+
.dmypy.json
|
|
161
|
+
dmypy.json
|
|
162
|
+
|
|
163
|
+
# Pyre type checker
|
|
164
|
+
.pyre/
|
|
165
|
+
|
|
166
|
+
# pytype static type analyzer
|
|
167
|
+
.pytype/
|
|
168
|
+
|
|
169
|
+
# Cython debug symbols
|
|
170
|
+
cython_debug/
|
|
171
|
+
|
|
172
|
+
# PyCharm
|
|
173
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
174
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
175
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
176
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
177
|
+
#.idea/
|
|
178
|
+
|
|
179
|
+
# Abstra
|
|
180
|
+
# Abstra is an AI-powered process automation framework.
|
|
181
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
182
|
+
# Learn more at https://abstra.io/docs
|
|
183
|
+
.abstra/
|
|
184
|
+
|
|
185
|
+
# Visual Studio Code
|
|
186
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
187
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
188
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
189
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
190
|
+
# .vscode/
|
|
191
|
+
|
|
192
|
+
# Ruff stuff:
|
|
193
|
+
.ruff_cache/
|
|
194
|
+
|
|
195
|
+
# PyPI configuration file
|
|
196
|
+
.pypirc
|
|
197
|
+
|
|
198
|
+
# Cursor
|
|
199
|
+
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
|
200
|
+
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
|
201
|
+
# refer to https://docs.cursor.com/context/ignore-files
|
|
202
|
+
.cursorignore
|
|
203
|
+
.cursorindexingignore
|
|
204
|
+
|
|
205
|
+
# Marimo
|
|
206
|
+
marimo/_static/
|
|
207
|
+
marimo/_lsp/
|
|
208
|
+
__marimo__/
|
|
209
|
+
|
|
210
|
+
# Dev Resources
|
|
211
|
+
*.txt
|
|
212
|
+
dev_scripts/
|
|
213
|
+
|
|
214
|
+
# Data
|
|
215
|
+
data/
|
|
216
|
+
|
|
217
|
+
# Benchmarking run outputs (regenerated by run_all.sh; not committed)
|
|
218
|
+
.benchmarking/results/full_pipeline/
|
|
219
|
+
.benchmarking/results/external_tagger_comparison
|