awx-zipline-ai 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of awx-zipline-ai might be problematic. Click here for more details.
- {awx_zipline_ai-0.2.0/awx_zipline_ai.egg-info → awx_zipline_ai-0.3.0}/PKG-INFO +47 -24
- awx_zipline_ai-0.3.0/pyproject.toml +23 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/agent/ttypes.py +6 -6
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/airflow_helpers.py +20 -23
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/column_hashing.py +40 -17
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/compile_context.py +13 -17
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/compiler.py +59 -36
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/conf_validator.py +251 -99
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/display/class_tracker.py +6 -16
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/display/compile_status.py +10 -10
- awx_zipline_ai-0.3.0/src/ai/chronon/cli/compile/display/diff_result.py +111 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/fill_templates.py +3 -8
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/parse_configs.py +10 -17
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/parse_teams.py +38 -34
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/serializer.py +3 -9
- awx_zipline_ai-0.3.0/src/ai/chronon/cli/compile/version_utils.py +42 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/git_utils.py +2 -13
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/logger.py +0 -2
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/constants.py +1 -1
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/group_by.py +47 -47
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/join.py +46 -32
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/logger.py +1 -2
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/model.py +9 -4
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/query.py +2 -2
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/__init__.py +1 -2
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/aws.py +17 -31
- awx_zipline_ai-0.3.0/src/ai/chronon/repo/cluster.py +136 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/compile.py +14 -8
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/constants.py +1 -1
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/default_runner.py +32 -54
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/explore.py +70 -73
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/extract_objects.py +6 -9
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/gcp.py +89 -88
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/gitpython_utils.py +3 -2
- awx_zipline_ai-0.3.0/src/ai/chronon/repo/hub_runner.py +261 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/hub_uploader.py +2 -1
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/init.py +12 -5
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/join_backfill.py +19 -5
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/run.py +42 -39
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/serializer.py +4 -12
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/utils.py +72 -63
- awx_zipline_ai-0.3.0/src/ai/chronon/repo/zipline.py +35 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/repo/zipline_hub.py +277 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/__init__.py +0 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/__init__.py +0 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/group_bys/__init__.py +0 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/group_bys/test/__init__.py +0 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/group_bys/test/data.py +30 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/joins/__init__.py +0 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/joins/test/__init__.py +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/resources/gcp/joins/test/data.py +4 -8
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/sources/__init__.py +0 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/sources/test/__init__.py +0 -0
- awx_zipline_ai-0.3.0/src/ai/chronon/resources/gcp/sources/test/data.py +26 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/resources/gcp/teams.py +9 -21
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/source.py +2 -4
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/staging_query.py +60 -19
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/types.py +3 -2
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/utils.py +21 -68
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/windows.py +2 -4
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src/awx_zipline_ai.egg-info}/PKG-INFO +47 -24
- awx_zipline_ai-0.3.0/src/awx_zipline_ai.egg-info/SOURCES.txt +99 -0
- awx_zipline_ai-0.3.0/src/awx_zipline_ai.egg-info/requires.txt +42 -0
- awx_zipline_ai-0.3.0/src/awx_zipline_ai.egg-info/top_level.txt +4 -0
- awx_zipline_ai-0.3.0/src/gen_thrift/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/api/ttypes.py +327 -197
- {awx_zipline_ai-0.2.0/ai/chronon/api → awx_zipline_ai-0.3.0/src/gen_thrift}/common/ttypes.py +9 -39
- awx_zipline_ai-0.3.0/src/gen_thrift/eval/ttypes.py +660 -0
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/hub/ttypes.py +12 -131
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/observability/ttypes.py +343 -180
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/planner/ttypes.py +326 -45
- awx_zipline_ai-0.2.0/LICENSE +0 -202
- awx_zipline_ai-0.2.0/MANIFEST.in +0 -5
- awx_zipline_ai-0.2.0/ai/chronon/cli/compile/display/diff_result.py +0 -46
- awx_zipline_ai-0.2.0/ai/chronon/eval/__init__.py +0 -122
- awx_zipline_ai-0.2.0/ai/chronon/eval/query_parsing.py +0 -19
- awx_zipline_ai-0.2.0/ai/chronon/eval/sample_tables.py +0 -100
- awx_zipline_ai-0.2.0/ai/chronon/eval/table_scan.py +0 -186
- awx_zipline_ai-0.2.0/ai/chronon/orchestration/ttypes.py +0 -4406
- awx_zipline_ai-0.2.0/ai/chronon/repo/cluster.py +0 -65
- awx_zipline_ai-0.2.0/ai/chronon/repo/hub_runner.py +0 -171
- awx_zipline_ai-0.2.0/ai/chronon/repo/zipline.py +0 -51
- awx_zipline_ai-0.2.0/ai/chronon/repo/zipline_hub.py +0 -105
- awx_zipline_ai-0.2.0/ai/chronon/resources/gcp/README.md +0 -174
- awx_zipline_ai-0.2.0/ai/chronon/resources/gcp/group_bys/test/data.py +0 -34
- awx_zipline_ai-0.2.0/ai/chronon/resources/gcp/sources/test/data.py +0 -23
- awx_zipline_ai-0.2.0/ai/chronon/resources/gcp/zipline-cli-install.sh +0 -54
- awx_zipline_ai-0.2.0/awx_zipline_ai.egg-info/SOURCES.txt +0 -111
- awx_zipline_ai-0.2.0/awx_zipline_ai.egg-info/not-zip-safe +0 -1
- awx_zipline_ai-0.2.0/awx_zipline_ai.egg-info/requires.txt +0 -15
- awx_zipline_ai-0.2.0/awx_zipline_ai.egg-info/top_level.txt +0 -3
- awx_zipline_ai-0.2.0/pyproject.toml +0 -41
- awx_zipline_ai-0.2.0/requirements/base.in +0 -12
- awx_zipline_ai-0.2.0/setup.py +0 -78
- awx_zipline_ai-0.2.0/test/test_compile.py +0 -58
- awx_zipline_ai-0.2.0/test/test_decorator.py +0 -52
- awx_zipline_ai-0.2.0/test/test_explore.py +0 -43
- awx_zipline_ai-0.2.0/test/test_git_utils.py +0 -94
- awx_zipline_ai-0.2.0/test/test_group_by.py +0 -331
- awx_zipline_ai-0.2.0/test/test_join.py +0 -49
- awx_zipline_ai-0.2.0/test/test_parse_teams.py +0 -391
- awx_zipline_ai-0.2.0/test/test_run.py +0 -341
- awx_zipline_ai-0.2.0/test/test_semantic_hashing.py +0 -1223
- awx_zipline_ai-0.2.0/test/test_teams.py +0 -26
- awx_zipline_ai-0.2.0/test/test_utils.py +0 -358
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0}/README.md +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0}/setup.cfg +0 -0
- {awx_zipline_ai-0.2.0/ai → awx_zipline_ai-0.3.0/src}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/agent/__init__.py +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/agent/constants.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/ai}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/resources/gcp/group_bys/test → awx_zipline_ai-0.3.0/src/ai/chronon}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/resources/gcp/joins/test → awx_zipline_ai-0.3.0/src/ai/chronon/cli}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/resources/gcp/sources/test → awx_zipline_ai-0.3.0/src/ai/chronon/cli/compile}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/jars → awx_zipline_ai-0.3.0/src/ai/chronon/cli/compile/display}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/display/compiled_obj.py +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/cli/compile/display/console.py +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/ai/chronon/repo/team_json_utils.py +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/awx_zipline_ai.egg-info/dependency_links.txt +0 -0
- {awx_zipline_ai-0.2.0 → awx_zipline_ai-0.3.0/src}/awx_zipline_ai.egg-info/entry_points.txt +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/api/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/api/common → awx_zipline_ai-0.3.0/src/gen_thrift/api}/constants.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/api → awx_zipline_ai-0.3.0/src/gen_thrift}/common/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/api → awx_zipline_ai-0.3.0/src/gen_thrift/common}/constants.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/fetcher → awx_zipline_ai-0.3.0/src/gen_thrift/eval}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/fetcher → awx_zipline_ai-0.3.0/src/gen_thrift/eval}/constants.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/hub → awx_zipline_ai-0.3.0/src/gen_thrift/fetcher}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/hub → awx_zipline_ai-0.3.0/src/gen_thrift/fetcher}/constants.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/fetcher/ttypes.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/observability → awx_zipline_ai-0.3.0/src/gen_thrift/hub}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/observability → awx_zipline_ai-0.3.0/src/gen_thrift/hub}/constants.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/orchestration → awx_zipline_ai-0.3.0/src/gen_thrift/observability}/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon/orchestration → awx_zipline_ai-0.3.0/src/gen_thrift/observability}/constants.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/planner/__init__.py +0 -0
- {awx_zipline_ai-0.2.0/ai/chronon → awx_zipline_ai-0.3.0/src/gen_thrift}/planner/constants.py +0 -0
|
@@ -1,33 +1,56 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: awx-zipline-ai
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Zipline
|
|
5
|
-
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: CLI tool for the Zipline AI platform
|
|
5
|
+
Author-email: Zipline AI <hello@zipline.ai>
|
|
6
|
+
License: Apache License 2.0
|
|
7
|
+
Project-URL: homepage, https://zipline.ai
|
|
8
|
+
Project-URL: documentation, https://docs.zipline.ai
|
|
9
|
+
Project-URL: github, https://github.com/zipline-ai/chronon/
|
|
6
10
|
Requires-Python: >=3.11
|
|
7
11
|
Description-Content-Type: text/markdown
|
|
8
|
-
|
|
9
|
-
Requires-Dist:
|
|
10
|
-
Requires-Dist:
|
|
11
|
-
Requires-Dist:
|
|
12
|
-
Requires-Dist:
|
|
12
|
+
Requires-Dist: boto3==1.40.26
|
|
13
|
+
Requires-Dist: botocore==1.40.26
|
|
14
|
+
Requires-Dist: cachetools==5.5.2
|
|
15
|
+
Requires-Dist: certifi==2025.8.3
|
|
16
|
+
Requires-Dist: charset-normalizer==3.4.3
|
|
17
|
+
Requires-Dist: click==8.2.1
|
|
13
18
|
Requires-Dist: crcmod==1.7
|
|
14
|
-
Requires-Dist:
|
|
15
|
-
Requires-Dist:
|
|
16
|
-
Requires-Dist:
|
|
17
|
-
Requires-Dist:
|
|
19
|
+
Requires-Dist: gitdb==4.0.12
|
|
20
|
+
Requires-Dist: gitpython==3.1.45
|
|
21
|
+
Requires-Dist: google-api-core[grpc]==2.25.1
|
|
22
|
+
Requires-Dist: google-auth==2.40.3
|
|
23
|
+
Requires-Dist: google-cloud-bigquery-storage==2.33.0
|
|
24
|
+
Requires-Dist: google-cloud-core==2.4.3
|
|
25
|
+
Requires-Dist: google-cloud-iam==2.19.1
|
|
18
26
|
Requires-Dist: google-cloud-storage==2.19.0
|
|
19
|
-
Requires-Dist: google-
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
Requires-Dist: google-crc32c==1.7.1
|
|
28
|
+
Requires-Dist: google-resumable-media==2.7.2
|
|
29
|
+
Requires-Dist: googleapis-common-protos[grpc]==1.70.0
|
|
30
|
+
Requires-Dist: grpc-google-iam-v1==0.14.2
|
|
31
|
+
Requires-Dist: grpcio==1.74.0
|
|
32
|
+
Requires-Dist: grpcio-status==1.74.0
|
|
33
|
+
Requires-Dist: idna==3.10
|
|
34
|
+
Requires-Dist: importlib-resources==6.5.2
|
|
35
|
+
Requires-Dist: jmespath==1.0.1
|
|
36
|
+
Requires-Dist: markdown-it-py==4.0.0
|
|
37
|
+
Requires-Dist: mdurl==0.1.2
|
|
38
|
+
Requires-Dist: proto-plus==1.26.1
|
|
39
|
+
Requires-Dist: protobuf==6.32.0
|
|
40
|
+
Requires-Dist: py4j==0.10.9.7
|
|
41
|
+
Requires-Dist: pyasn1==0.6.1
|
|
42
|
+
Requires-Dist: pyasn1-modules==0.4.2
|
|
43
|
+
Requires-Dist: pygments==2.19.2
|
|
44
|
+
Requires-Dist: pyspark==3.5.4
|
|
45
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
46
|
+
Requires-Dist: requests==2.32.5
|
|
47
|
+
Requires-Dist: rich==14.1.0
|
|
48
|
+
Requires-Dist: rsa==4.9.1
|
|
49
|
+
Requires-Dist: s3transfer==0.13.1
|
|
50
|
+
Requires-Dist: six==1.17.0
|
|
51
|
+
Requires-Dist: smmap==5.0.2
|
|
52
|
+
Requires-Dist: thrift==0.20.0
|
|
53
|
+
Requires-Dist: urllib3==2.5.0
|
|
31
54
|
|
|
32
55
|
### Chronon Python API
|
|
33
56
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name="awx-zipline-ai"
|
|
3
|
+
version="0.3.0"
|
|
4
|
+
description="CLI tool for the Zipline AI platform"
|
|
5
|
+
readme="README.md"
|
|
6
|
+
dependencies=["boto3==1.40.26", "botocore==1.40.26", "cachetools==5.5.2", "certifi==2025.8.3", "charset-normalizer==3.4.3", "click==8.2.1", "crcmod==1.7", "gitdb==4.0.12", "gitpython==3.1.45", "google-api-core[grpc]==2.25.1", "google-auth==2.40.3", "google-cloud-bigquery-storage==2.33.0", "google-cloud-core==2.4.3", "google-cloud-iam==2.19.1", "google-cloud-storage==2.19.0", "google-crc32c==1.7.1", "google-resumable-media==2.7.2", "googleapis-common-protos[grpc]==1.70.0", "grpc-google-iam-v1==0.14.2", "grpcio==1.74.0", "grpcio-status==1.74.0", "idna==3.10", "importlib-resources==6.5.2", "jmespath==1.0.1", "markdown-it-py==4.0.0", "mdurl==0.1.2", "proto-plus==1.26.1", "protobuf==6.32.0", "py4j==0.10.9.7", "pyasn1==0.6.1", "pyasn1-modules==0.4.2", "pygments==2.19.2", "pyspark==3.5.4", "python-dateutil==2.9.0.post0", "requests==2.32.5", "rich==14.1.0", "rsa==4.9.1", "s3transfer==0.13.1", "six==1.17.0", "smmap==5.0.2", "thrift==0.20.0", "urllib3==2.5.0"]
|
|
7
|
+
requires-python=">= 3.11"
|
|
8
|
+
license={text="Apache License 2.0"}
|
|
9
|
+
keywords=[]
|
|
10
|
+
classifiers=[]
|
|
11
|
+
authors=[{name="Zipline AI", email="hello@zipline.ai"}]
|
|
12
|
+
|
|
13
|
+
[project.urls]
|
|
14
|
+
"homepage"="https://zipline.ai"
|
|
15
|
+
"documentation"="https://docs.zipline.ai"
|
|
16
|
+
"github"="https://github.com/zipline-ai/chronon/"
|
|
17
|
+
|
|
18
|
+
[build-system]
|
|
19
|
+
requires=["setuptools"]
|
|
20
|
+
build-backend="setuptools.build_meta"
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
zipline = "ai.chronon.repo.zipline:zipline"
|
|
@@ -12,7 +12,7 @@ from thrift.TRecursive import fix_spec
|
|
|
12
12
|
from uuid import UUID
|
|
13
13
|
|
|
14
14
|
import sys
|
|
15
|
-
import
|
|
15
|
+
import gen_thrift.common.ttypes
|
|
16
16
|
|
|
17
17
|
from thrift.transport import TTransport
|
|
18
18
|
all_structs = []
|
|
@@ -39,7 +39,7 @@ class JobStatusType(object):
|
|
|
39
39
|
RUNNING = 2
|
|
40
40
|
SUCCEEDED = 3
|
|
41
41
|
FAILED = 4
|
|
42
|
-
|
|
42
|
+
CANCELLED = 5
|
|
43
43
|
|
|
44
44
|
_VALUES_TO_NAMES = {
|
|
45
45
|
0: "UNKNOWN",
|
|
@@ -47,7 +47,7 @@ class JobStatusType(object):
|
|
|
47
47
|
2: "RUNNING",
|
|
48
48
|
3: "SUCCEEDED",
|
|
49
49
|
4: "FAILED",
|
|
50
|
-
5: "
|
|
50
|
+
5: "CANCELLED",
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
_NAMES_TO_VALUES = {
|
|
@@ -56,7 +56,7 @@ class JobStatusType(object):
|
|
|
56
56
|
"RUNNING": 2,
|
|
57
57
|
"SUCCEEDED": 3,
|
|
58
58
|
"FAILED": 4,
|
|
59
|
-
"
|
|
59
|
+
"CANCELLED": 5,
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
|
|
@@ -1388,7 +1388,7 @@ class PartitionListingPutRequest(object):
|
|
|
1388
1388
|
_val84 = []
|
|
1389
1389
|
(_etype88, _size85) = iprot.readListBegin()
|
|
1390
1390
|
for _i89 in range(_size85):
|
|
1391
|
-
_elem90 =
|
|
1391
|
+
_elem90 = gen_thrift.common.ttypes.DateRange()
|
|
1392
1392
|
_elem90.read(iprot)
|
|
1393
1393
|
_val84.append(_elem90)
|
|
1394
1394
|
iprot.readListEnd()
|
|
@@ -1672,7 +1672,7 @@ JobInfo.thrift_spec = (
|
|
|
1672
1672
|
all_structs.append(PartitionListingPutRequest)
|
|
1673
1673
|
PartitionListingPutRequest.thrift_spec = (
|
|
1674
1674
|
None, # 0
|
|
1675
|
-
(1, TType.MAP, 'partitions', (TType.STRUCT, [PartitionListingJob, None], TType.LIST, (TType.STRUCT, [
|
|
1675
|
+
(1, TType.MAP, 'partitions', (TType.STRUCT, [PartitionListingJob, None], TType.LIST, (TType.STRUCT, [gen_thrift.common.ttypes.DateRange, None], False), False), None, ), # 1
|
|
1676
1676
|
(2, TType.MAP, 'errors', (TType.STRUCT, [PartitionListingJob, None], TType.STRING, 'UTF8', False), None, ), # 2
|
|
1677
1677
|
)
|
|
1678
1678
|
all_structs.append(JobInfoPutRequest)
|
|
@@ -2,9 +2,10 @@ import json
|
|
|
2
2
|
import math
|
|
3
3
|
from typing import OrderedDict
|
|
4
4
|
|
|
5
|
+
from gen_thrift.api.ttypes import GroupBy, Join
|
|
6
|
+
from gen_thrift.common.ttypes import TimeUnit
|
|
7
|
+
|
|
5
8
|
import ai.chronon.utils as utils
|
|
6
|
-
from ai.chronon.api.common.ttypes import TimeUnit
|
|
7
|
-
from ai.chronon.api.ttypes import GroupBy, Join
|
|
8
9
|
from ai.chronon.constants import (
|
|
9
10
|
AIRFLOW_DEPENDENCIES_KEY,
|
|
10
11
|
AIRFLOW_LABEL_DEPENDENCIES_KEY,
|
|
@@ -55,6 +56,7 @@ def _get_partition_col_from_query(query):
|
|
|
55
56
|
return query.partitionColumn
|
|
56
57
|
return None
|
|
57
58
|
|
|
59
|
+
|
|
58
60
|
def _get_additional_subPartitionsToWaitFor_from_query(query):
|
|
59
61
|
"""Gets additional subPartitionsToWaitFor from query if available"""
|
|
60
62
|
if query:
|
|
@@ -80,7 +82,8 @@ def _get_airflow_deps_from_source(source, partition_column=None):
|
|
|
80
82
|
tables = [source.events.table]
|
|
81
83
|
# Use partition column from query if available, otherwise use the provided one
|
|
82
84
|
source_partition_column, additional_partitions = (
|
|
83
|
-
_get_partition_col_from_query(source.events.query) or partition_column,
|
|
85
|
+
_get_partition_col_from_query(source.events.query) or partition_column,
|
|
86
|
+
_get_additional_subPartitionsToWaitFor_from_query(source.events.query),
|
|
84
87
|
)
|
|
85
88
|
|
|
86
89
|
elif source.entities:
|
|
@@ -89,7 +92,8 @@ def _get_airflow_deps_from_source(source, partition_column=None):
|
|
|
89
92
|
if source.entities.mutationTable:
|
|
90
93
|
tables.append(source.entities.mutationTable)
|
|
91
94
|
source_partition_column, additional_partitions = (
|
|
92
|
-
_get_partition_col_from_query(source.entities.query) or partition_column,
|
|
95
|
+
_get_partition_col_from_query(source.entities.query) or partition_column,
|
|
96
|
+
_get_additional_subPartitionsToWaitFor_from_query(source.entities.query),
|
|
93
97
|
)
|
|
94
98
|
elif source.joinSource:
|
|
95
99
|
# TODO: Handle joinSource -- it doesn't work right now because the metadata isn't set on joinSource at this point
|
|
@@ -99,15 +103,14 @@ def _get_airflow_deps_from_source(source, partition_column=None):
|
|
|
99
103
|
return []
|
|
100
104
|
|
|
101
105
|
return [
|
|
102
|
-
create_airflow_dependency(table, source_partition_column, additional_partitions)
|
|
106
|
+
create_airflow_dependency(table, source_partition_column, additional_partitions)
|
|
107
|
+
for table in tables
|
|
103
108
|
]
|
|
104
109
|
|
|
105
110
|
|
|
106
111
|
def extract_default_partition_column(obj):
|
|
107
112
|
try:
|
|
108
|
-
return obj.metaData.executionInfo.conf.common.get(
|
|
109
|
-
"spark.chronon.partition.column"
|
|
110
|
-
)
|
|
113
|
+
return obj.metaData.executionInfo.conf.common.get("spark.chronon.partition.column")
|
|
111
114
|
except Exception:
|
|
112
115
|
# Error handling occurs in `create_airflow_dependency`
|
|
113
116
|
return None
|
|
@@ -124,9 +127,9 @@ def _get_distinct_day_windows(group_by):
|
|
|
124
127
|
if time_unit == TimeUnit.DAYS:
|
|
125
128
|
windows.append(length)
|
|
126
129
|
elif time_unit == TimeUnit.HOURS:
|
|
127
|
-
windows.append(math.ceil(length/24))
|
|
130
|
+
windows.append(math.ceil(length / 24))
|
|
128
131
|
elif time_unit == TimeUnit.MINUTES:
|
|
129
|
-
windows.append(math.ceil(length/(24*60)))
|
|
132
|
+
windows.append(math.ceil(length / (24 * 60)))
|
|
130
133
|
return set(windows)
|
|
131
134
|
|
|
132
135
|
|
|
@@ -137,9 +140,7 @@ def _set_join_deps(join):
|
|
|
137
140
|
|
|
138
141
|
# Handle left source
|
|
139
142
|
left_query = utils.get_query(join.left)
|
|
140
|
-
left_partition_column = (
|
|
141
|
-
_get_partition_col_from_query(left_query) or default_partition_col
|
|
142
|
-
)
|
|
143
|
+
left_partition_column = _get_partition_col_from_query(left_query) or default_partition_col
|
|
143
144
|
deps.extend(_get_airflow_deps_from_source(join.left, left_partition_column))
|
|
144
145
|
|
|
145
146
|
# Handle right parts (join parts)
|
|
@@ -149,12 +150,9 @@ def _set_join_deps(join):
|
|
|
149
150
|
for source in join_part.groupBy.sources:
|
|
150
151
|
source_query = utils.get_query(source)
|
|
151
152
|
source_partition_column = (
|
|
152
|
-
_get_partition_col_from_query(source_query)
|
|
153
|
-
or default_partition_col
|
|
154
|
-
)
|
|
155
|
-
deps.extend(
|
|
156
|
-
_get_airflow_deps_from_source(source, source_partition_column)
|
|
153
|
+
_get_partition_col_from_query(source_query) or default_partition_col
|
|
157
154
|
)
|
|
155
|
+
deps.extend(_get_airflow_deps_from_source(source, source_partition_column))
|
|
158
156
|
|
|
159
157
|
label_deps = []
|
|
160
158
|
# Handle label parts
|
|
@@ -162,7 +160,6 @@ def _set_join_deps(join):
|
|
|
162
160
|
join_output_table = utils.output_table_name(join, full_name=True)
|
|
163
161
|
partition_column = join.metaData.executionInfo.conf.common[PARTITION_COLUMN_KEY]
|
|
164
162
|
|
|
165
|
-
|
|
166
163
|
# set the dependencies on the label sources
|
|
167
164
|
for label_part in join.labelParts.labels:
|
|
168
165
|
group_by = label_part.groupBy
|
|
@@ -171,21 +168,21 @@ def _set_join_deps(join):
|
|
|
171
168
|
windows = _get_distinct_day_windows(group_by)
|
|
172
169
|
for window in windows:
|
|
173
170
|
label_deps.append(
|
|
174
|
-
create_airflow_dependency(
|
|
171
|
+
create_airflow_dependency(
|
|
172
|
+
join_output_table, partition_column, offset=-1 * window
|
|
173
|
+
)
|
|
175
174
|
)
|
|
176
175
|
|
|
177
176
|
if group_by and group_by.sources:
|
|
178
177
|
for source in label_part.groupBy.sources:
|
|
179
178
|
source_query = utils.get_query(source)
|
|
180
179
|
source_partition_column = (
|
|
181
|
-
_get_partition_col_from_query(source_query)
|
|
182
|
-
or default_partition_col
|
|
180
|
+
_get_partition_col_from_query(source_query) or default_partition_col
|
|
183
181
|
)
|
|
184
182
|
label_deps.extend(
|
|
185
183
|
_get_airflow_deps_from_source(source, source_partition_column)
|
|
186
184
|
)
|
|
187
185
|
|
|
188
|
-
|
|
189
186
|
# Update the metadata customJson with dependencies
|
|
190
187
|
_dedupe_and_set_airflow_deps_json(join, deps, AIRFLOW_DEPENDENCIES_KEY)
|
|
191
188
|
|
|
@@ -3,12 +3,15 @@ import re
|
|
|
3
3
|
from collections import defaultdict
|
|
4
4
|
from typing import Dict, List
|
|
5
5
|
|
|
6
|
-
from
|
|
6
|
+
from gen_thrift.api.ttypes import Derivation, ExternalPart, GroupBy, Join, Source
|
|
7
|
+
|
|
7
8
|
from ai.chronon.group_by import get_output_col_names
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
# Returns a map of output column to semantic hash, including derivations
|
|
11
|
-
def compute_group_by_columns_hashes(
|
|
12
|
+
def compute_group_by_columns_hashes(
|
|
13
|
+
group_by: GroupBy, exclude_keys: bool = False
|
|
14
|
+
) -> Dict[str, str]:
|
|
12
15
|
"""
|
|
13
16
|
From the group_by object, get the final output columns after derivations.
|
|
14
17
|
"""
|
|
@@ -24,7 +27,7 @@ def compute_group_by_columns_hashes(group_by: GroupBy, exclude_keys: bool = Fals
|
|
|
24
27
|
group_by_minor_version_suffix = f"__{group_by.metaData.version}"
|
|
25
28
|
group_by_major_version = group_by.metaData.name
|
|
26
29
|
if group_by_major_version.endswith(group_by_minor_version_suffix):
|
|
27
|
-
group_by_major_version = group_by_major_version[
|
|
30
|
+
group_by_major_version = group_by_major_version[: -len(group_by_minor_version_suffix)]
|
|
28
31
|
base_semantics.append(f"group_by_name:{group_by_major_version}")
|
|
29
32
|
|
|
30
33
|
# Compute the semantic hash for each output column
|
|
@@ -87,7 +90,9 @@ def get_pre_derived_source_keys(source: Source) -> Dict[str, str]:
|
|
|
87
90
|
base_semantics = _extract_source_semantic_info(source)
|
|
88
91
|
source_keys_to_hashes = {}
|
|
89
92
|
for key, expression in extract_selects(source).items():
|
|
90
|
-
source_keys_to_hashes[key] = _compute_semantic_hash(
|
|
93
|
+
source_keys_to_hashes[key] = _compute_semantic_hash(
|
|
94
|
+
base_semantics + [f"select:{key}={expression}"]
|
|
95
|
+
)
|
|
91
96
|
return source_keys_to_hashes
|
|
92
97
|
|
|
93
98
|
|
|
@@ -101,7 +106,6 @@ def extract_selects(source: Source) -> Dict[str, str]:
|
|
|
101
106
|
|
|
102
107
|
|
|
103
108
|
def get_pre_derived_join_internal_features(join: Join) -> Dict[str, str]:
|
|
104
|
-
|
|
105
109
|
# Get the base semantic fields from join left side (without key columns)
|
|
106
110
|
join_base_semantic_fields = _extract_source_semantic_info(join.left)
|
|
107
111
|
|
|
@@ -109,7 +113,9 @@ def get_pre_derived_join_internal_features(join: Join) -> Dict[str, str]:
|
|
|
109
113
|
for jp in join.joinParts:
|
|
110
114
|
# Build key mapping semantics - include left side key expressions
|
|
111
115
|
if jp.keyMapping:
|
|
112
|
-
key_mapping_semantics = [
|
|
116
|
+
key_mapping_semantics = [
|
|
117
|
+
"join_keys:" + ",".join(f"{k}:{v}" for k, v in sorted(jp.keyMapping.items()))
|
|
118
|
+
]
|
|
113
119
|
else:
|
|
114
120
|
key_mapping_semantics = []
|
|
115
121
|
|
|
@@ -133,7 +139,9 @@ def get_pre_derived_join_internal_features(join: Join) -> Dict[str, str]:
|
|
|
133
139
|
# These semantics apply to all features in the joinPart
|
|
134
140
|
jp_base_semantics = key_mapping_semantics + left_key_expressions + join_base_semantic_fields
|
|
135
141
|
|
|
136
|
-
pre_derived_group_by_features = get_pre_derived_group_by_features(
|
|
142
|
+
pre_derived_group_by_features = get_pre_derived_group_by_features(
|
|
143
|
+
jp.groupBy, jp_base_semantics
|
|
144
|
+
)
|
|
137
145
|
|
|
138
146
|
if jp.groupBy.derivations:
|
|
139
147
|
derived_group_by_features = build_derived_columns(
|
|
@@ -164,7 +172,9 @@ def get_pre_derived_group_by_columns(group_by: GroupBy) -> Dict[str, str]:
|
|
|
164
172
|
return output_columns_to_hashes
|
|
165
173
|
|
|
166
174
|
|
|
167
|
-
def get_pre_derived_group_by_features(
|
|
175
|
+
def get_pre_derived_group_by_features(
|
|
176
|
+
group_by: GroupBy, additional_semantic_fields=None
|
|
177
|
+
) -> Dict[str, str]:
|
|
168
178
|
# Get the base semantic fields that apply to all aggs
|
|
169
179
|
if additional_semantic_fields is None:
|
|
170
180
|
additional_semantic_fields = []
|
|
@@ -174,9 +184,13 @@ def get_pre_derived_group_by_features(group_by: GroupBy, additional_semantic_fie
|
|
|
174
184
|
# For group_bys with aggregations, aggregated columns
|
|
175
185
|
if group_by.aggregations:
|
|
176
186
|
for agg in group_by.aggregations:
|
|
177
|
-
input_expression_str = ",".join(
|
|
187
|
+
input_expression_str = ",".join(
|
|
188
|
+
get_input_expression_across_sources(group_by, agg.inputColumn)
|
|
189
|
+
)
|
|
178
190
|
for output_col_name in get_output_col_names(agg):
|
|
179
|
-
output_columns[output_col_name] = _compute_semantic_hash(
|
|
191
|
+
output_columns[output_col_name] = _compute_semantic_hash(
|
|
192
|
+
base_semantics + [input_expression_str] + additional_semantic_fields
|
|
193
|
+
)
|
|
180
194
|
# For group_bys without aggregations, selected fields from query
|
|
181
195
|
else:
|
|
182
196
|
combined_selects = defaultdict(set)
|
|
@@ -190,7 +204,10 @@ def get_pre_derived_group_by_features(group_by: GroupBy, additional_semantic_fie
|
|
|
190
204
|
unified_selects = {key: ",".join(sorted(vals)) for key, vals in combined_selects.items()}
|
|
191
205
|
|
|
192
206
|
# now compute the hashes on base semantics + expression
|
|
193
|
-
selected_hashes = {
|
|
207
|
+
selected_hashes = {
|
|
208
|
+
key: _compute_semantic_hash(base_semantics + [val] + additional_semantic_fields)
|
|
209
|
+
for key, val in unified_selects.items()
|
|
210
|
+
}
|
|
194
211
|
output_columns.update(selected_hashes)
|
|
195
212
|
return output_columns
|
|
196
213
|
|
|
@@ -262,11 +279,13 @@ def _compute_semantic_hash(components: List[str]) -> str:
|
|
|
262
279
|
# Sort components to ensure consistent ordering
|
|
263
280
|
sorted_components = sorted(components)
|
|
264
281
|
hash_input = "|".join(sorted_components)
|
|
265
|
-
return hashlib.md5(hash_input.encode(
|
|
282
|
+
return hashlib.md5(hash_input.encode("utf-8")).hexdigest()
|
|
266
283
|
|
|
267
284
|
|
|
268
285
|
def build_derived_columns(
|
|
269
|
-
base_columns_to_hashes: Dict[str, str],
|
|
286
|
+
base_columns_to_hashes: Dict[str, str],
|
|
287
|
+
derivations: List[Derivation],
|
|
288
|
+
additional_semantic_fields: List[str],
|
|
270
289
|
) -> Dict[str, str]:
|
|
271
290
|
"""
|
|
272
291
|
Build the derived columns from pre-derived columns and derivations.
|
|
@@ -279,20 +298,24 @@ def build_derived_columns(
|
|
|
279
298
|
output_columns.update(base_columns_to_hashes)
|
|
280
299
|
for derivation in derivations:
|
|
281
300
|
if base_columns_to_hashes.get(derivation.expression):
|
|
282
|
-
|
|
283
|
-
|
|
301
|
+
# don't change the semantics if you're just passing a base column through derivations
|
|
302
|
+
output_columns[derivation.name] = base_columns_to_hashes[derivation.expression]
|
|
284
303
|
if derivation.name != "*":
|
|
285
304
|
# Identify base fields present within the derivation to include in the semantic hash
|
|
286
305
|
# We go long to short to avoid taking both a windowed feature and the unwindowed feature
|
|
287
306
|
# i.e. f_7d and f
|
|
288
307
|
derivation_expression = derivation.expression
|
|
289
308
|
base_col_semantic_fields = []
|
|
290
|
-
tokens = re.findall(r
|
|
309
|
+
tokens = re.findall(r"\b\w+\b", derivation_expression)
|
|
291
310
|
for token in tokens:
|
|
292
311
|
if token in base_columns_to_hashes:
|
|
293
312
|
base_col_semantic_fields.append(base_columns_to_hashes[token])
|
|
294
313
|
|
|
295
|
-
output_columns[derivation.name] = _compute_semantic_hash(
|
|
314
|
+
output_columns[derivation.name] = _compute_semantic_hash(
|
|
315
|
+
additional_semantic_fields
|
|
316
|
+
+ [f"derivation:{derivation.expression}"]
|
|
317
|
+
+ base_col_semantic_fields
|
|
318
|
+
)
|
|
296
319
|
return output_columns
|
|
297
320
|
|
|
298
321
|
|
|
@@ -2,14 +2,14 @@ import os
|
|
|
2
2
|
from dataclasses import dataclass
|
|
3
3
|
from typing import Any, Dict, List, Optional, Type
|
|
4
4
|
|
|
5
|
+
from gen_thrift.api.ttypes import ConfType, GroupBy, Join, MetaData, Model, StagingQuery, Team
|
|
6
|
+
|
|
5
7
|
import ai.chronon.cli.compile.parse_teams as teams
|
|
6
|
-
from ai.chronon.api.ttypes import GroupBy, Join, MetaData, Model, StagingQuery, Team
|
|
7
8
|
from ai.chronon.cli.compile.conf_validator import ConfValidator
|
|
8
9
|
from ai.chronon.cli.compile.display.compile_status import CompileStatus
|
|
9
10
|
from ai.chronon.cli.compile.display.compiled_obj import CompiledObj
|
|
10
11
|
from ai.chronon.cli.compile.serializer import file2thrift
|
|
11
12
|
from ai.chronon.cli.logger import get_logger, require
|
|
12
|
-
from ai.chronon.orchestration.ttypes import ConfType
|
|
13
13
|
|
|
14
14
|
logger = get_logger()
|
|
15
15
|
|
|
@@ -23,11 +23,11 @@ class ConfigInfo:
|
|
|
23
23
|
|
|
24
24
|
@dataclass
|
|
25
25
|
class CompileContext:
|
|
26
|
-
|
|
27
|
-
def __init__(self):
|
|
26
|
+
def __init__(self, ignore_python_errors: bool = False):
|
|
28
27
|
self.chronon_root: str = os.getenv("CHRONON_ROOT", os.getcwd())
|
|
29
28
|
self.teams_dict: Dict[str, Team] = teams.load_teams(self.chronon_root)
|
|
30
29
|
self.compile_dir: str = "compiled"
|
|
30
|
+
self.ignore_python_errors: bool = ignore_python_errors
|
|
31
31
|
|
|
32
32
|
self.config_infos: List[ConfigInfo] = [
|
|
33
33
|
ConfigInfo(folder_name="joins", cls=Join, config_type=ConfType.JOIN),
|
|
@@ -42,7 +42,9 @@ class CompileContext:
|
|
|
42
42
|
config_type=ConfType.STAGING_QUERY,
|
|
43
43
|
),
|
|
44
44
|
ConfigInfo(folder_name="models", cls=Model, config_type=ConfType.MODEL),
|
|
45
|
-
ConfigInfo(
|
|
45
|
+
ConfigInfo(
|
|
46
|
+
folder_name="teams_metadata", cls=MetaData, config_type=None
|
|
47
|
+
), # only for team metadata
|
|
46
48
|
]
|
|
47
49
|
|
|
48
50
|
self.compile_status = CompileStatus(use_live=False)
|
|
@@ -52,13 +54,12 @@ class CompileContext:
|
|
|
52
54
|
cls = config_info.cls
|
|
53
55
|
self.existing_confs[cls] = self._parse_existing_confs(cls)
|
|
54
56
|
|
|
55
|
-
|
|
56
|
-
|
|
57
57
|
self.validator: ConfValidator = ConfValidator(
|
|
58
58
|
input_root=self.chronon_root,
|
|
59
59
|
output_root=self.compile_dir,
|
|
60
60
|
existing_gbs=self.existing_confs[GroupBy],
|
|
61
61
|
existing_joins=self.existing_confs[Join],
|
|
62
|
+
existing_staging_queries=self.existing_confs[StagingQuery],
|
|
62
63
|
)
|
|
63
64
|
|
|
64
65
|
def input_dir(self, cls: type) -> str:
|
|
@@ -93,9 +94,7 @@ class CompileContext:
|
|
|
93
94
|
return os.path.join(self.chronon_root, self.compile_dir)
|
|
94
95
|
else:
|
|
95
96
|
config_info = self.config_info_for_class(cls)
|
|
96
|
-
return os.path.join(
|
|
97
|
-
self.chronon_root, self.compile_dir, config_info.folder_name
|
|
98
|
-
)
|
|
97
|
+
return os.path.join(self.chronon_root, self.compile_dir, config_info.folder_name)
|
|
99
98
|
|
|
100
99
|
def staging_output_path(self, compiled_obj: CompiledObj):
|
|
101
100
|
"""
|
|
@@ -121,7 +120,6 @@ class CompileContext:
|
|
|
121
120
|
require(False, f"Class {cls} not found in CONFIG_INFOS")
|
|
122
121
|
|
|
123
122
|
def _parse_existing_confs(self, obj_class: type) -> Dict[str, object]:
|
|
124
|
-
|
|
125
123
|
result = {}
|
|
126
124
|
|
|
127
125
|
output_dir = self.output_dir(obj_class)
|
|
@@ -131,9 +129,7 @@ class CompileContext:
|
|
|
131
129
|
return result
|
|
132
130
|
|
|
133
131
|
for sub_root, _sub_dirs, sub_files in os.walk(output_dir):
|
|
134
|
-
|
|
135
132
|
for f in sub_files:
|
|
136
|
-
|
|
137
133
|
if f.startswith("."): # ignore hidden files - such as .DS_Store
|
|
138
134
|
continue
|
|
139
135
|
|
|
@@ -155,7 +151,9 @@ class CompileContext:
|
|
|
155
151
|
)
|
|
156
152
|
self.compile_status.add_existing_object_update_display(compiled_obj)
|
|
157
153
|
elif isinstance(obj, MetaData):
|
|
158
|
-
team_metadata_name =
|
|
154
|
+
team_metadata_name = ".".join(
|
|
155
|
+
full_path.split("/")[-2:]
|
|
156
|
+
) # use the name of the file as team metadata won't have name
|
|
159
157
|
result[team_metadata_name] = obj
|
|
160
158
|
compiled_obj = CompiledObj(
|
|
161
159
|
name=team_metadata_name,
|
|
@@ -167,9 +165,7 @@ class CompileContext:
|
|
|
167
165
|
)
|
|
168
166
|
self.compile_status.add_existing_object_update_display(compiled_obj)
|
|
169
167
|
else:
|
|
170
|
-
logger.errors(
|
|
171
|
-
f"Parsed object from {full_path} has no metaData attribute"
|
|
172
|
-
)
|
|
168
|
+
logger.errors(f"Parsed object from {full_path} has no metaData attribute")
|
|
173
169
|
|
|
174
170
|
except Exception as e:
|
|
175
171
|
print(f"Failed to parse file {full_path}: {str(e)}", e)
|