spark-kindling-cli 0.14.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. kindling_cli/__init__.py +8 -0
  2. kindling_cli/_runner.py +334 -0
  3. kindling_cli/_tracing.py +104 -0
  4. kindling_cli/agent_files.py +306 -0
  5. kindling_cli/agent_skill/SKILL.md +83 -0
  6. kindling_cli/agent_skill/project_instructions.md +15 -0
  7. kindling_cli/agent_skill/references/apps.md +311 -0
  8. kindling_cli/agent_skill/references/config.md +291 -0
  9. kindling_cli/agent_skill/references/databricks.md +345 -0
  10. kindling_cli/agent_skill/references/entities.md +310 -0
  11. kindling_cli/agent_skill/references/pipes.md +420 -0
  12. kindling_cli/bundle.py +1344 -0
  13. kindling_cli/cli.py +9927 -0
  14. kindling_cli/scaffold.py +346 -0
  15. kindling_cli/templates/.devcontainer/devcontainer.json.j2 +59 -0
  16. kindling_cli/templates/.env.example.j2 +40 -0
  17. kindling_cli/templates/.github/workflows/ci.yml.j2 +46 -0
  18. kindling_cli/templates/.gitignore.j2 +10 -0
  19. kindling_cli/templates/app.batch.py.j2 +13 -0
  20. kindling_cli/templates/app.file-ingestion.py.j2 +14 -0
  21. kindling_cli/templates/app.py.j2 +13 -0
  22. kindling_cli/templates/app.streaming.py.j2 +13 -0
  23. kindling_cli/templates/app.yaml.j2 +2 -0
  24. kindling_cli/templates/bundle/databricks/README.md +23 -0
  25. kindling_cli/templates/bundle/databricks/databricks.yml.j2 +18 -0
  26. kindling_cli/templates/bundle/databricks/resources/__pipeline__.pipeline.yml.j2 +21 -0
  27. kindling_cli/templates/bundle/databricks/src/kindling_lakeflow.py +9 -0
  28. kindling_cli/templates/lake-reqs.txt.j2 +5 -0
  29. kindling_cli/templates/package/PACKAGE_QUICKSTART.md.j2 +39 -0
  30. kindling_cli/templates/package/QUICKSTART.md.j2 +94 -0
  31. kindling_cli/templates/pyproject.root.toml.j2 +26 -0
  32. kindling_cli/templates/pyproject.toml.j2 +81 -0
  33. kindling_cli/templates/scripts/setup-local-dev.sh.j2 +34 -0
  34. kindling_cli/templates/settings.local.yaml.j2 +43 -0
  35. kindling_cli/templates/settings.yaml.j2 +14 -0
  36. kindling_cli/templates/src/entities/records.medallion.py.j2 +56 -0
  37. kindling_cli/templates/src/entities/records.minimal.py.j2 +31 -0
  38. kindling_cli/templates/src/pipes/bronze_to_silver.py.j2 +24 -0
  39. kindling_cli/templates/src/pipes/process.py.j2 +17 -0
  40. kindling_cli/templates/src/transforms/quality.medallion.py.j2 +26 -0
  41. kindling_cli/templates/src/transforms/quality.minimal.py.j2 +12 -0
  42. kindling_cli/templates/tests/component/test_registration.py.j2 +96 -0
  43. kindling_cli/templates/tests/conftest.py.j2 +325 -0
  44. kindling_cli/templates/tests/integration/test_pipeline.py.j2 +8 -0
  45. kindling_cli/templates/tests/integration/test_pipeline_azure.py.j2 +78 -0
  46. kindling_cli/templates/tests/integration/test_pipeline_local.py.j2 +45 -0
  47. kindling_cli/templates/tests/unit/test_transforms.py.j2 +60 -0
  48. kindling_cli/test_runner.py +451 -0
  49. spark_kindling_cli-0.14.0.dist-info/METADATA +226 -0
  50. spark_kindling_cli-0.14.0.dist-info/RECORD +52 -0
  51. spark_kindling_cli-0.14.0.dist-info/WHEEL +4 -0
  52. spark_kindling_cli-0.14.0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,8 @@
1
+ """Command-line tooling for Kindling.
2
+
3
+ INVARIANT: this module must stay import-light (no click, no third-party
4
+ imports). ``python -m kindling_cli.test_runner`` is the frontend Poe tasks
5
+ use to run tests from any interpreter — importing the package must never
6
+ require CLI dependencies. Pinned by
7
+ ``tests/unit/test_test_runner.py::test_runner_module_imports_without_click``.
8
+ """
@@ -0,0 +1,334 @@
1
+ """Standalone app runner.
2
+
3
+ Invoked by `kindling app run` for local/standalone execution:
4
+ python -m kindling_cli._runner --env local --config settings.yaml [--config ...] app.py
5
+
6
+ Initializes the Kindling framework then execs the app entrypoint in-process,
7
+ mirroring how DataAppManager._execute_app() works for remote runs.
8
+ """
9
+
10
+ import argparse
11
+ import builtins
12
+ import json
13
+ import logging
14
+ import os
15
+ import sys
16
+ from pathlib import Path
17
+ from typing import Callable, List, Optional, Set
18
+
19
+ _runner_logger = logging.getLogger("kindling._runner")
20
+
21
+
22
+ def _normalize_distribution_name(name: str) -> str:
23
+ return (name or "").strip().replace("-", "_").lower()
24
+
25
+
26
+ def _discover_package_roots_from_pythonpath() -> Set[str]:
27
+ """Best-effort package root discovery from PYTHONPATH entries."""
28
+ raw_pythonpath = (os.getenv("PYTHONPATH") or "").strip()
29
+ if not raw_pythonpath:
30
+ return set()
31
+
32
+ discovered = set()
33
+ for entry in raw_pythonpath.split(os.pathsep):
34
+ if not entry:
35
+ continue
36
+ path = Path(entry).expanduser()
37
+ if not path.is_dir():
38
+ continue
39
+
40
+ try:
41
+ candidates = list(path.iterdir())
42
+ except OSError:
43
+ continue
44
+
45
+ for candidate in candidates:
46
+ if candidate.is_dir() and not candidate.name.startswith((".", "_")):
47
+ discovered.add(_normalize_distribution_name(candidate.name))
48
+
49
+ return discovered
50
+
51
+
52
+ def _load_local_package_modules() -> Set[str]:
53
+ raw_modules = (os.getenv("KINDLING_LOCAL_PACKAGE_MODULES") or "").strip()
54
+ discovered = _discover_package_roots_from_pythonpath()
55
+ if not raw_modules:
56
+ return discovered
57
+
58
+ try:
59
+ parsed = json.loads(raw_modules)
60
+ except json.JSONDecodeError:
61
+ return discovered
62
+
63
+ if not isinstance(parsed, list):
64
+ return discovered
65
+
66
+ normalized_modules = set()
67
+ for module_name in parsed:
68
+ if isinstance(module_name, str):
69
+ normalized = _normalize_distribution_name(module_name)
70
+ if normalized:
71
+ normalized_modules.add(normalized)
72
+ return normalized_modules | discovered
73
+
74
+
75
+ def _install_local_package_version_shim() -> Optional[Callable[[], None]]:
76
+ """Return a restore callback when local packages should bypass metadata version lookup."""
77
+ local_modules = _load_local_package_modules()
78
+ if not local_modules:
79
+ return None
80
+
81
+ from importlib import metadata as importlib_metadata
82
+
83
+ original_version = importlib_metadata.version
84
+ local_version = os.getenv("KINDLING_LOCAL_PACKAGE_VERSION", "0.0.0+local")
85
+
86
+ def _version_with_local_fallback(package_name: str) -> str:
87
+ if _normalize_distribution_name(package_name) in local_modules:
88
+ return local_version
89
+ return original_version(package_name)
90
+
91
+ importlib_metadata.version = _version_with_local_fallback
92
+
93
+ def _restore() -> None:
94
+ importlib_metadata.version = original_version
95
+
96
+ return _restore
97
+
98
+
99
+ def _dist_name_from_spec(spec: str) -> str:
100
+ """Extract the bare distribution name from a requirement spec (e.g. 'pkg==1.0' → 'pkg')."""
101
+ for sep in ("~=", ">=", "==", "<=", "!=", ">", "<"):
102
+ if sep in spec:
103
+ return spec.split(sep)[0].strip()
104
+ return spec.strip()
105
+
106
+
107
+ def _is_editable_install(dist_name: str) -> bool:
108
+ """Return True if dist_name is installed as an editable (development) install."""
109
+ import importlib.metadata as im
110
+
111
+ try:
112
+ dist = im.distribution(dist_name)
113
+ raw = dist.read_text("direct_url.json")
114
+ if raw:
115
+ import json as _json
116
+
117
+ info = _json.loads(raw)
118
+ return bool(info.get("dir_info", {}).get("editable", False))
119
+ except Exception:
120
+ pass
121
+ return False
122
+
123
+
124
+ def _is_available_in_env(dist_name: str) -> bool:
125
+ """Return True if the distribution is resolvable in the current environment.
126
+
127
+ This covers editable installs, regular pip installs, devcontainer pre-installs,
128
+ and packages published to a registry and installed — anything importlib.metadata
129
+ can find, regardless of how it got there.
130
+ """
131
+ import importlib.metadata as im
132
+
133
+ try:
134
+ im.distribution(dist_name)
135
+ return True
136
+ except im.PackageNotFoundError:
137
+ return False
138
+
139
+
140
+ def _filter_editable_packages(lake_requirements: List[str]) -> List[str]:
141
+ """Return only the lake requirements whose distributions are NOT editable installs.
142
+
143
+ Editable installs are skipped because:
144
+ - pip install over an editable install removes the editable link, breaking local dev.
145
+ - The package is already imported in the running process; reinstalling it mid-run
146
+ has no effect on the current execution.
147
+ """
148
+ filtered = []
149
+ for spec in lake_requirements:
150
+ dist_name = _dist_name_from_spec(spec)
151
+ if _is_editable_install(dist_name):
152
+ _runner_logger.debug(
153
+ "Skipping lake install of %r — editable install active (local source takes precedence)",
154
+ dist_name,
155
+ )
156
+ else:
157
+ filtered.append(spec)
158
+ return filtered
159
+
160
+
161
+ def _read_lake_requirements(app_dir: Path) -> List[str]:
162
+ """Read lake-reqs.txt from the app directory, returning non-comment lines."""
163
+ lake_reqs_path = app_dir / "lake-reqs.txt"
164
+ if not lake_reqs_path.exists():
165
+ return []
166
+ return [
167
+ line.strip()
168
+ for line in lake_reqs_path.read_text(encoding="utf-8").splitlines()
169
+ if line.strip() and not line.startswith("#")
170
+ ]
171
+
172
+
173
+ def _read_lake_requirement_package_names(app_dir: Path) -> List[str]:
174
+ """Return the importable (snake_case) package names declared in
175
+ lake-reqs.txt -- the packages this app depends on, which may include
176
+ the app's own domain package(s) as well as shared/extension packages.
177
+
178
+ Used to drive kindling.bootstrap's entity/pipe registration walker
179
+ (initialize_framework's `registration_packages`), so a package's
180
+ entities/pipes get registered without app.py needing to hand-import
181
+ them. Normalizes kebab-case to snake_case (hyphens to underscores) --
182
+ true by construction for a Kindling-scaffolded package (its
183
+ distribution name and top-level import name are always generated in
184
+ lockstep, see AppScaffoldConfig.package_kebab_name/package_snake_name
185
+ in scaffold.py), but not a general Python packaging guarantee for an
186
+ arbitrary third-party distribution, whose import name can differ
187
+ entirely from its distribution name. A mismatch here isn't fatal
188
+ either way: it just means that entry contributes nothing to
189
+ registration, same as a listed package with no entities/pipes/
190
+ ingestion namespace at all (e.g. a telemetry extension) -- the walker
191
+ already tolerates a missing/wrong namespace.
192
+ """
193
+ names = []
194
+ seen = set()
195
+ for spec in _read_lake_requirements(app_dir):
196
+ dist_name = _dist_name_from_spec(spec)
197
+ package_name = _normalize_distribution_name(dist_name)
198
+ if package_name and package_name not in seen:
199
+ seen.add(package_name)
200
+ names.append(package_name)
201
+ return names
202
+
203
+
204
+ def _install_lake_requirements(
205
+ lake_requirements: List[str], app_name: str, load_lake: bool = False
206
+ ) -> None:
207
+ """Install lake packages listed in lake-reqs.txt.
208
+
209
+ By default (load_lake=False), packages already installed locally are used as-is and
210
+ the lake is not contacted. Pass load_lake=True to force downloading from the lake
211
+ regardless of local install state (mirrors DataAppManager.run_app() on Synapse/Databricks).
212
+ """
213
+ if not lake_requirements:
214
+ return
215
+
216
+ installable = _filter_editable_packages(lake_requirements)
217
+ skipped = len(lake_requirements) - len(installable)
218
+ if skipped:
219
+ _runner_logger.debug(
220
+ "Skipped %d lake package(s) that are active editable installs", skipped
221
+ )
222
+ if not installable:
223
+ return
224
+
225
+ if not load_lake:
226
+ available = [s for s in installable if _is_available_in_env(_dist_name_from_spec(s))]
227
+ missing = [s for s in installable if s not in available]
228
+ if available:
229
+ _runner_logger.debug(
230
+ "Using %d locally installed lake package(s): %s",
231
+ len(available),
232
+ available,
233
+ )
234
+ if missing:
235
+ _runner_logger.warning(
236
+ "%d lake package(s) in lake-reqs.txt are not installed locally and will be "
237
+ "skipped — pass --load-lake to fetch them from the lake: %s",
238
+ len(missing),
239
+ missing,
240
+ )
241
+ return
242
+
243
+ try:
244
+ from kindling.data_apps import DataAppManager
245
+ from kindling.injection import get_kindling_service
246
+
247
+ manager = get_kindling_service(DataAppManager)
248
+ if not manager.artifacts_path:
249
+ _runner_logger.warning(
250
+ "lake-reqs.txt found but artifacts_storage_path is not configured — "
251
+ "lake packages cannot be fetched for local runs without it"
252
+ )
253
+ return
254
+ _runner_logger.info(
255
+ "Installing %d lake package(s) from lake-reqs.txt: %s",
256
+ len(installable),
257
+ installable,
258
+ )
259
+ manager._install_app_dependencies(app_name, [], installable)
260
+ except Exception as exc:
261
+ _runner_logger.warning("Could not install lake packages from lake-reqs.txt: %s", exc)
262
+
263
+
264
+ def main() -> None:
265
+ parser = argparse.ArgumentParser(prog="kindling_cli._runner", add_help=False)
266
+ parser.add_argument("app_path")
267
+ parser.add_argument("--env", default="local")
268
+ parser.add_argument(
269
+ "--config-dir",
270
+ dest="config_dir",
271
+ default=None,
272
+ help="Shared settings overlay directory; the app's own settings*.yaml are "
273
+ "always resolved from the app directory.",
274
+ )
275
+ parser.add_argument(
276
+ "--load-lake",
277
+ action="store_true",
278
+ default=False,
279
+ help="Download lake-reqs packages from the lake even if already installed locally.",
280
+ )
281
+ args = parser.parse_args()
282
+
283
+ app_path = Path(args.app_path).resolve()
284
+
285
+ restore_version_shim = _install_local_package_version_shim()
286
+
287
+ try:
288
+ from kindling.bootstrap import initialize_framework
289
+
290
+ # Install (or confirm locally available) lake-reqs.txt packages
291
+ # before bootstrapping, so the framework's registration walker
292
+ # (registration_packages below) finds them already importable
293
+ # rather than racing an install that hasn't happened yet.
294
+ lake_requirements = _read_lake_requirements(app_path.parent)
295
+ _install_lake_requirements(lake_requirements, app_path.stem, load_lake=args.load_lake)
296
+ registration_packages = _read_lake_requirement_package_names(app_path.parent)
297
+
298
+ raw_parameters = os.getenv("KINDLING_RUN_PARAMETERS", "")
299
+ parameters = json.loads(raw_parameters) if raw_parameters else {}
300
+ if not isinstance(parameters, dict):
301
+ raise ValueError("KINDLING_RUN_PARAMETERS must contain a JSON object")
302
+ initialize_framework(
303
+ {
304
+ **parameters,
305
+ "platform": "standalone",
306
+ "environment": args.env,
307
+ # Settings by convention: bootstrap resolves the hierarchy
308
+ # from these directories; no file list is built here.
309
+ "app_dir": str(app_path.parent),
310
+ **({"config_dir": args.config_dir} if args.config_dir else {}),
311
+ "registration_packages": registration_packages,
312
+ }
313
+ )
314
+
315
+ code = app_path.read_text(encoding="utf-8")
316
+ exec_globals = {
317
+ "__name__": "__main__",
318
+ "__file__": str(app_path),
319
+ "__spec__": None,
320
+ "__builtins__": builtins,
321
+ }
322
+ # Ensure imports inside the app resolve relative to its directory
323
+ app_dir = str(app_path.parent)
324
+ if app_dir not in sys.path:
325
+ sys.path.insert(0, app_dir)
326
+
327
+ exec(compile(code, str(app_path), "exec"), exec_globals) # noqa: S102
328
+ finally:
329
+ if restore_version_shim is not None:
330
+ restore_version_shim()
331
+
332
+
333
+ if __name__ == "__main__":
334
+ main()
@@ -0,0 +1,104 @@
1
+ """Design-time CLI tracing (gh#210).
2
+
3
+ The CLI is a separate process with no injector or ConfigService — the SDK
4
+ deliberately has zero kindling-core imports, and the CLI populates the
5
+ injector only when it loads the user's app. Spans here come from a lazily
6
+ built PlainPythonTraceProvider over a tiny dict-config shim, gated by
7
+ environment variables since no ConfigService exists pre-app-load:
8
+
9
+ - ``KINDLING_TRACE=1`` enables CLI spans.
10
+ - ``KINDLING_KINDLING__TELEMETRY__TRACING__PRINT=true`` prints them (the
11
+ same env override Dynaconf applies to kindling.telemetry.tracing.print
12
+ once an app loads).
13
+
14
+ CLI migrate/pipeline commands get real tracing for free via the app's
15
+ initialize(); this module only brackets design-time operations (package/
16
+ app/runtime deploys, workspace uploads, run submission) with kindling.cli
17
+ spans. Tracing must never break a CLI command: any import or provider
18
+ failure degrades to a no-op.
19
+ """
20
+
21
+ import functools
22
+ import os
23
+ from contextlib import nullcontext
24
+
25
+ # Hardcoded rather than imported from kindling.trace_ops: the constant is
26
+ # not worth importing the core (and transitively pyspark) for at CLI start.
27
+ CLI_COMPONENT = "kindling.cli"
28
+
29
+ _TRACE_ENV = "KINDLING_TRACE"
30
+ _PRINT_ENV = "KINDLING_KINDLING__TELEMETRY__TRACING__PRINT"
31
+
32
+ # Whitelisted command kwargs promoted to span attributes.
33
+ _DETAIL_KEYS = ("app_name", "platform", "env", "environment", "job_name", "package_name")
34
+
35
+ _provider = None
36
+
37
+
38
+ class _EnvConfigShim:
39
+ """Just enough ConfigService surface for PlainPythonTraceProvider."""
40
+
41
+ def __init__(self, values):
42
+ self._values = values
43
+
44
+ def get(self, key, default=None):
45
+ return self._values.get(key, default)
46
+
47
+
48
+ def _truthy(value) -> bool:
49
+ return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
50
+
51
+
52
+ def cli_tracing_enabled() -> bool:
53
+ return _truthy(os.environ.get(_TRACE_ENV))
54
+
55
+
56
+ def _get_provider():
57
+ global _provider
58
+ if _provider is None:
59
+ from kindling.plain_telemetry import (
60
+ PlainPythonLoggerProvider,
61
+ PlainPythonTraceProvider,
62
+ )
63
+
64
+ values = {}
65
+ if _truthy(os.environ.get(_PRINT_ENV)):
66
+ values["print_trace"] = True
67
+ shim = _EnvConfigShim(values)
68
+ _provider = PlainPythonTraceProvider(PlainPythonLoggerProvider(shim), shim)
69
+ return _provider
70
+
71
+
72
+ def cli_span(operation: str, details=None):
73
+ """A kindling.cli span, or a no-op CM when KINDLING_TRACE is unset."""
74
+ if not cli_tracing_enabled():
75
+ return nullcontext()
76
+ try:
77
+ return _get_provider().span(
78
+ operation=operation, component=CLI_COMPONENT, details=details, reraise=True
79
+ )
80
+ except Exception:
81
+ return nullcontext()
82
+
83
+
84
+ def traced_command(operation: str):
85
+ """Bracket a click command callback with a kindling.cli span.
86
+
87
+ Place immediately above ``def`` (below the click decorators) so the
88
+ span wraps the raw callback. Whitelisted kwargs become attributes.
89
+ """
90
+
91
+ def decorate(fn):
92
+ @functools.wraps(fn)
93
+ def wrapper(*args, **kwargs):
94
+ details = {
95
+ key: str(value)
96
+ for key, value in kwargs.items()
97
+ if key in _DETAIL_KEYS and value is not None
98
+ }
99
+ with cli_span(operation, details):
100
+ return fn(*args, **kwargs)
101
+
102
+ return wrapper
103
+
104
+ return decorate