weaverstack 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. weaver/__init__.py +59 -0
  2. weaver/build_bundle/__init__.py +109 -0
  3. weaver/build_bundle/aliases.py +325 -0
  4. weaver/build_bundle/bundle.py +359 -0
  5. weaver/build_bundle/catalogue_actions.py +275 -0
  6. weaver/build_bundle/changes.py +186 -0
  7. weaver/build_bundle/endpoints.py +83 -0
  8. weaver/build_bundle/executors/__init__.py +69 -0
  9. weaver/build_bundle/executors/alias.py +202 -0
  10. weaver/build_bundle/executors/base.py +132 -0
  11. weaver/build_bundle/executors/folder.py +71 -0
  12. weaver/build_bundle/executors/load_file.py +205 -0
  13. weaver/build_bundle/executors/spark_case.py +26 -0
  14. weaver/build_bundle/executors/spark_schema.py +60 -0
  15. weaver/build_bundle/executors/spark_sql.py +59 -0
  16. weaver/build_bundle/executors/spark_sql_batch.py +57 -0
  17. weaver/build_bundle/executors/spark_table.py +213 -0
  18. weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
  19. weaver/build_bundle/executors/tsql.py +81 -0
  20. weaver/build_bundle/incremental.py +288 -0
  21. weaver/build_bundle/installer.py +384 -0
  22. weaver/build_bundle/models.py +288 -0
  23. weaver/build_bundle/payloads.py +34 -0
  24. weaver/build_bundle/physical.py +625 -0
  25. weaver/build_bundle/planner.py +389 -0
  26. weaver/build_bundle/prune.py +620 -0
  27. weaver/build_bundle/report.py +108 -0
  28. weaver/build_bundle/stages.py +196 -0
  29. weaver/build_bundle/targets.py +272 -0
  30. weaver/build_bundle/workflow.py +585 -0
  31. weaver/catalogue/__init__.py +73 -0
  32. weaver/catalogue/builtin.py +238 -0
  33. weaver/catalogue/claims.py +121 -0
  34. weaver/catalogue/projection.py +437 -0
  35. weaver/catalogue/reader.py +152 -0
  36. weaver/catalogue/reconcile.py +231 -0
  37. weaver/catalogue/render.py +410 -0
  38. weaver/catalogue/state.py +660 -0
  39. weaver/catalogue/tables.py +648 -0
  40. weaver/config.py +178 -0
  41. weaver/declaration/__init__.py +171 -0
  42. weaver/declaration/columns.py +223 -0
  43. weaver/declaration/ddl.py +266 -0
  44. weaver/declaration/dependencies.py +544 -0
  45. weaver/declaration/graph.py +240 -0
  46. weaver/declaration/item_dependencies.py +292 -0
  47. weaver/declaration/load.py +191 -0
  48. weaver/declaration/metadata.py +1405 -0
  49. weaver/declaration/model.py +448 -0
  50. weaver/declaration/references.py +294 -0
  51. weaver/declaration/repository.py +959 -0
  52. weaver/declaration/schemas.py +135 -0
  53. weaver/declaration/source.py +674 -0
  54. weaver/declaration/spark_load.py +759 -0
  55. weaver/declaration/sql_shaping.py +591 -0
  56. weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
  57. weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
  58. weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
  59. weaver/declaration/templates/load/column_metadata.sql +40 -0
  60. weaver/declaration/templates/load/full_replace_body.sql +21 -0
  61. weaver/declaration/templates/load/install_load_procedure.sql +27 -0
  62. weaver/declaration/templates/load/load_procedure.sql +48 -0
  63. weaver/declaration/templates/load/primary_key_body.sql +113 -0
  64. weaver/declaration/tsql_ddl.py +468 -0
  65. weaver/declaration/tsql_load.py +417 -0
  66. weaver/declaration/warehouse_type_mapping.yml +93 -0
  67. weaver/diagnostics.py +247 -0
  68. weaver/errors.py +61 -0
  69. weaver/etl.py +469 -0
  70. weaver/fabric/__init__.py +107 -0
  71. weaver/fabric/auth.py +137 -0
  72. weaver/fabric/capacity.py +143 -0
  73. weaver/fabric/client.py +147 -0
  74. weaver/fabric/environment.py +460 -0
  75. weaver/fabric/livy.py +478 -0
  76. weaver/fabric/notebooks.py +201 -0
  77. weaver/fabric/onelake.py +263 -0
  78. weaver/fabric/resolution.py +344 -0
  79. weaver/fabric/resources.py +245 -0
  80. weaver/fabric/session.py +148 -0
  81. weaver/fabric/shortcuts.py +120 -0
  82. weaver/fabric/sql.py +118 -0
  83. weaver/fabric/store.py +198 -0
  84. weaver/initialise.py +209 -0
  85. weaver/lakehouse.py +386 -0
  86. weaver/load.py +474 -0
  87. weaver/load_execution.py +483 -0
  88. weaver/load_plan.py +912 -0
  89. weaver/load_report.py +330 -0
  90. weaver/load_resolution.py +386 -0
  91. weaver/locations.py +164 -0
  92. weaver/objects.py +392 -0
  93. weaver/operations.py +757 -0
  94. weaver/physical_wipe.py +369 -0
  95. weaver/push.py +76 -0
  96. weaver/resolution.py +292 -0
  97. weaver/runtime/__init__.py +30 -0
  98. weaver/runtime/folder_load.py +402 -0
  99. weaver/runtime/load_contract.py +245 -0
  100. weaver/runtime/load_result.py +104 -0
  101. weaver/runtime/spark_load.py +152 -0
  102. weaver/runtime/table_load.py +497 -0
  103. weaver/spark/__init__.py +49 -0
  104. weaver/spark/catalogue.py +245 -0
  105. weaver/spark/destination.py +195 -0
  106. weaver/spark/session.py +84 -0
  107. weaver/spark/tokens.py +138 -0
  108. weaver/sql/__init__.py +40 -0
  109. weaver/sql/authentication.py +38 -0
  110. weaver/sql/connection.py +90 -0
  111. weaver/sql/errors.py +25 -0
  112. weaver/sql/execution.py +123 -0
  113. weaver/sql/pool.py +174 -0
  114. weaver/sql/wipe.py +156 -0
  115. weaver/store.py +209 -0
  116. weaver/targets.py +257 -0
  117. weaver/task_logging.py +215 -0
  118. weaver/unbind.py +74 -0
  119. weaver/workspaces.py +175 -0
  120. weaver_cli/__init__.py +12 -0
  121. weaver_cli/__main__.py +7 -0
  122. weaver_cli/main.py +626 -0
  123. weaverstack-0.1.1.dist-info/METADATA +113 -0
  124. weaverstack-0.1.1.dist-info/RECORD +127 -0
  125. weaverstack-0.1.1.dist-info/WHEEL +4 -0
  126. weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
  127. weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
weaver/unbind.py ADDED
@@ -0,0 +1,74 @@
1
+ """Remove catalogue installations for explicitly named physical targets."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+ from .catalogue.reader import read_table
8
+ from .catalogue.reconcile import prune_installation
9
+ from .catalogue.render import InstallationScope
10
+ from .catalogue.tables import INSTALLATION
11
+ from .declaration.model import LAKEHOUSE, WAREHOUSE
12
+ from .targets import ItemRef
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class UnbindResult:
17
+ targets: tuple[str, ...]
18
+ logical_items: tuple[str, ...]
19
+ statements: tuple[str, ...]
20
+
21
+ def to_mapping(self) -> dict[str, object]:
22
+ return {
23
+ "targets": list(self.targets),
24
+ "logical_items": list(self.logical_items),
25
+ "statements": len(self.statements),
26
+ }
27
+
28
+
29
+ def plan_unbind(
30
+ catalogue,
31
+ *,
32
+ lakehouses=(),
33
+ warehouses=(),
34
+ ) -> UnbindResult:
35
+ """Render complete catalogue deletion without inspecting physical targets."""
36
+
37
+ selected = {
38
+ (LAKEHOUSE, ItemRef.parse(value).name) for value in lakehouses
39
+ } | {
40
+ (WAREHOUSE, ItemRef.parse(value).name) for value in warehouses
41
+ }
42
+ rows = read_table(catalogue, INSTALLATION)
43
+ scopes = sorted(
44
+ {
45
+ InstallationScope(
46
+ str(row["item_type"]), str(row["item_name"])
47
+ )
48
+ for row in rows
49
+ if (str(row["item_type"]), str(row["target_name"])) in selected
50
+ },
51
+ key=str,
52
+ )
53
+ statements = tuple(
54
+ statement
55
+ for scope in scopes
56
+ for statement in prune_installation(scope)
57
+ )
58
+ targets = tuple(f"{item_type}/{name}" for item_type, name in sorted(selected))
59
+ return UnbindResult(
60
+ targets=targets,
61
+ logical_items=tuple(map(str, scopes)),
62
+ statements=statements,
63
+ )
64
+
65
+
66
+ def unbind_targets(catalogue, *, lakehouses=(), warehouses=()) -> UnbindResult:
67
+ """Execute target-directed catalogue deletion and touch no physical target."""
68
+
69
+ result = plan_unbind(
70
+ catalogue, lakehouses=lakehouses, warehouses=warehouses
71
+ )
72
+ for statement in result.statements:
73
+ catalogue.sql(statement)
74
+ return result
weaver/workspaces.py ADDED
@@ -0,0 +1,175 @@
1
+ """Workspace configuration and the two execution environments Weaver supports.
2
+
3
+ A Workspace identifies where resources live. For Fabric that identity is a
4
+ Microsoft Fabric workspace name; for the local emulator it is the path to the
5
+ folder that stands in for one. It does not say where Weaver code executes.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass, field
11
+ from pathlib import Path
12
+ from types import MappingProxyType
13
+ from typing import Mapping
14
+
15
+ from .declaration.model import LAKEHOUSE, WAREHOUSE, WeaverItemId
16
+ from .errors import ConfigError
17
+ from .targets import validate_name
18
+
19
+ WEAVER_ITEMS_AREA = "weaver_items"
20
+ BUILD_BUNDLES_AREA = "build_bundles"
21
+ CLI_AREA = "cli"
22
+ FABRIC = "fabric"
23
+ LOCAL = "local"
24
+ WORKSPACE_TYPES = (FABRIC, LOCAL)
25
+
26
+
27
+ @dataclass(frozen=True)
28
+ class ExecutionSettings:
29
+ """Technical parallelism settings for a Workspace or one physical item."""
30
+
31
+ parallel_workers: int | None = None
32
+
33
+ def __post_init__(self) -> None:
34
+ workers = self.parallel_workers
35
+ if workers is not None and (
36
+ isinstance(workers, bool) or not isinstance(workers, int) or workers < 1
37
+ ):
38
+ raise ConfigError("parallel_workers must be a positive integer")
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class TargetDeclaration:
43
+ """A configured physical target and its default logical Weaver item."""
44
+
45
+ item: WeaverItemId
46
+ execution: ExecutionSettings = field(default_factory=ExecutionSettings)
47
+
48
+
49
+ def _target_declarations(
50
+ declarations: Mapping[str, TargetDeclaration], *, item_type: str, field_name: str
51
+ ) -> Mapping[str, TargetDeclaration]:
52
+ resolved: dict[str, TargetDeclaration] = {}
53
+ for physical_name, declaration in dict(declarations).items():
54
+ name = validate_name(physical_name, what=f"{field_name} key")
55
+ if not isinstance(declaration, TargetDeclaration):
56
+ raise ConfigError(f"{field_name}[{name!r}] must be a TargetDeclaration")
57
+ if declaration.item.item_type != item_type:
58
+ raise ConfigError(
59
+ f"{field_name}[{name!r}] must name a {item_type} item, "
60
+ f"got {declaration.item}"
61
+ )
62
+ resolved[name] = declaration
63
+ return MappingProxyType(resolved)
64
+
65
+
66
+ @dataclass(frozen=True, kw_only=True)
67
+ class Workspace:
68
+ """The common configuration for one Fabric workspace or local emulator."""
69
+
70
+ environment: str | None = None
71
+ weaver_lakehouse: str | None = None
72
+ execution: ExecutionSettings = field(default_factory=ExecutionSettings)
73
+ lakehouses: Mapping[str, TargetDeclaration] = field(default_factory=dict)
74
+ warehouses: Mapping[str, TargetDeclaration] = field(default_factory=dict)
75
+
76
+ def __post_init__(self) -> None:
77
+ if self.environment is not None:
78
+ object.__setattr__(
79
+ self,
80
+ "environment",
81
+ validate_name(self.environment, what="environment"),
82
+ )
83
+ if self.weaver_lakehouse is not None:
84
+ object.__setattr__(
85
+ self,
86
+ "weaver_lakehouse",
87
+ validate_name(self.weaver_lakehouse, what="weaver_lakehouse"),
88
+ )
89
+ if not isinstance(self.execution, ExecutionSettings):
90
+ raise ConfigError("execution must be ExecutionSettings")
91
+ object.__setattr__(
92
+ self,
93
+ "lakehouses",
94
+ _target_declarations(
95
+ self.lakehouses, item_type=LAKEHOUSE, field_name="lakehouses"
96
+ ),
97
+ )
98
+ object.__setattr__(
99
+ self,
100
+ "warehouses",
101
+ _target_declarations(
102
+ self.warehouses, item_type=WAREHOUSE, field_name="warehouses"
103
+ ),
104
+ )
105
+
106
+ @property
107
+ def workspace_type(self) -> str:
108
+ raise NotImplementedError
109
+
110
+ @property
111
+ def supports_sql(self) -> bool:
112
+ raise NotImplementedError
113
+
114
+ def declaration_for(self, item_type: str, physical_name: str) -> TargetDeclaration:
115
+ declarations = self.lakehouses if item_type == LAKEHOUSE else self.warehouses
116
+ try:
117
+ return declarations[physical_name]
118
+ except KeyError as exc:
119
+ plural = "Lakehouses" if item_type == LAKEHOUSE else "Warehouses"
120
+ raise ConfigError(
121
+ f"physical target {plural}/{physical_name} is not configured"
122
+ ) from exc
123
+
124
+
125
+ @dataclass(frozen=True, kw_only=True)
126
+ class FabricWorkspace(Workspace):
127
+ """One Microsoft Fabric workspace."""
128
+
129
+ workspace: str
130
+
131
+ def __post_init__(self) -> None:
132
+ super().__post_init__()
133
+ object.__setattr__(
134
+ self, "workspace", validate_name(self.workspace, what="workspace")
135
+ )
136
+
137
+ @property
138
+ def workspace_type(self) -> str:
139
+ return FABRIC
140
+
141
+ @property
142
+ def supports_sql(self) -> bool:
143
+ return True
144
+
145
+ def settings_for_warehouse(self, name: str) -> ExecutionSettings:
146
+ declaration = self.warehouses.get(name)
147
+ return declaration.execution if declaration else self.execution
148
+
149
+
150
+ @dataclass(frozen=True, kw_only=True)
151
+ class LocalWorkspace(Workspace):
152
+ """A filesystem folder standing in for one Fabric workspace."""
153
+
154
+ workspace: Path
155
+
156
+ def __post_init__(self) -> None:
157
+ super().__post_init__()
158
+ value = self.workspace
159
+ if isinstance(value, str):
160
+ if not value.strip():
161
+ raise ConfigError("workspace path must not be empty")
162
+ value = Path(value.strip())
163
+ elif not isinstance(value, Path):
164
+ raise ConfigError(
165
+ f"workspace must be a path, got {type(value).__name__}"
166
+ )
167
+ object.__setattr__(self, "workspace", value.expanduser())
168
+
169
+ @property
170
+ def workspace_type(self) -> str:
171
+ return LOCAL
172
+
173
+ @property
174
+ def supports_sql(self) -> bool:
175
+ return False
weaver_cli/__init__.py ADDED
@@ -0,0 +1,12 @@
1
+ """The optional desktop CLI — an adapter over :mod:`weaver`.
2
+
3
+ The dependency direction is one way: this package imports ``weaver``; the core
4
+ never imports this package. The CLI owns argument parsing and presentation
5
+ only. Build, load and catalogue semantics belong to the core.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from .main import main
11
+
12
+ __all__ = ["main"]
weaver_cli/__main__.py ADDED
@@ -0,0 +1,7 @@
1
+ """Allow ``python -m weaver_cli``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .main import main
6
+
7
+ raise SystemExit(main())