weaverstack 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- weaver/__init__.py +59 -0
- weaver/build_bundle/__init__.py +109 -0
- weaver/build_bundle/aliases.py +325 -0
- weaver/build_bundle/bundle.py +359 -0
- weaver/build_bundle/catalogue_actions.py +275 -0
- weaver/build_bundle/changes.py +186 -0
- weaver/build_bundle/endpoints.py +83 -0
- weaver/build_bundle/executors/__init__.py +69 -0
- weaver/build_bundle/executors/alias.py +202 -0
- weaver/build_bundle/executors/base.py +132 -0
- weaver/build_bundle/executors/folder.py +71 -0
- weaver/build_bundle/executors/load_file.py +205 -0
- weaver/build_bundle/executors/spark_case.py +26 -0
- weaver/build_bundle/executors/spark_schema.py +60 -0
- weaver/build_bundle/executors/spark_sql.py +59 -0
- weaver/build_bundle/executors/spark_sql_batch.py +57 -0
- weaver/build_bundle/executors/spark_table.py +213 -0
- weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
- weaver/build_bundle/executors/tsql.py +81 -0
- weaver/build_bundle/incremental.py +288 -0
- weaver/build_bundle/installer.py +384 -0
- weaver/build_bundle/models.py +288 -0
- weaver/build_bundle/payloads.py +34 -0
- weaver/build_bundle/physical.py +625 -0
- weaver/build_bundle/planner.py +389 -0
- weaver/build_bundle/prune.py +620 -0
- weaver/build_bundle/report.py +108 -0
- weaver/build_bundle/stages.py +196 -0
- weaver/build_bundle/targets.py +272 -0
- weaver/build_bundle/workflow.py +585 -0
- weaver/catalogue/__init__.py +73 -0
- weaver/catalogue/builtin.py +238 -0
- weaver/catalogue/claims.py +121 -0
- weaver/catalogue/projection.py +437 -0
- weaver/catalogue/reader.py +152 -0
- weaver/catalogue/reconcile.py +231 -0
- weaver/catalogue/render.py +410 -0
- weaver/catalogue/state.py +660 -0
- weaver/catalogue/tables.py +648 -0
- weaver/config.py +178 -0
- weaver/declaration/__init__.py +171 -0
- weaver/declaration/columns.py +223 -0
- weaver/declaration/ddl.py +266 -0
- weaver/declaration/dependencies.py +544 -0
- weaver/declaration/graph.py +240 -0
- weaver/declaration/item_dependencies.py +292 -0
- weaver/declaration/load.py +191 -0
- weaver/declaration/metadata.py +1405 -0
- weaver/declaration/model.py +448 -0
- weaver/declaration/references.py +294 -0
- weaver/declaration/repository.py +959 -0
- weaver/declaration/schemas.py +135 -0
- weaver/declaration/source.py +674 -0
- weaver/declaration/spark_load.py +759 -0
- weaver/declaration/sql_shaping.py +591 -0
- weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
- weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
- weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
- weaver/declaration/templates/load/column_metadata.sql +40 -0
- weaver/declaration/templates/load/full_replace_body.sql +21 -0
- weaver/declaration/templates/load/install_load_procedure.sql +27 -0
- weaver/declaration/templates/load/load_procedure.sql +48 -0
- weaver/declaration/templates/load/primary_key_body.sql +113 -0
- weaver/declaration/tsql_ddl.py +468 -0
- weaver/declaration/tsql_load.py +417 -0
- weaver/declaration/warehouse_type_mapping.yml +93 -0
- weaver/diagnostics.py +247 -0
- weaver/errors.py +61 -0
- weaver/etl.py +469 -0
- weaver/fabric/__init__.py +107 -0
- weaver/fabric/auth.py +137 -0
- weaver/fabric/capacity.py +143 -0
- weaver/fabric/client.py +147 -0
- weaver/fabric/environment.py +460 -0
- weaver/fabric/livy.py +478 -0
- weaver/fabric/notebooks.py +201 -0
- weaver/fabric/onelake.py +263 -0
- weaver/fabric/resolution.py +344 -0
- weaver/fabric/resources.py +245 -0
- weaver/fabric/session.py +148 -0
- weaver/fabric/shortcuts.py +120 -0
- weaver/fabric/sql.py +118 -0
- weaver/fabric/store.py +198 -0
- weaver/initialise.py +209 -0
- weaver/lakehouse.py +386 -0
- weaver/load.py +474 -0
- weaver/load_execution.py +483 -0
- weaver/load_plan.py +912 -0
- weaver/load_report.py +330 -0
- weaver/load_resolution.py +386 -0
- weaver/locations.py +164 -0
- weaver/objects.py +392 -0
- weaver/operations.py +757 -0
- weaver/physical_wipe.py +369 -0
- weaver/push.py +76 -0
- weaver/resolution.py +292 -0
- weaver/runtime/__init__.py +30 -0
- weaver/runtime/folder_load.py +402 -0
- weaver/runtime/load_contract.py +245 -0
- weaver/runtime/load_result.py +104 -0
- weaver/runtime/spark_load.py +152 -0
- weaver/runtime/table_load.py +497 -0
- weaver/spark/__init__.py +49 -0
- weaver/spark/catalogue.py +245 -0
- weaver/spark/destination.py +195 -0
- weaver/spark/session.py +84 -0
- weaver/spark/tokens.py +138 -0
- weaver/sql/__init__.py +40 -0
- weaver/sql/authentication.py +38 -0
- weaver/sql/connection.py +90 -0
- weaver/sql/errors.py +25 -0
- weaver/sql/execution.py +123 -0
- weaver/sql/pool.py +174 -0
- weaver/sql/wipe.py +156 -0
- weaver/store.py +209 -0
- weaver/targets.py +257 -0
- weaver/task_logging.py +215 -0
- weaver/unbind.py +74 -0
- weaver/workspaces.py +175 -0
- weaver_cli/__init__.py +12 -0
- weaver_cli/__main__.py +7 -0
- weaver_cli/main.py +626 -0
- weaverstack-0.1.1.dist-info/METADATA +113 -0
- weaverstack-0.1.1.dist-info/RECORD +127 -0
- weaverstack-0.1.1.dist-info/WHEEL +4 -0
- weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
- weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
weaver/unbind.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Remove catalogue installations for explicitly named physical targets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from .catalogue.reader import read_table
|
|
8
|
+
from .catalogue.reconcile import prune_installation
|
|
9
|
+
from .catalogue.render import InstallationScope
|
|
10
|
+
from .catalogue.tables import INSTALLATION
|
|
11
|
+
from .declaration.model import LAKEHOUSE, WAREHOUSE
|
|
12
|
+
from .targets import ItemRef
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class UnbindResult:
|
|
17
|
+
targets: tuple[str, ...]
|
|
18
|
+
logical_items: tuple[str, ...]
|
|
19
|
+
statements: tuple[str, ...]
|
|
20
|
+
|
|
21
|
+
def to_mapping(self) -> dict[str, object]:
|
|
22
|
+
return {
|
|
23
|
+
"targets": list(self.targets),
|
|
24
|
+
"logical_items": list(self.logical_items),
|
|
25
|
+
"statements": len(self.statements),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def plan_unbind(
|
|
30
|
+
catalogue,
|
|
31
|
+
*,
|
|
32
|
+
lakehouses=(),
|
|
33
|
+
warehouses=(),
|
|
34
|
+
) -> UnbindResult:
|
|
35
|
+
"""Render complete catalogue deletion without inspecting physical targets."""
|
|
36
|
+
|
|
37
|
+
selected = {
|
|
38
|
+
(LAKEHOUSE, ItemRef.parse(value).name) for value in lakehouses
|
|
39
|
+
} | {
|
|
40
|
+
(WAREHOUSE, ItemRef.parse(value).name) for value in warehouses
|
|
41
|
+
}
|
|
42
|
+
rows = read_table(catalogue, INSTALLATION)
|
|
43
|
+
scopes = sorted(
|
|
44
|
+
{
|
|
45
|
+
InstallationScope(
|
|
46
|
+
str(row["item_type"]), str(row["item_name"])
|
|
47
|
+
)
|
|
48
|
+
for row in rows
|
|
49
|
+
if (str(row["item_type"]), str(row["target_name"])) in selected
|
|
50
|
+
},
|
|
51
|
+
key=str,
|
|
52
|
+
)
|
|
53
|
+
statements = tuple(
|
|
54
|
+
statement
|
|
55
|
+
for scope in scopes
|
|
56
|
+
for statement in prune_installation(scope)
|
|
57
|
+
)
|
|
58
|
+
targets = tuple(f"{item_type}/{name}" for item_type, name in sorted(selected))
|
|
59
|
+
return UnbindResult(
|
|
60
|
+
targets=targets,
|
|
61
|
+
logical_items=tuple(map(str, scopes)),
|
|
62
|
+
statements=statements,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def unbind_targets(catalogue, *, lakehouses=(), warehouses=()) -> UnbindResult:
|
|
67
|
+
"""Execute target-directed catalogue deletion and touch no physical target."""
|
|
68
|
+
|
|
69
|
+
result = plan_unbind(
|
|
70
|
+
catalogue, lakehouses=lakehouses, warehouses=warehouses
|
|
71
|
+
)
|
|
72
|
+
for statement in result.statements:
|
|
73
|
+
catalogue.sql(statement)
|
|
74
|
+
return result
|
weaver/workspaces.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Workspace configuration and the two execution environments Weaver supports.
|
|
2
|
+
|
|
3
|
+
A Workspace identifies where resources live. For Fabric that identity is a
|
|
4
|
+
Microsoft Fabric workspace name; for the local emulator it is the path to the
|
|
5
|
+
folder that stands in for one. It does not say where Weaver code executes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from types import MappingProxyType
|
|
13
|
+
from typing import Mapping
|
|
14
|
+
|
|
15
|
+
from .declaration.model import LAKEHOUSE, WAREHOUSE, WeaverItemId
|
|
16
|
+
from .errors import ConfigError
|
|
17
|
+
from .targets import validate_name
|
|
18
|
+
|
|
19
|
+
WEAVER_ITEMS_AREA = "weaver_items"
|
|
20
|
+
BUILD_BUNDLES_AREA = "build_bundles"
|
|
21
|
+
CLI_AREA = "cli"
|
|
22
|
+
FABRIC = "fabric"
|
|
23
|
+
LOCAL = "local"
|
|
24
|
+
WORKSPACE_TYPES = (FABRIC, LOCAL)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class ExecutionSettings:
|
|
29
|
+
"""Technical parallelism settings for a Workspace or one physical item."""
|
|
30
|
+
|
|
31
|
+
parallel_workers: int | None = None
|
|
32
|
+
|
|
33
|
+
def __post_init__(self) -> None:
|
|
34
|
+
workers = self.parallel_workers
|
|
35
|
+
if workers is not None and (
|
|
36
|
+
isinstance(workers, bool) or not isinstance(workers, int) or workers < 1
|
|
37
|
+
):
|
|
38
|
+
raise ConfigError("parallel_workers must be a positive integer")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class TargetDeclaration:
|
|
43
|
+
"""A configured physical target and its default logical Weaver item."""
|
|
44
|
+
|
|
45
|
+
item: WeaverItemId
|
|
46
|
+
execution: ExecutionSettings = field(default_factory=ExecutionSettings)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _target_declarations(
|
|
50
|
+
declarations: Mapping[str, TargetDeclaration], *, item_type: str, field_name: str
|
|
51
|
+
) -> Mapping[str, TargetDeclaration]:
|
|
52
|
+
resolved: dict[str, TargetDeclaration] = {}
|
|
53
|
+
for physical_name, declaration in dict(declarations).items():
|
|
54
|
+
name = validate_name(physical_name, what=f"{field_name} key")
|
|
55
|
+
if not isinstance(declaration, TargetDeclaration):
|
|
56
|
+
raise ConfigError(f"{field_name}[{name!r}] must be a TargetDeclaration")
|
|
57
|
+
if declaration.item.item_type != item_type:
|
|
58
|
+
raise ConfigError(
|
|
59
|
+
f"{field_name}[{name!r}] must name a {item_type} item, "
|
|
60
|
+
f"got {declaration.item}"
|
|
61
|
+
)
|
|
62
|
+
resolved[name] = declaration
|
|
63
|
+
return MappingProxyType(resolved)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True, kw_only=True)
|
|
67
|
+
class Workspace:
|
|
68
|
+
"""The common configuration for one Fabric workspace or local emulator."""
|
|
69
|
+
|
|
70
|
+
environment: str | None = None
|
|
71
|
+
weaver_lakehouse: str | None = None
|
|
72
|
+
execution: ExecutionSettings = field(default_factory=ExecutionSettings)
|
|
73
|
+
lakehouses: Mapping[str, TargetDeclaration] = field(default_factory=dict)
|
|
74
|
+
warehouses: Mapping[str, TargetDeclaration] = field(default_factory=dict)
|
|
75
|
+
|
|
76
|
+
def __post_init__(self) -> None:
|
|
77
|
+
if self.environment is not None:
|
|
78
|
+
object.__setattr__(
|
|
79
|
+
self,
|
|
80
|
+
"environment",
|
|
81
|
+
validate_name(self.environment, what="environment"),
|
|
82
|
+
)
|
|
83
|
+
if self.weaver_lakehouse is not None:
|
|
84
|
+
object.__setattr__(
|
|
85
|
+
self,
|
|
86
|
+
"weaver_lakehouse",
|
|
87
|
+
validate_name(self.weaver_lakehouse, what="weaver_lakehouse"),
|
|
88
|
+
)
|
|
89
|
+
if not isinstance(self.execution, ExecutionSettings):
|
|
90
|
+
raise ConfigError("execution must be ExecutionSettings")
|
|
91
|
+
object.__setattr__(
|
|
92
|
+
self,
|
|
93
|
+
"lakehouses",
|
|
94
|
+
_target_declarations(
|
|
95
|
+
self.lakehouses, item_type=LAKEHOUSE, field_name="lakehouses"
|
|
96
|
+
),
|
|
97
|
+
)
|
|
98
|
+
object.__setattr__(
|
|
99
|
+
self,
|
|
100
|
+
"warehouses",
|
|
101
|
+
_target_declarations(
|
|
102
|
+
self.warehouses, item_type=WAREHOUSE, field_name="warehouses"
|
|
103
|
+
),
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def workspace_type(self) -> str:
|
|
108
|
+
raise NotImplementedError
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def supports_sql(self) -> bool:
|
|
112
|
+
raise NotImplementedError
|
|
113
|
+
|
|
114
|
+
def declaration_for(self, item_type: str, physical_name: str) -> TargetDeclaration:
|
|
115
|
+
declarations = self.lakehouses if item_type == LAKEHOUSE else self.warehouses
|
|
116
|
+
try:
|
|
117
|
+
return declarations[physical_name]
|
|
118
|
+
except KeyError as exc:
|
|
119
|
+
plural = "Lakehouses" if item_type == LAKEHOUSE else "Warehouses"
|
|
120
|
+
raise ConfigError(
|
|
121
|
+
f"physical target {plural}/{physical_name} is not configured"
|
|
122
|
+
) from exc
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass(frozen=True, kw_only=True)
|
|
126
|
+
class FabricWorkspace(Workspace):
|
|
127
|
+
"""One Microsoft Fabric workspace."""
|
|
128
|
+
|
|
129
|
+
workspace: str
|
|
130
|
+
|
|
131
|
+
def __post_init__(self) -> None:
|
|
132
|
+
super().__post_init__()
|
|
133
|
+
object.__setattr__(
|
|
134
|
+
self, "workspace", validate_name(self.workspace, what="workspace")
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def workspace_type(self) -> str:
|
|
139
|
+
return FABRIC
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def supports_sql(self) -> bool:
|
|
143
|
+
return True
|
|
144
|
+
|
|
145
|
+
def settings_for_warehouse(self, name: str) -> ExecutionSettings:
|
|
146
|
+
declaration = self.warehouses.get(name)
|
|
147
|
+
return declaration.execution if declaration else self.execution
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True, kw_only=True)
|
|
151
|
+
class LocalWorkspace(Workspace):
|
|
152
|
+
"""A filesystem folder standing in for one Fabric workspace."""
|
|
153
|
+
|
|
154
|
+
workspace: Path
|
|
155
|
+
|
|
156
|
+
def __post_init__(self) -> None:
|
|
157
|
+
super().__post_init__()
|
|
158
|
+
value = self.workspace
|
|
159
|
+
if isinstance(value, str):
|
|
160
|
+
if not value.strip():
|
|
161
|
+
raise ConfigError("workspace path must not be empty")
|
|
162
|
+
value = Path(value.strip())
|
|
163
|
+
elif not isinstance(value, Path):
|
|
164
|
+
raise ConfigError(
|
|
165
|
+
f"workspace must be a path, got {type(value).__name__}"
|
|
166
|
+
)
|
|
167
|
+
object.__setattr__(self, "workspace", value.expanduser())
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def workspace_type(self) -> str:
|
|
171
|
+
return LOCAL
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def supports_sql(self) -> bool:
|
|
175
|
+
return False
|
weaver_cli/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""The optional desktop CLI — an adapter over :mod:`weaver`.
|
|
2
|
+
|
|
3
|
+
The dependency direction is one way: this package imports ``weaver``; the core
|
|
4
|
+
never imports this package. The CLI owns argument parsing and presentation
|
|
5
|
+
only. Build, load and catalogue semantics belong to the core.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from .main import main
|
|
11
|
+
|
|
12
|
+
__all__ = ["main"]
|