harness-toolbox 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harness_toolbox-0.2.2/.gitignore +45 -0
- harness_toolbox-0.2.2/Makefile +16 -0
- harness_toolbox-0.2.2/PKG-INFO +14 -0
- harness_toolbox-0.2.2/README.md +58 -0
- harness_toolbox-0.2.2/docs/lifecycle.md +198 -0
- harness_toolbox-0.2.2/harness_toolbox/AGENTS.md +40 -0
- harness_toolbox-0.2.2/harness_toolbox/__init__.py +19 -0
- harness_toolbox-0.2.2/harness_toolbox/_mysql_values.py +46 -0
- harness_toolbox-0.2.2/harness_toolbox/address.py +170 -0
- harness_toolbox-0.2.2/harness_toolbox/client.py +8 -0
- harness_toolbox-0.2.2/harness_toolbox/diagnostics.py +73 -0
- harness_toolbox-0.2.2/harness_toolbox/environment.py +31 -0
- harness_toolbox-0.2.2/harness_toolbox/errors.py +56 -0
- harness_toolbox-0.2.2/harness_toolbox/host.py +44 -0
- harness_toolbox-0.2.2/harness_toolbox/kube/__init__.py +11 -0
- harness_toolbox-0.2.2/harness_toolbox/kube/client.py +784 -0
- harness_toolbox-0.2.2/harness_toolbox/kube/model.py +82 -0
- harness_toolbox-0.2.2/harness_toolbox/kube/resources.py +138 -0
- harness_toolbox-0.2.2/harness_toolbox/kube/selector.py +22 -0
- harness_toolbox-0.2.2/harness_toolbox/mysql.py +430 -0
- harness_toolbox-0.2.2/harness_toolbox/opensearch.py +331 -0
- harness_toolbox-0.2.2/harness_toolbox/pod_log.py +213 -0
- harness_toolbox-0.2.2/harness_toolbox/process.py +116 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_address.py +263 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_diagnostics.py +279 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_environment.py +63 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_host.py +34 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_kube.py +360 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_kube_manifests.py +157 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_kube_resources.py +153 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_mysql.py +81 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_mysql_queries.py +282 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_opensearch.py +111 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_pod_log.py +92 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_process.py +25 -0
- harness_toolbox-0.2.2/harness_toolbox/tests/test_transport.py +50 -0
- harness_toolbox-0.2.2/harness_toolbox/transport.py +167 -0
- harness_toolbox-0.2.2/pyproject.toml +23 -0
- harness_toolbox-0.2.2/ruff.toml +4 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
.venv/
|
|
8
|
+
*.egg
|
|
9
|
+
.pytest_cache/
|
|
10
|
+
.mypy_cache/
|
|
11
|
+
.ruff_cache/
|
|
12
|
+
htmlcov/
|
|
13
|
+
.coverage
|
|
14
|
+
*.whl
|
|
15
|
+
*.tar.gz
|
|
16
|
+
|
|
17
|
+
# Go
|
|
18
|
+
/sdks/go/bin/
|
|
19
|
+
|
|
20
|
+
# TypeScript
|
|
21
|
+
node_modules/
|
|
22
|
+
|
|
23
|
+
# IDE
|
|
24
|
+
.idea/
|
|
25
|
+
.vscode/
|
|
26
|
+
*.swp
|
|
27
|
+
*.swo
|
|
28
|
+
|
|
29
|
+
# OS
|
|
30
|
+
.DS_Store
|
|
31
|
+
Thumbs.db
|
|
32
|
+
|
|
33
|
+
# uv
|
|
34
|
+
sdks/python/.venv/
|
|
35
|
+
|
|
36
|
+
# DevOps plugin
|
|
37
|
+
.devops/
|
|
38
|
+
|
|
39
|
+
# Runtime artifacts (agent tier)
|
|
40
|
+
runtime/
|
|
41
|
+
runs/
|
|
42
|
+
|
|
43
|
+
# devloop
|
|
44
|
+
.devloop/
|
|
45
|
+
.worktrees/
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
.PHONY: test lint fix build
|
|
2
|
+
|
|
3
|
+
CHECK_PATHS ?= $(if $(filter %.py,$(LINT_FILES)),$(filter %.py,$(LINT_FILES)),harness_toolbox)
|
|
4
|
+
|
|
5
|
+
test:
|
|
6
|
+
uv run --project .. pytest -v harness_toolbox/tests $(TEST_FILES)
|
|
7
|
+
|
|
8
|
+
lint:
|
|
9
|
+
uv run --project .. ruff check $(CHECK_PATHS)
|
|
10
|
+
|
|
11
|
+
fix:
|
|
12
|
+
uv run --project .. ruff check --fix $(CHECK_PATHS)
|
|
13
|
+
uv run --project .. ruff format $(CHECK_PATHS)
|
|
14
|
+
|
|
15
|
+
build:
|
|
16
|
+
uv build --out-dir ../../../dist
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: harness-toolbox
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Async infrastructure clients and execution-scoped resource sharing
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Requires-Dist: harness-common<0.4,>=0.3.1
|
|
8
|
+
Provides-Extra: kube
|
|
9
|
+
Requires-Dist: kubernetes-asyncio<37,>=36.1; extra == 'kube'
|
|
10
|
+
Provides-Extra: mysql
|
|
11
|
+
Requires-Dist: asyncmy<1,>=0.2.9; extra == 'mysql'
|
|
12
|
+
Requires-Dist: sqlalchemy[asyncio]<3,>=2.0; extra == 'mysql'
|
|
13
|
+
Provides-Extra: opensearch
|
|
14
|
+
Requires-Dist: httpx<1,>=0.27; extra == 'opensearch'
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Harness Toolbox for Python
|
|
2
|
+
|
|
3
|
+
Async infrastructure clients for test runners, diagnostic commands, and operational Skills.
|
|
4
|
+
The same execution can create test Pods, inspect their state, run commands, query databases,
|
|
5
|
+
and collect logs without rebuilding clients for each nested operation.
|
|
6
|
+
|
|
7
|
+
Install only the protocols you use:
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
pip install 'harness-toolbox[kube,opensearch,mysql]'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
`harness-common` owns `Client`, `DataSource`, `ClientProvider` and `ClientManager`;
|
|
14
|
+
toolbox re-exports these names. `Client` owns initialization and disposal. A `DataSource` identifies its configuration;
|
|
15
|
+
`ClientManager` shares it across concurrent or nested work and cleans up at root exit.
|
|
16
|
+
`ConnectionSource` and `Transport` separate caller-owned configuration from the path used to reach it.
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import asyncio
|
|
20
|
+
from harness_toolbox import ClientManager
|
|
21
|
+
from harness_toolbox.kube import KubernetesDataSource, Options, PodSpec
|
|
22
|
+
|
|
23
|
+
async def main():
|
|
24
|
+
source = KubernetesDataSource(
|
|
25
|
+
options=Options(namespace="quality", request_timeout_s=15, connection_pool_maxsize=4),
|
|
26
|
+
kubeconfig="/path/to/kubeconfig",
|
|
27
|
+
)
|
|
28
|
+
async with ClientManager() as clients:
|
|
29
|
+
kube = await clients.get(source)
|
|
30
|
+
pod = await kube.create_pod(PodSpec("toolbox-demo", "busybox:1.37",
|
|
31
|
+
command=("sleep", "300")))
|
|
32
|
+
try:
|
|
33
|
+
await kube.wait_ready(pod.ref(), timeout_s=60, interval_s=1)
|
|
34
|
+
result = await kube.execute(pod.ref(), ["echo", "hello"])
|
|
35
|
+
print(result.stdout.decode(), result.exit_code)
|
|
36
|
+
finally:
|
|
37
|
+
await kube.delete_pod(pod.ref())
|
|
38
|
+
await kube.wait_deleted(pod.ref(), timeout_s=60, interval_s=1)
|
|
39
|
+
|
|
40
|
+
asyncio.run(main())
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Existing workloads can be observed with `kube.list_service_pods(name)` or
|
|
44
|
+
`kube.list_deployment_pods(name)`. Both use the resource's label selector, return stable Pod
|
|
45
|
+
observations, and leave readiness/sample selection to the caller. Missing resources raise
|
|
46
|
+
`ResourceNotFoundError`; permission and network errors remain failures. Selectorless resources
|
|
47
|
+
cannot be used to enumerate all Pods accidentally.
|
|
48
|
+
|
|
49
|
+
The caller chooses cluster, namespace, credentials, SQL, indexes, collection windows, and authorization.
|
|
50
|
+
The package does not read product-specific environment variables or registries. Kubernetes control uses
|
|
51
|
+
`kubernetes-asyncio`; exec, port-forward and log streams require `kubectl` on the host and use the same
|
|
52
|
+
kubeconfig/context. Pod Python access additionally requires Python and the protocol driver in the target Pod.
|
|
53
|
+
|
|
54
|
+
See [lifecycle and connection examples](docs/lifecycle.md) for shared clients, database routes, TLS,
|
|
55
|
+
log budgets, and cancellation behavior.
|
|
56
|
+
|
|
57
|
+
`list_workload_pods` resolves Deployment, StatefulSet, DaemonSet or an explicit Pod.
|
|
58
|
+
Kubernetes Service endpoints retain their separate `list_service_pods` operation.
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# Execution-scoped infrastructure
|
|
2
|
+
|
|
3
|
+
## Ownership and identity
|
|
4
|
+
|
|
5
|
+
One root operation owns a `ClientManager`. Nested commands receive a `ClientProvider`, reuse its
|
|
6
|
+
clients, and join their work before the root exits. A cancelled waiter does not cancel shared
|
|
7
|
+
initialization. Root disposal stops pending initialization, drains it, and disposes ready clients
|
|
8
|
+
in reverse dependency order. A failed initialization is cleaned before retry; failed cleanup keeps
|
|
9
|
+
that identity failed and is reported again at finalize. Disposal does not delete remote workloads.
|
|
10
|
+
|
|
11
|
+
The manager is local to one Python execution and event loop. It cannot share live connections across
|
|
12
|
+
separately launched scripts. Synchronous CLIs call `asyncio.run` at their outer boundary; domain
|
|
13
|
+
operations remain async and receive the provider explicitly.
|
|
14
|
+
|
|
15
|
+
A DataSource key must cover everything that changes reuse: target, protocol, credentials, configuration,
|
|
16
|
+
and capacity. ConnectionSource accepts a caller-supplied stable key because environment resolution is
|
|
17
|
+
external. Use `data_source_key` to hash those inputs. Do not key only by a friendly service name when
|
|
18
|
+
multiple clusters or credentials can coexist. Configuration and credentials remain fixed for a root
|
|
19
|
+
execution; key new configurations separately.
|
|
20
|
+
|
|
21
|
+
## Database and HTTP paths
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
from dataclasses import asdict
|
|
25
|
+
from harness_toolbox import ClientManager, data_source_key
|
|
26
|
+
from harness_toolbox.mysql import MySQLDataSource, MySQLTarget
|
|
27
|
+
from harness_toolbox.transport import ConnectionSource, DirectTransport
|
|
28
|
+
|
|
29
|
+
async def query(clients, settings):
|
|
30
|
+
target = MySQLTarget(**settings)
|
|
31
|
+
async def resolve():
|
|
32
|
+
return target
|
|
33
|
+
source = MySQLDataSource(ConnectionSource(
|
|
34
|
+
data_source_key("db-config", asdict(target)), resolve, (DirectTransport(),)
|
|
35
|
+
))
|
|
36
|
+
database = await clients.get(source)
|
|
37
|
+
return await database.query("SELECT id FROM message WHERE id = %(id)s", {"id": "example"})
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
MySQL uses SQLAlchemy with asyncmy for direct and forwarded access. It selects a route while establishing
|
|
41
|
+
a connection, before user SQL. Authentication errors may advance to another declared address, but
|
|
42
|
+
do not retry the same address through another transport. Statement failures never cause replay. PodPythonTransport sends explicit connection settings and query parameters over stdin and uses
|
|
43
|
+
PyMySQL in the Pod; it shares the resolved route/client, but opens a remote DB connection per exec.
|
|
44
|
+
Statements use DBAPI `%(name)s` parameters on both paths. Queries are bounded by concurrency, timeout,
|
|
45
|
+
and row limit. A statement timeout does not prove the server rolled back; callers decide transaction
|
|
46
|
+
and retry semantics.
|
|
47
|
+
|
|
48
|
+
With `params=None`, SQL is passed literally, so `DATE_FORMAT(ts, '%Y-%m-%d')` needs no escaping.
|
|
49
|
+
Supplying a parameter mapping, including `{}`, enables DBAPI interpolation: bind values with
|
|
50
|
+
`%(name)s` and write literal percent signs as `%%`. Do not interpolate values in the caller.
|
|
51
|
+
|
|
52
|
+
`QueryResult.rows` and `mappings()` preserve native Python values on both paths, including
|
|
53
|
+
`Decimal`, `datetime`, `date`, `time`, `timedelta` and `bytes`. The Pod wire format preserves
|
|
54
|
+
types and precision for parameters and results; it does not require toolbox installed in the Pod.
|
|
55
|
+
Consumers choose their own JSON presentation rather than relying on transport-specific string conversion.
|
|
56
|
+
For row-returning statements, `affected_rows` is `-1` (not a record count); use `len(result.rows)`.
|
|
57
|
+
Empty results retain their columns. Non-row statements retain the driver's affected-row count.
|
|
58
|
+
|
|
59
|
+
OpenSearchDataSource uses an HTTP pool and bounded response decoding. TLS verification is enabled;
|
|
60
|
+
private CAs use `ca_file`, and an intentional insecure environment must explicitly set
|
|
61
|
+
`insecure_skip_verify`. Forwarded connections preserve the original TLS server name. `scroll` yields
|
|
62
|
+
pages; use `contextlib.aclosing` when stopping early so the cursor is cleared before the client closes.
|
|
63
|
+
Consumers own index names, queries, and output files.
|
|
64
|
+
|
|
65
|
+
### Failures and connection evidence
|
|
66
|
+
|
|
67
|
+
Catch `ToolboxError` for translated access failures, or a protocol/operation subclass:
|
|
68
|
+
`MySQLError → MySQLConnectionError / MySQLQueryError` and
|
|
69
|
+
`OpenSearchError → OpenSearchConnectionError / OpenSearchRequestError`.
|
|
70
|
+
|
|
71
|
+
Every toolbox exception has three fields: `kind` is a stable `ErrorKind`, `code` is the optional
|
|
72
|
+
native DB/HTTP error code, and `message` is a safe contextual explanation, also returned by
|
|
73
|
+
`str(error)`. Interpret codes in the context of the protocol class, never by parsing messages.
|
|
74
|
+
Protocol adapters translate native failures once and preserve the original exception with
|
|
75
|
+
`raise ... from error`; already-translated exceptions propagate unchanged.
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from harness_toolbox import ToolboxError, ErrorKind
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
database = await clients.get(source)
|
|
82
|
+
result = await database.query("SELECT 1")
|
|
83
|
+
except ToolboxError as error:
|
|
84
|
+
if error.kind == ErrorKind.AUTHENTICATION_FAILED:
|
|
85
|
+
handle_authentication_failure(error.message)
|
|
86
|
+
report({"kind": error.kind, "code": error.code, "message": error.message})
|
|
87
|
+
else:
|
|
88
|
+
report({"connection": database.diagnostics, "rows": len(result.rows)})
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Consumers choose report fields and serialization. Exception messages exclude native driver text,
|
|
92
|
+
SQL/parameters, credentials and URL paths/query strings. Native causes and full tracebacks are
|
|
93
|
+
debugging material, **not** safe report output. Pod Python carries only safe error categories and
|
|
94
|
+
numeric MySQL codes; the host reconstructs the same public exception hierarchy without fabricating
|
|
95
|
+
an in-process driver cause.
|
|
96
|
+
|
|
97
|
+
`client.diagnostics` is independent connection evidence: protocol, resolved target, ordered
|
|
98
|
+
attempts and selected transport (`direct`, `port-forward` or `pod-python`). Snapshots are detached.
|
|
99
|
+
A target without a selected transport does not prove connection success. MySQL targets also expose
|
|
100
|
+
`target.diagnostics` before connecting. When initialization fails before a caller obtains the
|
|
101
|
+
client, the exception message still identifies the target and failed action; exceptions do not
|
|
102
|
+
carry the connection snapshot.
|
|
103
|
+
|
|
104
|
+
Arguments/programming errors and cancellation are not translated. Connection configuration
|
|
105
|
+
resolution belongs to the caller and its failures propagate unchanged. Result limits and invalid
|
|
106
|
+
remote responses are operational failures. Successful empty queries are not errors. Classification
|
|
107
|
+
does not authorize retries: only connection probes can use configured fallback routes; statements
|
|
108
|
+
and user requests are never automatically replayed.
|
|
109
|
+
|
|
110
|
+
## Kubernetes and logs
|
|
111
|
+
|
|
112
|
+
KubernetesDataSource owns namespace, API capacity, exec capacity and timeouts. KubernetesClient offers
|
|
113
|
+
Pod create/get/list/delete, readiness/replacement/deletion waits, Event collection, exec, and
|
|
114
|
+
port-forward. Exec returns stdout, stderr and exit code separately. Nonzero process exit is a result;
|
|
115
|
+
connection failure, timeout, and output limit violations are exceptions. Created Pods are explicit
|
|
116
|
+
consumer-owned workloads, so tests use `try/finally` to delete their own physical Pod instance.
|
|
117
|
+
|
|
118
|
+
Delete uses a server-side UID precondition. Kubernetes does not support that condition for exec or
|
|
119
|
+
logs; the client checks identity before and after access and rejects a changed instance. This detects
|
|
120
|
+
replacement but cannot guarantee an atomic exec against a UID. Port-forward subprocesses belong to the
|
|
121
|
+
Kubernetes client and close at finalize.
|
|
122
|
+
|
|
123
|
+
PodLogDataSource depends on KubernetesDataSource through the same provider. It owns one capture pool,
|
|
124
|
+
one total byte budget and a temporary directory. Physical Pod UID, container, restart count, previous
|
|
125
|
+
selection, and absolute timezone-aware `[since, until)` window identify a capture. Concurrent consumers
|
|
126
|
+
join the same capture; different trace IDs filter its file locally. Truncation is explicit in
|
|
127
|
+
PodLogCapture; timeout, transport failure or identity change do not publish a reusable capture.
|
|
128
|
+
Capture paths are borrowed and cease to exist after root disposal: copy evidence before root exit.
|
|
129
|
+
|
|
130
|
+
The capacities on a shared source apply to every user of that source. Use one configured log source
|
|
131
|
+
for all child commands rather than creating sources with conflicting limits. Defaults are bounded;
|
|
132
|
+
products should choose limits appropriate to their clusters.
|
|
133
|
+
|
|
134
|
+
## Common runtime identities
|
|
135
|
+
|
|
136
|
+
A logical Service declares its `workloads`; deployment configuration supplies the mapping. Workloads
|
|
137
|
+
are stable references, while Pod snapshots and UIDs are obtained during execution. The mapping can
|
|
138
|
+
change without changing the logical Service identity. Kubernetes Service endpoints are separate
|
|
139
|
+
from workload controllers and Pods.
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from harness_common import KubernetesWorkload
|
|
143
|
+
from harness_toolbox.environment import kubernetes_source
|
|
144
|
+
from harness_toolbox.kube import Options
|
|
145
|
+
|
|
146
|
+
async def observe(clients, service):
|
|
147
|
+
pods = []
|
|
148
|
+
for workload in service.workloads:
|
|
149
|
+
if not isinstance(workload, KubernetesWorkload):
|
|
150
|
+
raise TypeError("This observer requires Kubernetes workloads")
|
|
151
|
+
source = kubernetes_source(service.environment, Options(workload.namespace, 15, 4))
|
|
152
|
+
kube = await clients.get(source)
|
|
153
|
+
pods.extend(await kube.list_workload_pods(workload.kind, workload.name))
|
|
154
|
+
return pods
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Workload operations are one toolbox capability, alongside database, OpenSearch, process and transport
|
|
158
|
+
operations. Those capabilities do not require Service or Workload objects. ServiceDataSource can
|
|
159
|
+
associate logical context with independently configured sources without changing source keys.
|
|
160
|
+
|
|
161
|
+
PodPythonTransport accepts an explicit `container` for multi-container Pods. It is part of the
|
|
162
|
+
transport identity so clients using different containers cannot share a route. Credentials and
|
|
163
|
+
query parameters continue to travel over stdin.
|
|
164
|
+
|
|
165
|
+
## Environment-scoped address selection
|
|
166
|
+
|
|
167
|
+
Generic client ownership and service associations live in `harness_common`; toolbox
|
|
168
|
+
re-exports the original lifecycle names. `harness_toolbox.environment.kubernetes_source`
|
|
169
|
+
is the Kubernetes-specific factory. The independent packages depend in one direction:
|
|
170
|
+
`quality-harness` → `harness-toolbox` → `harness-common`.
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
from harness_common import KubernetesEnvironment
|
|
174
|
+
from harness_toolbox.address import AddressPolicy
|
|
175
|
+
|
|
176
|
+
addresses = AddressPolicy(
|
|
177
|
+
environment=KubernetesEnvironment("smoke", "/configs/smoke", "smoke-context"),
|
|
178
|
+
namespace="runtime",
|
|
179
|
+
fallback_hosts=("mysql-replica.storage.svc",),
|
|
180
|
+
timeout_s=10,
|
|
181
|
+
connection_pool_maxsize=8,
|
|
182
|
+
)
|
|
183
|
+
# Pass addresses=addresses to ConnectionSource for MySQL or OpenSearch.
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Resolution is lazy and shares Kubernetes clients through the root ClientProvider.
|
|
187
|
+
A Service lookup uses this environment's kubeconfig/context (empty kubeconfig means
|
|
188
|
+
in-cluster identity, never ambient local kubeconfig). A confirmed Service never falls
|
|
189
|
+
back to local short-name DNS. Cross-namespace names use `service.namespace.svc`;
|
|
190
|
+
ordinary domains retain DNS semantics. ClusterIP, ready headless addresses, external
|
|
191
|
+
DNS addresses and explicit alternatives are attempted sequentially and deduplicated.
|
|
192
|
+
|
|
193
|
+
Only initialization can switch addresses: authentication/database failures advance to
|
|
194
|
+
the next address, network failures may also advance to the next transport for the same
|
|
195
|
+
address. Queries are never replayed. The first successful client is reused; failed
|
|
196
|
+
attempt resources are closed. HTTPS keeps the configured SNI and HTTP Host even when
|
|
197
|
+
connecting by IP. Diagnostics distinguish configured target, candidate origin, mapped
|
|
198
|
+
endpoint and selected transport, without credentials or query data.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Python 平台工具箱
|
|
2
|
+
|
|
3
|
+
## 项目定位与边界
|
|
4
|
+
|
|
5
|
+
独立分发包 `harness-toolbox` 提供异步基础设施客户端。测试环境操作与诊断采集共享机制;
|
|
6
|
+
环境台账、凭据来源、业务查询、动作授权和结果判定由消费方拥有;环境内的网络地址解析由 toolbox 拥有。
|
|
7
|
+
|
|
8
|
+
## 代码地图与核心模块
|
|
9
|
+
|
|
10
|
+
```text
|
|
11
|
+
harness_toolbox/
|
|
12
|
+
├── client.py # common 生命周期类型的兼容导出
|
|
13
|
+
├── environment.py # KubernetesEnvironment → KubernetesDataSource
|
|
14
|
+
├── address.py # IP/DNS/环境内 Service 地址候选与解析失败
|
|
15
|
+
├── errors.py # ToolboxError 契约与协议子类;适配器负责转换原生异常
|
|
16
|
+
├── diagnostics.py # 客户端连接路径快照,与异常独立
|
|
17
|
+
├── transport.py # 连接解析、直连、port-forward、Pod Python 路径
|
|
18
|
+
├── process.py # 有界子进程 I/O、退出结果与取消清理
|
|
19
|
+
├── kube/ # Pod 增删、状态等待、Event、exec 与隧道
|
|
20
|
+
├── opensearch.py # HTTP 池、有界响应与 scroll
|
|
21
|
+
├── mysql.py # SQLAlchemy async 池与 Pod Python 查询
|
|
22
|
+
├── pod_log.py # 按物理实例/窗口共享采集,依赖 Kubernetes 客户端
|
|
23
|
+
└── tests/ # 协议 stub 和本机 HTTP/进程测试
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## 关键约定
|
|
27
|
+
|
|
28
|
+
- 根调用方通过 `async with ClientManager()` 管理执行期,子调用方只接收 ClientProvider。
|
|
29
|
+
- DataSource 创建不做外部 I/O;initialize 完成依赖获取后才发布成功,dispose 必须幂等。
|
|
30
|
+
- Service 名称必须用对应 Environment 的 kubeconfig/context 解析,不回退本机短名 DNS;仅初始化阶段切换地址,业务请求不重放。
|
|
31
|
+
- 同 key 的配置、凭据和容量必须一致;失败资源清理完成才允许重试。禁止日志输出凭据和查询内容。
|
|
32
|
+
- 协议依赖仅通过 extras 引入;通用生命周期归 harness-common,只使用标准库。Skill 必须显式安装所用协议 extra。
|
|
33
|
+
- Pod 删除通过 UID precondition 保证实例身份;exec/log API 没有原子 UID 条件,只能前后核验。
|
|
34
|
+
- ClientManager 释放连接、临时采集文件和子进程,不隐式删除消费方创建的远端 Pod。
|
|
35
|
+
|
|
36
|
+
## References
|
|
37
|
+
|
|
38
|
+
- [使用指南](../README.md)
|
|
39
|
+
- [生命周期与访问边界](../docs/lifecycle.md)
|
|
40
|
+
- [跨语言工具箱](../../../../../docs/toolbox.md)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Infrastructure mechanisms without business or environment configuration."""
|
|
2
|
+
|
|
3
|
+
from harness_toolbox.client import (
|
|
4
|
+
Client as Client,
|
|
5
|
+
)
|
|
6
|
+
from harness_toolbox.client import (
|
|
7
|
+
ClientManager as ClientManager,
|
|
8
|
+
)
|
|
9
|
+
from harness_toolbox.client import (
|
|
10
|
+
ClientProvider as ClientProvider,
|
|
11
|
+
)
|
|
12
|
+
from harness_toolbox.client import (
|
|
13
|
+
DataSource as DataSource,
|
|
14
|
+
)
|
|
15
|
+
from harness_toolbox.client import (
|
|
16
|
+
data_source_key as data_source_key,
|
|
17
|
+
)
|
|
18
|
+
from harness_toolbox.errors import ErrorKind as ErrorKind
|
|
19
|
+
from harness_toolbox.errors import ToolboxError as ToolboxError
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Private, stdlib-only wire codec, also shipped as source to Pod Python.
|
|
2
|
+
|
|
3
|
+
Every cell is tagged so SQL text cannot be mistaken for a serialized native value.
|
|
4
|
+
Keep this module independent of package imports: Pods need only Python and PyMySQL.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from base64 import b64decode, b64encode
|
|
8
|
+
from datetime import date, datetime, time, timedelta
|
|
9
|
+
from decimal import Decimal
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def encode_value(value: object) -> list[object]:
|
|
13
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
14
|
+
return ["scalar", value]
|
|
15
|
+
if isinstance(value, bytes):
|
|
16
|
+
return ["bytes", b64encode(value).decode("ascii")]
|
|
17
|
+
if isinstance(value, Decimal):
|
|
18
|
+
return ["decimal", str(value)]
|
|
19
|
+
if isinstance(value, datetime):
|
|
20
|
+
return ["datetime", value.isoformat()]
|
|
21
|
+
if isinstance(value, date):
|
|
22
|
+
return ["date", value.isoformat()]
|
|
23
|
+
if isinstance(value, time):
|
|
24
|
+
return ["time", value.isoformat()]
|
|
25
|
+
if isinstance(value, timedelta):
|
|
26
|
+
return ["timedelta", [value.days, value.seconds, value.microseconds]]
|
|
27
|
+
raise TypeError(f"Unsupported MySQL value type: {type(value).__name__}")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def decode_value(cell: list[object]) -> object:
|
|
31
|
+
kind, value = cell
|
|
32
|
+
if kind == "scalar":
|
|
33
|
+
return value
|
|
34
|
+
if kind == "bytes":
|
|
35
|
+
return b64decode(value)
|
|
36
|
+
if kind == "decimal":
|
|
37
|
+
return Decimal(value)
|
|
38
|
+
if kind == "datetime":
|
|
39
|
+
return datetime.fromisoformat(value)
|
|
40
|
+
if kind == "date":
|
|
41
|
+
return date.fromisoformat(value)
|
|
42
|
+
if kind == "time":
|
|
43
|
+
return time.fromisoformat(value)
|
|
44
|
+
if kind == "timedelta":
|
|
45
|
+
return timedelta(*value)
|
|
46
|
+
raise ValueError("Unknown MySQL wire value type")
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""Resolve access addresses within the declared environment, without probing services."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import ipaddress
|
|
7
|
+
import socket
|
|
8
|
+
from collections.abc import AsyncIterator
|
|
9
|
+
from dataclasses import asdict, dataclass
|
|
10
|
+
|
|
11
|
+
from harness_common import Environment, KubernetesEnvironment
|
|
12
|
+
|
|
13
|
+
from harness_toolbox.client import ClientProvider, data_source_key
|
|
14
|
+
from harness_toolbox.errors import ErrorKind, ToolboxError
|
|
15
|
+
from harness_toolbox.transport import Endpoint
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class AddressResolutionError(ToolboxError):
|
|
19
|
+
"""An address could not be resolved in its declared environment."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class AddressPolicy:
|
|
24
|
+
"""Environment and ordered alternatives for one logical connection target.
|
|
25
|
+
|
|
26
|
+
Short names in Kubernetes environments are Services in ``namespace``. Qualified
|
|
27
|
+
Service names use ``name.namespace.svc`` (optionally a cluster DNS suffix).
|
|
28
|
+
Two-label names are Services only when their namespace matches the explicit
|
|
29
|
+
scope. Other domains use DNS; no arbitrary domain is guessed into a Service.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
environment: Environment | None = None
|
|
33
|
+
namespace: str = ""
|
|
34
|
+
fallback_hosts: tuple[str, ...] = ()
|
|
35
|
+
timeout_s: float = 10
|
|
36
|
+
connection_pool_maxsize: int = 8
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def key(self) -> str:
|
|
40
|
+
config = asdict(self)
|
|
41
|
+
# Logical aliases must not fragment clients using the same physical access.
|
|
42
|
+
config["environment"] = (
|
|
43
|
+
{"kubeconfig": self.environment.kubeconfig, "context": self.environment.context}
|
|
44
|
+
if isinstance(self.environment, KubernetesEnvironment)
|
|
45
|
+
else None
|
|
46
|
+
)
|
|
47
|
+
return data_source_key("address-policy", config)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class AddressCandidate:
|
|
52
|
+
endpoint: Endpoint
|
|
53
|
+
source: str
|
|
54
|
+
error: AddressResolutionError | None = None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _service_name(host: str, namespace: str) -> tuple[str, str] | None:
|
|
58
|
+
parts = host.rstrip(".").split(".")
|
|
59
|
+
if len(parts) == 1:
|
|
60
|
+
if not namespace:
|
|
61
|
+
raise ValueError("Kubernetes short Service names require an explicit namespace")
|
|
62
|
+
return host, namespace
|
|
63
|
+
if len(parts) >= 3 and parts[2] == "svc":
|
|
64
|
+
return parts[0], parts[1]
|
|
65
|
+
if len(parts) == 2 and parts[1] == namespace:
|
|
66
|
+
return parts[0], namespace
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
async def _dns(host: str, port: int, timeout_s: float) -> tuple[str, ...]:
|
|
71
|
+
async with asyncio.timeout(timeout_s):
|
|
72
|
+
records = await asyncio.get_running_loop().getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
|
73
|
+
# Keep resolver preference (including IPv6), but remove duplicate socket records.
|
|
74
|
+
return tuple(dict.fromkeys(record[4][0] for record in records))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def _hosts(
|
|
78
|
+
policy: AddressPolicy, endpoint: Endpoint, clients: ClientProvider | None
|
|
79
|
+
) -> tuple[tuple[str, ...], str]:
|
|
80
|
+
try:
|
|
81
|
+
ipaddress.ip_address(endpoint.host)
|
|
82
|
+
except ValueError:
|
|
83
|
+
pass
|
|
84
|
+
else:
|
|
85
|
+
return (endpoint.host,), "ip"
|
|
86
|
+
environment = policy.environment
|
|
87
|
+
service = (
|
|
88
|
+
_service_name(endpoint.host, policy.namespace)
|
|
89
|
+
if isinstance(environment, KubernetesEnvironment)
|
|
90
|
+
else None
|
|
91
|
+
)
|
|
92
|
+
if service is None:
|
|
93
|
+
return await _dns(endpoint.host, endpoint.port, policy.timeout_s), "dns"
|
|
94
|
+
assert isinstance(environment, KubernetesEnvironment)
|
|
95
|
+
if clients is None:
|
|
96
|
+
raise ValueError("Kubernetes address resolution requires a ClientProvider")
|
|
97
|
+
from harness_toolbox.environment import kubernetes_source
|
|
98
|
+
from harness_toolbox.kube import Options
|
|
99
|
+
|
|
100
|
+
name, namespace = service
|
|
101
|
+
from aiohttp import ClientError
|
|
102
|
+
from kubernetes_asyncio.config.config_exception import ConfigException
|
|
103
|
+
|
|
104
|
+
try:
|
|
105
|
+
kube = await clients.get(
|
|
106
|
+
kubernetes_source(
|
|
107
|
+
environment,
|
|
108
|
+
Options(
|
|
109
|
+
namespace=namespace,
|
|
110
|
+
request_timeout_s=policy.timeout_s,
|
|
111
|
+
connection_pool_maxsize=policy.connection_pool_maxsize,
|
|
112
|
+
),
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
# why: local short-name DNS can resolve a healthy database in a different cluster.
|
|
116
|
+
# Service identity is authoritative; never fall back to ambient DNS for this name.
|
|
117
|
+
return await kube.service_addresses(name), "kubernetes-service"
|
|
118
|
+
except (ClientError, ConfigException) as error:
|
|
119
|
+
raise AddressResolutionError(
|
|
120
|
+
f"Cannot access Kubernetes environment {environment.name!r} for Service {name!r}",
|
|
121
|
+
kind=ErrorKind.CONNECTION_FAILED,
|
|
122
|
+
) from error
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
async def address_candidates(
|
|
126
|
+
endpoint: Endpoint, policy: AddressPolicy | None, clients: ClientProvider | None
|
|
127
|
+
) -> AsyncIterator[AddressCandidate]:
|
|
128
|
+
"""Yield ordered candidates; resolution failures remain visible before alternatives.
|
|
129
|
+
|
|
130
|
+
Without a policy, transports keep ownership of name resolution (e.g. a remote
|
|
131
|
+
Pod or a caller-supplied tunnel). No network work occurs while creating a source.
|
|
132
|
+
"""
|
|
133
|
+
if policy is None:
|
|
134
|
+
yield AddressCandidate(endpoint, "configured")
|
|
135
|
+
return
|
|
136
|
+
if policy.timeout_s <= 0 or policy.connection_pool_maxsize < 1:
|
|
137
|
+
raise ValueError("Address resolution limits must be positive")
|
|
138
|
+
seen: set[Endpoint] = set()
|
|
139
|
+
for host in dict.fromkeys((endpoint.host, *policy.fallback_hosts)):
|
|
140
|
+
configured = Endpoint(host, endpoint.port, endpoint.servername or endpoint.host)
|
|
141
|
+
try:
|
|
142
|
+
hosts, source = await _hosts(policy, configured, clients)
|
|
143
|
+
if not hosts:
|
|
144
|
+
raise AddressResolutionError(
|
|
145
|
+
f"No addresses for {host!r}", kind=ErrorKind.RESOURCE_NOT_FOUND
|
|
146
|
+
)
|
|
147
|
+
except (OSError, TimeoutError, ToolboxError) as error:
|
|
148
|
+
kind = (
|
|
149
|
+
error.kind
|
|
150
|
+
if isinstance(error, ToolboxError)
|
|
151
|
+
else (
|
|
152
|
+
ErrorKind.TIMEOUT
|
|
153
|
+
if isinstance(error, TimeoutError)
|
|
154
|
+
else ErrorKind.CONNECTION_FAILED
|
|
155
|
+
)
|
|
156
|
+
)
|
|
157
|
+
failure = AddressResolutionError(
|
|
158
|
+
f"Resolve {host!r} in environment "
|
|
159
|
+
f"{policy.environment.name if policy.environment else 'local'!r}: {kind.value}",
|
|
160
|
+
kind=kind,
|
|
161
|
+
code=error.code if isinstance(error, ToolboxError) else None,
|
|
162
|
+
)
|
|
163
|
+
failure.__cause__ = error
|
|
164
|
+
yield AddressCandidate(configured, "resolution", failure)
|
|
165
|
+
continue
|
|
166
|
+
for address in hosts:
|
|
167
|
+
resolved = Endpoint(address, endpoint.port, configured.servername)
|
|
168
|
+
if resolved not in seen:
|
|
169
|
+
seen.add(resolved)
|
|
170
|
+
yield AddressCandidate(resolved, source)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Compatibility imports; generic client ownership lives in harness-common."""
|
|
2
|
+
|
|
3
|
+
from harness_common.client import Client as Client
|
|
4
|
+
from harness_common.client import ClientManager as ClientManager
|
|
5
|
+
from harness_common.client import ClientProvider as ClientProvider
|
|
6
|
+
from harness_common.client import DataSource as DataSource
|
|
7
|
+
from harness_common.client import ServiceDataSource as ServiceDataSource
|
|
8
|
+
from harness_common.client import data_source_key as data_source_key
|