ecosemantic-core 0.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ecosemantic_core-0.1.8/.gitignore +24 -0
- ecosemantic_core-0.1.8/LICENSE +73 -0
- ecosemantic_core-0.1.8/NOTICE +13 -0
- ecosemantic_core-0.1.8/PKG-INFO +62 -0
- ecosemantic_core-0.1.8/README.md +27 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/__init__.py +0 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/activity.py +103 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/installs.py +81 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/records.py +183 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/run_folder.py +55 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/runs.py +95 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/storage.py +267 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/filesystem/watching.py +45 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/__init__.py +0 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/calculation.py +446 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/catalog.py +287 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/install.py +126 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/inventory.py +197 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/search.py +230 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/store.py +439 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/native/terms.py +43 -0
- ecosemantic_core-0.1.8/ecosemantic/backends/retry.py +54 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/__init__.py +0 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/accounts.py +52 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/consequences.py +66 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/edits.py +564 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/links.py +170 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/montecarlo.py +110 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/overrides.py +80 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/parameters.py +155 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/render.py +76 -0
- ecosemantic_core-0.1.8/ecosemantic/compute/results.py +105 -0
- ecosemantic_core-0.1.8/ecosemantic/events/__init__.py +267 -0
- ecosemantic_core-0.1.8/ecosemantic/events/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/geometry/__init__.py +175 -0
- ecosemantic_core-0.1.8/ecosemantic/geometry/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/layout/__init__.py +304 -0
- ecosemantic_core-0.1.8/ecosemantic/layout/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/package/__init__.py +0 -0
- ecosemantic_core-0.1.8/ecosemantic/package/background.py +510 -0
- ecosemantic_core-0.1.8/ecosemantic/package/base.py +159 -0
- ecosemantic_core-0.1.8/ecosemantic/package/base_writer.py +321 -0
- ecosemantic_core-0.1.8/ecosemantic/package/compile.py +57 -0
- ecosemantic_core-0.1.8/ecosemantic/package/gates.py +58 -0
- ecosemantic_core-0.1.8/ecosemantic/package/jsonld.py +699 -0
- ecosemantic_core-0.1.8/ecosemantic/package/log.py +165 -0
- ecosemantic_core-0.1.8/ecosemantic/package/manifest.py +112 -0
- ecosemantic_core-0.1.8/ecosemantic/package/model_package.py +164 -0
- ecosemantic_core-0.1.8/ecosemantic/package/olca.py +135 -0
- ecosemantic_core-0.1.8/ecosemantic/package/package.py +180 -0
- ecosemantic_core-0.1.8/ecosemantic/package/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/package/release.py +266 -0
- ecosemantic_core-0.1.8/ecosemantic/package/requests.py +55 -0
- ecosemantic_core-0.1.8/ecosemantic/package/resolver.py +56 -0
- ecosemantic_core-0.1.8/ecosemantic/package/sidecar.py +122 -0
- ecosemantic_core-0.1.8/ecosemantic/package/wholeness.py +148 -0
- ecosemantic_core-0.1.8/ecosemantic/package/working.py +275 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/__init__.py +0 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/activity.py +25 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/calculation.py +64 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/graph.py +14 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/install.py +37 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/inventory.py +41 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/resolver.py +43 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/run.py +37 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/search.py +37 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/storage.py +76 -0
- ecosemantic_core-0.1.8/ecosemantic/ports/types.py +938 -0
- ecosemantic_core-0.1.8/ecosemantic/quantity/__init__.py +15 -0
- ecosemantic_core-0.1.8/ecosemantic/quantity/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/routes/__init__.py +233 -0
- ecosemantic_core-0.1.8/ecosemantic/routes/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/scaling/__init__.py +69 -0
- ecosemantic_core-0.1.8/ecosemantic/scaling/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/tables/__init__.py +1 -0
- ecosemantic_core-0.1.8/ecosemantic/tables/folder.py +147 -0
- ecosemantic_core-0.1.8/ecosemantic/tables/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/tables/reader.py +259 -0
- ecosemantic_core-0.1.8/ecosemantic/tables/results.py +328 -0
- ecosemantic_core-0.1.8/ecosemantic/textmap/__init__.py +231 -0
- ecosemantic_core-0.1.8/ecosemantic/textmap/py.typed +0 -0
- ecosemantic_core-0.1.8/ecosemantic/troubles/__init__.py +188 -0
- ecosemantic_core-0.1.8/ecosemantic/troubles/py.typed +0 -0
- ecosemantic_core-0.1.8/pyproject.toml +46 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
venv/
|
|
5
|
+
*.egg-info/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
node_modules/
|
|
9
|
+
.pytest_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.mypy_cache/
|
|
12
|
+
.coverage
|
|
13
|
+
coverage.json
|
|
14
|
+
htmlcov/
|
|
15
|
+
|
|
16
|
+
.env
|
|
17
|
+
.env.*
|
|
18
|
+
!.env.example
|
|
19
|
+
|
|
20
|
+
.DS_Store
|
|
21
|
+
graphify-out
|
|
22
|
+
knowledge/
|
|
23
|
+
.okf-cache.json
|
|
24
|
+
**/ecosemantic/view/static/
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# PolyForm Noncommercial License 1.0.0
|
|
2
|
+
|
|
3
|
+
<https://polyformproject.org/licenses/noncommercial/1.0.0>
|
|
4
|
+
|
|
5
|
+
## Acceptance
|
|
6
|
+
|
|
7
|
+
In order to get any license under these terms, you must agree to them as both strict obligations and conditions to all your licenses.
|
|
8
|
+
|
|
9
|
+
## Copyright License
|
|
10
|
+
|
|
11
|
+
The licensor grants you a copyright license for the software to do everything you might do with the software that would otherwise infringe the licensor's copyright in it for any permitted purpose. However, you may only distribute the software according to [Distribution License](#distribution-license) and make changes or new works based on the software according to [Changes and New Works License](#changes-and-new-works-license).
|
|
12
|
+
|
|
13
|
+
## Distribution License
|
|
14
|
+
|
|
15
|
+
The licensor grants you an additional copyright license to distribute copies of the software. Your license to distribute covers distributing the software with changes and new works permitted by [Changes and New Works License](#changes-and-new-works-license).
|
|
16
|
+
|
|
17
|
+
## Notices
|
|
18
|
+
|
|
19
|
+
You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms or the URL for them above, as well as copies of any plain-text lines beginning with `Required Notice:` that the licensor provided with the software. For example:
|
|
20
|
+
|
|
21
|
+
> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
|
|
22
|
+
|
|
23
|
+
## Changes and New Works License
|
|
24
|
+
|
|
25
|
+
The licensor grants you an additional copyright license to make changes and new works based on the software for any permitted purpose.
|
|
26
|
+
|
|
27
|
+
## Patent License
|
|
28
|
+
|
|
29
|
+
The licensor grants you a patent license for the software that covers patent claims the licensor can license, or becomes able to license, that you would infringe by using the software.
|
|
30
|
+
|
|
31
|
+
## Noncommercial Purposes
|
|
32
|
+
|
|
33
|
+
Any noncommercial purpose is a permitted purpose.
|
|
34
|
+
|
|
35
|
+
## Personal Uses
|
|
36
|
+
|
|
37
|
+
Personal use for research, experiment, and testing for the benefit of public knowledge, personal study, private entertainment, hobby projects, amateur pursuits, or religious observance, without any anticipated commercial application, is use for a permitted purpose.
|
|
38
|
+
|
|
39
|
+
## Noncommercial Organizations
|
|
40
|
+
|
|
41
|
+
Use by any charitable organization, educational institution, public research organization, public safety or health organization, environmental protection organization, or government institution is use for a permitted purpose regardless of the source of funding or obligations resulting from the funding.
|
|
42
|
+
|
|
43
|
+
## Fair Use
|
|
44
|
+
|
|
45
|
+
You may have "fair use" rights for the software under the law. These terms do not limit them.
|
|
46
|
+
|
|
47
|
+
## No Other Rights
|
|
48
|
+
|
|
49
|
+
These terms do not allow you to sublicense or transfer any of your licenses to anyone else, or prevent the licensor from granting licenses to anyone else. These terms do not imply any other licenses.
|
|
50
|
+
|
|
51
|
+
## Patent Defense
|
|
52
|
+
|
|
53
|
+
If you make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
|
|
54
|
+
|
|
55
|
+
## Violations
|
|
56
|
+
|
|
57
|
+
The first time you are notified in writing that you have violated any of these terms, or done anything with the software not covered by your licenses, your licenses can nonetheless continue if you come into full compliance with these terms, and take practical steps to correct past violations, within 32 days of receiving notice. Otherwise, all your licenses end immediately.
|
|
58
|
+
|
|
59
|
+
## No Liability
|
|
60
|
+
|
|
61
|
+
***As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.***
|
|
62
|
+
|
|
63
|
+
## Definitions
|
|
64
|
+
|
|
65
|
+
The **licensor** is the individual or entity offering these terms, and the **software** is the software the licensor makes available under these terms.
|
|
66
|
+
|
|
67
|
+
**You** refers to the individual or entity agreeing to these terms.
|
|
68
|
+
|
|
69
|
+
**Your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. **Control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect.
|
|
70
|
+
|
|
71
|
+
**Your licenses** are all the licenses granted to you for the software under these terms.
|
|
72
|
+
|
|
73
|
+
**Use** means anything you do with the software requiring one of your licenses.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
EcoSemantic
|
|
2
|
+
Required Notice: Copyright 2026 Carlos Gaete-Morales (https://github.com/cdgaete/ecosemantic)
|
|
3
|
+
|
|
4
|
+
EcoSemantic is licensed under the PolyForm Noncommercial License 1.0.0, in LICENSE
|
|
5
|
+
and at https://polyformproject.org/licenses/noncommercial/1.0.0. Any noncommercial
|
|
6
|
+
purpose is permitted. Commercial use needs a separate licence from Carlos
|
|
7
|
+
Gaete-Morales: cdgaete@gmail.com.
|
|
8
|
+
|
|
9
|
+
The page that ecosemantic-view carries bundles third-party software under its
|
|
10
|
+
own licences; their notices are in ecosemantic/view/static/THIRD_PARTY_NOTICES.txt
|
|
11
|
+
inside that distribution. Components of the page adapt shadcn/ui, Copyright (c)
|
|
12
|
+
2023 shadcn, under the MIT licence, as page/PROVENANCE.md in the source
|
|
13
|
+
repository records.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ecosemantic-core
|
|
3
|
+
Version: 0.1.8
|
|
4
|
+
Summary: The EcoSemantic ports, pure computations, portable format and filesystem storage.
|
|
5
|
+
Project-URL: Homepage, https://github.com/cdgaete/ecosemantic
|
|
6
|
+
Project-URL: Repository, https://github.com/cdgaete/ecosemantic
|
|
7
|
+
Project-URL: Issues, https://github.com/cdgaete/ecosemantic/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/cdgaete/ecosemantic/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Carlos Gaete-Morales <cdgaete@gmail.com>
|
|
10
|
+
License-Expression: PolyForm-Noncommercial-1.0.0
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
License-File: NOTICE
|
|
13
|
+
Keywords: JSON-LD,LCA,life cycle assessment,openLCA,product system
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.12
|
|
24
|
+
Requires-Dist: asteval<2,>=1.0.10
|
|
25
|
+
Requires-Dist: duckdb<2,>=1.5
|
|
26
|
+
Requires-Dist: ecosemantic-engine==0.1.8
|
|
27
|
+
Requires-Dist: filelock<4,>=3.29.5
|
|
28
|
+
Requires-Dist: numpy>=2.5
|
|
29
|
+
Requires-Dist: olca-schema==2.4.0
|
|
30
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
31
|
+
Requires-Dist: pyyaml<7,>=6.0.1
|
|
32
|
+
Requires-Dist: scipy>=1.16
|
|
33
|
+
Requires-Dist: watchfiles<2,>=1.0
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# ecosemantic-core
|
|
37
|
+
|
|
38
|
+
The ports of EcoSemantic and the types they carry, the pure computations over a product system, the portable package a model travels in, the tables folder, the event bus, the drawing the page renders, and the two backends that serve the ports from installed database bundles and from a folder of product systems.
|
|
39
|
+
|
|
40
|
+
## Names it publishes
|
|
41
|
+
|
|
42
|
+
- `ecosemantic.ports` — the protocols every backend answers and the types they carry: `activity`, `calculation`, `graph`, `install`, `inventory`, `resolver`, `run`, `search`, `storage` and `types`.
|
|
43
|
+
- `ecosemantic.compute` — the pure computations, with no store behind them: `parameters`, `overrides`, `links`, `consequences`, `edits`, `results`, `render`, `montecarlo` and `accounts`.
|
|
44
|
+
- `ecosemantic.package` — the portable package a product system travels in, openLCA's JSON-LD among its forms, with the change log, the manifest, the gates and the compilation that reads it.
|
|
45
|
+
- `ecosemantic.tables` — a product system as a folder of text: a YAML model, CSV inputs and CSV results.
|
|
46
|
+
- `ecosemantic.events` — one bus per server process, fed by the storage every write goes through and by the watcher of each run.
|
|
47
|
+
- `ecosemantic.geometry`, `layout`, `routes`, `textmap` and `troubles` — the drawing, in the pixels the page renders it at: each card's rows, where a process sits, every arrow as an orthogonal route, the drawing as characters an agent reads, and what is wrong with it.
|
|
48
|
+
- `ecosemantic.quantity` and `scaling` — a quantity as the canvas writes it, and each card's exchanges scaled to one unit of the functional unit.
|
|
49
|
+
- `ecosemantic.backends.filesystem` — product systems, runs, installs and their records in a folder, and the watcher that announces what changes.
|
|
50
|
+
- `ecosemantic.backends.native` — the databases installed under one data directory: their registry entries, their catalogs and their compiled backgrounds, serving the search, inventory and calculation ports.
|
|
51
|
+
|
|
52
|
+
## What it depends on
|
|
53
|
+
|
|
54
|
+
`ecosemantic-engine` at the same version, and pydantic, olca-schema, asteval, duckdb, numpy, scipy, filelock, watchfiles and pyyaml.
|
|
55
|
+
|
|
56
|
+
## Installing it
|
|
57
|
+
|
|
58
|
+
`pip install ecosemantic` installs this distribution with the other five and the `ecosemantic` command. `pip install ecosemantic-core` installs it alone, for a backend or a tool that depends on the ports rather than on the server.
|
|
59
|
+
|
|
60
|
+
## Licence
|
|
61
|
+
|
|
62
|
+
PolyForm Noncommercial License 1.0.0, copyright Carlos Gaete-Morales: `LICENSE` and `NOTICE` in this distribution. Any noncommercial purpose is permitted, by individuals and by charities, schools, public research and government bodies; commercial use needs a separate licence from him (cdgaete@gmail.com). EcoSemantic is his trademark, and the licence grants no right to use that name (`TRADEMARK.md` in the repository). The code is published from https://github.com/cdgaete/ecosemantic.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# ecosemantic-core
|
|
2
|
+
|
|
3
|
+
The ports of EcoSemantic and the types they carry, the pure computations over a product system, the portable package a model travels in, the tables folder, the event bus, the drawing the page renders, and the two backends that serve the ports from installed database bundles and from a folder of product systems.
|
|
4
|
+
|
|
5
|
+
## Names it publishes
|
|
6
|
+
|
|
7
|
+
- `ecosemantic.ports` — the protocols every backend answers and the types they carry: `activity`, `calculation`, `graph`, `install`, `inventory`, `resolver`, `run`, `search`, `storage` and `types`.
|
|
8
|
+
- `ecosemantic.compute` — the pure computations, with no store behind them: `parameters`, `overrides`, `links`, `consequences`, `edits`, `results`, `render`, `montecarlo` and `accounts`.
|
|
9
|
+
- `ecosemantic.package` — the portable package a product system travels in, openLCA's JSON-LD among its forms, with the change log, the manifest, the gates and the compilation that reads it.
|
|
10
|
+
- `ecosemantic.tables` — a product system as a folder of text: a YAML model, CSV inputs and CSV results.
|
|
11
|
+
- `ecosemantic.events` — one bus per server process, fed by the storage every write goes through and by the watcher of each run.
|
|
12
|
+
- `ecosemantic.geometry`, `layout`, `routes`, `textmap` and `troubles` — the drawing, in the pixels the page renders it at: each card's rows, where a process sits, every arrow as an orthogonal route, the drawing as characters an agent reads, and what is wrong with it.
|
|
13
|
+
- `ecosemantic.quantity` and `scaling` — a quantity as the canvas writes it, and each card's exchanges scaled to one unit of the functional unit.
|
|
14
|
+
- `ecosemantic.backends.filesystem` — product systems, runs, installs and their records in a folder, and the watcher that announces what changes.
|
|
15
|
+
- `ecosemantic.backends.native` — the databases installed under one data directory: their registry entries, their catalogs and their compiled backgrounds, serving the search, inventory and calculation ports.
|
|
16
|
+
|
|
17
|
+
## What it depends on
|
|
18
|
+
|
|
19
|
+
`ecosemantic-engine` at the same version, and pydantic, olca-schema, asteval, duckdb, numpy, scipy, filelock, watchfiles and pyyaml.
|
|
20
|
+
|
|
21
|
+
## Installing it
|
|
22
|
+
|
|
23
|
+
`pip install ecosemantic` installs this distribution with the other five and the `ecosemantic` command. `pip install ecosemantic-core` installs it alone, for a backend or a tool that depends on the ports rather than on the server.
|
|
24
|
+
|
|
25
|
+
## Licence
|
|
26
|
+
|
|
27
|
+
PolyForm Noncommercial License 1.0.0, copyright Carlos Gaete-Morales: `LICENSE` and `NOTICE` in this distribution. Any noncommercial purpose is permitted, by individuals and by charities, schools, public research and government bodies; commercial use needs a separate licence from him (cdgaete@gmail.com). EcoSemantic is his trademark, and the licence grants no right to use that name (`TRADEMARK.md` in the repository). The code is published from https://github.com/cdgaete/ecosemantic.
|
|
File without changes
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""The activity port over a directory: one JSON Lines file per agent under the systems
|
|
2
|
+
directory."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import uuid
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from filelock import FileLock
|
|
12
|
+
from pydantic import ValidationError
|
|
13
|
+
|
|
14
|
+
from ecosemantic.errors import NotFound, Unavailable
|
|
15
|
+
from ecosemantic.ports.types import (
|
|
16
|
+
AgentSummary,
|
|
17
|
+
Call,
|
|
18
|
+
CallDraft,
|
|
19
|
+
agents_of,
|
|
20
|
+
checked_agent,
|
|
21
|
+
selected,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
ACTIVITY_DIR = "activity"
|
|
25
|
+
SUFFIX = ".jsonl"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class FilesystemActivity:
|
|
29
|
+
"""Recorded calls as one line each, appended to the file of the agent that made them.
|
|
30
|
+
|
|
31
|
+
Two processes over one directory may serve agents of one name, so a file is appended to and
|
|
32
|
+
read under a lock both take.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, root: Path) -> None:
|
|
36
|
+
self.root = root
|
|
37
|
+
|
|
38
|
+
def _folder(self) -> Path:
|
|
39
|
+
return self.root / ACTIVITY_DIR
|
|
40
|
+
|
|
41
|
+
def _lock(self, agent: str) -> FileLock:
|
|
42
|
+
return FileLock(self._folder() / f"{agent}.lock")
|
|
43
|
+
|
|
44
|
+
def _append(self, call: Call) -> Call:
|
|
45
|
+
self._folder().mkdir(parents=True, exist_ok=True)
|
|
46
|
+
with self._lock(call.agent):
|
|
47
|
+
with (self._folder() / f"{call.agent}{SUFFIX}").open("a", encoding="utf-8") as handle:
|
|
48
|
+
handle.write(call.model_dump_json() + "\n")
|
|
49
|
+
return call
|
|
50
|
+
|
|
51
|
+
def _read(self, agent: str | None) -> list[Call]:
|
|
52
|
+
if not self._folder().is_dir():
|
|
53
|
+
return []
|
|
54
|
+
if agent is not None:
|
|
55
|
+
files = [self._folder() / f"{checked_agent(agent)}{SUFFIX}"]
|
|
56
|
+
else:
|
|
57
|
+
files = sorted(self._folder().glob(f"*{SUFFIX}"))
|
|
58
|
+
calls: list[Call] = []
|
|
59
|
+
for file in files:
|
|
60
|
+
if file.is_file():
|
|
61
|
+
with self._lock(file.name.removesuffix(SUFFIX)):
|
|
62
|
+
calls.extend(_parsed(file))
|
|
63
|
+
return calls
|
|
64
|
+
|
|
65
|
+
async def record(self, draft: CallDraft) -> Call:
|
|
66
|
+
call = Call(id=uuid.uuid4(), **draft.model_dump())
|
|
67
|
+
return await asyncio.to_thread(self._append, call)
|
|
68
|
+
|
|
69
|
+
async def list_calls(
|
|
70
|
+
self,
|
|
71
|
+
agent: str | None = None,
|
|
72
|
+
product_system_id: uuid.UUID | None = None,
|
|
73
|
+
after: datetime | None = None,
|
|
74
|
+
) -> list[Call]:
|
|
75
|
+
calls = await asyncio.to_thread(self._read, agent)
|
|
76
|
+
return selected(calls, agent, product_system_id, after)
|
|
77
|
+
|
|
78
|
+
async def get_call(self, call_id: uuid.UUID) -> Call:
|
|
79
|
+
for call in await asyncio.to_thread(self._read, None):
|
|
80
|
+
if call.id == call_id:
|
|
81
|
+
return call
|
|
82
|
+
raise NotFound(f"no call {call_id}")
|
|
83
|
+
|
|
84
|
+
async def list_agents(self) -> list[AgentSummary]:
|
|
85
|
+
return agents_of(await asyncio.to_thread(self._read, None))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _parsed(file: Path) -> list[Call]:
|
|
89
|
+
"""A file's calls, or Unavailable naming the file, the line and the first field this version
|
|
90
|
+
cannot read in it."""
|
|
91
|
+
calls: list[Call] = []
|
|
92
|
+
for number, line in enumerate(file.read_text(encoding="utf-8").splitlines(), start=1):
|
|
93
|
+
try:
|
|
94
|
+
calls.append(Call.model_validate_json(line))
|
|
95
|
+
except ValidationError as error:
|
|
96
|
+
first = error.errors(include_url=False)[0]
|
|
97
|
+
field = ".".join(str(part) for part in first["loc"])
|
|
98
|
+
raise Unavailable(
|
|
99
|
+
f"{file.name} line {number} cannot be read by this version: "
|
|
100
|
+
f"{field}: {first['msg']}",
|
|
101
|
+
hint=f"remove {file} to read the activity again",
|
|
102
|
+
) from error
|
|
103
|
+
return calls
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""The install port over a directory: one folder per install under the data directory."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import uuid
|
|
7
|
+
from collections.abc import AsyncIterator
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from filelock import AsyncFileLock
|
|
12
|
+
|
|
13
|
+
from ecosemantic.backends.filesystem.records import RecordFolder, newest_first
|
|
14
|
+
from ecosemantic.ports.types import Install, InstallDraft, InstallPatch
|
|
15
|
+
|
|
16
|
+
INSTALLS_DIR = "installs"
|
|
17
|
+
INSTALL_FILE = "install.json"
|
|
18
|
+
UPLOADS_DIR = "uploads"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def install_dir(root: Path, install_id: uuid.UUID) -> Path:
|
|
22
|
+
"""The directory of one install under the data directory, whether or not it exists yet."""
|
|
23
|
+
return root / INSTALLS_DIR / str(install_id)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def uploads_dir(root: Path) -> Path:
|
|
27
|
+
"""The directory a package uploaded for an install is written to before that install reads
|
|
28
|
+
it."""
|
|
29
|
+
return root / INSTALLS_DIR / UPLOADS_DIR
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class FilesystemInstalls:
|
|
33
|
+
"""Install records as one JSON file per install, newest first by the time they were posted.
|
|
34
|
+
|
|
35
|
+
The install process writes a record from its own process while the server reads it, so a
|
|
36
|
+
record is replaced whole and every change is read and written under a lock both take.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, root: Path) -> None:
|
|
40
|
+
self.root = root
|
|
41
|
+
self.records = RecordFolder(root, INSTALLS_DIR, INSTALL_FILE, Install, "install")
|
|
42
|
+
|
|
43
|
+
def read_record(self, install_id: uuid.UUID) -> Install:
|
|
44
|
+
"""One install, read synchronously; the install process's callback runs off the event
|
|
45
|
+
loop."""
|
|
46
|
+
return self.records.read(install_id)
|
|
47
|
+
|
|
48
|
+
def apply_patch(self, install_id: uuid.UUID, patch: InstallPatch) -> Install:
|
|
49
|
+
"""The record with the patch's named fields applied, written synchronously."""
|
|
50
|
+
return self.records.patch(install_id, patch.applied_to)
|
|
51
|
+
|
|
52
|
+
async def post_install(self, draft: InstallDraft) -> Install:
|
|
53
|
+
def construct(posted_at: datetime) -> Install:
|
|
54
|
+
return Install(id=uuid.uuid4(), draft=draft, posted_at=posted_at)
|
|
55
|
+
|
|
56
|
+
return await asyncio.to_thread(self.records.post, construct)
|
|
57
|
+
|
|
58
|
+
async def get_install(self, install_id: uuid.UUID) -> Install:
|
|
59
|
+
return await asyncio.to_thread(self.read_record, install_id)
|
|
60
|
+
|
|
61
|
+
async def list_installs(self) -> list[Install]:
|
|
62
|
+
return newest_first(await asyncio.to_thread(self.records.all))
|
|
63
|
+
|
|
64
|
+
async def cancel_install(self, install_id: uuid.UUID) -> Install:
|
|
65
|
+
def ask() -> Install:
|
|
66
|
+
return self.records.patch(
|
|
67
|
+
install_id, lambda install: install.cancellation().applied_to(install)
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
return await asyncio.to_thread(ask)
|
|
71
|
+
|
|
72
|
+
async def update_install(self, install_id: uuid.UUID, patch: InstallPatch) -> Install:
|
|
73
|
+
return await asyncio.to_thread(self.apply_patch, install_id, patch)
|
|
74
|
+
|
|
75
|
+
def slot(self) -> AsyncFileLock:
|
|
76
|
+
"""The workspace's one-install slot, which every process over this directory shares."""
|
|
77
|
+
return self.records.slot()
|
|
78
|
+
|
|
79
|
+
def changes(self) -> AsyncIterator[uuid.UUID]:
|
|
80
|
+
"""Every install any process wrote, from the moment this is called."""
|
|
81
|
+
return self.records.changed()
|
|
File without changes
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""A record folder: one directory per record under its kind's directory, the record replaced
|
|
2
|
+
whole under a lock both processes take."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import uuid
|
|
8
|
+
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable
|
|
9
|
+
from datetime import UTC, datetime, timedelta
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Protocol, cast
|
|
12
|
+
|
|
13
|
+
from filelock import AsyncFileLock, FileLock
|
|
14
|
+
from pydantic import BaseModel, ValidationError
|
|
15
|
+
|
|
16
|
+
from ecosemantic.backends.filesystem.watching import touched
|
|
17
|
+
from ecosemantic.backends.retry import read_bytes, replace
|
|
18
|
+
from ecosemantic.errors import NotFound, Unavailable
|
|
19
|
+
|
|
20
|
+
SLOT_FILE = "slot.lock"
|
|
21
|
+
POST_LOCK = "post.lock"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Posted(Protocol):
|
|
25
|
+
"""What a record carries for a listing: its id, and the time it was posted."""
|
|
26
|
+
|
|
27
|
+
id: uuid.UUID
|
|
28
|
+
posted_at: datetime
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def newest_first[T: Posted](records: Iterable[T]) -> list[T]:
|
|
32
|
+
"""Records newest first, the id breaking a tie between two posted at once."""
|
|
33
|
+
return sorted(records, key=lambda record: (record.posted_at, str(record.id)), reverse=True)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def now() -> datetime:
|
|
37
|
+
"""The current instant in UTC, the clock a record's posted time is measured from."""
|
|
38
|
+
return datetime.now(UTC)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def next_posted_at(now: datetime, newest: datetime | None) -> datetime:
|
|
42
|
+
"""A new record's posted time, newest-first independent of the clock's resolution: the
|
|
43
|
+
clock, or one microsecond past the store's newest when the clock would tie two."""
|
|
44
|
+
if newest is None:
|
|
45
|
+
return now
|
|
46
|
+
return max(now, newest + timedelta(microseconds=1))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class RecordFolder[T: BaseModel]:
|
|
50
|
+
"""The records of one kind, as one JSON file per record in a folder of its own.
|
|
51
|
+
|
|
52
|
+
A record is written from another process while this one reads it, so it is replaced whole,
|
|
53
|
+
never rewritten in place, and every change is read and written under a lock both take.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, root: Path, kind: str, record_file: str, model: type[T], noun: str) -> None:
|
|
57
|
+
self.root = root
|
|
58
|
+
self.kind = kind
|
|
59
|
+
self.record_file = record_file
|
|
60
|
+
self.model = model
|
|
61
|
+
self.noun = noun
|
|
62
|
+
|
|
63
|
+
def folder(self, record_id: uuid.UUID) -> Path:
|
|
64
|
+
"""The directory of one record under its kind's directory, whether or not it exists."""
|
|
65
|
+
return self.root / self.kind / str(record_id)
|
|
66
|
+
|
|
67
|
+
def lock(self, record_id: uuid.UUID) -> FileLock:
|
|
68
|
+
"""The record's file lock, taken by every read-and-write of it, from either process."""
|
|
69
|
+
folder = self.root / self.kind
|
|
70
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
return FileLock(folder / f"{record_id}.lock")
|
|
72
|
+
|
|
73
|
+
def post_lock(self) -> FileLock:
|
|
74
|
+
"""The lock every poster of this kind takes, so two cannot mint the same posted time."""
|
|
75
|
+
folder = self.root / self.kind
|
|
76
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
77
|
+
return FileLock(folder / POST_LOCK)
|
|
78
|
+
|
|
79
|
+
def read(self, record_id: uuid.UUID) -> T:
|
|
80
|
+
"""One record, or NotFound when no folder holds it."""
|
|
81
|
+
record = self.folder(record_id) / self.record_file
|
|
82
|
+
if not record.is_file():
|
|
83
|
+
raise NotFound(f"no {self.noun} {record_id}")
|
|
84
|
+
return self._parsed(record)
|
|
85
|
+
|
|
86
|
+
def write(self, record_id: uuid.UUID, record: T) -> T:
|
|
87
|
+
"""The record written, staged beside its file and moved over it within the bounded
|
|
88
|
+
retry, so a reader sees the whole of one version or the whole of the other."""
|
|
89
|
+
folder = self.folder(record_id)
|
|
90
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
staged = folder / f"{self.record_file}.tmp"
|
|
92
|
+
staged.write_bytes(record.model_dump_json(indent=1).encode("utf-8"))
|
|
93
|
+
replace(staged, folder / self.record_file)
|
|
94
|
+
return record
|
|
95
|
+
|
|
96
|
+
def patch(self, record_id: uuid.UUID, change: Callable[[T], T]) -> T:
|
|
97
|
+
"""The record with the change applied, read and written under its lock."""
|
|
98
|
+
with self.lock(record_id):
|
|
99
|
+
return self.write(record_id, change(self.read(record_id)))
|
|
100
|
+
|
|
101
|
+
def post(self, construct: Callable[[datetime], T]) -> T:
|
|
102
|
+
"""A new record posted under the kind's lock, its posted time advanced past the newest
|
|
103
|
+
stored so two posters cannot mint the same instant."""
|
|
104
|
+
with self.post_lock():
|
|
105
|
+
record = construct(self._next_posted_at())
|
|
106
|
+
return self.write(cast(Posted, record).id, record)
|
|
107
|
+
|
|
108
|
+
def _next_posted_at(self) -> datetime:
|
|
109
|
+
"""The posted time of the next record of this kind: the clock, advanced past the newest."""
|
|
110
|
+
newest = max(
|
|
111
|
+
(record.posted_at for record in cast(Iterable[Posted], self.all())), default=None
|
|
112
|
+
)
|
|
113
|
+
return next_posted_at(now(), newest)
|
|
114
|
+
|
|
115
|
+
def all(self) -> list[T]:
|
|
116
|
+
"""Every record of this kind."""
|
|
117
|
+
folder = self.root / self.kind
|
|
118
|
+
if not folder.is_dir():
|
|
119
|
+
return []
|
|
120
|
+
return [
|
|
121
|
+
self._parsed(path / self.record_file)
|
|
122
|
+
for path in folder.iterdir()
|
|
123
|
+
if (path / self.record_file).is_file()
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
def changed(self) -> AsyncGenerator[uuid.UUID]:
|
|
127
|
+
"""Every record any process wrote under this kind's directory, from the moment this is
|
|
128
|
+
called: the watch is armed eagerly, so a write between this call and the first read is
|
|
129
|
+
announced."""
|
|
130
|
+
queue: asyncio.Queue[uuid.UUID] = asyncio.Queue()
|
|
131
|
+
watching = asyncio.create_task(self._watch(touched(self.root / self.kind), queue))
|
|
132
|
+
|
|
133
|
+
async def iterate() -> AsyncGenerator[uuid.UUID]:
|
|
134
|
+
try:
|
|
135
|
+
while True:
|
|
136
|
+
yield await queue.get()
|
|
137
|
+
finally:
|
|
138
|
+
watching.cancel()
|
|
139
|
+
|
|
140
|
+
return iterate()
|
|
141
|
+
|
|
142
|
+
async def _watch(
|
|
143
|
+
self, batches: AsyncIterator[set[Path]], queue: asyncio.Queue[uuid.UUID]
|
|
144
|
+
) -> None:
|
|
145
|
+
async for batch in batches:
|
|
146
|
+
found = {record for path in batch if (record := self._record_of(path)) is not None}
|
|
147
|
+
for record_id in sorted(found, key=str):
|
|
148
|
+
if (self.folder(record_id) / self.record_file).is_file():
|
|
149
|
+
queue.put_nowait(record_id)
|
|
150
|
+
|
|
151
|
+
def _record_of(self, path: Path) -> uuid.UUID | None:
|
|
152
|
+
"""The record a written path belongs to, or none for a path that names no record."""
|
|
153
|
+
folder = self.root / self.kind
|
|
154
|
+
try:
|
|
155
|
+
first = path.resolve().relative_to(folder.resolve()).parts[0]
|
|
156
|
+
except (ValueError, IndexError):
|
|
157
|
+
return None
|
|
158
|
+
try:
|
|
159
|
+
return uuid.UUID(first)
|
|
160
|
+
except ValueError:
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
def slot(self) -> AsyncFileLock:
|
|
164
|
+
"""The single slot every process over this directory takes, released when its holder's
|
|
165
|
+
process ends. Each try is non-blocking on the event loop, so a caller cancelled while it
|
|
166
|
+
waits holds nothing."""
|
|
167
|
+
folder = self.root / self.kind
|
|
168
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
169
|
+
return AsyncFileLock(folder / SLOT_FILE, run_in_executor=False)
|
|
170
|
+
|
|
171
|
+
def _parsed(self, record: Path) -> T:
|
|
172
|
+
"""The record parsed, or Unavailable naming it and the first field this version cannot
|
|
173
|
+
read in it."""
|
|
174
|
+
try:
|
|
175
|
+
return self.model.model_validate_json(read_bytes(record))
|
|
176
|
+
except ValidationError as error:
|
|
177
|
+
first = error.errors(include_url=False)[0]
|
|
178
|
+
field = ".".join(str(part) for part in first["loc"])
|
|
179
|
+
raise Unavailable(
|
|
180
|
+
f"{self.noun} {record.parent.name} cannot be read by this version: "
|
|
181
|
+
f"{field}: {first['msg']}",
|
|
182
|
+
hint=f"remove {record.parent} to list and start {self.noun}s again",
|
|
183
|
+
) from error
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Where a run lives on disk: one directory per run under the systems directory, and the
|
|
2
|
+
heartbeat its worker beats there."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import time
|
|
7
|
+
import uuid
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from contextlib import contextmanager
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from threading import Event, Thread
|
|
12
|
+
|
|
13
|
+
RUNS_DIR = "runs"
|
|
14
|
+
RECORD_FILE = "run.json"
|
|
15
|
+
LOG_FILE = "worker.log"
|
|
16
|
+
HEARTBEAT_FILE = "heartbeat"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def run_dir(root: Path, run_id: uuid.UUID) -> Path:
|
|
20
|
+
"""The directory of one run under a systems directory, whether or not it exists yet."""
|
|
21
|
+
return root / RUNS_DIR / str(run_id)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def beat(folder: Path) -> None:
|
|
25
|
+
"""Mark the worker carrying the run in this folder as alive now, by the heartbeat file's
|
|
26
|
+
modification time."""
|
|
27
|
+
(folder / HEARTBEAT_FILE).touch()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def heartbeat_age(folder: Path) -> float | None:
|
|
31
|
+
"""Seconds since the worker carrying the run in this folder last beat, or None when it never
|
|
32
|
+
has."""
|
|
33
|
+
heartbeat = folder / HEARTBEAT_FILE
|
|
34
|
+
if not heartbeat.is_file():
|
|
35
|
+
return None
|
|
36
|
+
return time.time() - heartbeat.stat().st_mtime
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@contextmanager
|
|
40
|
+
def beating(folder: Path, every: float) -> Iterator[None]:
|
|
41
|
+
"""Beat this folder's heartbeat every `every` seconds until the block ends, so a step that
|
|
42
|
+
reports nothing does not look like a process that is gone."""
|
|
43
|
+
stop = Event()
|
|
44
|
+
|
|
45
|
+
def keep_beating() -> None:
|
|
46
|
+
while not stop.wait(every):
|
|
47
|
+
beat(folder)
|
|
48
|
+
|
|
49
|
+
beater = Thread(target=keep_beating, daemon=True)
|
|
50
|
+
beater.start()
|
|
51
|
+
try:
|
|
52
|
+
yield
|
|
53
|
+
finally:
|
|
54
|
+
stop.set()
|
|
55
|
+
beater.join()
|