appm 0.0.2__tar.gz → 0.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- appm-0.0.3/PKG-INFO +76 -0
- appm-0.0.3/README.md +67 -0
- appm-0.0.3/appm/__version__.py +1 -0
- appm-0.0.3/appm/default.py +36 -0
- {appm-0.0.2 → appm-0.0.3}/appm/exceptions.py +6 -0
- appm-0.0.3/appm/manager.py +184 -0
- appm-0.0.3/appm/model.py +316 -0
- appm-0.0.3/appm/utils.py +72 -0
- {appm-0.0.2 → appm-0.0.3}/pyproject.toml +4 -3
- appm-0.0.2/PKG-INFO +0 -9
- appm-0.0.2/README.md +0 -0
- appm-0.0.2/appm/__version__.py +0 -1
- appm-0.0.2/appm/manager.py +0 -187
- appm-0.0.2/appm/model.py +0 -82
- appm-0.0.2/appm/utils.py +0 -27
- {appm-0.0.2 → appm-0.0.3}/appm/__init__.py +0 -0
- {appm-0.0.2 → appm-0.0.3}/appm/py.typed +0 -0
appm-0.0.3/PKG-INFO
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
Metadata-Version: 2.3
|
2
|
+
Name: appm
|
3
|
+
Version: 0.0.3
|
4
|
+
Summary: APPN Phenomate Project Manager
|
5
|
+
Requires-Dist: pydantic>=2.11.7
|
6
|
+
Requires-Dist: ruamel-yaml>=0.18.14
|
7
|
+
Requires-Python: >=3.13
|
8
|
+
Description-Content-Type: text/markdown
|
9
|
+
|
10
|
+
# APPN Phenomate Project Manager
|
11
|
+
|
12
|
+
A Python package for managing project templates, metadata, and file organization using flexible YAML schemas. Designed for research and data projects that require consistent file naming, metadata, and directory structures.
|
13
|
+
|
14
|
+
## Install
|
15
|
+
|
16
|
+
```bash
|
17
|
+
pip install appm
|
18
|
+
```
|
19
|
+
|
20
|
+
## Features
|
21
|
+
|
22
|
+
- Template-driven project structure: Define project layouts, file naming conventions, and metadata in YAML.
|
23
|
+
- Automatic project initialization: Create new projects with standardized folders and metadata files.
|
24
|
+
- File placement and matching: Automatically determine where files belong based on their names and template rules.
|
25
|
+
- Extensible and validated: Uses Pydantic for schema validation and ruamel.yaml for YAML parsing.
|
26
|
+
Installation
|
27
|
+
Or for development:
|
28
|
+
|
29
|
+
## Usage
|
30
|
+
1. Define a Template
|
31
|
+
|
32
|
+
Create a YAML template describing your project's structure, naming conventions, and file formats. See `examples/template.yaml` for the default template.
|
33
|
+
|
34
|
+
2. Initialize a Project
|
35
|
+
|
36
|
+
```py
|
37
|
+
from appm import ProjectManager
|
38
|
+
|
39
|
+
pm = ProjectManager.from_template(
|
40
|
+
root="projects",
|
41
|
+
year=2024,
|
42
|
+
summary="Wheat yield trial",
|
43
|
+
internal=True,
|
44
|
+
researcherName="Jane Doe",
|
45
|
+
organisationName="Plant Research Org",
|
46
|
+
template="examples/template.yaml"
|
47
|
+
)
|
48
|
+
pm.init_project()
|
49
|
+
|
50
|
+
```
|
51
|
+
|
52
|
+
3. Add Files
|
53
|
+
|
54
|
+
Files are automatically placed in the correct directory based on the template.
|
55
|
+
|
56
|
+
```py
|
57
|
+
pm.copy_file("data/20240601-120000_SiteA_SensorX_Trial1_T0-raw.csv")
|
58
|
+
```
|
59
|
+
|
60
|
+
## Project Structure
|
61
|
+
- appm – Core package (template parsing, project management, utilities)
|
62
|
+
- examples – Example YAML templates
|
63
|
+
- schema – JSON schema for template validation
|
64
|
+
- tests – Unit tests and fixtures
|
65
|
+
|
66
|
+
## Development
|
67
|
+
- Python 3.13+
|
68
|
+
- Pydantic
|
69
|
+
- ruamel.yaml
|
70
|
+
- pytest for testing
|
71
|
+
|
72
|
+
## Run tests:
|
73
|
+
|
74
|
+
```
|
75
|
+
pytest
|
76
|
+
```
|
appm-0.0.3/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
# APPN Phenomate Project Manager
|
2
|
+
|
3
|
+
A Python package for managing project templates, metadata, and file organization using flexible YAML schemas. Designed for research and data projects that require consistent file naming, metadata, and directory structures.
|
4
|
+
|
5
|
+
## Install
|
6
|
+
|
7
|
+
```bash
|
8
|
+
pip install appm
|
9
|
+
```
|
10
|
+
|
11
|
+
## Features
|
12
|
+
|
13
|
+
- Template-driven project structure: Define project layouts, file naming conventions, and metadata in YAML.
|
14
|
+
- Automatic project initialization: Create new projects with standardized folders and metadata files.
|
15
|
+
- File placement and matching: Automatically determine where files belong based on their names and template rules.
|
16
|
+
- Extensible and validated: Uses Pydantic for schema validation and ruamel.yaml for YAML parsing.
|
17
|
+
Installation
|
18
|
+
Or for development:
|
19
|
+
|
20
|
+
## Usage
|
21
|
+
1. Define a Template
|
22
|
+
|
23
|
+
Create a YAML template describing your project's structure, naming conventions, and file formats. See `examples/template.yaml` for the default template.
|
24
|
+
|
25
|
+
2. Initialize a Project
|
26
|
+
|
27
|
+
```py
|
28
|
+
from appm import ProjectManager
|
29
|
+
|
30
|
+
pm = ProjectManager.from_template(
|
31
|
+
root="projects",
|
32
|
+
year=2024,
|
33
|
+
summary="Wheat yield trial",
|
34
|
+
internal=True,
|
35
|
+
researcherName="Jane Doe",
|
36
|
+
organisationName="Plant Research Org",
|
37
|
+
template="examples/template.yaml"
|
38
|
+
)
|
39
|
+
pm.init_project()
|
40
|
+
|
41
|
+
```
|
42
|
+
|
43
|
+
3. Add Files
|
44
|
+
|
45
|
+
Files are automatically placed in the correct directory based on the template.
|
46
|
+
|
47
|
+
```py
|
48
|
+
pm.copy_file("data/20240601-120000_SiteA_SensorX_Trial1_T0-raw.csv")
|
49
|
+
```
|
50
|
+
|
51
|
+
## Project Structure
|
52
|
+
- appm – Core package (template parsing, project management, utilities)
|
53
|
+
- examples – Example YAML templates
|
54
|
+
- schema – JSON schema for template validation
|
55
|
+
- tests – Unit tests and fixtures
|
56
|
+
|
57
|
+
## Development
|
58
|
+
- Python 3.13+
|
59
|
+
- Pydantic
|
60
|
+
- ruamel.yaml
|
61
|
+
- pytest for testing
|
62
|
+
|
63
|
+
## Run tests:
|
64
|
+
|
65
|
+
```
|
66
|
+
pytest
|
67
|
+
```
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = "0.0.3"
|
@@ -0,0 +1,36 @@
|
|
1
|
+
DEFAULT_TEMPLATE = {
|
2
|
+
"version": "0.0.3",
|
3
|
+
"naming_convention": {
|
4
|
+
"sep": "_",
|
5
|
+
"structure": [
|
6
|
+
"year",
|
7
|
+
"summary",
|
8
|
+
"internal",
|
9
|
+
"researcherName",
|
10
|
+
"organisationName",
|
11
|
+
],
|
12
|
+
},
|
13
|
+
"layout": {
|
14
|
+
"structure": ["site", "sensor", "date", "trial", "procLevel"],
|
15
|
+
"mapping": {
|
16
|
+
"procLevel": {"raw": "T0-raw", "proc": "T1-proc", "trait": "T2-trait"}
|
17
|
+
},
|
18
|
+
},
|
19
|
+
"file": {
|
20
|
+
"*": {
|
21
|
+
"sep": "_",
|
22
|
+
"default": {"procLevel": "raw"},
|
23
|
+
"components": [
|
24
|
+
{"sep": "-", "components": [["date", r"\d{8}"], ["time", r"\d{6}"]]},
|
25
|
+
["site", "[^_.]+"],
|
26
|
+
["sensor", "[^_.]+"],
|
27
|
+
["trial", "[^_.]+"],
|
28
|
+
{
|
29
|
+
"name": "procLevel",
|
30
|
+
"pattern": "T0-raw|T1-proc|T2-trait|raw|proc|trait",
|
31
|
+
"required": False,
|
32
|
+
},
|
33
|
+
],
|
34
|
+
}
|
35
|
+
},
|
36
|
+
}
|
@@ -0,0 +1,184 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import shutil
|
4
|
+
from copy import deepcopy
|
5
|
+
from pathlib import Path
|
6
|
+
from typing import Any
|
7
|
+
|
8
|
+
from ruamel.yaml import YAML
|
9
|
+
|
10
|
+
from appm.__version__ import __version__
|
11
|
+
from appm.default import DEFAULT_TEMPLATE
|
12
|
+
from appm.exceptions import (
|
13
|
+
UnsupportedFileExtension,
|
14
|
+
)
|
15
|
+
from appm.model import Project
|
16
|
+
from appm.utils import to_flow_style, validate_path
|
17
|
+
|
18
|
+
yaml = YAML()
|
19
|
+
yaml.indent(mapping=2, sequence=4, offset=2)
|
20
|
+
yaml.preserve_quotes = True # optional, if you want to preserve quotes
|
21
|
+
|
22
|
+
|
23
|
+
class ProjectManager:
|
24
|
+
METADATA_NAME: str = "metadata.yaml"
|
25
|
+
|
26
|
+
def __init__(
|
27
|
+
self,
|
28
|
+
metadata: dict[str, Any],
|
29
|
+
root: str | Path,
|
30
|
+
) -> None:
|
31
|
+
self.root = Path(root)
|
32
|
+
self.metadata = Project.model_validate(metadata)
|
33
|
+
self.handlers = {ext: handler for ext, handler in self.metadata.file.items()}
|
34
|
+
|
35
|
+
@property
|
36
|
+
def location(self) -> Path:
|
37
|
+
return self.root / self.metadata.project_name
|
38
|
+
|
39
|
+
def match(self, name: str) -> dict[str, str | None]:
|
40
|
+
"""Match a file name and separate into format defined field components
|
41
|
+
|
42
|
+
The result contains a * which captures all non-captured values.
|
43
|
+
|
44
|
+
Args:
|
45
|
+
name (str): file name
|
46
|
+
|
47
|
+
Raises:
|
48
|
+
UnsupportedFileExtension: the metadata does not define an
|
49
|
+
extension declaration for the file's extension.
|
50
|
+
|
51
|
+
Returns:
|
52
|
+
dict[str, str]: key value dictionary of the field component
|
53
|
+
defined using the format field.
|
54
|
+
"""
|
55
|
+
ext = name.split(".")[-1]
|
56
|
+
if ext in self.handlers:
|
57
|
+
return self.handlers[ext].match(name)
|
58
|
+
if "*" in self.handlers:
|
59
|
+
return self.handlers["*"].match(name)
|
60
|
+
raise UnsupportedFileExtension(str(ext))
|
61
|
+
|
62
|
+
def get_file_placement(self, name: str) -> str:
|
63
|
+
"""Find location where a file should be placed.
|
64
|
+
|
65
|
+
Determination is based on the metadata's layout field,
|
66
|
+
the file extension format definition, and the file name.
|
67
|
+
More concretely, field component - values are matched using the
|
68
|
+
RegEx defined in format. Fields that match layout values will be
|
69
|
+
extracted and path-appended in the order they appear in layout.
|
70
|
+
|
71
|
+
Args:
|
72
|
+
name (str): file name
|
73
|
+
|
74
|
+
Returns:
|
75
|
+
str: file placement directory
|
76
|
+
"""
|
77
|
+
layout = self.metadata.parsed_layout
|
78
|
+
groups = self.match(name)
|
79
|
+
return layout.get_path(groups)
|
80
|
+
|
81
|
+
def init_project(self) -> None:
|
82
|
+
"""Create a project:
|
83
|
+
|
84
|
+
- Determine the project's name from nameing_convention and metadata
|
85
|
+
- Create a folder based on project's root and project name
|
86
|
+
- Create a metadata file in the project's location
|
87
|
+
"""
|
88
|
+
self.location.mkdir(exist_ok=True, parents=True)
|
89
|
+
self.save_metadata()
|
90
|
+
|
91
|
+
def save_metadata(self) -> None:
|
92
|
+
"""Save the current metadata to the project location"""
|
93
|
+
metadata_path = self.location / self.METADATA_NAME
|
94
|
+
with metadata_path.open("w") as file:
|
95
|
+
data = self.metadata.model_dump(mode="json")
|
96
|
+
data["version"] = __version__
|
97
|
+
yaml.dump(
|
98
|
+
to_flow_style(data),
|
99
|
+
file,
|
100
|
+
)
|
101
|
+
|
102
|
+
def copy_file(self, src_path: str | Path) -> None:
|
103
|
+
"""Copy a file located at `src_path` to an appropriate
|
104
|
+
location in the project.
|
105
|
+
|
106
|
+
Args:
|
107
|
+
src_path (str | Path): path to where src data is found
|
108
|
+
"""
|
109
|
+
src_path = validate_path(src_path)
|
110
|
+
dst_path = self.location / self.get_file_placement(src_path.name)
|
111
|
+
dst_path.mkdir(parents=True, exist_ok=True)
|
112
|
+
shutil.copy2(src_path, dst_path)
|
113
|
+
|
114
|
+
@classmethod
|
115
|
+
def from_template(
|
116
|
+
cls,
|
117
|
+
root: str | Path,
|
118
|
+
year: int,
|
119
|
+
summary: str,
|
120
|
+
internal: bool = True,
|
121
|
+
template: str | Path | dict[str, Any] | None = None,
|
122
|
+
researcherName: str | None = None,
|
123
|
+
organisationName: str | None = None,
|
124
|
+
) -> ProjectManager:
|
125
|
+
"""Create a ProjectManager based on template and meta information
|
126
|
+
|
127
|
+
Args:
|
128
|
+
root (str | Path): parent directory - where project is stored
|
129
|
+
template (str | Path | dict[str, Any]): path to template file or the template content.
|
130
|
+
year (int): meta information - year
|
131
|
+
summary (str): meta information - summary
|
132
|
+
internal (bool, optional): meta information - internal. Defaults to True.
|
133
|
+
researcher (str | None, optional): meta information - researcherName. Defaults to None.
|
134
|
+
organisation (str | None, optional): meta information - organisationName. Defaults to None.
|
135
|
+
|
136
|
+
Returns:
|
137
|
+
ProjectManager: ProjectManager object
|
138
|
+
"""
|
139
|
+
if isinstance(template, str | Path):
|
140
|
+
metadata_path = Path(template)
|
141
|
+
metadata_path = validate_path(template)
|
142
|
+
with metadata_path.open("r") as file:
|
143
|
+
metadata = yaml.load(file)
|
144
|
+
elif isinstance(template, dict):
|
145
|
+
metadata = template
|
146
|
+
elif not template:
|
147
|
+
metadata = deepcopy(DEFAULT_TEMPLATE)
|
148
|
+
else:
|
149
|
+
raise TypeError(
|
150
|
+
f"Unexpected type for template: {type(template)}. Accepts str, dict or None"
|
151
|
+
)
|
152
|
+
metadata["meta"] = {
|
153
|
+
"year": year,
|
154
|
+
"summary": summary,
|
155
|
+
"internal": internal,
|
156
|
+
"researcherName": researcherName,
|
157
|
+
"organisationName": organisationName,
|
158
|
+
}
|
159
|
+
return cls(root=root, metadata=metadata)
|
160
|
+
|
161
|
+
@classmethod
|
162
|
+
def load_project(
|
163
|
+
cls, project_path: Path | str, metadata_name: str | None = None
|
164
|
+
) -> ProjectManager:
|
165
|
+
"""Load a project from project's path
|
166
|
+
|
167
|
+
Args:
|
168
|
+
project_path (Path | str): path to project to open
|
169
|
+
metadata_name (str | None, optional): name for metadata file. If not provided, use "metadata.yaml". Defaults to None.
|
170
|
+
|
171
|
+
Returns:
|
172
|
+
ProjectManager: ProjectManager object
|
173
|
+
"""
|
174
|
+
project_path = validate_path(project_path)
|
175
|
+
metadata_path = (
|
176
|
+
project_path / cls.METADATA_NAME
|
177
|
+
if not metadata_name
|
178
|
+
else project_path / metadata_name
|
179
|
+
)
|
180
|
+
metadata_path = validate_path(metadata_path)
|
181
|
+
|
182
|
+
with metadata_path.open("r") as file:
|
183
|
+
metadata = yaml.load(file)
|
184
|
+
return cls(metadata=metadata, root=project_path.parent)
|
appm-0.0.3/appm/model.py
ADDED
@@ -0,0 +1,316 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import re
|
4
|
+
from collections import Counter
|
5
|
+
from typing import Self
|
6
|
+
|
7
|
+
from pydantic import BaseModel, model_validator
|
8
|
+
from ruamel.yaml import YAML
|
9
|
+
|
10
|
+
from appm.__version__ import __version__
|
11
|
+
from appm.exceptions import FileFormatMismatch
|
12
|
+
from appm.utils import slugify
|
13
|
+
|
14
|
+
yaml = YAML()
|
15
|
+
|
16
|
+
with open("examples/template.yaml") as file:
|
17
|
+
data = yaml.load(file)
|
18
|
+
|
19
|
+
STRUCTURES = {"year", "summary", "internal", "researcherName", "organisationName"}
|
20
|
+
|
21
|
+
|
22
|
+
class Field(BaseModel):
|
23
|
+
name: str
|
24
|
+
pattern: str
|
25
|
+
required: bool = True
|
26
|
+
|
27
|
+
@property
|
28
|
+
def regex(self) -> str:
|
29
|
+
return f"(?P<{self.name}>{self.pattern})"
|
30
|
+
|
31
|
+
@classmethod
|
32
|
+
def from_tuple(cls, value: tuple[str, str] | list[str]) -> Field:
|
33
|
+
assert len(value) == 2
|
34
|
+
return Field(name=value[0], pattern=value[1])
|
35
|
+
|
36
|
+
|
37
|
+
class Group(BaseModel):
|
38
|
+
components: list[tuple[str, str] | Field | Group]
|
39
|
+
sep: str = "-"
|
40
|
+
|
41
|
+
def validate_components(self) -> Self:
|
42
|
+
if not self.components:
|
43
|
+
raise ValueError(f"Components cannot be empty: {self.components}")
|
44
|
+
self._fields: list[Field | Group] = []
|
45
|
+
self._normalised_fields: list[Field] = []
|
46
|
+
for field in self.components:
|
47
|
+
if isinstance(field, tuple | list):
|
48
|
+
f = Field.from_tuple(field)
|
49
|
+
self._fields.append(f)
|
50
|
+
self._normalised_fields.append(f)
|
51
|
+
elif isinstance(field, Field):
|
52
|
+
self._fields.append(field)
|
53
|
+
self._normalised_fields.append(field)
|
54
|
+
else:
|
55
|
+
self._fields.append(field)
|
56
|
+
self._normalised_fields.extend(field.normalised_fields)
|
57
|
+
return self
|
58
|
+
|
59
|
+
def validate_names(self) -> Self:
|
60
|
+
self._names: list[str] = []
|
61
|
+
self._optional_names: set[str] = set()
|
62
|
+
for field in self.fields:
|
63
|
+
if isinstance(field, Field):
|
64
|
+
self._names.append(field.name)
|
65
|
+
if not field.required:
|
66
|
+
self._optional_names.add(field.name)
|
67
|
+
else:
|
68
|
+
self._names.extend(field.names)
|
69
|
+
self._optional_names.update(field.optional_names)
|
70
|
+
return self
|
71
|
+
|
72
|
+
def validate_regex(self) -> Self:
|
73
|
+
regex_str = []
|
74
|
+
for i, field in enumerate(self.fields):
|
75
|
+
is_optional = isinstance(field, Field) and not field.required
|
76
|
+
pattern = field.regex
|
77
|
+
|
78
|
+
if i == 0:
|
79
|
+
if is_optional:
|
80
|
+
# First field, no separator; make only field optional
|
81
|
+
regex_str.append(f"(?:{pattern})?")
|
82
|
+
else:
|
83
|
+
regex_str.append(pattern)
|
84
|
+
else:
|
85
|
+
if is_optional:
|
86
|
+
# Wrap separator + field together as optional
|
87
|
+
regex_str.append(f"(?:{self.sep}{pattern})?")
|
88
|
+
else:
|
89
|
+
regex_str.append(f"{self.sep}{pattern}")
|
90
|
+
self._regex = "".join(regex_str)
|
91
|
+
return self
|
92
|
+
|
93
|
+
@model_validator(mode="after")
|
94
|
+
def validate_group(self) -> Self:
|
95
|
+
return self.validate_components().validate_names().validate_regex()
|
96
|
+
|
97
|
+
@property
|
98
|
+
def normalised_fields(self) -> list[Field]:
|
99
|
+
return self._normalised_fields
|
100
|
+
|
101
|
+
@property
|
102
|
+
def fields(self) -> list[Field | Group]:
|
103
|
+
return self._fields
|
104
|
+
|
105
|
+
@property
|
106
|
+
def names(self) -> list[str]:
|
107
|
+
return self._names
|
108
|
+
|
109
|
+
@property
|
110
|
+
def optional_names(self) -> set[str]:
|
111
|
+
return self._optional_names
|
112
|
+
|
113
|
+
@property
|
114
|
+
def regex(self) -> str:
|
115
|
+
return self._regex
|
116
|
+
|
117
|
+
|
118
|
+
class Extension(Group):
|
119
|
+
default: dict[str, str] | None = None
|
120
|
+
|
121
|
+
@property
|
122
|
+
def default_names(self) -> set[str]:
|
123
|
+
return set() if not self.default else set(self.default.keys())
|
124
|
+
|
125
|
+
@property
|
126
|
+
def all_names(self) -> set[str]:
|
127
|
+
return set(self.names) | self.default_names
|
128
|
+
|
129
|
+
def validate_regex(self) -> Self:
|
130
|
+
self = super().validate_regex()
|
131
|
+
self._regex = f"^{self._regex}(?P<rest>.*)$"
|
132
|
+
return self
|
133
|
+
|
134
|
+
def validate_unique_names(self) -> Self:
|
135
|
+
count = Counter(self.names)
|
136
|
+
non_uniques = {k: v for k, v in count.items() if v > 1}
|
137
|
+
if non_uniques:
|
138
|
+
raise ValueError(f"Non-unique field name: {non_uniques}")
|
139
|
+
return self
|
140
|
+
|
141
|
+
def validate_reserved_name(self) -> Self:
|
142
|
+
if "rest" in self.names:
|
143
|
+
raise ValueError("Field component must not contain reserved key: rest")
|
144
|
+
return self
|
145
|
+
|
146
|
+
def validate_first_field_must_be_required(self) -> Self:
|
147
|
+
if not (field := self.normalised_fields[0]).required:
|
148
|
+
raise ValueError(f"First component must be required: {field.name}")
|
149
|
+
return self
|
150
|
+
|
151
|
+
@model_validator(mode="after")
|
152
|
+
def validate_extension(self) -> Self:
|
153
|
+
return (
|
154
|
+
self.validate_components()
|
155
|
+
.validate_names()
|
156
|
+
.validate_regex()
|
157
|
+
.validate_unique_names()
|
158
|
+
.validate_reserved_name()
|
159
|
+
.validate_first_field_must_be_required()
|
160
|
+
)
|
161
|
+
|
162
|
+
def match(self, name: str) -> dict[str, str | None]:
|
163
|
+
m = re.match(self.regex, name)
|
164
|
+
if not m:
|
165
|
+
raise FileFormatMismatch(f"Name: {name}. Pattern: {self.regex}")
|
166
|
+
result = m.groupdict()
|
167
|
+
if self.default:
|
168
|
+
for k, v in self.default.items():
|
169
|
+
if result.get(k) is None:
|
170
|
+
result[k] = v
|
171
|
+
return result
|
172
|
+
|
173
|
+
|
174
|
+
type File = dict[str, Extension]
|
175
|
+
|
176
|
+
|
177
|
+
class Layout(BaseModel):
|
178
|
+
structure: list[str]
|
179
|
+
mapping: dict[str, dict[str, str]] | None = None
|
180
|
+
|
181
|
+
@classmethod
|
182
|
+
def from_list(cls, value: list[str]) -> Layout:
|
183
|
+
return cls(structure=value)
|
184
|
+
|
185
|
+
@property
|
186
|
+
def structure_set(self) -> set[str]:
|
187
|
+
return self._structure_set
|
188
|
+
|
189
|
+
@model_validator(mode="after")
|
190
|
+
def validate_layout(self) -> Self:
|
191
|
+
self._structure_set = set(self.structure)
|
192
|
+
if self.mapping:
|
193
|
+
if not set(self.mapping.keys()).issubset(self._structure_set):
|
194
|
+
raise ValueError(
|
195
|
+
f"Mapping keys must be a subset of structure. Mapping keys: {set(self.mapping.keys())}, structure: {self.structure}"
|
196
|
+
)
|
197
|
+
return self
|
198
|
+
|
199
|
+
def get_path(self, components: dict[str, str | None]) -> str:
|
200
|
+
result: list[str] = []
|
201
|
+
for key in self.structure:
|
202
|
+
value = components.get(key, None)
|
203
|
+
if self.mapping and key in self.mapping and value in self.mapping[key]:
|
204
|
+
value = self.mapping[key][value]
|
205
|
+
if value is None:
|
206
|
+
raise ValueError(
|
207
|
+
f"None value for key: {key}. Either set a default for Extension definition, change Extension pattern to capture key value, or rename file."
|
208
|
+
)
|
209
|
+
result.append(value)
|
210
|
+
return "/".join(result)
|
211
|
+
|
212
|
+
|
213
|
+
class NamingConv(BaseModel):
|
214
|
+
sep: str = "_"
|
215
|
+
structure: list[str] = [
|
216
|
+
"year",
|
217
|
+
"summary",
|
218
|
+
"internal",
|
219
|
+
"researcherName",
|
220
|
+
"organisationName",
|
221
|
+
]
|
222
|
+
|
223
|
+
@model_validator(mode="after")
|
224
|
+
def validate_naming_convention(self) -> Self:
|
225
|
+
"""Validate structure value
|
226
|
+
|
227
|
+
structure:
|
228
|
+
- cannot be empty
|
229
|
+
- cannot have repeated component(s)
|
230
|
+
- cannot have a field component that is not one of the metadata fields.
|
231
|
+
"""
|
232
|
+
counter: dict[str, int] = {}
|
233
|
+
if len(self.structure) == 0:
|
234
|
+
raise ValueError("Invalid naming structure - empty structure")
|
235
|
+
for field in self.structure:
|
236
|
+
counter[field] = counter.get(field, 0) + 1
|
237
|
+
if counter[field] > 1:
|
238
|
+
raise ValueError(f"Invalid naming structure - repetition: {field}")
|
239
|
+
if field not in STRUCTURES:
|
240
|
+
raise ValueError(
|
241
|
+
f"Invalid naming structure - invalid field: {field}. Structure must be a non empty permutation of {STRUCTURES}"
|
242
|
+
)
|
243
|
+
return self
|
244
|
+
|
245
|
+
|
246
|
+
class Template(BaseModel):
|
247
|
+
layout: Layout | list[str]
|
248
|
+
file: File
|
249
|
+
naming_convention: NamingConv = NamingConv()
|
250
|
+
version: str = __version__
|
251
|
+
|
252
|
+
def validate_layout(self) -> Self:
|
253
|
+
if isinstance(self.layout, list):
|
254
|
+
self._layout = Layout.from_list(self.layout)
|
255
|
+
else:
|
256
|
+
self._layout = self.layout
|
257
|
+
return self
|
258
|
+
|
259
|
+
def validate_file_non_empty(self) -> Self:
|
260
|
+
if not self.file:
|
261
|
+
raise ValueError("Empty extension")
|
262
|
+
return self
|
263
|
+
|
264
|
+
def validate_file_name_subset_layout(self) -> Self:
|
265
|
+
for ext, decl in self.file.items():
|
266
|
+
for field in self.parsed_layout.structure_set:
|
267
|
+
if field not in decl.all_names:
|
268
|
+
raise ValueError(
|
269
|
+
f"Component fields must be a superset of layout fields: {field}. Ext: {ext}"
|
270
|
+
)
|
271
|
+
if field in decl.optional_names and field not in decl.default_names:
|
272
|
+
raise ValueError(
|
273
|
+
f"Optional field that is also a layout field must have a default value: {field}. Ext: {ext}"
|
274
|
+
)
|
275
|
+
return self
|
276
|
+
|
277
|
+
@property
|
278
|
+
def parsed_layout(self) -> Layout:
|
279
|
+
return self._layout
|
280
|
+
|
281
|
+
@model_validator(mode="after")
|
282
|
+
def validate_template(self) -> Self:
|
283
|
+
return (
|
284
|
+
self.validate_layout()
|
285
|
+
.validate_file_non_empty()
|
286
|
+
.validate_file_name_subset_layout()
|
287
|
+
)
|
288
|
+
|
289
|
+
|
290
|
+
class Metadata(BaseModel):
|
291
|
+
year: int
|
292
|
+
summary: str
|
293
|
+
internal: bool = True
|
294
|
+
researcherName: str | None = None
|
295
|
+
organisationName: str | None = None
|
296
|
+
|
297
|
+
|
298
|
+
class Project(Template):
|
299
|
+
meta: Metadata
|
300
|
+
|
301
|
+
@property
|
302
|
+
def project_name(self) -> str:
|
303
|
+
"""Project name based on metadata and naming convention definiton"""
|
304
|
+
fields = self.naming_convention.structure
|
305
|
+
name: list[str] = []
|
306
|
+
for field in fields:
|
307
|
+
value = getattr(self.meta, field)
|
308
|
+
if value is not None:
|
309
|
+
if isinstance(value, str):
|
310
|
+
name.append(slugify(value))
|
311
|
+
elif field == "year":
|
312
|
+
name.append(str(value))
|
313
|
+
elif field == "internal":
|
314
|
+
value = "internal" if value else "external"
|
315
|
+
name.append(value)
|
316
|
+
return self.naming_convention.sep.join(name)
|
appm-0.0.3/appm/utils.py
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
import re
|
2
|
+
from collections.abc import Mapping, Sequence
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import Any
|
5
|
+
|
6
|
+
from ruamel.yaml import CommentedMap, CommentedSeq
|
7
|
+
|
8
|
+
from appm.exceptions import NotAFileErr, NotFoundErr
|
9
|
+
|
10
|
+
|
11
|
+
def slugify(text: str) -> str:
|
12
|
+
"""Generate a slug from a text
|
13
|
+
|
14
|
+
Used for generating project name and url slug
|
15
|
+
|
16
|
+
https://developer.mozilla.org/en-US/docs/Glossary/Slug
|
17
|
+
|
18
|
+
Example:
|
19
|
+
- The Plant Accelerator -> the-plant-accelerator
|
20
|
+
|
21
|
+
- APPN -> appn
|
22
|
+
|
23
|
+
Args:
|
24
|
+
text (str): source text
|
25
|
+
|
26
|
+
Returns:
|
27
|
+
str: slug
|
28
|
+
"""
|
29
|
+
text = text.lower()
|
30
|
+
# Replace non slug characters
|
31
|
+
text = re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE)
|
32
|
+
# Replace spaces with hyphens
|
33
|
+
text = re.sub(r"[\s\-]+", "-", text)
|
34
|
+
return text.strip("-")
|
35
|
+
|
36
|
+
|
37
|
+
def to_flow_style(obj: Any) -> Any:
|
38
|
+
"""Recursively convert dict/list to ruamel structures with ALL lists using flow-style."""
|
39
|
+
if isinstance(obj, Mapping):
|
40
|
+
cm = CommentedMap()
|
41
|
+
for k, v in obj.items():
|
42
|
+
cm[k] = to_flow_style(v)
|
43
|
+
return cm
|
44
|
+
if isinstance(obj, Sequence) and not isinstance(obj, str):
|
45
|
+
cs = CommentedSeq()
|
46
|
+
for item in obj:
|
47
|
+
cs.append(to_flow_style(item))
|
48
|
+
cs.fa.set_flow_style()
|
49
|
+
return cs
|
50
|
+
return obj
|
51
|
+
|
52
|
+
|
53
|
+
def validate_path(path: str | Path, is_file: bool = False) -> Path:
|
54
|
+
"""Verify that path describes an existing file/folder
|
55
|
+
|
56
|
+
Args:
|
57
|
+
path (str | Path): path to validate
|
58
|
+
is_file (bool): whether path is a file. Defaults to False.
|
59
|
+
|
60
|
+
Raises:
|
61
|
+
NotFoundErr: path item doesnt exist
|
62
|
+
NotAFileErr: path doesn't describe a file
|
63
|
+
|
64
|
+
Returns:
|
65
|
+
Path: validated path
|
66
|
+
"""
|
67
|
+
_path = Path(path)
|
68
|
+
if not _path.exists():
|
69
|
+
raise NotFoundErr(str(path))
|
70
|
+
if is_file and not _path.is_file():
|
71
|
+
raise NotAFileErr(str(path))
|
72
|
+
return _path
|
@@ -1,7 +1,7 @@
|
|
1
1
|
[project]
|
2
2
|
name = "appm"
|
3
|
-
version = "0.0.
|
4
|
-
description = "APPN Project Manager
|
3
|
+
version = "0.0.3"
|
4
|
+
description = "APPN Phenomate Project Manager"
|
5
5
|
readme = "README.md"
|
6
6
|
requires-python = ">=3.13"
|
7
7
|
dependencies = [
|
@@ -16,6 +16,7 @@ analysis = [
|
|
16
16
|
]
|
17
17
|
dev = [
|
18
18
|
"pytest>=8.4.1",
|
19
|
+
"pytest-cov>=6.2.1",
|
19
20
|
]
|
20
21
|
|
21
22
|
[build-system]
|
@@ -24,4 +25,4 @@ build-backend = "uv_build"
|
|
24
25
|
|
25
26
|
[tool.uv.build-backend]
|
26
27
|
module-name = "appm"
|
27
|
-
module-root = ""
|
28
|
+
module-root = ""
|
appm-0.0.2/PKG-INFO
DELETED
appm-0.0.2/README.md
DELETED
File without changes
|
appm-0.0.2/appm/__version__.py
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
__version__ = "0.0.2"
|
appm-0.0.2/appm/manager.py
DELETED
@@ -1,187 +0,0 @@
|
|
1
|
-
from __future__ import annotations
|
2
|
-
|
3
|
-
import re
|
4
|
-
import shutil
|
5
|
-
from collections.abc import Mapping
|
6
|
-
from pathlib import Path
|
7
|
-
from typing import Any, Sequence, overload
|
8
|
-
|
9
|
-
from ruamel.yaml import YAML, CommentedMap, CommentedSeq
|
10
|
-
|
11
|
-
from appm.__version__ import __version__
|
12
|
-
from appm.exceptions import (
|
13
|
-
FileFormatMismatch,
|
14
|
-
MetadataFileNotFoundErr,
|
15
|
-
ProjectNotFoundErr,
|
16
|
-
UnsupportedFileExtension,
|
17
|
-
)
|
18
|
-
from appm.model import ExtDecl, ProjectMetadata
|
19
|
-
|
20
|
-
yaml = YAML()
|
21
|
-
yaml.indent(mapping=2, sequence=4, offset=2)
|
22
|
-
yaml.preserve_quotes = True # optional, if you want to preserve quotes
|
23
|
-
|
24
|
-
|
25
|
-
def to_flow_style(obj: Any) -> Any:
|
26
|
-
"""Recursively convert dict/list to ruamel structures with ALL lists using flow-style."""
|
27
|
-
if isinstance(obj, Mapping):
|
28
|
-
cm = CommentedMap()
|
29
|
-
for k, v in obj.items():
|
30
|
-
cm[k] = to_flow_style(v)
|
31
|
-
return cm
|
32
|
-
if isinstance(obj, Sequence) and not isinstance(obj, str):
|
33
|
-
cs = CommentedSeq()
|
34
|
-
for item in obj:
|
35
|
-
cs.append(to_flow_style(item))
|
36
|
-
cs.fa.set_flow_style()
|
37
|
-
return cs
|
38
|
-
return obj
|
39
|
-
|
40
|
-
|
41
|
-
class ExtManager:
|
42
|
-
def __init__(self, ext: str, decl: ExtDecl) -> None:
|
43
|
-
self.ext = ext
|
44
|
-
self.decl = decl
|
45
|
-
|
46
|
-
@property
|
47
|
-
def pattern(self) -> str:
|
48
|
-
return (
|
49
|
-
r"^"
|
50
|
-
+ self.decl.sep.join([f"({p})" for _, p in self.decl.format])
|
51
|
-
+ r"(.*)$"
|
52
|
-
)
|
53
|
-
|
54
|
-
def match(self, name: str) -> dict[str, str]:
|
55
|
-
match = re.match(self.pattern, name)
|
56
|
-
if not match:
|
57
|
-
raise FileFormatMismatch(f"Name: {name}. Pattern: {self.pattern}")
|
58
|
-
groups = match.groups()
|
59
|
-
result = {}
|
60
|
-
for i, (field, _) in enumerate(self.decl.format):
|
61
|
-
result[field] = groups[i]
|
62
|
-
result["*"] = groups[-1]
|
63
|
-
return result
|
64
|
-
|
65
|
-
|
66
|
-
class ProjectManager:
|
67
|
-
METADATA_NAME: str = "metadata.yaml"
|
68
|
-
|
69
|
-
def __init__(
|
70
|
-
self,
|
71
|
-
metadata: dict[str, Any],
|
72
|
-
root: str | Path,
|
73
|
-
) -> None:
|
74
|
-
self.root = Path(root)
|
75
|
-
self.metadata = ProjectMetadata.model_validate(metadata)
|
76
|
-
self.handlers = {
|
77
|
-
ext: ExtManager(ext, ext_decl)
|
78
|
-
for ext, ext_decl in self.metadata.file.items()
|
79
|
-
}
|
80
|
-
|
81
|
-
def match(self, name: str) -> dict[str, str]:
|
82
|
-
ext = name.split(".")[-1]
|
83
|
-
if ext not in self.handlers:
|
84
|
-
raise UnsupportedFileExtension(ext)
|
85
|
-
return self.handlers[ext].match(name)
|
86
|
-
|
87
|
-
def get_file_placement(self, name: str) -> str:
|
88
|
-
layout = self.metadata.layout
|
89
|
-
groups = self.match(name)
|
90
|
-
values = [groups[component] for component in layout]
|
91
|
-
return "/".join(values)
|
92
|
-
|
93
|
-
def init_project(self) -> None:
|
94
|
-
self.root.mkdir(exist_ok=True, parents=True)
|
95
|
-
self.save_metadata()
|
96
|
-
|
97
|
-
def save_metadata(self) -> None:
|
98
|
-
metadata_path = self.root / self.METADATA_NAME
|
99
|
-
with metadata_path.open("w") as file:
|
100
|
-
data = self.metadata.model_dump(mode="json")
|
101
|
-
data["version"] = __version__
|
102
|
-
yaml.dump(
|
103
|
-
to_flow_style(data),
|
104
|
-
file,
|
105
|
-
)
|
106
|
-
|
107
|
-
def copy_file(self, src_path: Path) -> None:
|
108
|
-
if src_path.exists():
|
109
|
-
raise FileNotFoundError(str(src_path))
|
110
|
-
location = self.get_file_placement(src_path.name)
|
111
|
-
dst_path = self.root / location
|
112
|
-
dst_path.mkdir(parents=True, exist_ok=True)
|
113
|
-
shutil.copy2(src_path, dst_path)
|
114
|
-
|
115
|
-
@overload
|
116
|
-
@classmethod
|
117
|
-
def from_metadata(
|
118
|
-
cls,
|
119
|
-
root: str | Path,
|
120
|
-
metadata: str | Path,
|
121
|
-
year: int,
|
122
|
-
summary: str,
|
123
|
-
internal: bool = True,
|
124
|
-
researcher: str | None = None,
|
125
|
-
organisation: str | None = None,
|
126
|
-
) -> ProjectManager: ...
|
127
|
-
|
128
|
-
@overload
|
129
|
-
@classmethod
|
130
|
-
def from_metadata(
|
131
|
-
cls,
|
132
|
-
root: str | Path,
|
133
|
-
metadata: dict[str, Any],
|
134
|
-
year: int,
|
135
|
-
summary: str,
|
136
|
-
internal: bool = True,
|
137
|
-
researcher: str | None = None,
|
138
|
-
organisation: str | None = None,
|
139
|
-
) -> ProjectManager: ...
|
140
|
-
|
141
|
-
@classmethod
|
142
|
-
def from_metadata(
|
143
|
-
cls,
|
144
|
-
root: str | Path,
|
145
|
-
metadata: str | Path | dict[str, Any],
|
146
|
-
year: int,
|
147
|
-
summary: str,
|
148
|
-
internal: bool = True,
|
149
|
-
researcher: str | None = None,
|
150
|
-
organisation: str | None = None,
|
151
|
-
) -> ProjectManager:
|
152
|
-
if isinstance(metadata, str | Path):
|
153
|
-
metadata = Path(metadata)
|
154
|
-
if not metadata.exists():
|
155
|
-
raise MetadataFileNotFoundErr(str(metadata))
|
156
|
-
with metadata.open("r") as file:
|
157
|
-
_metadata = yaml.load(file)
|
158
|
-
else:
|
159
|
-
_metadata = metadata
|
160
|
-
_metadata.update(
|
161
|
-
{
|
162
|
-
"year": year,
|
163
|
-
"summary": summary,
|
164
|
-
"internal": internal,
|
165
|
-
"researcher": researcher,
|
166
|
-
"organisation": organisation,
|
167
|
-
}
|
168
|
-
)
|
169
|
-
return cls(root=root, metadata=_metadata)
|
170
|
-
|
171
|
-
@classmethod
|
172
|
-
def load_project(
|
173
|
-
cls, project_path: Path | str, metadata_name: str | None = None
|
174
|
-
) -> ProjectManager:
|
175
|
-
project_path = Path(project_path)
|
176
|
-
if not project_path.exists():
|
177
|
-
raise ProjectNotFoundErr(str(project_path))
|
178
|
-
metadata_path = (
|
179
|
-
project_path / cls.METADATA_NAME
|
180
|
-
if not metadata_name
|
181
|
-
else project_path / metadata_name
|
182
|
-
)
|
183
|
-
if not metadata_path.exists():
|
184
|
-
raise MetadataFileNotFoundErr(str(metadata_path))
|
185
|
-
with metadata_path.open("r") as file:
|
186
|
-
metadata = yaml.load(file)
|
187
|
-
return cls(metadata=metadata, root=project_path)
|
appm-0.0.2/appm/model.py
DELETED
@@ -1,82 +0,0 @@
|
|
1
|
-
from typing import Self
|
2
|
-
|
3
|
-
from pydantic import BaseModel, model_validator
|
4
|
-
|
5
|
-
from appm.__version__ import __version__
|
6
|
-
from appm.utils import slugify
|
7
|
-
|
8
|
-
STRUCTURES = {"year", "summary", "internal", "researcher", "organisation"}
|
9
|
-
|
10
|
-
|
11
|
-
class ExtDecl(BaseModel):
|
12
|
-
sep: str = "_"
|
13
|
-
format: list[tuple[str, str]]
|
14
|
-
|
15
|
-
@property
|
16
|
-
def fields(self) -> set[str]:
|
17
|
-
return {item[0] for item in self.format}
|
18
|
-
|
19
|
-
|
20
|
-
class NamingConventionDecl(BaseModel):
|
21
|
-
sep: str = "_"
|
22
|
-
structure: list[str] = ["year", "summary", "internal", "researcher", "organisation"]
|
23
|
-
|
24
|
-
@model_validator(mode="after")
|
25
|
-
def validate_structure_values(self) -> Self:
|
26
|
-
counter: dict[str, int] = {}
|
27
|
-
if len(self.structure) == 0:
|
28
|
-
raise ValueError("Invalid naming structure - empty structure")
|
29
|
-
for field in self.structure:
|
30
|
-
counter[field] = counter.get(field, 0) + 1
|
31
|
-
if counter[field] > 1:
|
32
|
-
raise ValueError(f"Invalid naming structure - repetition: {field}")
|
33
|
-
if field not in STRUCTURES:
|
34
|
-
raise ValueError(
|
35
|
-
f"Invalid naming structure - invalid field: {field}. Structure must be a non empty permutation of {STRUCTURES}"
|
36
|
-
)
|
37
|
-
return self
|
38
|
-
|
39
|
-
|
40
|
-
class ProjectDecl(BaseModel):
|
41
|
-
layout: list[str]
|
42
|
-
file: dict[str, ExtDecl]
|
43
|
-
naming_convention: NamingConventionDecl = NamingConventionDecl()
|
44
|
-
|
45
|
-
@property
|
46
|
-
def layout_set(self) -> set[str]:
|
47
|
-
return set(self.layout)
|
48
|
-
|
49
|
-
@model_validator(mode="after")
|
50
|
-
def validate_format_and_layout(self) -> Self:
|
51
|
-
for ext, decl in self.file.items():
|
52
|
-
if not self.layout_set.issubset(decl.fields):
|
53
|
-
raise ValueError(
|
54
|
-
f"""Format fields must be a superset of layout fields.
|
55
|
-
Extension: {ext}. Format fields: {decl.fields}. Layout fields: {self.layout_set}"""
|
56
|
-
)
|
57
|
-
return self
|
58
|
-
|
59
|
-
|
60
|
-
class ProjectMetadata(ProjectDecl):
|
61
|
-
year: int
|
62
|
-
summary: str
|
63
|
-
internal: bool
|
64
|
-
researcher: str | None = None
|
65
|
-
organisation: str | None = None
|
66
|
-
version: str | None = __version__
|
67
|
-
|
68
|
-
@property
|
69
|
-
def name(self) -> str:
|
70
|
-
fields = self.naming_convention.structure
|
71
|
-
name: list[str] = []
|
72
|
-
for field in fields:
|
73
|
-
value = getattr(self, field)
|
74
|
-
if value is not None:
|
75
|
-
if isinstance(value, str):
|
76
|
-
name.append(slugify(value))
|
77
|
-
elif field == "year":
|
78
|
-
name.append(str(value))
|
79
|
-
elif field == "internal":
|
80
|
-
value = "internal" if value else "external"
|
81
|
-
name.append(value)
|
82
|
-
return self.naming_convention.sep.join(name)
|
appm-0.0.2/appm/utils.py
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
import re
|
2
|
-
|
3
|
-
|
4
|
-
def slugify(text: str) -> str:
|
5
|
-
"""Generate a slug from a text
|
6
|
-
|
7
|
-
Used for generating project name and url slug
|
8
|
-
|
9
|
-
https://developer.mozilla.org/en-US/docs/Glossary/Slug
|
10
|
-
|
11
|
-
Example:
|
12
|
-
- The Plant Accelerator -> the-plant-accelerator
|
13
|
-
|
14
|
-
- APPN -> appn
|
15
|
-
|
16
|
-
Args:
|
17
|
-
text (str): source text
|
18
|
-
|
19
|
-
Returns:
|
20
|
-
str: slug
|
21
|
-
"""
|
22
|
-
text = text.lower()
|
23
|
-
# Replace non slug characters
|
24
|
-
text = re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE)
|
25
|
-
# Replace spaces with hyphens
|
26
|
-
text = re.sub(r"[\s\-]+", "-", text)
|
27
|
-
return text.strip("-")
|
File without changes
|
File without changes
|