appm 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- appm/__version__.py +1 -1
- appm/default.py +36 -0
- appm/exceptions.py +6 -0
- appm/manager.py +118 -121
- appm/model.py +264 -30
- appm/utils.py +45 -0
- appm-0.0.3.dist-info/METADATA +76 -0
- appm-0.0.3.dist-info/RECORD +11 -0
- appm-0.0.2.dist-info/METADATA +0 -9
- appm-0.0.2.dist-info/RECORD +0 -10
- {appm-0.0.2.dist-info → appm-0.0.3.dist-info}/WHEEL +0 -0
appm/__version__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.0.
|
1
|
+
__version__ = "0.0.3"
|
appm/default.py
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
DEFAULT_TEMPLATE = {
|
2
|
+
"version": "0.0.3",
|
3
|
+
"naming_convention": {
|
4
|
+
"sep": "_",
|
5
|
+
"structure": [
|
6
|
+
"year",
|
7
|
+
"summary",
|
8
|
+
"internal",
|
9
|
+
"researcherName",
|
10
|
+
"organisationName",
|
11
|
+
],
|
12
|
+
},
|
13
|
+
"layout": {
|
14
|
+
"structure": ["site", "sensor", "date", "trial", "procLevel"],
|
15
|
+
"mapping": {
|
16
|
+
"procLevel": {"raw": "T0-raw", "proc": "T1-proc", "trait": "T2-trait"}
|
17
|
+
},
|
18
|
+
},
|
19
|
+
"file": {
|
20
|
+
"*": {
|
21
|
+
"sep": "_",
|
22
|
+
"default": {"procLevel": "raw"},
|
23
|
+
"components": [
|
24
|
+
{"sep": "-", "components": [["date", r"\d{8}"], ["time", r"\d{6}"]]},
|
25
|
+
["site", "[^_.]+"],
|
26
|
+
["sensor", "[^_.]+"],
|
27
|
+
["trial", "[^_.]+"],
|
28
|
+
{
|
29
|
+
"name": "procLevel",
|
30
|
+
"pattern": "T0-raw|T1-proc|T2-trait|raw|proc|trait",
|
31
|
+
"required": False,
|
32
|
+
},
|
33
|
+
],
|
34
|
+
}
|
35
|
+
},
|
36
|
+
}
|
appm/exceptions.py
CHANGED
appm/manager.py
CHANGED
@@ -1,68 +1,25 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
import re
|
4
3
|
import shutil
|
5
|
-
from
|
4
|
+
from copy import deepcopy
|
6
5
|
from pathlib import Path
|
7
|
-
from typing import Any
|
6
|
+
from typing import Any
|
8
7
|
|
9
|
-
from ruamel.yaml import YAML
|
8
|
+
from ruamel.yaml import YAML
|
10
9
|
|
11
10
|
from appm.__version__ import __version__
|
11
|
+
from appm.default import DEFAULT_TEMPLATE
|
12
12
|
from appm.exceptions import (
|
13
|
-
FileFormatMismatch,
|
14
|
-
MetadataFileNotFoundErr,
|
15
|
-
ProjectNotFoundErr,
|
16
13
|
UnsupportedFileExtension,
|
17
14
|
)
|
18
|
-
from appm.model import
|
15
|
+
from appm.model import Project
|
16
|
+
from appm.utils import to_flow_style, validate_path
|
19
17
|
|
20
18
|
yaml = YAML()
|
21
19
|
yaml.indent(mapping=2, sequence=4, offset=2)
|
22
20
|
yaml.preserve_quotes = True # optional, if you want to preserve quotes
|
23
21
|
|
24
22
|
|
25
|
-
def to_flow_style(obj: Any) -> Any:
|
26
|
-
"""Recursively convert dict/list to ruamel structures with ALL lists using flow-style."""
|
27
|
-
if isinstance(obj, Mapping):
|
28
|
-
cm = CommentedMap()
|
29
|
-
for k, v in obj.items():
|
30
|
-
cm[k] = to_flow_style(v)
|
31
|
-
return cm
|
32
|
-
if isinstance(obj, Sequence) and not isinstance(obj, str):
|
33
|
-
cs = CommentedSeq()
|
34
|
-
for item in obj:
|
35
|
-
cs.append(to_flow_style(item))
|
36
|
-
cs.fa.set_flow_style()
|
37
|
-
return cs
|
38
|
-
return obj
|
39
|
-
|
40
|
-
|
41
|
-
class ExtManager:
|
42
|
-
def __init__(self, ext: str, decl: ExtDecl) -> None:
|
43
|
-
self.ext = ext
|
44
|
-
self.decl = decl
|
45
|
-
|
46
|
-
@property
|
47
|
-
def pattern(self) -> str:
|
48
|
-
return (
|
49
|
-
r"^"
|
50
|
-
+ self.decl.sep.join([f"({p})" for _, p in self.decl.format])
|
51
|
-
+ r"(.*)$"
|
52
|
-
)
|
53
|
-
|
54
|
-
def match(self, name: str) -> dict[str, str]:
|
55
|
-
match = re.match(self.pattern, name)
|
56
|
-
if not match:
|
57
|
-
raise FileFormatMismatch(f"Name: {name}. Pattern: {self.pattern}")
|
58
|
-
groups = match.groups()
|
59
|
-
result = {}
|
60
|
-
for i, (field, _) in enumerate(self.decl.format):
|
61
|
-
result[field] = groups[i]
|
62
|
-
result["*"] = groups[-1]
|
63
|
-
return result
|
64
|
-
|
65
|
-
|
66
23
|
class ProjectManager:
|
67
24
|
METADATA_NAME: str = "metadata.yaml"
|
68
25
|
|
@@ -72,30 +29,68 @@ class ProjectManager:
|
|
72
29
|
root: str | Path,
|
73
30
|
) -> None:
|
74
31
|
self.root = Path(root)
|
75
|
-
self.metadata =
|
76
|
-
self.handlers = {
|
77
|
-
|
78
|
-
|
79
|
-
|
32
|
+
self.metadata = Project.model_validate(metadata)
|
33
|
+
self.handlers = {ext: handler for ext, handler in self.metadata.file.items()}
|
34
|
+
|
35
|
+
@property
|
36
|
+
def location(self) -> Path:
|
37
|
+
return self.root / self.metadata.project_name
|
80
38
|
|
81
|
-
def match(self, name: str) -> dict[str, str]:
|
39
|
+
def match(self, name: str) -> dict[str, str | None]:
|
40
|
+
"""Match a file name and separate into format defined field components
|
41
|
+
|
42
|
+
The result contains a * which captures all non-captured values.
|
43
|
+
|
44
|
+
Args:
|
45
|
+
name (str): file name
|
46
|
+
|
47
|
+
Raises:
|
48
|
+
UnsupportedFileExtension: the metadata does not define an
|
49
|
+
extension declaration for the file's extension.
|
50
|
+
|
51
|
+
Returns:
|
52
|
+
dict[str, str]: key value dictionary of the field component
|
53
|
+
defined using the format field.
|
54
|
+
"""
|
82
55
|
ext = name.split(".")[-1]
|
83
|
-
if ext
|
84
|
-
|
85
|
-
|
56
|
+
if ext in self.handlers:
|
57
|
+
return self.handlers[ext].match(name)
|
58
|
+
if "*" in self.handlers:
|
59
|
+
return self.handlers["*"].match(name)
|
60
|
+
raise UnsupportedFileExtension(str(ext))
|
86
61
|
|
87
62
|
def get_file_placement(self, name: str) -> str:
|
88
|
-
|
63
|
+
"""Find location where a file should be placed.
|
64
|
+
|
65
|
+
Determination is based on the metadata's layout field,
|
66
|
+
the file extension format definition, and the file name.
|
67
|
+
More concretely, field component - values are matched using the
|
68
|
+
RegEx defined in format. Fields that match layout values will be
|
69
|
+
extracted and path-appended in the order they appear in layout.
|
70
|
+
|
71
|
+
Args:
|
72
|
+
name (str): file name
|
73
|
+
|
74
|
+
Returns:
|
75
|
+
str: file placement directory
|
76
|
+
"""
|
77
|
+
layout = self.metadata.parsed_layout
|
89
78
|
groups = self.match(name)
|
90
|
-
|
91
|
-
return "/".join(values)
|
79
|
+
return layout.get_path(groups)
|
92
80
|
|
93
81
|
def init_project(self) -> None:
|
94
|
-
|
82
|
+
"""Create a project:
|
83
|
+
|
84
|
+
- Determine the project's name from nameing_convention and metadata
|
85
|
+
- Create a folder based on project's root and project name
|
86
|
+
- Create a metadata file in the project's location
|
87
|
+
"""
|
88
|
+
self.location.mkdir(exist_ok=True, parents=True)
|
95
89
|
self.save_metadata()
|
96
90
|
|
97
91
|
def save_metadata(self) -> None:
|
98
|
-
|
92
|
+
"""Save the current metadata to the project location"""
|
93
|
+
metadata_path = self.location / self.METADATA_NAME
|
99
94
|
with metadata_path.open("w") as file:
|
100
95
|
data = self.metadata.model_dump(mode="json")
|
101
96
|
data["version"] = __version__
|
@@ -104,84 +99,86 @@ class ProjectManager:
|
|
104
99
|
file,
|
105
100
|
)
|
106
101
|
|
107
|
-
def copy_file(self, src_path: Path) -> None:
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
102
|
+
def copy_file(self, src_path: str | Path) -> None:
|
103
|
+
"""Copy a file located at `src_path` to an appropriate
|
104
|
+
location in the project.
|
105
|
+
|
106
|
+
Args:
|
107
|
+
src_path (str | Path): path to where src data is found
|
108
|
+
"""
|
109
|
+
src_path = validate_path(src_path)
|
110
|
+
dst_path = self.location / self.get_file_placement(src_path.name)
|
112
111
|
dst_path.mkdir(parents=True, exist_ok=True)
|
113
112
|
shutil.copy2(src_path, dst_path)
|
114
113
|
|
115
|
-
@overload
|
116
114
|
@classmethod
|
117
|
-
def
|
115
|
+
def from_template(
|
118
116
|
cls,
|
119
117
|
root: str | Path,
|
120
|
-
metadata: str | Path,
|
121
118
|
year: int,
|
122
119
|
summary: str,
|
123
120
|
internal: bool = True,
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
@overload
|
129
|
-
@classmethod
|
130
|
-
def from_metadata(
|
131
|
-
cls,
|
132
|
-
root: str | Path,
|
133
|
-
metadata: dict[str, Any],
|
134
|
-
year: int,
|
135
|
-
summary: str,
|
136
|
-
internal: bool = True,
|
137
|
-
researcher: str | None = None,
|
138
|
-
organisation: str | None = None,
|
139
|
-
) -> ProjectManager: ...
|
140
|
-
|
141
|
-
@classmethod
|
142
|
-
def from_metadata(
|
143
|
-
cls,
|
144
|
-
root: str | Path,
|
145
|
-
metadata: str | Path | dict[str, Any],
|
146
|
-
year: int,
|
147
|
-
summary: str,
|
148
|
-
internal: bool = True,
|
149
|
-
researcher: str | None = None,
|
150
|
-
organisation: str | None = None,
|
121
|
+
template: str | Path | dict[str, Any] | None = None,
|
122
|
+
researcherName: str | None = None,
|
123
|
+
organisationName: str | None = None,
|
151
124
|
) -> ProjectManager:
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
125
|
+
"""Create a ProjectManager based on template and meta information
|
126
|
+
|
127
|
+
Args:
|
128
|
+
root (str | Path): parent directory - where project is stored
|
129
|
+
template (str | Path | dict[str, Any]): path to template file or the template content.
|
130
|
+
year (int): meta information - year
|
131
|
+
summary (str): meta information - summary
|
132
|
+
internal (bool, optional): meta information - internal. Defaults to True.
|
133
|
+
researcher (str | None, optional): meta information - researcherName. Defaults to None.
|
134
|
+
organisation (str | None, optional): meta information - organisationName. Defaults to None.
|
135
|
+
|
136
|
+
Returns:
|
137
|
+
ProjectManager: ProjectManager object
|
138
|
+
"""
|
139
|
+
if isinstance(template, str | Path):
|
140
|
+
metadata_path = Path(template)
|
141
|
+
metadata_path = validate_path(template)
|
142
|
+
with metadata_path.open("r") as file:
|
143
|
+
metadata = yaml.load(file)
|
144
|
+
elif isinstance(template, dict):
|
145
|
+
metadata = template
|
146
|
+
elif not template:
|
147
|
+
metadata = deepcopy(DEFAULT_TEMPLATE)
|
158
148
|
else:
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
return cls(root=root, metadata=
|
149
|
+
raise TypeError(
|
150
|
+
f"Unexpected type for template: {type(template)}. Accepts str, dict or None"
|
151
|
+
)
|
152
|
+
metadata["meta"] = {
|
153
|
+
"year": year,
|
154
|
+
"summary": summary,
|
155
|
+
"internal": internal,
|
156
|
+
"researcherName": researcherName,
|
157
|
+
"organisationName": organisationName,
|
158
|
+
}
|
159
|
+
return cls(root=root, metadata=metadata)
|
170
160
|
|
171
161
|
@classmethod
|
172
162
|
def load_project(
|
173
163
|
cls, project_path: Path | str, metadata_name: str | None = None
|
174
164
|
) -> ProjectManager:
|
175
|
-
|
176
|
-
|
177
|
-
|
165
|
+
"""Load a project from project's path
|
166
|
+
|
167
|
+
Args:
|
168
|
+
project_path (Path | str): path to project to open
|
169
|
+
metadata_name (str | None, optional): name for metadata file. If not provided, use "metadata.yaml". Defaults to None.
|
170
|
+
|
171
|
+
Returns:
|
172
|
+
ProjectManager: ProjectManager object
|
173
|
+
"""
|
174
|
+
project_path = validate_path(project_path)
|
178
175
|
metadata_path = (
|
179
176
|
project_path / cls.METADATA_NAME
|
180
177
|
if not metadata_name
|
181
178
|
else project_path / metadata_name
|
182
179
|
)
|
183
|
-
|
184
|
-
|
180
|
+
metadata_path = validate_path(metadata_path)
|
181
|
+
|
185
182
|
with metadata_path.open("r") as file:
|
186
183
|
metadata = yaml.load(file)
|
187
|
-
return cls(metadata=metadata, root=project_path)
|
184
|
+
return cls(metadata=metadata, root=project_path.parent)
|
appm/model.py
CHANGED
@@ -1,28 +1,234 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import re
|
4
|
+
from collections import Counter
|
1
5
|
from typing import Self
|
2
6
|
|
3
7
|
from pydantic import BaseModel, model_validator
|
8
|
+
from ruamel.yaml import YAML
|
4
9
|
|
5
10
|
from appm.__version__ import __version__
|
11
|
+
from appm.exceptions import FileFormatMismatch
|
6
12
|
from appm.utils import slugify
|
7
13
|
|
8
|
-
|
14
|
+
yaml = YAML()
|
9
15
|
|
16
|
+
with open("examples/template.yaml") as file:
|
17
|
+
data = yaml.load(file)
|
10
18
|
|
11
|
-
|
12
|
-
|
13
|
-
|
19
|
+
STRUCTURES = {"year", "summary", "internal", "researcherName", "organisationName"}
|
20
|
+
|
21
|
+
|
22
|
+
class Field(BaseModel):
|
23
|
+
name: str
|
24
|
+
pattern: str
|
25
|
+
required: bool = True
|
26
|
+
|
27
|
+
@property
|
28
|
+
def regex(self) -> str:
|
29
|
+
return f"(?P<{self.name}>{self.pattern})"
|
30
|
+
|
31
|
+
@classmethod
|
32
|
+
def from_tuple(cls, value: tuple[str, str] | list[str]) -> Field:
|
33
|
+
assert len(value) == 2
|
34
|
+
return Field(name=value[0], pattern=value[1])
|
35
|
+
|
36
|
+
|
37
|
+
class Group(BaseModel):
|
38
|
+
components: list[tuple[str, str] | Field | Group]
|
39
|
+
sep: str = "-"
|
40
|
+
|
41
|
+
def validate_components(self) -> Self:
|
42
|
+
if not self.components:
|
43
|
+
raise ValueError(f"Components cannot be empty: {self.components}")
|
44
|
+
self._fields: list[Field | Group] = []
|
45
|
+
self._normalised_fields: list[Field] = []
|
46
|
+
for field in self.components:
|
47
|
+
if isinstance(field, tuple | list):
|
48
|
+
f = Field.from_tuple(field)
|
49
|
+
self._fields.append(f)
|
50
|
+
self._normalised_fields.append(f)
|
51
|
+
elif isinstance(field, Field):
|
52
|
+
self._fields.append(field)
|
53
|
+
self._normalised_fields.append(field)
|
54
|
+
else:
|
55
|
+
self._fields.append(field)
|
56
|
+
self._normalised_fields.extend(field.normalised_fields)
|
57
|
+
return self
|
58
|
+
|
59
|
+
def validate_names(self) -> Self:
|
60
|
+
self._names: list[str] = []
|
61
|
+
self._optional_names: set[str] = set()
|
62
|
+
for field in self.fields:
|
63
|
+
if isinstance(field, Field):
|
64
|
+
self._names.append(field.name)
|
65
|
+
if not field.required:
|
66
|
+
self._optional_names.add(field.name)
|
67
|
+
else:
|
68
|
+
self._names.extend(field.names)
|
69
|
+
self._optional_names.update(field.optional_names)
|
70
|
+
return self
|
71
|
+
|
72
|
+
def validate_regex(self) -> Self:
|
73
|
+
regex_str = []
|
74
|
+
for i, field in enumerate(self.fields):
|
75
|
+
is_optional = isinstance(field, Field) and not field.required
|
76
|
+
pattern = field.regex
|
77
|
+
|
78
|
+
if i == 0:
|
79
|
+
if is_optional:
|
80
|
+
# First field, no separator; make only field optional
|
81
|
+
regex_str.append(f"(?:{pattern})?")
|
82
|
+
else:
|
83
|
+
regex_str.append(pattern)
|
84
|
+
else:
|
85
|
+
if is_optional:
|
86
|
+
# Wrap separator + field together as optional
|
87
|
+
regex_str.append(f"(?:{self.sep}{pattern})?")
|
88
|
+
else:
|
89
|
+
regex_str.append(f"{self.sep}{pattern}")
|
90
|
+
self._regex = "".join(regex_str)
|
91
|
+
return self
|
92
|
+
|
93
|
+
@model_validator(mode="after")
|
94
|
+
def validate_group(self) -> Self:
|
95
|
+
return self.validate_components().validate_names().validate_regex()
|
96
|
+
|
97
|
+
@property
|
98
|
+
def normalised_fields(self) -> list[Field]:
|
99
|
+
return self._normalised_fields
|
100
|
+
|
101
|
+
@property
|
102
|
+
def fields(self) -> list[Field | Group]:
|
103
|
+
return self._fields
|
104
|
+
|
105
|
+
@property
|
106
|
+
def names(self) -> list[str]:
|
107
|
+
return self._names
|
108
|
+
|
109
|
+
@property
|
110
|
+
def optional_names(self) -> set[str]:
|
111
|
+
return self._optional_names
|
112
|
+
|
113
|
+
@property
|
114
|
+
def regex(self) -> str:
|
115
|
+
return self._regex
|
116
|
+
|
117
|
+
|
118
|
+
class Extension(Group):
|
119
|
+
default: dict[str, str] | None = None
|
120
|
+
|
121
|
+
@property
|
122
|
+
def default_names(self) -> set[str]:
|
123
|
+
return set() if not self.default else set(self.default.keys())
|
124
|
+
|
125
|
+
@property
|
126
|
+
def all_names(self) -> set[str]:
|
127
|
+
return set(self.names) | self.default_names
|
128
|
+
|
129
|
+
def validate_regex(self) -> Self:
|
130
|
+
self = super().validate_regex()
|
131
|
+
self._regex = f"^{self._regex}(?P<rest>.*)$"
|
132
|
+
return self
|
133
|
+
|
134
|
+
def validate_unique_names(self) -> Self:
|
135
|
+
count = Counter(self.names)
|
136
|
+
non_uniques = {k: v for k, v in count.items() if v > 1}
|
137
|
+
if non_uniques:
|
138
|
+
raise ValueError(f"Non-unique field name: {non_uniques}")
|
139
|
+
return self
|
140
|
+
|
141
|
+
def validate_reserved_name(self) -> Self:
|
142
|
+
if "rest" in self.names:
|
143
|
+
raise ValueError("Field component must not contain reserved key: rest")
|
144
|
+
return self
|
145
|
+
|
146
|
+
def validate_first_field_must_be_required(self) -> Self:
|
147
|
+
if not (field := self.normalised_fields[0]).required:
|
148
|
+
raise ValueError(f"First component must be required: {field.name}")
|
149
|
+
return self
|
150
|
+
|
151
|
+
@model_validator(mode="after")
|
152
|
+
def validate_extension(self) -> Self:
|
153
|
+
return (
|
154
|
+
self.validate_components()
|
155
|
+
.validate_names()
|
156
|
+
.validate_regex()
|
157
|
+
.validate_unique_names()
|
158
|
+
.validate_reserved_name()
|
159
|
+
.validate_first_field_must_be_required()
|
160
|
+
)
|
161
|
+
|
162
|
+
def match(self, name: str) -> dict[str, str | None]:
|
163
|
+
m = re.match(self.regex, name)
|
164
|
+
if not m:
|
165
|
+
raise FileFormatMismatch(f"Name: {name}. Pattern: {self.regex}")
|
166
|
+
result = m.groupdict()
|
167
|
+
if self.default:
|
168
|
+
for k, v in self.default.items():
|
169
|
+
if result.get(k) is None:
|
170
|
+
result[k] = v
|
171
|
+
return result
|
172
|
+
|
173
|
+
|
174
|
+
type File = dict[str, Extension]
|
175
|
+
|
176
|
+
|
177
|
+
class Layout(BaseModel):
|
178
|
+
structure: list[str]
|
179
|
+
mapping: dict[str, dict[str, str]] | None = None
|
180
|
+
|
181
|
+
@classmethod
|
182
|
+
def from_list(cls, value: list[str]) -> Layout:
|
183
|
+
return cls(structure=value)
|
14
184
|
|
15
185
|
@property
|
16
|
-
def
|
17
|
-
return
|
186
|
+
def structure_set(self) -> set[str]:
|
187
|
+
return self._structure_set
|
188
|
+
|
189
|
+
@model_validator(mode="after")
|
190
|
+
def validate_layout(self) -> Self:
|
191
|
+
self._structure_set = set(self.structure)
|
192
|
+
if self.mapping:
|
193
|
+
if not set(self.mapping.keys()).issubset(self._structure_set):
|
194
|
+
raise ValueError(
|
195
|
+
f"Mapping keys must be a subset of structure. Mapping keys: {set(self.mapping.keys())}, structure: {self.structure}"
|
196
|
+
)
|
197
|
+
return self
|
18
198
|
|
199
|
+
def get_path(self, components: dict[str, str | None]) -> str:
|
200
|
+
result: list[str] = []
|
201
|
+
for key in self.structure:
|
202
|
+
value = components.get(key, None)
|
203
|
+
if self.mapping and key in self.mapping and value in self.mapping[key]:
|
204
|
+
value = self.mapping[key][value]
|
205
|
+
if value is None:
|
206
|
+
raise ValueError(
|
207
|
+
f"None value for key: {key}. Either set a default for Extension definition, change Extension pattern to capture key value, or rename file."
|
208
|
+
)
|
209
|
+
result.append(value)
|
210
|
+
return "/".join(result)
|
19
211
|
|
20
|
-
|
212
|
+
|
213
|
+
class NamingConv(BaseModel):
|
21
214
|
sep: str = "_"
|
22
|
-
structure: list[str] = [
|
215
|
+
structure: list[str] = [
|
216
|
+
"year",
|
217
|
+
"summary",
|
218
|
+
"internal",
|
219
|
+
"researcherName",
|
220
|
+
"organisationName",
|
221
|
+
]
|
23
222
|
|
24
223
|
@model_validator(mode="after")
|
25
|
-
def
|
224
|
+
def validate_naming_convention(self) -> Self:
|
225
|
+
"""Validate structure value
|
226
|
+
|
227
|
+
structure:
|
228
|
+
- cannot be empty
|
229
|
+
- cannot have repeated component(s)
|
230
|
+
- cannot have a field component that is not one of the metadata fields.
|
231
|
+
"""
|
26
232
|
counter: dict[str, int] = {}
|
27
233
|
if len(self.structure) == 0:
|
28
234
|
raise ValueError("Invalid naming structure - empty structure")
|
@@ -37,40 +243,68 @@ class NamingConventionDecl(BaseModel):
|
|
37
243
|
return self
|
38
244
|
|
39
245
|
|
40
|
-
class
|
41
|
-
layout: list[str]
|
42
|
-
file:
|
43
|
-
naming_convention:
|
246
|
+
class Template(BaseModel):
|
247
|
+
layout: Layout | list[str]
|
248
|
+
file: File
|
249
|
+
naming_convention: NamingConv = NamingConv()
|
250
|
+
version: str = __version__
|
44
251
|
|
45
|
-
|
46
|
-
|
47
|
-
|
252
|
+
def validate_layout(self) -> Self:
|
253
|
+
if isinstance(self.layout, list):
|
254
|
+
self._layout = Layout.from_list(self.layout)
|
255
|
+
else:
|
256
|
+
self._layout = self.layout
|
257
|
+
return self
|
48
258
|
|
49
|
-
|
50
|
-
|
259
|
+
def validate_file_non_empty(self) -> Self:
|
260
|
+
if not self.file:
|
261
|
+
raise ValueError("Empty extension")
|
262
|
+
return self
|
263
|
+
|
264
|
+
def validate_file_name_subset_layout(self) -> Self:
|
51
265
|
for ext, decl in self.file.items():
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
266
|
+
for field in self.parsed_layout.structure_set:
|
267
|
+
if field not in decl.all_names:
|
268
|
+
raise ValueError(
|
269
|
+
f"Component fields must be a superset of layout fields: {field}. Ext: {ext}"
|
270
|
+
)
|
271
|
+
if field in decl.optional_names and field not in decl.default_names:
|
272
|
+
raise ValueError(
|
273
|
+
f"Optional field that is also a layout field must have a default value: {field}. Ext: {ext}"
|
274
|
+
)
|
57
275
|
return self
|
58
276
|
|
277
|
+
@property
|
278
|
+
def parsed_layout(self) -> Layout:
|
279
|
+
return self._layout
|
59
280
|
|
60
|
-
|
281
|
+
@model_validator(mode="after")
|
282
|
+
def validate_template(self) -> Self:
|
283
|
+
return (
|
284
|
+
self.validate_layout()
|
285
|
+
.validate_file_non_empty()
|
286
|
+
.validate_file_name_subset_layout()
|
287
|
+
)
|
288
|
+
|
289
|
+
|
290
|
+
class Metadata(BaseModel):
|
61
291
|
year: int
|
62
292
|
summary: str
|
63
|
-
internal: bool
|
64
|
-
|
65
|
-
|
66
|
-
|
293
|
+
internal: bool = True
|
294
|
+
researcherName: str | None = None
|
295
|
+
organisationName: str | None = None
|
296
|
+
|
297
|
+
|
298
|
+
class Project(Template):
|
299
|
+
meta: Metadata
|
67
300
|
|
68
301
|
@property
|
69
|
-
def
|
302
|
+
def project_name(self) -> str:
|
303
|
+
"""Project name based on metadata and naming convention definiton"""
|
70
304
|
fields = self.naming_convention.structure
|
71
305
|
name: list[str] = []
|
72
306
|
for field in fields:
|
73
|
-
value = getattr(self, field)
|
307
|
+
value = getattr(self.meta, field)
|
74
308
|
if value is not None:
|
75
309
|
if isinstance(value, str):
|
76
310
|
name.append(slugify(value))
|
appm/utils.py
CHANGED
@@ -1,4 +1,11 @@
|
|
1
1
|
import re
|
2
|
+
from collections.abc import Mapping, Sequence
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import Any
|
5
|
+
|
6
|
+
from ruamel.yaml import CommentedMap, CommentedSeq
|
7
|
+
|
8
|
+
from appm.exceptions import NotAFileErr, NotFoundErr
|
2
9
|
|
3
10
|
|
4
11
|
def slugify(text: str) -> str:
|
@@ -25,3 +32,41 @@ def slugify(text: str) -> str:
|
|
25
32
|
# Replace spaces with hyphens
|
26
33
|
text = re.sub(r"[\s\-]+", "-", text)
|
27
34
|
return text.strip("-")
|
35
|
+
|
36
|
+
|
37
|
+
def to_flow_style(obj: Any) -> Any:
|
38
|
+
"""Recursively convert dict/list to ruamel structures with ALL lists using flow-style."""
|
39
|
+
if isinstance(obj, Mapping):
|
40
|
+
cm = CommentedMap()
|
41
|
+
for k, v in obj.items():
|
42
|
+
cm[k] = to_flow_style(v)
|
43
|
+
return cm
|
44
|
+
if isinstance(obj, Sequence) and not isinstance(obj, str):
|
45
|
+
cs = CommentedSeq()
|
46
|
+
for item in obj:
|
47
|
+
cs.append(to_flow_style(item))
|
48
|
+
cs.fa.set_flow_style()
|
49
|
+
return cs
|
50
|
+
return obj
|
51
|
+
|
52
|
+
|
53
|
+
def validate_path(path: str | Path, is_file: bool = False) -> Path:
|
54
|
+
"""Verify that path describes an existing file/folder
|
55
|
+
|
56
|
+
Args:
|
57
|
+
path (str | Path): path to validate
|
58
|
+
is_file (bool): whether path is a file. Defaults to False.
|
59
|
+
|
60
|
+
Raises:
|
61
|
+
NotFoundErr: path item doesnt exist
|
62
|
+
NotAFileErr: path doesn't describe a file
|
63
|
+
|
64
|
+
Returns:
|
65
|
+
Path: validated path
|
66
|
+
"""
|
67
|
+
_path = Path(path)
|
68
|
+
if not _path.exists():
|
69
|
+
raise NotFoundErr(str(path))
|
70
|
+
if is_file and not _path.is_file():
|
71
|
+
raise NotAFileErr(str(path))
|
72
|
+
return _path
|
@@ -0,0 +1,76 @@
|
|
1
|
+
Metadata-Version: 2.3
|
2
|
+
Name: appm
|
3
|
+
Version: 0.0.3
|
4
|
+
Summary: APPN Phenomate Project Manager
|
5
|
+
Requires-Dist: pydantic>=2.11.7
|
6
|
+
Requires-Dist: ruamel-yaml>=0.18.14
|
7
|
+
Requires-Python: >=3.13
|
8
|
+
Description-Content-Type: text/markdown
|
9
|
+
|
10
|
+
# APPN Phenomate Project Manager
|
11
|
+
|
12
|
+
A Python package for managing project templates, metadata, and file organization using flexible YAML schemas. Designed for research and data projects that require consistent file naming, metadata, and directory structures.
|
13
|
+
|
14
|
+
## Install
|
15
|
+
|
16
|
+
```bash
|
17
|
+
pip install appm
|
18
|
+
```
|
19
|
+
|
20
|
+
## Features
|
21
|
+
|
22
|
+
- Template-driven project structure: Define project layouts, file naming conventions, and metadata in YAML.
|
23
|
+
- Automatic project initialization: Create new projects with standardized folders and metadata files.
|
24
|
+
- File placement and matching: Automatically determine where files belong based on their names and template rules.
|
25
|
+
- Extensible and validated: Uses Pydantic for schema validation and ruamel.yaml for YAML parsing.
|
26
|
+
Installation
|
27
|
+
Or for development:
|
28
|
+
|
29
|
+
## Usage
|
30
|
+
1. Define a Template
|
31
|
+
|
32
|
+
Create a YAML template describing your project's structure, naming conventions, and file formats. See `examples/template.yaml` for the default template.
|
33
|
+
|
34
|
+
2. Initialize a Project
|
35
|
+
|
36
|
+
```py
|
37
|
+
from appm import ProjectManager
|
38
|
+
|
39
|
+
pm = ProjectManager.from_template(
|
40
|
+
root="projects",
|
41
|
+
year=2024,
|
42
|
+
summary="Wheat yield trial",
|
43
|
+
internal=True,
|
44
|
+
researcherName="Jane Doe",
|
45
|
+
organisationName="Plant Research Org",
|
46
|
+
template="examples/template.yaml"
|
47
|
+
)
|
48
|
+
pm.init_project()
|
49
|
+
|
50
|
+
```
|
51
|
+
|
52
|
+
3. Add Files
|
53
|
+
|
54
|
+
Files are automatically placed in the correct directory based on the template.
|
55
|
+
|
56
|
+
```py
|
57
|
+
pm.copy_file("data/20240601-120000_SiteA_SensorX_Trial1_T0-raw.csv")
|
58
|
+
```
|
59
|
+
|
60
|
+
## Project Structure
|
61
|
+
- appm – Core package (template parsing, project management, utilities)
|
62
|
+
- examples – Example YAML templates
|
63
|
+
- schema – JSON schema for template validation
|
64
|
+
- tests – Unit tests and fixtures
|
65
|
+
|
66
|
+
## Development
|
67
|
+
- Python 3.13+
|
68
|
+
- Pydantic
|
69
|
+
- ruamel.yaml
|
70
|
+
- pytest for testing
|
71
|
+
|
72
|
+
## Run tests:
|
73
|
+
|
74
|
+
```
|
75
|
+
pytest
|
76
|
+
```
|
@@ -0,0 +1,11 @@
|
|
1
|
+
appm/__init__.py,sha256=70a098d1ac000a7e4f9aa72fa542e63b165b884837cf4e5c31a56af7df6fc6db,71
|
2
|
+
appm/__version__.py,sha256=e0664a8b5de50d30f6e5806419a9216721106564c447f39640c34fa07fd43367,22
|
3
|
+
appm/default.py,sha256=0d5daf7fd877203abcb73b3beb47a9de841b1dfc6c2566caefc156a8897414ca,1018
|
4
|
+
appm/exceptions.py,sha256=9fadce3fa4b8ed97fb98b745f441097d4f7b4f7582e592e56e22edb689f4f139,385
|
5
|
+
appm/manager.py,sha256=8450711204a71d54d073a5fba44dd21a147c8c3da400e285c7ca7f46e0492cfe,6437
|
6
|
+
appm/model.py,sha256=92d1db4bac02b535e7e3b1a6b15783db475bf85ba992b2b3f2e6882fe6a03a1f,10264
|
7
|
+
appm/py.typed,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
|
8
|
+
appm/utils.py,sha256=c481c81c6ecdfbb53047a0babf3fc5b1ecbfc64645c06ff49f96a5896c1afa61,1871
|
9
|
+
appm-0.0.3.dist-info/WHEEL,sha256=b70116f4076fa664af162441d2ba3754dbb4ec63e09d563bdc1e9ab023cce400,78
|
10
|
+
appm-0.0.3.dist-info/METADATA,sha256=4b49db0bfb6a4fb7bacbd8a4546116ab6923e7e9ce86eb487962c1f57b6b0f0e,2001
|
11
|
+
appm-0.0.3.dist-info/RECORD,,
|
appm-0.0.2.dist-info/METADATA
DELETED
appm-0.0.2.dist-info/RECORD
DELETED
@@ -1,10 +0,0 @@
|
|
1
|
-
appm/__init__.py,sha256=70a098d1ac000a7e4f9aa72fa542e63b165b884837cf4e5c31a56af7df6fc6db,71
|
2
|
-
appm/__version__.py,sha256=42f95587825397724bee34008daefac8ab53f88bc5e3adf50128d6635c12e804,22
|
3
|
-
appm/exceptions.py,sha256=3be0e11cfa9cef94fbf2c442f81fc571760f33e26b6fc0b1201b1961057d4d9d,285
|
4
|
-
appm/manager.py,sha256=7e1add7d357300aa5e9287bf7683f4ec623ca02cd683f252526f0f416646a594,5633
|
5
|
-
appm/model.py,sha256=d5a928f0c07b4927933de7b93fb2efaa3dc631318fe0dcd9b2b14b98d0a4aa1b,2708
|
6
|
-
appm/py.typed,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
|
7
|
-
appm/utils.py,sha256=f72c5864d51ce2e37734c7724ca076fb5e5a5ce6d427ca6eeebd10d27511351e,578
|
8
|
-
appm-0.0.2.dist-info/WHEEL,sha256=b70116f4076fa664af162441d2ba3754dbb4ec63e09d563bdc1e9ab023cce400,78
|
9
|
-
appm-0.0.2.dist-info/METADATA,sha256=cab58410a5a23a225f8a7065a38f92431cb56cfc66877e42c2f04a5c6766ffb3,219
|
10
|
-
appm-0.0.2.dist-info/RECORD,,
|
File without changes
|