senfd 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {senfd-0.2.0/src/senfd.egg-info → senfd-0.2.2}/PKG-INFO +1 -1
- {senfd-0.2.0 → senfd-0.2.2}/pyproject.toml +1 -1
- senfd-0.2.2/src/senfd/__init__.py +1 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/cli.py +16 -1
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/documents/base.py +5 -4
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/documents/enriched.py +2 -1
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/documents/plain.py +17 -15
- senfd-0.2.2/src/senfd/errors.py +28 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/pipeline.py +3 -1
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/schemas/enriched.figure.document.schema.json +1 -1
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/schemas/plain.figure.document.schema.json +1 -1
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/tables.py +11 -7
- {senfd-0.2.0 → senfd-0.2.2/src/senfd.egg-info}/PKG-INFO +1 -1
- senfd-0.2.0/src/senfd/__init__.py +0 -1
- senfd-0.2.0/src/senfd/errors.py +0 -36
- {senfd-0.2.0 → senfd-0.2.2}/LICENSE +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/README.rst +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/setup.cfg +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/documents/__init__.py +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/schemas/__init__.py +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/templates/__init__.py +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/templates/enriched.figure.document.html.jinja2 +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/templates/plain.figure.document.html.jinja2 +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd/utils.py +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd.egg-info/SOURCES.txt +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd.egg-info/dependency_links.txt +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd.egg-info/entry_points.txt +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd.egg-info/requires.txt +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/src/senfd.egg-info/top_level.txt +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/tests/test_case_conversion.py +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/tests/test_cli.py +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/tests/test_module.py +0 -0
- {senfd-0.2.0 → senfd-0.2.2}/tests/test_schema.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.2"
|
|
@@ -5,14 +5,29 @@ Command-Line Interface
|
|
|
5
5
|
Produces organized and semantically enriched ``.json`` documents from
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
import json
|
|
8
9
|
from argparse import ArgumentParser, Namespace
|
|
9
10
|
from pathlib import Path
|
|
11
|
+
from typing import List
|
|
10
12
|
|
|
11
13
|
import senfd
|
|
12
14
|
import senfd.errors
|
|
13
15
|
import senfd.pipeline
|
|
14
16
|
import senfd.schemas
|
|
15
17
|
from senfd.documents import get_document_classes
|
|
18
|
+
from senfd.documents.base import to_file
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def to_log_file(
|
|
22
|
+
errors: List[senfd.errors.TableError], filename: str, output: Path
|
|
23
|
+
) -> Path:
|
|
24
|
+
|
|
25
|
+
content = json.dumps(
|
|
26
|
+
[{"type": type(error).__name__, **error.model_dump()} for error in errors],
|
|
27
|
+
indent=4,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
return to_file(content, f"{filename}.error.log", output)
|
|
16
31
|
|
|
17
32
|
|
|
18
33
|
def parse_args() -> Namespace:
|
|
@@ -62,6 +77,6 @@ def main() -> int:
|
|
|
62
77
|
|
|
63
78
|
for path in args.document:
|
|
64
79
|
errors = senfd.pipeline.process(path, args.output)
|
|
65
|
-
|
|
80
|
+
to_log_file(errors, path.stem, args.output)
|
|
66
81
|
|
|
67
82
|
return 0
|
|
@@ -11,13 +11,14 @@ import importlib.resources as pkg_resources
|
|
|
11
11
|
import json
|
|
12
12
|
from abc import ABC, abstractmethod
|
|
13
13
|
from pathlib import Path
|
|
14
|
-
from typing import Any, ClassVar, Dict, List,
|
|
14
|
+
from typing import Any, ClassVar, Dict, List, Optional, Tuple
|
|
15
15
|
|
|
16
16
|
from jinja2 import Environment, PackageLoader, select_autoescape
|
|
17
17
|
from pydantic import BaseModel, Field, ValidationError
|
|
18
18
|
|
|
19
19
|
import senfd.schemas
|
|
20
20
|
import senfd.tables
|
|
21
|
+
from senfd.errors import TableError
|
|
21
22
|
|
|
22
23
|
TRANSLATION_TABLE: Dict[int, str] = str.maketrans(
|
|
23
24
|
{
|
|
@@ -119,12 +120,12 @@ class Document(BaseModel):
|
|
|
119
120
|
return f"{self.meta.stem}{self.SUFFIX_JSON}"
|
|
120
121
|
|
|
121
122
|
def to_json(self) -> str:
|
|
122
|
-
"""Returns the document as a JSON-
|
|
123
|
+
"""Returns the document as a JSON-formatted string"""
|
|
123
124
|
|
|
124
125
|
return self.model_dump_json(indent=4)
|
|
125
126
|
|
|
126
127
|
def to_json_file(self, path: Optional[Path] = None) -> Path:
|
|
127
|
-
"""Writes the document,
|
|
128
|
+
"""Writes the document, formatted as JSON, to file at the given 'path'"""
|
|
128
129
|
|
|
129
130
|
return to_file(self.to_json(), self.json_filename(), path)
|
|
130
131
|
|
|
@@ -168,5 +169,5 @@ class Converter(ABC):
|
|
|
168
169
|
|
|
169
170
|
@staticmethod
|
|
170
171
|
@abstractmethod
|
|
171
|
-
def convert(path: Path) -> Tuple[Document, List[
|
|
172
|
+
def convert(path: Path) -> Tuple[Document, List[TableError]]:
|
|
172
173
|
pass
|
|
@@ -5,6 +5,7 @@ from typing import ClassVar, List, NamedTuple, Optional, Tuple
|
|
|
5
5
|
|
|
6
6
|
from pydantic import Field
|
|
7
7
|
|
|
8
|
+
import senfd.errors
|
|
8
9
|
import senfd.schemas
|
|
9
10
|
import senfd.tables
|
|
10
11
|
from senfd.documents.base import (
|
|
@@ -212,7 +213,7 @@ class FromFigureDocument(Converter):
|
|
|
212
213
|
return "".join(path.suffixes).lower() == ".plain.figure.document.json"
|
|
213
214
|
|
|
214
215
|
@staticmethod
|
|
215
|
-
def convert(path: Path) -> Tuple[Document, List[
|
|
216
|
+
def convert(path: Path) -> Tuple[Document, List[senfd.errors.TableError]]:
|
|
216
217
|
"""Instantiate an 'organized' Document from a 'figure' document"""
|
|
217
218
|
|
|
218
219
|
errors = []
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import re
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import ClassVar, List,
|
|
3
|
+
from typing import ClassVar, List, Optional, Tuple
|
|
4
4
|
|
|
5
5
|
import docx
|
|
6
6
|
from pydantic import BaseModel, Field
|
|
@@ -74,8 +74,7 @@ class FromDocx(Converter):
|
|
|
74
74
|
return path.suffix.lower() == ".docx"
|
|
75
75
|
|
|
76
76
|
@staticmethod
|
|
77
|
-
def convert(path: Path) -> Tuple[FigureDocument, List[
|
|
78
|
-
|
|
77
|
+
def convert(path: Path) -> Tuple[FigureDocument, List[senfd.errors.TableError]]:
|
|
79
78
|
def docx_table_to_table(docx_table: docx.table.Table) -> Table:
|
|
80
79
|
table = Table()
|
|
81
80
|
|
|
@@ -96,7 +95,7 @@ class FromDocx(Converter):
|
|
|
96
95
|
return table
|
|
97
96
|
|
|
98
97
|
figures = {}
|
|
99
|
-
errors: List[
|
|
98
|
+
errors: List[senfd.errors.TableError] = []
|
|
100
99
|
|
|
101
100
|
docx_document = docx.Document(path)
|
|
102
101
|
|
|
@@ -108,8 +107,8 @@ class FromDocx(Converter):
|
|
|
108
107
|
if not figure:
|
|
109
108
|
errors.append(
|
|
110
109
|
senfd.errors.TableCaptionError(
|
|
111
|
-
"Does not match figure caption assumptions",
|
|
112
|
-
caption,
|
|
110
|
+
message="Does not match figure caption assumptions",
|
|
111
|
+
caption=caption,
|
|
113
112
|
)
|
|
114
113
|
)
|
|
115
114
|
continue
|
|
@@ -117,8 +116,8 @@ class FromDocx(Converter):
|
|
|
117
116
|
if figure.figure_nr in figures:
|
|
118
117
|
errors.append(
|
|
119
118
|
senfd.errors.TableCaptionError(
|
|
120
|
-
"Duplicate caption",
|
|
121
|
-
caption,
|
|
119
|
+
message="Duplicate caption",
|
|
120
|
+
caption=caption,
|
|
122
121
|
)
|
|
123
122
|
)
|
|
124
123
|
continue
|
|
@@ -127,7 +126,7 @@ class FromDocx(Converter):
|
|
|
127
126
|
figures[figure.figure_nr] = figure
|
|
128
127
|
|
|
129
128
|
# Update tabular figures with page_nr
|
|
130
|
-
# Add non-
|
|
129
|
+
# Add non-tabular figures
|
|
131
130
|
# Check table-of-figure description validity
|
|
132
131
|
prev = cur = None
|
|
133
132
|
for paragraph in docx_document.paragraphs:
|
|
@@ -148,15 +147,17 @@ class FromDocx(Converter):
|
|
|
148
147
|
if not figure:
|
|
149
148
|
errors.append(
|
|
150
149
|
senfd.errors.TableOfFiguresError(
|
|
151
|
-
"Does not match figure assumptions",
|
|
152
|
-
caption,
|
|
150
|
+
message="Does not match figure assumptions",
|
|
151
|
+
caption=caption,
|
|
153
152
|
)
|
|
154
153
|
)
|
|
155
154
|
continue
|
|
156
155
|
|
|
157
156
|
if not figure.page_nr:
|
|
158
157
|
errors.append(
|
|
159
|
-
senfd.errors.TableOfFiguresError(
|
|
158
|
+
senfd.errors.TableOfFiguresError(
|
|
159
|
+
message="Is missing <page_nr>", caption=caption
|
|
160
|
+
)
|
|
160
161
|
)
|
|
161
162
|
continue
|
|
162
163
|
|
|
@@ -166,8 +167,8 @@ class FromDocx(Converter):
|
|
|
166
167
|
if figure.description not in existing.description:
|
|
167
168
|
errors.append(
|
|
168
169
|
senfd.errors.TableOfFiguresError(
|
|
169
|
-
f"({existing.description}) != {figure.description}",
|
|
170
|
-
caption,
|
|
170
|
+
message=f"({existing.description}) != {figure.description}",
|
|
171
|
+
caption=caption,
|
|
171
172
|
)
|
|
172
173
|
)
|
|
173
174
|
else:
|
|
@@ -175,7 +176,8 @@ class FromDocx(Converter):
|
|
|
175
176
|
|
|
176
177
|
return (
|
|
177
178
|
FigureDocument(
|
|
178
|
-
meta=DocumentMeta(stem=path.stem
|
|
179
|
+
meta=DocumentMeta(stem=path.stem.replace(".", "-")),
|
|
180
|
+
figures=list(figures.values()),
|
|
179
181
|
),
|
|
180
182
|
errors,
|
|
181
183
|
)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from typing import Any, List
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TableError(BaseModel):
|
|
7
|
+
message: str
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TableOfFiguresError(TableError):
|
|
11
|
+
caption: str
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TableCaptionError(TableError):
|
|
15
|
+
caption: str
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TableHeaderError(TableError):
|
|
19
|
+
caption: str
|
|
20
|
+
cells: List[Any]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class IrregularTableError(TableError):
|
|
24
|
+
lengths: List[int]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class NonTableHeaderError(TableError):
|
|
28
|
+
pass
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
+
from typing import List
|
|
2
3
|
|
|
3
4
|
from senfd.documents.enriched import FromFigureDocument
|
|
4
5
|
from senfd.documents.plain import FromDocx
|
|
6
|
+
from senfd.errors import TableError
|
|
5
7
|
|
|
6
8
|
CONVERTERS = [FromDocx, FromFigureDocument]
|
|
7
9
|
|
|
8
10
|
|
|
9
|
-
def process(input: Path, output: Path):
|
|
11
|
+
def process(input: Path, output: Path) -> List[TableError]:
|
|
10
12
|
all_errors = []
|
|
11
13
|
|
|
12
14
|
for converter in CONVERTERS:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import ClassVar, List,
|
|
1
|
+
from typing import ClassVar, List, Optional, Tuple
|
|
2
2
|
|
|
3
3
|
from pydantic import BaseModel, Field
|
|
4
4
|
|
|
@@ -41,16 +41,20 @@ class HeaderTable(Table):
|
|
|
41
41
|
headers: List[str] = Field(default_factory=list)
|
|
42
42
|
|
|
43
43
|
@classmethod
|
|
44
|
-
def from_table(
|
|
44
|
+
def from_table(
|
|
45
|
+
cls, table: Table
|
|
46
|
+
) -> Tuple[Optional["HeaderTable"], Optional[senfd.errors.TableError]]:
|
|
45
47
|
|
|
46
48
|
lengths = list(set([len(row.cells) for row in table.rows]))
|
|
47
49
|
if len(lengths) != 1:
|
|
48
50
|
return None, senfd.errors.IrregularTableError(
|
|
49
|
-
f"Varying row lengths({lengths})", lengths
|
|
51
|
+
message=f"Varying row lengths({lengths})", lengths=lengths
|
|
50
52
|
)
|
|
51
53
|
|
|
52
54
|
if len(table.rows) < 2:
|
|
53
|
-
return None, senfd.errors.NonTableHeaderError(
|
|
55
|
+
return None, senfd.errors.NonTableHeaderError(
|
|
56
|
+
message="Insufficent number of rows"
|
|
57
|
+
)
|
|
54
58
|
|
|
55
59
|
headers = [
|
|
56
60
|
cell.text.strip()
|
|
@@ -59,9 +63,9 @@ class HeaderTable(Table):
|
|
|
59
63
|
]
|
|
60
64
|
if len(headers) != len(table.rows[1].cells):
|
|
61
65
|
return None, senfd.errors.TableHeaderError(
|
|
62
|
-
"Unsupported names",
|
|
63
|
-
table.rows[0].cells[0].text,
|
|
64
|
-
[cell.text for cell in table.rows[1].cells],
|
|
66
|
+
message="Unsupported names",
|
|
67
|
+
caption=table.rows[0].cells[0].text,
|
|
68
|
+
cells=[cell.text for cell in table.rows[1].cells],
|
|
65
69
|
)
|
|
66
70
|
|
|
67
71
|
data = table.dict()
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.2.0"
|
senfd-0.2.0/src/senfd/errors.py
DELETED
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from collections import namedtuple
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
from typing import Any, Dict, List, NamedTuple
|
|
5
|
-
|
|
6
|
-
from pydantic import BaseModel
|
|
7
|
-
|
|
8
|
-
from senfd.documents.base import to_file
|
|
9
|
-
|
|
10
|
-
TableOfFiguresError = namedtuple("TableOfFiguresError", ["message", "caption"])
|
|
11
|
-
|
|
12
|
-
TableCaptionError = namedtuple("TableCaptionError", ["message", "caption"])
|
|
13
|
-
|
|
14
|
-
TableHeaderError = namedtuple("TableHeaderError", ["message", "caption", "cells"])
|
|
15
|
-
|
|
16
|
-
IrregularTableError = namedtuple("IrregularTableError", ["message", "lengths"])
|
|
17
|
-
|
|
18
|
-
NonTableHeaderError = namedtuple("NonTableHeaderError", ["message"])
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def error_to_dict(error: NamedTuple) -> Dict[str, List[Any]]:
|
|
22
|
-
error_dict = error._asdict()
|
|
23
|
-
for key, value in error_dict.items():
|
|
24
|
-
if isinstance(value, BaseModel):
|
|
25
|
-
error_dict[key] = value.dict()
|
|
26
|
-
return error_dict
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def to_log_file(errors: List[NamedTuple], filename: str, output: Path) -> Path:
|
|
30
|
-
|
|
31
|
-
content = json.dumps(
|
|
32
|
-
[{"type": type(error).__name__, **error_to_dict(error)} for error in errors],
|
|
33
|
-
indent=4,
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
return to_file(content, f"{filename}.error.log", output)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|