flatdata-generator 0.4.10__tar.gz → 0.4.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/.gitignore +0 -4
  2. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/PKG-INFO +39 -9
  3. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/README.md +36 -7
  4. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/app.py +34 -13
  5. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/engine.py +37 -16
  6. flatdata_generator-0.4.12/flatdata/generator/generators/__init__.py +109 -0
  7. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/generators/cpp.py +32 -13
  8. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/generators/dot.py +15 -6
  9. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/generators/flatdata.py +12 -6
  10. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/generators/python.py +27 -15
  11. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/generators/rust.py +83 -14
  12. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/grammar.py +20 -7
  13. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/cpp/cpp.jinja2 +3 -0
  14. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/py/python.jinja2 +1 -0
  15. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/rust.jinja2 +6 -1
  16. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/builder.py +129 -37
  17. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/errors.py +68 -26
  18. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/helpers/basictype.py +13 -13
  19. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/helpers/enumtype.py +5 -5
  20. flatdata_generator-0.4.12/flatdata/generator/tree/importer.py +127 -0
  21. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/archive.py +14 -8
  22. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/explicit_reference.py +9 -5
  23. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/node.py +98 -42
  24. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/references.py +16 -10
  25. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/resources/archive.py +10 -4
  26. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/resources/base.py +22 -13
  27. flatdata_generator-0.4.12/flatdata/generator/tree/nodes/resources/bound_resource.py +28 -0
  28. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/resources/instance.py +10 -4
  29. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/resources/multivector.py +18 -11
  30. flatdata_generator-0.4.12/flatdata/generator/tree/nodes/resources/rawdata.py +18 -0
  31. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/resources/vector.py +9 -3
  32. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/root.py +1 -1
  33. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/trivial/constant.py +14 -6
  34. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/trivial/enumeration.py +22 -17
  35. flatdata_generator-0.4.12/flatdata/generator/tree/nodes/trivial/enumeration_value.py +32 -0
  36. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/trivial/field.py +24 -15
  37. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/trivial/namespace.py +5 -1
  38. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/trivial/structure.py +15 -9
  39. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/resolver.py +13 -6
  40. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/syntax_tree.py +60 -22
  41. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/traversal.py +39 -23
  42. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/pyproject.toml +21 -2
  43. flatdata_generator-0.4.10/flatdata/generator/generators/__init__.py +0 -86
  44. flatdata_generator-0.4.10/flatdata/generator/generators/go.py +0 -95
  45. flatdata_generator-0.4.10/flatdata/generator/templates/go/archive.jinja2 +0 -176
  46. flatdata_generator-0.4.10/flatdata/generator/templates/go/base.jinja2 +0 -33
  47. flatdata_generator-0.4.10/flatdata/generator/templates/go/go.jinja2 +0 -35
  48. flatdata_generator-0.4.10/flatdata/generator/templates/go/instance.jinja2 +0 -41
  49. flatdata_generator-0.4.10/flatdata/generator/templates/go/multivector.jinja2 +0 -93
  50. flatdata_generator-0.4.10/flatdata/generator/templates/go/rawdata.jinja2 +0 -38
  51. flatdata_generator-0.4.10/flatdata/generator/templates/go/structure.jinja2 +0 -39
  52. flatdata_generator-0.4.10/flatdata/generator/templates/go/vector.jinja2 +0 -61
  53. flatdata_generator-0.4.10/flatdata/generator/tree/nodes/resources/bound_resource.py +0 -22
  54. flatdata_generator-0.4.10/flatdata/generator/tree/nodes/resources/rawdata.py +0 -13
  55. flatdata_generator-0.4.10/flatdata/generator/tree/nodes/trivial/enumeration_value.py +0 -24
  56. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/__init__.py +0 -0
  57. /flatdata_generator-0.4.10/flatdata/generator/templates/__init__.py → /flatdata_generator-0.4.12/flatdata/generator/py.typed +0 -0
  58. {flatdata_generator-0.4.10/flatdata/generator/templates/cpp → flatdata_generator-0.4.12/flatdata/generator/templates}/__init__.py +0 -0
  59. {flatdata_generator-0.4.10/flatdata/generator/templates/dot → flatdata_generator-0.4.12/flatdata/generator/templates/cpp}/__init__.py +0 -0
  60. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/cpp/archive.jinja2 +0 -0
  61. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/cpp/constant.jinja2 +0 -0
  62. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/cpp/enumeration.jinja2 +0 -0
  63. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/cpp/namespace.jinja2 +0 -0
  64. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/cpp/resource.jinja2 +0 -0
  65. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/cpp/structure.jinja2 +0 -0
  66. {flatdata_generator-0.4.10/flatdata/generator/templates/flatdata → flatdata_generator-0.4.12/flatdata/generator/templates/dot}/__init__.py +0 -0
  67. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/archive.jinja2 +0 -0
  68. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/dot.jinja2 +0 -0
  69. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/namespace.jinja2 +0 -0
  70. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/references.jinja2 +0 -0
  71. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/render.jinja2 +0 -0
  72. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/resource.jinja2 +0 -0
  73. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/structure.jinja2 +0 -0
  74. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/dot/styles.jinja2 +0 -0
  75. {flatdata_generator-0.4.10/flatdata/generator/templates/go → flatdata_generator-0.4.12/flatdata/generator/templates/flatdata}/__init__.py +0 -0
  76. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/flatdata/archive.jinja2 +0 -0
  77. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/flatdata/constant.jinja2 +0 -0
  78. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/flatdata/enumeration.jinja2 +0 -0
  79. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/flatdata/flatdata.jinja2 +0 -0
  80. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/flatdata/namespace.jinja2 +0 -0
  81. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/flatdata/resource.jinja2 +0 -0
  82. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/flatdata/structure.jinja2 +0 -0
  83. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/py/__init__.py +0 -0
  84. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/__init__.py +0 -0
  85. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/archive.jinja2 +0 -0
  86. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/constant.jinja2 +0 -0
  87. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/enumeration.jinja2 +0 -0
  88. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/index.jinja2 +0 -0
  89. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/structure.jinja2 +0 -0
  90. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/templates/rust/variadic.jinja2 +0 -0
  91. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/__init__.py +0 -0
  92. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/helpers/__init__.py +0 -0
  93. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/__init__.py +0 -0
  94. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/resources/__init__.py +0 -0
  95. {flatdata_generator-0.4.10 → flatdata_generator-0.4.12}/flatdata/generator/tree/nodes/trivial/__init__.py +0 -0
@@ -4,10 +4,6 @@ build
4
4
  venv*
5
5
  **/.vscode/**
6
6
  **/.idea/**
7
- **/*_generated.go
8
- **/coverage.out
9
- **/flatdata-fuzz.zip
10
- **/corpus/**
11
7
  **/dist/**
12
8
  *.egg-info
13
9
  .DS_Store
@@ -1,12 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flatdata-generator
3
- Version: 0.4.10
4
- Summary: Generate source code for C++, Rust, Go or Python from a Flatdata schema file
3
+ Version: 0.4.12
4
+ Summary: Generate source code for C++, Rust, or Python from a Flatdata schema file
5
5
  Project-URL: Homepage, https://github.com/heremaps/flatdata
6
6
  Author: Flatdata Developers
7
7
  Classifier: License :: OSI Approved :: Apache Software License
8
8
  Classifier: Operating System :: OS Independent
9
9
  Classifier: Programming Language :: Python :: 3
10
+ Requires-Python: >=3.8
10
11
  Requires-Dist: jinja2>=2.2
11
12
  Requires-Dist: pyparsing>=2.0
12
13
  Description-Content-Type: text/markdown
@@ -29,12 +30,31 @@ pip3 install flatdata-generator
29
30
  flatdata-generator -s locations.flatdata -g cpp -O locations.hpp
30
31
  ```
31
32
 
33
+ ### Multi-file Schemas
34
+
35
+ When a schema uses `import` statements, each file should be generated
36
+ separately. Imported types are referenced via include/import directives rather
37
+ than being re-emitted:
38
+
39
+ ```sh
40
+ # Generate shared types
41
+ flatdata-generator -s schema/types.flatdata -g cpp -O schema/types.h
42
+
43
+ # Generate main schema (will #include "types.h")
44
+ flatdata-generator -s schema/main.flatdata -g cpp -O schema/main.h
45
+ ```
46
+
47
+ For Rust, the same approach applies — each imported file becomes its own module
48
+ with `pub use` re-exports connecting the namespaces.
49
+
50
+ Python and Dot generators emit all types monolithically (no separate generation
51
+ needed for the root file — all imported definitions are included in the output).
52
+
32
53
  Currently supported target languages:
33
54
 
34
55
  * C++
35
56
  * Rust
36
57
  * Python
37
- * Go
38
58
  * Dot (graph of the schema)
39
59
  * Flatdata (normalized stable schema)
40
60
 
@@ -44,9 +64,14 @@ Currently supported target languages:
44
64
 
45
65
  The `flatdata` generator works in several stages which are clearly separated from one another and can be extended/tested in isolation:
46
66
 
47
- 1. **Parse the source schema** file using `pyparsing` library. Grammar
67
+ 1. **Resolve imports** starting from the root schema file. The importer
68
+ (`importer.py`) performs a depth-first traversal of import statements,
69
+ deduplicating files and handling cyclic imports. The result is an ordered
70
+ list of resolved files with their parsed content.
71
+
72
+ 2. **Parse the source schema** file using `pyparsing` library. Grammar
48
73
  for the schema is defined in `grammar.py`
49
- 2. **Construct a node tree** out of `pyparsing.ParseResults`. The node tree
74
+ 3. **Construct a node tree** out of `pyparsing.ParseResults`. The node tree
50
75
  contains entities for every construct of flatdata grammar, organized
51
76
  in hierarchical order, allowing non-tree references between nodes:
52
77
 
@@ -63,7 +88,7 @@ The `flatdata` generator works in several stages which are clearly separated fro
63
88
  - `TypeReference` - model type dependencies, which are used during
64
89
  topological sorting at a later stage and for schema resolution.
65
90
 
66
- 3. **Augment the tree** with structures and references that are not
91
+ 4. **Augment the tree** with structures and references that are not
67
92
  directly corresponding to `pyparsing.ParseResults` or needed to
68
93
  implement advanced features. Among these:
69
94
 
@@ -73,17 +98,17 @@ The `flatdata` generator works in several stages which are clearly separated fro
73
98
  - **Add constant references** to all archives so that constants are
74
99
  available for schema resolution.
75
100
 
76
- 4. **Resolve references** iterates through all references and tries to
101
+ 5. **Resolve references** iterates through all references and tries to
77
102
  find a node they refer to, either in:
78
103
 
79
104
  - Parent scopes until (inclusive) innermost parent namespace.
80
105
  - Root node if path is fully qualified.
81
106
 
82
- 5. **Perform topological sorting** to detect cycles in between entities
107
+ 6. **Perform topological sorting** to detect cycles in between entities
83
108
  and to determine the order of serialization for targets that depend
84
109
  on one.
85
110
 
86
- 6. **Generate the source code** using nodes in topological order *and/or*
111
+ 7. **Generate the source code** using nodes in topological order *and/or*
87
112
  the tree (depending on the generator architecture - recursive descent
88
113
  or iterative).
89
114
 
@@ -101,6 +126,11 @@ Node tree enforces several properties of the flatdata schema:
101
126
  participate in topological sorting of the DAG formed by the tree
102
127
  edges and edges between source and target of a `TypeReference`
103
128
 
129
+ When building a tree from multiple files, each node is tagged with its
130
+ `source_file` (the file it was defined in) and an `is_local` flag
131
+ (whether it belongs to the root file being generated). This allows
132
+ generators to filter nodes for separate compilation.
133
+
104
134
  ### References
105
135
 
106
136
  Reference names are mangled so they are not ambiguous with other paths
@@ -16,12 +16,31 @@ pip3 install flatdata-generator
16
16
  flatdata-generator -s locations.flatdata -g cpp -O locations.hpp
17
17
  ```
18
18
 
19
+ ### Multi-file Schemas
20
+
21
+ When a schema uses `import` statements, each file should be generated
22
+ separately. Imported types are referenced via include/import directives rather
23
+ than being re-emitted:
24
+
25
+ ```sh
26
+ # Generate shared types
27
+ flatdata-generator -s schema/types.flatdata -g cpp -O schema/types.h
28
+
29
+ # Generate main schema (will #include "types.h")
30
+ flatdata-generator -s schema/main.flatdata -g cpp -O schema/main.h
31
+ ```
32
+
33
+ For Rust, the same approach applies — each imported file becomes its own module
34
+ with `pub use` re-exports connecting the namespaces.
35
+
36
+ Python and Dot generators emit all types monolithically (no separate generation
37
+ needed for the root file — all imported definitions are included in the output).
38
+
19
39
  Currently supported target languages:
20
40
 
21
41
  * C++
22
42
  * Rust
23
43
  * Python
24
- * Go
25
44
  * Dot (graph of the schema)
26
45
  * Flatdata (normalized stable schema)
27
46
 
@@ -31,9 +50,14 @@ Currently supported target languages:
31
50
 
32
51
  The `flatdata` generator works in several stages which are clearly separated from one another and can be extended/tested in isolation:
33
52
 
34
- 1. **Parse the source schema** file using `pyparsing` library. Grammar
53
+ 1. **Resolve imports** starting from the root schema file. The importer
54
+ (`importer.py`) performs a depth-first traversal of import statements,
55
+ deduplicating files and handling cyclic imports. The result is an ordered
56
+ list of resolved files with their parsed content.
57
+
58
+ 2. **Parse the source schema** file using `pyparsing` library. Grammar
35
59
  for the schema is defined in `grammar.py`
36
- 2. **Construct a node tree** out of `pyparsing.ParseResults`. The node tree
60
+ 3. **Construct a node tree** out of `pyparsing.ParseResults`. The node tree
37
61
  contains entities for every construct of flatdata grammar, organized
38
62
  in hierarchical order, allowing non-tree references between nodes:
39
63
 
@@ -50,7 +74,7 @@ The `flatdata` generator works in several stages which are clearly separated fro
50
74
  - `TypeReference` - model type dependencies, which are used during
51
75
  topological sorting at a later stage and for schema resolution.
52
76
 
53
- 3. **Augment the tree** with structures and references that are not
77
+ 4. **Augment the tree** with structures and references that are not
54
78
  directly corresponding to `pyparsing.ParseResults` or needed to
55
79
  implement advanced features. Among these:
56
80
 
@@ -60,17 +84,17 @@ The `flatdata` generator works in several stages which are clearly separated fro
60
84
  - **Add constant references** to all archives so that constants are
61
85
  available for schema resolution.
62
86
 
63
- 4. **Resolve references** iterates through all references and tries to
87
+ 5. **Resolve references** iterates through all references and tries to
64
88
  find a node they refer to, either in:
65
89
 
66
90
  - Parent scopes until (inclusive) innermost parent namespace.
67
91
  - Root node if path is fully qualified.
68
92
 
69
- 5. **Perform topological sorting** to detect cycles in between entities
93
+ 6. **Perform topological sorting** to detect cycles in between entities
70
94
  and to determine the order of serialization for targets that depend
71
95
  on one.
72
96
 
73
- 6. **Generate the source code** using nodes in topological order *and/or*
97
+ 7. **Generate the source code** using nodes in topological order *and/or*
74
98
  the tree (depending on the generator architecture - recursive descent
75
99
  or iterative).
76
100
 
@@ -88,6 +112,11 @@ Node tree enforces several properties of the flatdata schema:
88
112
  participate in topological sorting of the DAG formed by the tree
89
113
  edges and edges between source and target of a `TypeReference`
90
114
 
115
+ When building a tree from multiple files, each node is tagged with its
116
+ `source_file` (the file it was defined in) and an `is_local` flag
117
+ (whether it belongs to the root file being generated). This allows
118
+ generators to filter nodes for separate compilation.
119
+
91
120
  ### References
92
121
 
93
122
  Reference names are mangled so they are not ambiguous with other paths
@@ -19,9 +19,10 @@ except ModuleNotFoundError as exc:
19
19
 
20
20
  from flatdata.generator.engine import Engine
21
21
  from flatdata.generator.tree.errors import FlatdataSyntaxError
22
+ from flatdata.generator.tree.syntax_tree import SyntaxTree
22
23
 
23
24
 
24
- def _parse_command_line():
25
+ def _parse_command_line() -> argparse.Namespace:
25
26
  parser = argparse.ArgumentParser(
26
27
  description="Generates code for a given flatdata schema file.")
27
28
  parser.add_argument("-s", "--schema", type=str, required=True,
@@ -32,6 +33,8 @@ def _parse_command_line():
32
33
  parser.add_argument("-O", "--output-file", type=str, required=True,
33
34
  default=None,
34
35
  help="Destination file. Forces all output to be stored in one file")
36
+ parser.add_argument("-d", "--depfile", type=str, default=None,
37
+ help="Write a Makefile-style dependency file listing all imported schemas")
35
38
  parser.add_argument("-v", "--verbose", action="store_true",
36
39
  help="Enable verbose mode")
37
40
  parser.add_argument("--debug", action="store_true",
@@ -39,7 +42,7 @@ def _parse_command_line():
39
42
  return parser.parse_args()
40
43
 
41
44
 
42
- def _setup_logging(args):
45
+ def _setup_logging(args: argparse.Namespace) -> None:
43
46
  level = logging.WARNING
44
47
  if args.debug:
45
48
  level = logging.DEBUG
@@ -52,24 +55,22 @@ def _setup_logging(args):
52
55
  level=level)
53
56
 
54
57
 
55
- def _check_args(args):
58
+ def _check_args(args: argparse.Namespace) -> None:
56
59
  if not os.path.isfile(args.schema):
57
60
  logging.fatal("Cannot find schema file at %s", args.schema)
58
61
  sys.exit(1)
59
62
 
60
63
 
61
- def _run(args):
64
+ def _run(args: argparse.Namespace) -> None:
62
65
  _setup_logging(args)
63
66
  _check_args(args)
64
67
 
65
- with open(args.schema, 'r') as input_file:
66
- schema = input_file.read()
67
- try:
68
- engine = Engine(schema)
69
- logging.debug("Tree: %s", engine.tree)
70
- except FlatdataSyntaxError as ex:
71
- logging.fatal("Error reading schema: %s ", ex)
72
- sys.exit(1)
68
+ try:
69
+ engine = Engine.from_file(args.schema)
70
+ logging.debug("Tree: %s", engine.tree)
71
+ except FlatdataSyntaxError as ex:
72
+ logging.fatal("Error reading schema: %s ", ex)
73
+ sys.exit(1)
73
74
 
74
75
  try:
75
76
  logging.info("Generating %s...", args.gen)
@@ -85,7 +86,27 @@ def _run(args):
85
86
  output.write(output_content)
86
87
  logging.info("Code for %s is written to %s", args.gen, args.output_file)
87
88
 
89
+ if args.depfile:
90
+ _write_depfile(args.depfile, args.output_file, args.schema, engine.tree)
91
+
92
+
93
+ def _write_depfile(depfile_path: str, output_file: str, schema_file: str,
94
+ tree: 'SyntaxTree') -> None:
95
+ """Write a Makefile-style depfile listing all schema dependencies."""
96
+ deps = [os.path.abspath(schema_file)]
97
+ # source_file_map keys are absolute paths of all imported files
98
+ deps.extend(sorted(tree.source_file_map.keys()))
99
+
100
+ # Escape spaces in paths for Make syntax
101
+ def escape(p: str) -> str:
102
+ return p.replace(" ", "\\ ")
103
+
104
+ dep_str = " ".join(escape(d) for d in deps)
105
+ with open(depfile_path, "w") as f:
106
+ f.write(f"{escape(output_file)}: {dep_str}\n")
107
+ logging.info("Depfile written to %s", depfile_path)
108
+
88
109
 
89
- def main():
110
+ def main() -> None:
90
111
  """Entrypoint"""
91
112
  _run(_parse_command_line())
@@ -3,18 +3,22 @@
3
3
  See the LICENSE file in the root of this project for license details.
4
4
  '''
5
5
 
6
+ from __future__ import annotations
7
+
6
8
  import types
9
+ from typing import overload
7
10
 
8
- from flatdata.generator.tree.builder import build_ast
11
+ from flatdata.generator.tree.builder import build_ast, build_ast_from_file
9
12
  from flatdata.generator.tree.nodes.trivial.namespace import Namespace
10
13
  from flatdata.generator.tree.nodes.node import Node
14
+ from flatdata.generator.tree.syntax_tree import SyntaxTree
11
15
 
12
16
  from .generators.cpp import CppGenerator
13
17
  from .generators.dot import DotGenerator
14
- from .generators.go import GoGenerator
15
18
  from .generators.python import PythonGenerator
16
19
  from .generators.rust import RustGenerator
17
20
  from .generators.flatdata import FlatdataGenerator
21
+ from .generators import BaseGenerator
18
22
 
19
23
 
20
24
  class Engine:
@@ -23,23 +27,33 @@ class Engine:
23
27
  Implements code generation from the given flatdata schema.
24
28
  """
25
29
 
26
- _GENERATORS = {
30
+ _GENERATORS: dict[str, type[BaseGenerator]] = {
27
31
  "cpp": CppGenerator,
28
32
  "dot": DotGenerator,
29
- "go": GoGenerator,
30
33
  "py": PythonGenerator,
31
34
  "rust": RustGenerator,
32
35
  "flatdata" : FlatdataGenerator
33
36
  }
34
37
 
35
38
  @classmethod
36
- def available_generators(cls):
39
+ def available_generators(cls) -> list[str]:
37
40
  """
38
41
  Lists names of available code generators.
39
42
  """
40
43
  return list(cls._GENERATORS.keys())
41
44
 
42
- def __init__(self, schema):
45
+ @classmethod
46
+ def from_file(cls, path: str) -> 'Engine':
47
+ """
48
+ Create Engine from a schema file, resolving imports.
49
+ :raises FlatdataSyntaxError
50
+ """
51
+ engine = cls.__new__(cls)
52
+ engine.tree = build_ast_from_file(path)
53
+ engine.schema = engine.tree.root_schema or ""
54
+ return engine
55
+
56
+ def __init__(self, schema: str) -> None:
43
57
  """
44
58
  Instantiates generator engine for a given schema.
45
59
  :raises FlatdataSyntaxError
@@ -47,7 +61,7 @@ class Engine:
47
61
  self.schema = schema
48
62
  self.tree = build_ast(schema)
49
63
 
50
- def render(self, generator_name):
64
+ def render(self, generator_name: str) -> str:
51
65
  """
52
66
  Render schema with a given generator
53
67
  :param generator_name:
@@ -60,9 +74,16 @@ class Engine:
60
74
  )
61
75
 
62
76
  output_content = generator.render(self.tree)
63
- return output_content
77
+ return str(output_content)
78
+
79
+ @overload
80
+ def render_python_module(self, module_name: str | None, archive_name: str, root_namespace: str | None = None) -> tuple[types.ModuleType, type]: ...
81
+ @overload
82
+ def render_python_module(self, *, archive_name: str, root_namespace: str | None = None) -> tuple[types.ModuleType, type]: ...
83
+ @overload
84
+ def render_python_module(self, module_name: str | None = None, archive_name: None = None, root_namespace: str | None = None) -> types.ModuleType: ...
64
85
 
65
- def render_python_module(self, module_name=None, archive_name=None, root_namespace=None):
86
+ def render_python_module(self, module_name: str | None = None, archive_name: str | None = None, root_namespace: str | None = None) -> types.ModuleType | tuple[types.ModuleType, type]:
66
87
  """
67
88
  Render python module.
68
89
  :param module_name: Module name to use. If none, root namespace name is used.
@@ -70,28 +91,28 @@ class Engine:
70
91
  if specified, archive type is returned along with the model
71
92
  :param root_namespace: Root namespace to pick in case of multiple top level namespaces.
72
93
  """
73
- root_namespace = self._find_root_namespace(self.tree, archive_name, root_namespace)
94
+ ns = self._find_root_namespace(self.tree, archive_name, root_namespace)
74
95
  module_code = self.render("py")
75
- module = types.ModuleType(module_name if module_name is not None else root_namespace.name)
96
+ module = types.ModuleType(module_name if module_name is not None else ns.name)
76
97
  #pylint: disable=exec-used
77
98
  exec(module_code, module.__dict__)
78
99
  if archive_name is None:
79
100
  return module
80
101
 
81
- name = root_namespace.name + "_" + archive_name
82
- archive_type = getattr(module, name) if archive_name else None
102
+ name = ns.name + "_" + archive_name
103
+ archive_type = getattr(module, name)
83
104
  return module, archive_type
84
105
 
85
106
  @classmethod
86
- def _create_generator(cls, name):
107
+ def _create_generator(cls, name: str) -> BaseGenerator | None:
87
108
  generator_type = cls._GENERATORS.get(name, None)
88
109
  if generator_type is None:
89
110
  return None
90
111
 
91
- return generator_type()
112
+ return generator_type() # type: ignore[call-arg] # dict values are concrete subclasses with zero-arg __init__
92
113
 
93
114
  @staticmethod
94
- def _find_root_namespace(tree, archive_name, root_namespace=None):
115
+ def _find_root_namespace(tree: SyntaxTree, archive_name: str | None, root_namespace: str | None = None) -> Namespace:
95
116
  root_children = tree.root.children
96
117
  root_namespaces = [
97
118
  child for child in root_children
@@ -0,0 +1,109 @@
1
+ '''
2
+ Copyright (c) 2017 HERE Europe B.V.
3
+ See the LICENSE file in the root of this project for license details.
4
+ '''
5
+
6
+ from __future__ import annotations
7
+
8
+ from abc import ABCMeta, abstractmethod
9
+ from typing import NoReturn
10
+
11
+ from jinja2 import Environment, PackageLoader
12
+ from jinja2 import nodes
13
+ from jinja2.ext import Extension
14
+ from jinja2.exceptions import TemplateRuntimeError
15
+ from jinja2.parser import Parser
16
+
17
+ from flatdata.generator.tree.nodes.archive import Archive
18
+ from flatdata.generator.tree.nodes.node import Node
19
+ from flatdata.generator.tree.nodes.trivial import Structure, Enumeration, Constant, Namespace
20
+ from flatdata.generator.tree.nodes.references import EnumerationReference
21
+ from flatdata.generator.tree.nodes.resources import ResourceBase, BoundResource, Archive as \
22
+ ArchiveResource, Vector, Multivector, Instance, RawData
23
+ from flatdata.generator.tree.syntax_tree import SyntaxTree
24
+ from flatdata.generator.tree.traversal import DfsTraversal
25
+
26
+
27
+ class BaseGenerator(metaclass=ABCMeta):
28
+ """Abstract base class for Flatdata generators"""
29
+
30
+ def __init__(self, template: str) -> None:
31
+ self._template = template
32
+ self._env: Environment | None = None
33
+
34
+ @abstractmethod
35
+ def supported_nodes(self) -> list[type]:
36
+ """List of supported nodes by this generator"""
37
+ raise RuntimeError(
38
+ "Derived generators must implement _supported_nodes")
39
+
40
+ @abstractmethod
41
+ def _populate_environment(self, env: Environment, tree: SyntaxTree) -> None:
42
+ raise RuntimeError(
43
+ "Derived generators must implement _populate_filters")
44
+
45
+ def filter_nodes(self, nodes: list[Node], tree: SyntaxTree) -> list[Node]:
46
+ """Filter nodes for rendering. Override for separate compilation."""
47
+ return nodes
48
+
49
+ def get_import_directives(self, tree: SyntaxTree) -> list[str]:
50
+ """Return language-specific import directives. Override in subclasses."""
51
+ return []
52
+
53
+ def _get_environment(self, tree: SyntaxTree) -> Environment:
54
+ if self._env is None:
55
+ env = Environment(loader=PackageLoader('flatdata.generator', 'templates'), lstrip_blocks=True,
56
+ trim_blocks=True, autoescape=False, extensions=[RaiseExtension])
57
+ env.filters['is_archive'] = lambda n: isinstance(n, Archive)
58
+ env.filters['is_instance'] = lambda n: isinstance(n, Instance)
59
+ env.filters['is_raw_data'] = lambda n: isinstance(n, RawData)
60
+ env.filters['is_archive_resource'] = lambda n: isinstance(
61
+ n, ArchiveResource)
62
+ env.filters['is_structure'] = lambda n: isinstance(n, Structure)
63
+ env.filters['is_enumeration_reference'] = lambda n: isinstance(n, EnumerationReference)
64
+ env.filters['is_enumeration'] = lambda n: isinstance(n, Enumeration)
65
+ env.filters['is_constant'] = lambda n: isinstance(n, Constant)
66
+ env.filters['is_namespace'] = lambda n: isinstance(n, Namespace)
67
+ env.filters['is_resource'] = lambda n: isinstance(n, ResourceBase)
68
+ env.filters['is_bound_resource'] = lambda n: isinstance(
69
+ n, BoundResource)
70
+ env.filters['is_vector'] = lambda n: isinstance(n, Vector)
71
+ env.filters['is_multivector'] = lambda n: isinstance(n, Multivector)
72
+ env.filters['is_multivector_index'] = lambda n: (isinstance(
73
+ n, Structure) and "_builtin.multivector" in SyntaxTree.namespace_path(n))
74
+ env.filters['namespaces'] = SyntaxTree.namespaces
75
+ env.filters['not_auto_generated'] = lambda n: [ x for x in n if not x.auto_generated]
76
+ self._populate_environment(env, tree)
77
+ self._env = env
78
+ return self._env
79
+
80
+ def render(self, tree: SyntaxTree) -> str:
81
+ """Generate the language implementation from the AST"""
82
+ env = self._get_environment(tree)
83
+ template = env.get_template(self._template)
84
+
85
+ flatdata_nodes = [n for n, _ in DfsTraversal(tree).dependency_order() if
86
+ any([isinstance(n, t) for t in self.supported_nodes()])]
87
+ filtered_nodes = self.filter_nodes(flatdata_nodes, tree)
88
+ imports = self.get_import_directives(tree)
89
+ return template.render(nodes=filtered_nodes, tree=tree, imports=imports)
90
+
91
+
92
+ class RaiseExtension(Extension):
93
+ """Nicer error formatting for jinja2"""
94
+
95
+ tags = set(['raise'])
96
+
97
+ def parse(self, parser: Parser) -> nodes.CallBlock:
98
+ """The first token is the line number, followed by the expression"""
99
+ lineno = next(parser.stream).lineno
100
+ message_node = parser.parse_expression()
101
+ return nodes.CallBlock(
102
+ self.call_method(name='_raise', args=[message_node], lineno=lineno), [], [], [],
103
+ lineno=lineno
104
+ )
105
+
106
+ #pylint: disable=no-self-use
107
+ def _raise(self, msg: str, caller: object) -> NoReturn:
108
+ """Helper callback."""
109
+ raise TemplateRuntimeError(msg)
@@ -3,31 +3,50 @@
3
3
  See the LICENSE file in the root of this project for license details.
4
4
  '''
5
5
 
6
+ from __future__ import annotations
7
+
8
+ import posixpath
9
+
10
+ from jinja2 import Environment
11
+
12
+ from flatdata.generator.tree.helpers.basictype import BasicType
13
+ from flatdata.generator.tree.helpers.enumtype import EnumType
14
+ from flatdata.generator.tree.nodes.node import Node
15
+ from flatdata.generator.tree.nodes.references import BuiltinStructureReference, StructureReference
6
16
  from flatdata.generator.tree.nodes.resources import Vector, Multivector, Instance, RawData, BoundResource, \
7
17
  ResourceBase, Archive as ArchiveResource
8
18
  from flatdata.generator.tree.nodes.trivial import Structure, Enumeration, Constant, Field
9
19
  from flatdata.generator.tree.nodes.archive import Archive
20
+ from flatdata.generator.tree.syntax_tree import SyntaxTree
10
21
  from . import BaseGenerator
11
22
 
12
23
 
13
24
  class CppGenerator(BaseGenerator):
14
25
  """Flatdata to C++ header file generator"""
15
26
 
16
- def __init__(self):
27
+ def __init__(self) -> None:
17
28
  BaseGenerator.__init__(self, "cpp/cpp.jinja2")
18
29
 
19
- def supported_nodes(self):
30
+ def supported_nodes(self) -> list[type]:
20
31
  return [Structure, Archive, Constant, Enumeration]
21
32
 
22
- def _populate_environment(self, env):
33
+ def filter_nodes(self, nodes: list[Node], tree: SyntaxTree) -> list[Node]:
34
+ if not tree.imports:
35
+ return nodes
36
+ return [n for n in nodes if tree.is_local_node(n)]
37
+
38
+ def get_import_directives(self, tree: SyntaxTree) -> list[str]:
39
+ return [posixpath.normpath(imp.path).replace('.flatdata', '.h') for imp in tree.imports]
40
+
41
+ def _populate_environment(self, env: Environment, tree: SyntaxTree) -> None:
23
42
  env.filters["cpp_doc"] = lambda value: value
24
43
 
25
- def _safe_cpp_string_line(value):
44
+ def _safe_cpp_string_line(value: str) -> str:
26
45
  return value.replace('\\', '\\\\').replace('"', r'\"')
27
46
 
28
47
  env.filters["safe_cpp_string_line"] = _safe_cpp_string_line
29
48
 
30
- def _cpp_base_type(flatdata_type):
49
+ def _cpp_base_type(flatdata_type: BasicType | EnumType | Node) -> str:
31
50
  type_map = {
32
51
  "bool": "bool",
33
52
  "i8": "int8_t",
@@ -41,28 +60,28 @@ class CppGenerator(BaseGenerator):
41
60
  }
42
61
  if flatdata_type.name in type_map:
43
62
  return type_map[flatdata_type.name]
44
- return flatdata_type.name.replace("@@", "::").replace("@", "::")
63
+ return str(flatdata_type.name.replace("@@", "::").replace("@", "::"))
45
64
 
46
65
  env.filters["cpp_base_type"] = _cpp_base_type
47
66
 
48
- def _to_type_params(refs):
67
+ def _to_type_params(refs: list[BuiltinStructureReference | StructureReference]) -> str:
49
68
  return ', '.join([ref.node.path_with("::") for ref in refs])
50
69
 
51
70
  env.filters["to_type_params"] = _to_type_params
52
71
 
53
- def _snake_to_upper_camel_case(expr):
72
+ def _snake_to_upper_camel_case(expr: str) -> str:
54
73
  return ''.join(p.title() for p in expr.split('_'))
55
74
 
56
75
  env.filters["snake_to_upper_camel_case"] = _snake_to_upper_camel_case
57
76
 
58
- def _typedef_name(entity, extra_suffix=""):
77
+ def _typedef_name(entity: Field | ResourceBase, extra_suffix: str = "") -> str:
59
78
  assert isinstance(entity, (Field, ResourceBase)), "Got: %s" % entity.__class__
60
79
  return _snake_to_upper_camel_case(entity.name) + extra_suffix + "Type"
61
80
 
62
81
  env.filters["typedef_name"] = _typedef_name
63
82
 
64
- def _optional_typedef_usage(resource, extra_suffix=""):
65
- def _wrap_in_optional(declaration):
83
+ def _optional_typedef_usage(resource: ResourceBase, extra_suffix: str = "") -> str:
84
+ def _wrap_in_optional(declaration: str) -> str:
66
85
  if resource.optional:
67
86
  return "boost::optional< %s >" % declaration
68
87
  return declaration
@@ -71,7 +90,7 @@ class CppGenerator(BaseGenerator):
71
90
 
72
91
  env.filters["archive_typedef_usage"] = _optional_typedef_usage
73
92
 
74
- def _resource_provides_incremental_builder(resource):
93
+ def _resource_provides_incremental_builder(resource: ResourceBase) -> bool:
75
94
  assert isinstance(resource, ResourceBase)
76
95
  if isinstance(resource, Instance):
77
96
  return False
@@ -86,7 +105,7 @@ class CppGenerator(BaseGenerator):
86
105
  env.filters[
87
106
  "resource_provides_incremental_builder"] = _resource_provides_incremental_builder
88
107
 
89
- def provides_setter(resource):
108
+ def provides_setter(resource: ResourceBase) -> bool:
90
109
  assert isinstance(resource, ResourceBase)
91
110
  if isinstance(resource, Instance):
92
111
  return True