prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,569 @@
1
+ """Cypher value types, as far as the SQL translator can know them.
2
+
3
+ SQL coerces where Cypher does not: `1 = '1'` is true on DuckDB and false in
4
+ Cypher, `size()` of a string is its length and of a list its size, and
5
+ `7 / 2` between integers is 3. The translator cannot get these right from
6
+ the expression alone; it needs the operand types. They come from literals,
7
+ from the data source's column types (recorded on the schema by the
8
+ engine), and from what operators and functions return. Anything not known
9
+ is UNKNOWN, and the translator then does what it did before.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import datetime
15
+ import decimal
16
+ import math
17
+ import re
18
+ from dataclasses import dataclass, field, replace
19
+ from typing import Any, Dict, Optional, Tuple
20
+ from prograph.rules import home
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class CypherType:
25
+ """A value type. `element` is a list's element type."""
26
+ kind: str
27
+ element: Optional['CypherType'] = None
28
+ # A DECIMAL: a float to Cypher, and equal to FLOAT, but its SQL type is
29
+ # its own and the text must not turn it into a double. Set by a DECIMAL
30
+ # column; kept by + - * / % with an integer or a decimal, a unary
31
+ # minus, sum, min, max, avg and a name a WITH gives it; lost to
32
+ # toFloat() and to arithmetic with a double (ExprTranslator._static_type).
33
+ exact: bool = field(default=False, compare=False)
34
+ # A fixed-length CHAR(n) column: SQL holds its value padded with spaces to n (R2RML 10.2: the
35
+ # literal is that value), which a cast to text drops on PostgreSQL.
36
+ padded: bool = field(default=False, compare=False)
37
+ # A 32-bit float column (REAL, FLOAT4, DuckDB's FLOAT, Spark's float): its value is the double its
38
+ # shortest text names (70.22, not 70.22000122070312: R2RML 10.2 and W3C R2RMLTC0016b).
39
+ single: bool = field(default=False, compare=False)
40
+ # A uuid column (PostgreSQL uuid, DuckDB UUID, SQL Server uniqueidentifier): a string whose text is the
41
+ # database's own (rule uuid-text); read through UUID_TEXT, compared on the native column.
42
+ uuid: bool = field(default=False, compare=False)
43
+ # A STRING column whose SQL equality is the equality of its bytes: its engine and collation compare
44
+ # strings exactly (`compares_bytes`). False where unknown.
45
+ bytewise: bool = field(default=False, compare=False)
46
+ # A Spark or Databricks string under a collation that is not binary (`string collate UTF8_LCASE`):
47
+ # its comparisons and string functions follow the collation, and Spark refuses RLIKE on it. A plan
48
+ # reads it under the binary collation (reader BINARY_TEXT): SPARQL and Cypher compare code points.
49
+ collated: bool = field(default=False, compare=False)
50
+ # A column of a database type with no natural RDF datatype (jsonb, inet, interval, money, an enum,
51
+ # xml, a struct, a map, an array, Spark's void): R2RML 10.2 gives its values a plain literal of the
52
+ # database's own text, so a plan reads it through a cast to text (reader CAST_TEXT, `entry._held`).
53
+ textual: bool = field(default=False, compare=False)
54
+ # A number's declared SQL size, (precision, scale), where the data
55
+ # source gives it: a DECIMAL(15, 2) column, a HUGEINT (39, 0). What
56
+ # chooses a decimal quotient's SQL form reads it
57
+ # (ExprTranslator.decimal_digits).
58
+ digits: Optional[Tuple[int, int]] = field(default=None, compare=False)
59
+ # A DATETIME that is an instant, written with a zone: a DuckDB (and
60
+ # PostgreSQL, Oracle) TIMESTAMP WITH TIME ZONE, a pandas column with a
61
+ # tz. Equal to DATETIME; its text is written in UTC with its offset,
62
+ # `Z` (ExprTranslator._as_text).
63
+ zoned: bool = field(default=False, compare=False)
64
+ # A date-time type's fractional-second digits where its name states them (TIMESTAMP(3), DuckDB's
65
+ # TIMESTAMP_MS and TIMESTAMP_NS); None for the engine's default. Two date-time columns of one zone and
66
+ # one precision hold equal values as one text (rewrite._one_text, R13).
67
+ precision: Optional[int] = field(default=None, compare=False)
68
+ # An INTEGER the database holds in 32 bits or fewer: an INT, SMALLINT
69
+ # or TINYINT column (an int32 frame column). Equal to INTEGER, whose
70
+ # Cypher and SPARQL value is 64-bit; +, - and * read such an operand as
71
+ # a BIGINT, so the result is not held to 32 bits
72
+ # (ExprTranslator._held_wide_enough; rules.md, "An integer constant is
73
+ # 64-bit"). A sum of it is not narrow.
74
+ narrow: bool = field(default=False, compare=False)
75
+ # A map whose fields are known: (name, type) for every key it may
76
+ # have, the type UNKNOWN where it is not known (`map_of`). The one
77
+ # record of a map's fields (rules.md, "A field of a map"):
78
+ # - `exact_keys`: the names are the exact key set of the struct the
79
+ # SQL builds (a map literal, properties() of a node or relationship),
80
+ # so a key it does not have reads as null and not as a field the
81
+ # database refuses. A map joined over several (a
82
+ # list, a CASE, a coalesce: `common`) has its keys' types but not one
83
+ # struct's key set.
84
+ # - `null_is_absent`: a null field is a key the map does not have, as
85
+ # in properties() of a node (a struct has the same fields on
86
+ # every row) and not in a map literal, whose null value is a key.
87
+ fields: Optional[Tuple[Tuple[str, 'CypherType'], ...]] = field(default=None, compare=False)
88
+ exact_keys: bool = field(default=False, compare=False)
89
+ null_is_absent: bool = field(default=False, compare=False)
90
+
91
+ @property
92
+ def is_number(self) -> bool:
93
+ return self.kind in ('integer', 'float')
94
+
95
+ def field_type(self, key: str) -> Optional['CypherType']:
96
+ """The type of field `key` of a map whose fields are known: NULL
97
+ for a key it does not have; None where they are not known."""
98
+ if self.fields is None:
99
+ return None
100
+ return dict(self.fields).get(key, NULL)
101
+
102
+ @property
103
+ def struct_keys(self) -> Optional[Tuple[str, ...]]:
104
+ """The exact key set of the struct the map is built as, where it is
105
+ known (`exact_keys`); None otherwise."""
106
+ if self.fields is None or not self.exact_keys:
107
+ return None
108
+ return tuple(name for name, _ in self.fields)
109
+
110
+ def family(self) -> Optional[str]:
111
+ """The family values compare within: numbers with numbers, and
112
+ every other known kind with itself alone. A date, a date-time and a
113
+ time are each a family of their own, so a date compared with a
114
+ string or a number is a comparison across types, which openCypher
115
+ never makes (`=` false, `<>` true, an ordering null; TCK
116
+ Comparison2 [3], Comparison1 [9]). They had none, and SQL cast the
117
+ other side: `n.seen > '1985'` was a cast error and `n.d <
118
+ '1985-01-01'` compared as dates."""
119
+ if self.kind in ('integer', 'float'):
120
+ return 'number'
121
+ if self.kind in ('unknown', 'null'):
122
+ return None
123
+ return self.kind
124
+
125
+ def __repr__(self) -> str:
126
+ return f"LIST<{self.element!r}>" if self.kind == 'list' else self.kind.upper()
127
+
128
+
129
+ INTEGER = CypherType('integer')
130
+ # An integer held in 32 bits or fewer (`CypherType.narrow`).
131
+ NARROW_INTEGER = CypherType('integer', narrow=True)
132
+ FLOAT = CypherType('float')
133
+ DECIMAL = CypherType('float', exact=True)
134
+ STRING = CypherType('string')
135
+ BOOLEAN = CypherType('boolean')
136
+ MAP = CypherType('map')
137
+ DATE = CypherType('date')
138
+ DATETIME = CypherType('datetime')
139
+ ZONED_DATETIME = CypherType('datetime', zoned=True)
140
+ # A time of day, a TIME column or a Python `datetime.time`.
141
+ TIME = CypherType('time')
142
+ # A TIME WITH TIME ZONE (TIMETZ): a time and its offset; `zoned` (not part of equality) tells it apart.
143
+ ZONED_TIME = CypherType('time', zoned=True)
144
+ # A duration: months, days, seconds and nanoseconds.
145
+ DURATION = CypherType('duration')
146
+ # Cypher temporals no engine column type holds exactly, each a struct of its integer parts (`temporal_struct`,
147
+ # rule cypher-source-temporal): a date-time at an offset or in a named zone, a local date-time, a time
148
+ # with its offset and a local time, each to the nanosecond.
149
+ DATETIME_STRUCT = CypherType('datetime_struct')
150
+ LOCALDATETIME_STRUCT = CypherType('localdatetime_struct')
151
+ TIME_STRUCT = CypherType('time_struct')
152
+ LOCALTIME_STRUCT = CypherType('localtime_struct')
153
+ # Bytes: a BLOB, BYTEA, BINARY or VARBINARY column, xsd:hexBinary (R2RML 10.2).
154
+ BINARY = CypherType('binary')
155
+ NULL = CypherType('null')
156
+ UNKNOWN = CypherType('unknown')
157
+ # A column of a type with no natural datatype, read as the database's text (`CypherType.textual`).
158
+ TEXTUAL = CypherType('string', textual=True)
159
+
160
+
161
+ @home("missing-values", "Missing values",
162
+ terms=("is_missing", "NaN", "None", "NaT", "pd.NA", "null"))
163
+ def is_missing(value: Any) -> bool:
164
+ """Whether a value is a missing value: None, a float NaN (numpy's
165
+ included), pandas' NaT and NA. A frame holds a missing cell as one of
166
+ these; a query reads it as null. Lists, maps and strings are never
167
+ missing themselves. The home of the rule (docs/architecture/rules.md);
168
+ the other copies are still to be folded in."""
169
+ if value is None:
170
+ return True
171
+ if isinstance(value, float):
172
+ return math.isnan(value)
173
+ if isinstance(value, (str, bytes, list, tuple, dict, set)):
174
+ return False
175
+ try:
176
+ import pandas as pd
177
+ except ImportError: # pragma: no cover - pandas is a dependency
178
+ return False
179
+ try:
180
+ return bool(pd.isna(value))
181
+ except (TypeError, ValueError):
182
+ return False
183
+
184
+
185
+ def missing_mask(column: Any) -> Any:
186
+ """:func:`is_missing` over a pandas column, as a boolean mask: the rows
187
+ holding None, NaN, NaT or pd.NA. The column form of the rule."""
188
+ return column.isna()
189
+
190
+
191
+ def list_of(element: CypherType) -> CypherType:
192
+ return CypherType('list', element)
193
+
194
+
195
+ def map_of(fields: Dict[str, CypherType], exact_keys: bool = True,
196
+ null_is_absent: bool = False) -> CypherType:
197
+ """A map whose fields are known: each key's type (`CypherType.fields`),
198
+ and whether they are the struct's exact key set (`exact_keys`)."""
199
+ return CypherType('map', fields=tuple(fields.items()), exact_keys=exact_keys,
200
+ null_is_absent=null_is_absent and exact_keys)
201
+
202
+
203
+ def _join_maps(maps) -> CypherType:
204
+ """The one map type several maps share: a field's type is the type its
205
+ values share, a map without the key giving null there, which does not
206
+ count. A field whose values are of different kinds, an integer beside a
207
+ float included, has no type: a struct field holds one SQL type, and a
208
+ FLOAT or an INTEGER claimed for it would be wrong for some rows. Where
209
+ any map's fields are not known, none are."""
210
+ if any(m.fields is None for m in maps):
211
+ return MAP
212
+ keys = list(dict.fromkeys(k for m in maps for k, _ in m.fields))
213
+ joined = {}
214
+ for key in keys:
215
+ present = [t for t in (m.field_type(key) for m in maps) if t.kind != 'null']
216
+ if not present:
217
+ joined[key] = NULL
218
+ elif all(t.kind in ('map', 'list') for t in present):
219
+ joined[key] = common(present)
220
+ elif all(type(t) is type(present[0]) and t == present[0]
221
+ and t.exact == present[0].exact for t in present):
222
+ joined[key] = present[0]
223
+ else:
224
+ joined[key] = UNKNOWN
225
+ # Several structs' keys joined are no one struct's key set.
226
+ return map_of(joined, exact_keys=False)
227
+
228
+
229
+ def fit_one_array(types) -> bool:
230
+ """Whether values of these types fit one SQL array, which has one
231
+ element type: at most one family among the types that are known, and,
232
+ where they are lists, element types that fit one array in turn. A
233
+ Cypher list may hold anything, `[1, 'a', [2]]`; such a list has no SQL
234
+ array form."""
235
+ known = [t for t in types if t.kind not in ('unknown', 'null')]
236
+ if len({'number' if t.is_number else t.kind for t in known}) > 1:
237
+ return False
238
+ inner = [t.element for t in known if t.kind == 'list' and t.element is not None]
239
+ return fit_one_array(inner) if inner else True
240
+
241
+
242
+ def mixes_integer_and_float(types) -> bool:
243
+ """Whether a list of these element types holds an integer beside a
244
+ float, itself or inside its lists. A SQL array of them is an array of
245
+ DOUBLE, so `[1, 2.5]` comes back `[1.0, 2.5]`: right where the list is
246
+ only tested (1 = 1.0), a different value where its elements are part
247
+ of the result."""
248
+ kinds = {t.kind for t in types}
249
+ if 'integer' in kinds and 'float' in kinds:
250
+ return True
251
+ inner = [t.element for t in types if t.kind == 'list' and t.element is not None]
252
+ return mixes_integer_and_float(inner) if inner else False
253
+
254
+
255
+ def of_value(value: Any) -> CypherType:
256
+ """The type of a Python value, as a literal or a column sample."""
257
+ if value is None or (isinstance(value, float) and math.isnan(value)):
258
+ return NULL
259
+ if isinstance(value, bool):
260
+ return BOOLEAN
261
+ if isinstance(value, int):
262
+ return INTEGER
263
+ if isinstance(value, float):
264
+ return FLOAT
265
+ if getattr(value, 'datatype', None) is not None and isinstance(value, str):
266
+ return UNKNOWN # a literal with no Python value (prograph.schema.xsd.Lexical)
267
+ if isinstance(value, str):
268
+ return STRING
269
+ if isinstance(value, datetime.datetime) or hasattr(value, 'to_pydatetime'):
270
+ return DATETIME
271
+ if isinstance(value, datetime.date):
272
+ return DATE
273
+ if isinstance(value, datetime.time):
274
+ return TIME
275
+ if isinstance(value, decimal.Decimal):
276
+ # An exact number, as a DECIMAL column's (a frame column of
277
+ # Decimals is one to DuckDB).
278
+ return DECIMAL
279
+ if isinstance(value, dict):
280
+ return MAP
281
+ if isinstance(value, (list, tuple)):
282
+ return list_of(common([of_value(v) for v in value]))
283
+ if hasattr(value, 'tolist'):
284
+ return of_value(value.tolist())
285
+ if hasattr(value, 'item'):
286
+ return of_value(value.item())
287
+ return UNKNOWN
288
+
289
+
290
+ def differ(types, numbers_widen: bool = False) -> bool:
291
+ """Whether values of these types have no one SQL type: two known types
292
+ that are not one (an integer beside a float included), or maps whose
293
+ one key has two such types, or lists whose elements do. A struct field
294
+ holds one SQL type, and the database casts one side to the other's (a
295
+ UNION of `{key: '1'}` and `{key: 1}` gave '1' twice) or refuses. Unknown types and nulls do not count.
296
+
297
+ `numbers_widen`: numbers of different kinds share the type `common`
298
+ gives them, a float or a decimal, and do not differ, however deep (a
299
+ list's elements, a map's fields): a Cypher union of an integer and a
300
+ float is a float, as the Python executor's answer is."""
301
+ known = [t for t in types if t.kind not in ('unknown', 'null')]
302
+ if numbers_widen and len(known) > 1 and all(t.is_number for t in known):
303
+ return False
304
+ if numbers_widen and len(known) > 1 and all(t.kind == 'list' for t in known):
305
+ return differ([t.element or NULL for t in known], numbers_widen)
306
+ if len(set(known)) > 1:
307
+ return True
308
+ if len(known) > 1 and known[0].kind == 'list':
309
+ return differ([t.element or NULL for t in known], numbers_widen)
310
+ maps = [t for t in known if t.kind == 'map' and t.fields is not None]
311
+ if len(maps) > 1:
312
+ keys = dict.fromkeys(k for m in maps for k, _ in m.fields)
313
+ return any(differ([m.field_type(k) for m in maps], numbers_widen) for k in keys)
314
+ return False
315
+
316
+
317
+ def common(types) -> CypherType:
318
+ """The one type several values share: nulls do not count, integer and
319
+ float give float, anything else mixed is UNKNOWN."""
320
+ kinds = [t for t in types if t.kind != 'null']
321
+ if not kinds:
322
+ return NULL
323
+ first = kinds[0]
324
+ if any(t.exact for t in kinds) and all(t.is_number for t in kinds):
325
+ # A decimal among integers and decimals is a decimal; beside a
326
+ # double it is a double (DECIMAL equals FLOAT, so it is read here).
327
+ return DECIMAL if all(t.exact for t in kinds if t.kind == 'float') else FLOAT
328
+ if all(t.kind == 'map' for t in kinds) and any(t.fields is not None for t in kinds):
329
+ # Maps compare equal whatever their fields: the fields' types are
330
+ # joined, and the result is no one struct's key set, so one map's
331
+ # fields never stand for another's.
332
+ return _join_maps(kinds)
333
+ if first.kind == 'string' and all(t.kind == 'string' for t in kinds):
334
+ # Texts: the digits each claims (`ExprTranslator.text_size`)
335
+ # joined, the widest whole part and scale; a text that claims none
336
+ # known (None) leaves none known.
337
+ claims = [t.digits for t in kinds]
338
+ if None in claims:
339
+ return STRING
340
+ from dataclasses import replace as _replace
341
+ return _replace(STRING, digits=(max(w for w, _ in claims), max(s for _, s in claims)))
342
+ if all(t == first for t in kinds):
343
+ return first
344
+ if all(t.is_number for t in kinds):
345
+ return FLOAT
346
+ if all(t.kind == 'list' for t in kinds):
347
+ # Lists share the type their elements share: an empty list and a
348
+ # list of nulls have no element type and do not count, as a null
349
+ # does not among values. One empty list in a column used to leave
350
+ # the whole column UNKNOWN.
351
+ element = common([t.element or NULL for t in kinds])
352
+ return list_of(element) if element != UNKNOWN else UNKNOWN
353
+ return UNKNOWN
354
+
355
+
356
+ def _precision(t: str) -> Optional[int]:
357
+ """A date-time type name's fractional-second digits (`CypherType.precision`): TIMESTAMP(3),
358
+ DATETIME2(7), DuckDB's TIMESTAMP_S, _MS and _NS; None where the name states none."""
359
+ unit = {'TIMESTAMP_S': 0, 'TIMESTAMP_MS': 3, 'TIMESTAMP_NS': 9}.get(t)
360
+ if unit is not None:
361
+ return unit
362
+ found = re.match(r'(?:TIMESTAMP|DATETIME2?)\s*\(\s*(\d+)\s*\)', t)
363
+ return int(found.group(1)) if found else None
364
+
365
+
366
+ _INTEGER_FIELD = ('BIGINT', 'LONG', 'INTEGER', 'INT')
367
+ _TEMPORAL_STRUCTS = (
368
+ (('MONTHS', 'DAYS', 'SECONDS', 'NANOSECONDS'), DURATION),
369
+ (('EPOCH_SECONDS', 'NANOSECONDS', 'OFFSET_SECONDS', 'ZONE'), DATETIME_STRUCT),
370
+ (('LOCAL_SECONDS', 'NANOSECONDS'), LOCALDATETIME_STRUCT),
371
+ (('NANOSECONDS_OF_DAY', 'OFFSET_SECONDS'), TIME_STRUCT),
372
+ (('NANOSECONDS_OF_DAY',), LOCALTIME_STRUCT),
373
+ )
374
+
375
+
376
+ def temporal_struct_fields(kind: CypherType) -> Tuple[str, ...]:
377
+ """The fields, lower-cased and in order, of the struct `temporal_struct` reads as `kind`."""
378
+ return next(tuple(n.lower() for n in names) for names, k in _TEMPORAL_STRUCTS if k == kind)
379
+
380
+
381
+ def temporal_struct(t: str) -> Optional[CypherType]:
382
+ """The Cypher temporal an (upper-cased) SQL struct type holds by its fields, exactly these in this order,
383
+ integers but a date-time's ZONE (text, null for an offset alone) (rule cypher-source-temporal):
384
+ months, days, seconds, nanoseconds a duration; epoch_seconds, nanoseconds, offset_seconds, zone a
385
+ date-time; local_seconds, nanoseconds a local date-time; nanoseconds_of_day, offset_seconds a time;
386
+ nanoseconds_of_day a local time. None for any other type. DuckDB writes `STRUCT("MONTHS" BIGINT, ...)`,
387
+ Spark `STRUCT<MONTHS:BIGINT,...>`."""
388
+ m = re.fullmatch(r'STRUCT\s*[(<](.*)[)>]', t.strip())
389
+ if m is None:
390
+ return None
391
+ fields = [re.split(r'[\s:]+', f.strip(), maxsplit=1) for f in m.group(1).split(',')]
392
+ if any(len(f) != 2 for f in fields):
393
+ return None
394
+ names = tuple(f[0].strip('"`') for f in fields)
395
+ for wanted, kind in _TEMPORAL_STRUCTS:
396
+ if names == wanted and all(f[1].strip() in (('VARCHAR', 'STRING', 'TEXT') if n == 'ZONE' else _INTEGER_FIELD)
397
+ for n, f in zip(names, fields)):
398
+ return kind
399
+ return None
400
+
401
+
402
+ # The database types known to have no natural XSD datatype (R2RML 10.2), by name as the catalogs write them: a
403
+ # column of one is a plain literal of the database's text. A name not here and not typed above stays UNKNOWN
404
+ # (a column nobody has typed is refused by name, never guessed to be text). PostgreSQL's catalog calls an
405
+ # enum, a composite and an extension's type USER-DEFINED; Spark's all-null column is VOID.
406
+ _TEXTUAL_NAMES = frozenset({
407
+ 'JSON', 'JSONB', 'INET', 'CIDR', 'MACADDR', 'MACADDR8', 'MONEY', 'XML', 'XMLTYPE', 'TSVECTOR', 'TSQUERY',
408
+ 'POINT', 'LINE', 'LSEG', 'BOX', 'PATH', 'POLYGON', 'CIRCLE', 'USER-DEFINED', 'VOID', 'VARIANT', 'GEOMETRY',
409
+ 'GEOGRAPHY', 'HIERARCHYID', 'SQL_VARIANT', 'SET', 'INTERVAL', 'PG_LSN', 'TXID_SNAPSHOT', 'OID'})
410
+ _TEXTUAL_PREFIXES = ('INTERVAL', 'ENUM', 'UNION', 'STRUCT', 'MAP')
411
+
412
+
413
+ @home("no-natural-datatype-text", "A column of a type with no natural RDF datatype",
414
+ terms=("no natural datatype", "textual column", "jsonb", "inet", "interval", "struct column", "all-null column",
415
+ "CAST_TEXT"))
416
+ def of_sql_type(sql_type: str, timestamp_is_an_instant: bool = False) -> CypherType:
417
+ """A column's type from a database type name (DuckDB, Spark, SQL).
418
+
419
+ `timestamp_is_an_instant`: the database's bare TIMESTAMP is an instant
420
+ shown in the session's zone (Spark and Databricks, whose type with no
421
+ zone is TIMESTAMP_NTZ), so it is read as zoned and its
422
+ text is its instant in UTC with `Z`. Elsewhere (DuckDB,
423
+ PostgreSQL) a bare TIMESTAMP has no zone."""
424
+ t = sql_type.upper().strip()
425
+ if t.endswith('[]'):
426
+ return list_of(of_sql_type(t[:-2], timestamp_is_an_instant))
427
+ if t.startswith('ARRAY<') and t.endswith('>'):
428
+ return list_of(of_sql_type(t[6:-1], timestamp_is_an_instant))
429
+ if t in ('MEDIUMINT', 'YEAR'):
430
+ return NARROW_INTEGER # MySQL's 24-bit integer and its year
431
+ # DuckDB's BIGNUM (VARINT) is not typed: its values reach the plan as the engine's binary form, which
432
+ # no integer reader decodes, so such a column is refused by name, never guessed.
433
+ if t in ('HUGEINT', 'UHUGEINT', 'UBIGINT'):
434
+ # Wider than 64 bits: a value can have 39 (or 20) digits.
435
+ return CypherType('integer', digits=(20 if t == 'UBIGINT' else 39, 0))
436
+ if t in ('TINYINT', 'SMALLINT', 'INTEGER', 'INT', 'SHORT', 'BYTE', 'UTINYINT',
437
+ 'USMALLINT', 'UINTEGER', 'INT32'):
438
+ return NARROW_INTEGER
439
+ if t in ('BIGINT', 'LONG', 'INT64'):
440
+ return INTEGER
441
+ if t.startswith('DECIMAL') or t.startswith('NUMERIC') or t.startswith('NUMBER'):
442
+ sized = re.fullmatch(r'(?:DECIMAL|NUMERIC)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)', t)
443
+ if sized:
444
+ return CypherType('float', exact=True, digits=(int(sized.group(1)), int(sized.group(2))))
445
+ return DECIMAL
446
+ if t in ('BLOB', 'BYTEA', 'BINARY', 'VARBINARY', 'LONGBLOB', 'RAW') or t.startswith('VARBINARY') \
447
+ or t.startswith('BINARY(') or t.startswith('BLOB') \
448
+ or t in ('IMAGE', 'TINYBLOB', 'MEDIUMBLOB'):
449
+ return BINARY
450
+ if t in ('REAL', 'FLOAT4', 'FLOAT'):
451
+ # 32 bits on DuckDB, Spark, Databricks and PostgreSQL (REAL); SQL Server's FLOAT is 64 bits,
452
+ # and there the dialect reads the column as it is (FLOAT32_VALUE).
453
+ return replace(FLOAT, single=True)
454
+ if t in ('FLOAT', 'DOUBLE', 'REAL', 'FLOAT4', 'FLOAT8', 'DOUBLE PRECISION',
455
+ 'BINARY_DOUBLE', 'BINARY_FLOAT'):
456
+ # Oracle's NUMBER with a scale; one that holds integers only is
457
+ # reported as INTEGER (oracle_graph.number_is_integer). Oracle's
458
+ # own float types, BINARY_DOUBLE and BINARY_FLOAT, were missing:
459
+ # a column of one had no type, and nothing that chooses a SQL form
460
+ # by a float's type did so on a connected Oracle.
461
+ return FLOAT
462
+ if t in ('CHAR', 'CHARACTER', 'BPCHAR', 'NCHAR') or re.fullmatch(r'N?CHAR(ACTER)?\s*\(\s*\d+\s*\)', t):
463
+ return replace(STRING, padded=True)
464
+ # A Databricks or Spark string under a collation (`string collate UTF8_LCASE_RTRIM`) is a string: its
465
+ # values are text whatever the collation. Whether it compares bytes is the collation's question
466
+ # (`collations.compares_bytes`, read from the same type text); it was left untyped, and a key
467
+ # column of one could not be built into an IRI at all.
468
+ collation = re.search(r'\s+COLLATE\s+([A-Za-z0-9_.]+)$', t)
469
+ t = re.sub(r'\s+COLLATE\s+[A-Za-z0-9_.]+$', '', t)
470
+ if t in ('TINYTEXT', 'MEDIUMTEXT', 'LONGTEXT', 'NTEXT', 'CLOB', 'NCLOB', 'LONG', 'ROWID', 'UROWID'):
471
+ return STRING # MySQL's, SQL Server's and Oracle's long text and Oracle's row addresses
472
+ if t in ('VARCHAR', 'STRING', 'TEXT') or t.startswith('VARCHAR') \
473
+ or t.startswith('CHAR') or t.startswith('NVARCHAR'):
474
+ if collation is not None and collation.group(1).upper() != 'UTF8_BINARY':
475
+ return replace(STRING, collated=True)
476
+ return STRING
477
+ if t in ('UUID', 'UNIQUEIDENTIFIER'):
478
+ # A uuid has no XSD datatype: a plain string of the database's own text (R2RML 10.2), lower case
479
+ # 8-4-4-4-12 on PostgreSQL and DuckDB, upper case on SQL Server (rule uuid-text).
480
+ return replace(STRING, uuid=True)
481
+ if t == 'CITEXT':
482
+ # PostgreSQL's case-insensitive text (the citext extension): its values are text, compared
483
+ # without case, so it is read as text under the binary collation (reader BINARY_TEXT).
484
+ return replace(STRING, collated=True)
485
+ bits = re.fullmatch(r'BIT\s*\(\s*(\d+)\s*\)', t)
486
+ if bits is not None or t in ('BIT VARYING', 'VARBIT') \
487
+ or t.startswith('BIT VARYING') or t.startswith('VARBIT'):
488
+ return TEXTUAL # a bit string is not a boolean: its database text
489
+ if t in ('BOOLEAN', 'BOOL', 'BIT'):
490
+ return BOOLEAN
491
+ if t == 'DATE':
492
+ return DATE
493
+ if t.startswith('TIMESTAMP') and (t.endswith('WITH TIME ZONE') or t == 'TIMESTAMPTZ'):
494
+ # DuckDB's TIMESTAMPTZ, PostgreSQL's and Oracle's TIMESTAMP WITH
495
+ # TIME ZONE: an instant.
496
+ return replace(ZONED_DATETIME, precision=_precision(t))
497
+ if t == 'TIMESTAMP_LTZ' or (timestamp_is_an_instant and t == 'TIMESTAMP'):
498
+ # Spark's and Databricks' TIMESTAMP (TIMESTAMP_LTZ): an instant;
499
+ # their TIMESTAMP_NTZ is a local date-time.
500
+ return ZONED_DATETIME
501
+ if t == 'SMALLDATETIME':
502
+ return DATETIME
503
+ if t == 'SMALLMONEY':
504
+ return CypherType('float', exact=True, digits=(10, 4))
505
+ if t.startswith('TIMESTAMP') or t.startswith('DATETIME'):
506
+ return replace(DATETIME, precision=_precision(t))
507
+ if t == 'TIMETZ' or (t.startswith('TIME ') and 'WITH TIME ZONE' in t and 'WITHOUT' not in t):
508
+ return ZONED_TIME
509
+ if t == 'TIME' or t.startswith('TIME '):
510
+ return TIME
511
+ held = temporal_struct(t)
512
+ if held is not None:
513
+ return held
514
+ if t.startswith('STRUCT') or t.startswith('MAP') or t == 'JSON' or t == 'JSONB':
515
+ return MAP
516
+ if t in _TEXTUAL_NAMES or t.startswith(_TEXTUAL_PREFIXES):
517
+ return TEXTUAL
518
+ return UNKNOWN
519
+
520
+
521
+ def of_frame(frame) -> Dict[str, CypherType]:
522
+ """Each column's type in a pandas DataFrame. An object column is typed
523
+ by its non-null values when they share one; an integer column with a
524
+ NULL, which pandas holds as float64, is still a float here (the values
525
+ alone cannot say otherwise). A float column with no value at all is
526
+ UNKNOWN: float64 is what pandas gives an empty column
527
+ (`pd.DataFrame({'id': []})`), so it says nothing about what the column
528
+ will hold, and a text key written to such a table was refused as "not a
529
+ float". A frame of no rows types such a column STRING: no value can
530
+ contradict it, and its maps give no term, so a query over every map (a
531
+ whole-graph pattern) is built rather than refused over a column it never
532
+ reads a value of. An object column whose every row is None is typed STRING
533
+ for the same reason: it holds no value, so its maps give no term (an
534
+ OPTIONAL over it is unbound, a join on it has no row)."""
535
+ out: Dict[str, CypherType] = {}
536
+ for column in frame.columns:
537
+ kind = frame[column].dtype.kind
538
+ arrow = getattr(frame[column].dtype, 'pyarrow_dtype', None)
539
+ if arrow is not None and str(arrow).startswith('date'):
540
+ # An Arrow date (date32[pyarrow], date64) reports kind 'M', as datetime64 does.
541
+ out[column] = DATE
542
+ continue
543
+ if kind in 'iu':
544
+ out[column] = NARROW_INTEGER if frame[column].dtype.itemsize < 8 else INTEGER
545
+ elif kind == 'f':
546
+ out[column] = FLOAT if frame[column].notna().any() else (STRING if len(frame) == 0 else UNKNOWN)
547
+ elif kind == 'b':
548
+ out[column] = BOOLEAN
549
+ elif kind == 'M':
550
+ zoned = getattr(frame[column].dtype, 'tz', None) is not None
551
+ out[column] = ZONED_DATETIME if zoned else DATETIME
552
+ elif kind in 'OSU':
553
+ sample = [of_value(v) for v in frame[column].dropna().head(200)]
554
+ out[column] = common(sample) if sample else STRING
555
+ else:
556
+ name = str(frame[column].dtype)
557
+ if name in ('Int8', 'Int16', 'Int32', 'UInt8', 'UInt16', 'UInt32'):
558
+ out[column] = NARROW_INTEGER
559
+ elif name in ('Int64', 'UInt64'):
560
+ out[column] = INTEGER
561
+ elif name in ('Float32', 'Float64'):
562
+ out[column] = FLOAT
563
+ elif name == 'boolean':
564
+ out[column] = BOOLEAN
565
+ elif name in ('string', 'str'):
566
+ out[column] = STRING
567
+ else:
568
+ out[column] = UNKNOWN
569
+ return out