prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/schema/types.py
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
"""Cypher value types, as far as the SQL translator can know them.
|
|
2
|
+
|
|
3
|
+
SQL coerces where Cypher does not: `1 = '1'` is true on DuckDB and false in
|
|
4
|
+
Cypher, `size()` of a string is its length and of a list its size, and
|
|
5
|
+
`7 / 2` between integers is 3. The translator cannot get these right from
|
|
6
|
+
the expression alone; it needs the operand types. They come from literals,
|
|
7
|
+
from the data source's column types (recorded on the schema by the
|
|
8
|
+
engine), and from what operators and functions return. Anything not known
|
|
9
|
+
is UNKNOWN, and the translator then does what it did before.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import datetime
|
|
15
|
+
import decimal
|
|
16
|
+
import math
|
|
17
|
+
import re
|
|
18
|
+
from dataclasses import dataclass, field, replace
|
|
19
|
+
from typing import Any, Dict, Optional, Tuple
|
|
20
|
+
from prograph.rules import home
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class CypherType:
|
|
25
|
+
"""A value type. `element` is a list's element type."""
|
|
26
|
+
kind: str
|
|
27
|
+
element: Optional['CypherType'] = None
|
|
28
|
+
# A DECIMAL: a float to Cypher, and equal to FLOAT, but its SQL type is
|
|
29
|
+
# its own and the text must not turn it into a double. Set by a DECIMAL
|
|
30
|
+
# column; kept by + - * / % with an integer or a decimal, a unary
|
|
31
|
+
# minus, sum, min, max, avg and a name a WITH gives it; lost to
|
|
32
|
+
# toFloat() and to arithmetic with a double (ExprTranslator._static_type).
|
|
33
|
+
exact: bool = field(default=False, compare=False)
|
|
34
|
+
# A fixed-length CHAR(n) column: SQL holds its value padded with spaces to n (R2RML 10.2: the
|
|
35
|
+
# literal is that value), which a cast to text drops on PostgreSQL.
|
|
36
|
+
padded: bool = field(default=False, compare=False)
|
|
37
|
+
# A 32-bit float column (REAL, FLOAT4, DuckDB's FLOAT, Spark's float): its value is the double its
|
|
38
|
+
# shortest text names (70.22, not 70.22000122070312: R2RML 10.2 and W3C R2RMLTC0016b).
|
|
39
|
+
single: bool = field(default=False, compare=False)
|
|
40
|
+
# A uuid column (PostgreSQL uuid, DuckDB UUID, SQL Server uniqueidentifier): a string whose text is the
|
|
41
|
+
# database's own (rule uuid-text); read through UUID_TEXT, compared on the native column.
|
|
42
|
+
uuid: bool = field(default=False, compare=False)
|
|
43
|
+
# A STRING column whose SQL equality is the equality of its bytes: its engine and collation compare
|
|
44
|
+
# strings exactly (`compares_bytes`). False where unknown.
|
|
45
|
+
bytewise: bool = field(default=False, compare=False)
|
|
46
|
+
# A Spark or Databricks string under a collation that is not binary (`string collate UTF8_LCASE`):
|
|
47
|
+
# its comparisons and string functions follow the collation, and Spark refuses RLIKE on it. A plan
|
|
48
|
+
# reads it under the binary collation (reader BINARY_TEXT): SPARQL and Cypher compare code points.
|
|
49
|
+
collated: bool = field(default=False, compare=False)
|
|
50
|
+
# A column of a database type with no natural RDF datatype (jsonb, inet, interval, money, an enum,
|
|
51
|
+
# xml, a struct, a map, an array, Spark's void): R2RML 10.2 gives its values a plain literal of the
|
|
52
|
+
# database's own text, so a plan reads it through a cast to text (reader CAST_TEXT, `entry._held`).
|
|
53
|
+
textual: bool = field(default=False, compare=False)
|
|
54
|
+
# A number's declared SQL size, (precision, scale), where the data
|
|
55
|
+
# source gives it: a DECIMAL(15, 2) column, a HUGEINT (39, 0). What
|
|
56
|
+
# chooses a decimal quotient's SQL form reads it
|
|
57
|
+
# (ExprTranslator.decimal_digits).
|
|
58
|
+
digits: Optional[Tuple[int, int]] = field(default=None, compare=False)
|
|
59
|
+
# A DATETIME that is an instant, written with a zone: a DuckDB (and
|
|
60
|
+
# PostgreSQL, Oracle) TIMESTAMP WITH TIME ZONE, a pandas column with a
|
|
61
|
+
# tz. Equal to DATETIME; its text is written in UTC with its offset,
|
|
62
|
+
# `Z` (ExprTranslator._as_text).
|
|
63
|
+
zoned: bool = field(default=False, compare=False)
|
|
64
|
+
# A date-time type's fractional-second digits where its name states them (TIMESTAMP(3), DuckDB's
|
|
65
|
+
# TIMESTAMP_MS and TIMESTAMP_NS); None for the engine's default. Two date-time columns of one zone and
|
|
66
|
+
# one precision hold equal values as one text (rewrite._one_text, R13).
|
|
67
|
+
precision: Optional[int] = field(default=None, compare=False)
|
|
68
|
+
# An INTEGER the database holds in 32 bits or fewer: an INT, SMALLINT
|
|
69
|
+
# or TINYINT column (an int32 frame column). Equal to INTEGER, whose
|
|
70
|
+
# Cypher and SPARQL value is 64-bit; +, - and * read such an operand as
|
|
71
|
+
# a BIGINT, so the result is not held to 32 bits
|
|
72
|
+
# (ExprTranslator._held_wide_enough; rules.md, "An integer constant is
|
|
73
|
+
# 64-bit"). A sum of it is not narrow.
|
|
74
|
+
narrow: bool = field(default=False, compare=False)
|
|
75
|
+
# A map whose fields are known: (name, type) for every key it may
|
|
76
|
+
# have, the type UNKNOWN where it is not known (`map_of`). The one
|
|
77
|
+
# record of a map's fields (rules.md, "A field of a map"):
|
|
78
|
+
# - `exact_keys`: the names are the exact key set of the struct the
|
|
79
|
+
# SQL builds (a map literal, properties() of a node or relationship),
|
|
80
|
+
# so a key it does not have reads as null and not as a field the
|
|
81
|
+
# database refuses. A map joined over several (a
|
|
82
|
+
# list, a CASE, a coalesce: `common`) has its keys' types but not one
|
|
83
|
+
# struct's key set.
|
|
84
|
+
# - `null_is_absent`: a null field is a key the map does not have, as
|
|
85
|
+
# in properties() of a node (a struct has the same fields on
|
|
86
|
+
# every row) and not in a map literal, whose null value is a key.
|
|
87
|
+
fields: Optional[Tuple[Tuple[str, 'CypherType'], ...]] = field(default=None, compare=False)
|
|
88
|
+
exact_keys: bool = field(default=False, compare=False)
|
|
89
|
+
null_is_absent: bool = field(default=False, compare=False)
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def is_number(self) -> bool:
|
|
93
|
+
return self.kind in ('integer', 'float')
|
|
94
|
+
|
|
95
|
+
def field_type(self, key: str) -> Optional['CypherType']:
|
|
96
|
+
"""The type of field `key` of a map whose fields are known: NULL
|
|
97
|
+
for a key it does not have; None where they are not known."""
|
|
98
|
+
if self.fields is None:
|
|
99
|
+
return None
|
|
100
|
+
return dict(self.fields).get(key, NULL)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def struct_keys(self) -> Optional[Tuple[str, ...]]:
|
|
104
|
+
"""The exact key set of the struct the map is built as, where it is
|
|
105
|
+
known (`exact_keys`); None otherwise."""
|
|
106
|
+
if self.fields is None or not self.exact_keys:
|
|
107
|
+
return None
|
|
108
|
+
return tuple(name for name, _ in self.fields)
|
|
109
|
+
|
|
110
|
+
def family(self) -> Optional[str]:
|
|
111
|
+
"""The family values compare within: numbers with numbers, and
|
|
112
|
+
every other known kind with itself alone. A date, a date-time and a
|
|
113
|
+
time are each a family of their own, so a date compared with a
|
|
114
|
+
string or a number is a comparison across types, which openCypher
|
|
115
|
+
never makes (`=` false, `<>` true, an ordering null; TCK
|
|
116
|
+
Comparison2 [3], Comparison1 [9]). They had none, and SQL cast the
|
|
117
|
+
other side: `n.seen > '1985'` was a cast error and `n.d <
|
|
118
|
+
'1985-01-01'` compared as dates."""
|
|
119
|
+
if self.kind in ('integer', 'float'):
|
|
120
|
+
return 'number'
|
|
121
|
+
if self.kind in ('unknown', 'null'):
|
|
122
|
+
return None
|
|
123
|
+
return self.kind
|
|
124
|
+
|
|
125
|
+
def __repr__(self) -> str:
|
|
126
|
+
return f"LIST<{self.element!r}>" if self.kind == 'list' else self.kind.upper()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
INTEGER = CypherType('integer')
|
|
130
|
+
# An integer held in 32 bits or fewer (`CypherType.narrow`).
|
|
131
|
+
NARROW_INTEGER = CypherType('integer', narrow=True)
|
|
132
|
+
FLOAT = CypherType('float')
|
|
133
|
+
DECIMAL = CypherType('float', exact=True)
|
|
134
|
+
STRING = CypherType('string')
|
|
135
|
+
BOOLEAN = CypherType('boolean')
|
|
136
|
+
MAP = CypherType('map')
|
|
137
|
+
DATE = CypherType('date')
|
|
138
|
+
DATETIME = CypherType('datetime')
|
|
139
|
+
ZONED_DATETIME = CypherType('datetime', zoned=True)
|
|
140
|
+
# A time of day, a TIME column or a Python `datetime.time`.
|
|
141
|
+
TIME = CypherType('time')
|
|
142
|
+
# A TIME WITH TIME ZONE (TIMETZ): a time and its offset; `zoned` (not part of equality) tells it apart.
|
|
143
|
+
ZONED_TIME = CypherType('time', zoned=True)
|
|
144
|
+
# A duration: months, days, seconds and nanoseconds.
|
|
145
|
+
DURATION = CypherType('duration')
|
|
146
|
+
# Cypher temporals no engine column type holds exactly, each a struct of its integer parts (`temporal_struct`,
|
|
147
|
+
# rule cypher-source-temporal): a date-time at an offset or in a named zone, a local date-time, a time
|
|
148
|
+
# with its offset and a local time, each to the nanosecond.
|
|
149
|
+
DATETIME_STRUCT = CypherType('datetime_struct')
|
|
150
|
+
LOCALDATETIME_STRUCT = CypherType('localdatetime_struct')
|
|
151
|
+
TIME_STRUCT = CypherType('time_struct')
|
|
152
|
+
LOCALTIME_STRUCT = CypherType('localtime_struct')
|
|
153
|
+
# Bytes: a BLOB, BYTEA, BINARY or VARBINARY column, xsd:hexBinary (R2RML 10.2).
|
|
154
|
+
BINARY = CypherType('binary')
|
|
155
|
+
NULL = CypherType('null')
|
|
156
|
+
UNKNOWN = CypherType('unknown')
|
|
157
|
+
# A column of a type with no natural datatype, read as the database's text (`CypherType.textual`).
|
|
158
|
+
TEXTUAL = CypherType('string', textual=True)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@home("missing-values", "Missing values",
|
|
162
|
+
terms=("is_missing", "NaN", "None", "NaT", "pd.NA", "null"))
|
|
163
|
+
def is_missing(value: Any) -> bool:
|
|
164
|
+
"""Whether a value is a missing value: None, a float NaN (numpy's
|
|
165
|
+
included), pandas' NaT and NA. A frame holds a missing cell as one of
|
|
166
|
+
these; a query reads it as null. Lists, maps and strings are never
|
|
167
|
+
missing themselves. The home of the rule (docs/architecture/rules.md);
|
|
168
|
+
the other copies are still to be folded in."""
|
|
169
|
+
if value is None:
|
|
170
|
+
return True
|
|
171
|
+
if isinstance(value, float):
|
|
172
|
+
return math.isnan(value)
|
|
173
|
+
if isinstance(value, (str, bytes, list, tuple, dict, set)):
|
|
174
|
+
return False
|
|
175
|
+
try:
|
|
176
|
+
import pandas as pd
|
|
177
|
+
except ImportError: # pragma: no cover - pandas is a dependency
|
|
178
|
+
return False
|
|
179
|
+
try:
|
|
180
|
+
return bool(pd.isna(value))
|
|
181
|
+
except (TypeError, ValueError):
|
|
182
|
+
return False
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def missing_mask(column: Any) -> Any:
|
|
186
|
+
""":func:`is_missing` over a pandas column, as a boolean mask: the rows
|
|
187
|
+
holding None, NaN, NaT or pd.NA. The column form of the rule."""
|
|
188
|
+
return column.isna()
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def list_of(element: CypherType) -> CypherType:
|
|
192
|
+
return CypherType('list', element)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def map_of(fields: Dict[str, CypherType], exact_keys: bool = True,
|
|
196
|
+
null_is_absent: bool = False) -> CypherType:
|
|
197
|
+
"""A map whose fields are known: each key's type (`CypherType.fields`),
|
|
198
|
+
and whether they are the struct's exact key set (`exact_keys`)."""
|
|
199
|
+
return CypherType('map', fields=tuple(fields.items()), exact_keys=exact_keys,
|
|
200
|
+
null_is_absent=null_is_absent and exact_keys)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _join_maps(maps) -> CypherType:
|
|
204
|
+
"""The one map type several maps share: a field's type is the type its
|
|
205
|
+
values share, a map without the key giving null there, which does not
|
|
206
|
+
count. A field whose values are of different kinds, an integer beside a
|
|
207
|
+
float included, has no type: a struct field holds one SQL type, and a
|
|
208
|
+
FLOAT or an INTEGER claimed for it would be wrong for some rows. Where
|
|
209
|
+
any map's fields are not known, none are."""
|
|
210
|
+
if any(m.fields is None for m in maps):
|
|
211
|
+
return MAP
|
|
212
|
+
keys = list(dict.fromkeys(k for m in maps for k, _ in m.fields))
|
|
213
|
+
joined = {}
|
|
214
|
+
for key in keys:
|
|
215
|
+
present = [t for t in (m.field_type(key) for m in maps) if t.kind != 'null']
|
|
216
|
+
if not present:
|
|
217
|
+
joined[key] = NULL
|
|
218
|
+
elif all(t.kind in ('map', 'list') for t in present):
|
|
219
|
+
joined[key] = common(present)
|
|
220
|
+
elif all(type(t) is type(present[0]) and t == present[0]
|
|
221
|
+
and t.exact == present[0].exact for t in present):
|
|
222
|
+
joined[key] = present[0]
|
|
223
|
+
else:
|
|
224
|
+
joined[key] = UNKNOWN
|
|
225
|
+
# Several structs' keys joined are no one struct's key set.
|
|
226
|
+
return map_of(joined, exact_keys=False)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def fit_one_array(types) -> bool:
|
|
230
|
+
"""Whether values of these types fit one SQL array, which has one
|
|
231
|
+
element type: at most one family among the types that are known, and,
|
|
232
|
+
where they are lists, element types that fit one array in turn. A
|
|
233
|
+
Cypher list may hold anything, `[1, 'a', [2]]`; such a list has no SQL
|
|
234
|
+
array form."""
|
|
235
|
+
known = [t for t in types if t.kind not in ('unknown', 'null')]
|
|
236
|
+
if len({'number' if t.is_number else t.kind for t in known}) > 1:
|
|
237
|
+
return False
|
|
238
|
+
inner = [t.element for t in known if t.kind == 'list' and t.element is not None]
|
|
239
|
+
return fit_one_array(inner) if inner else True
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def mixes_integer_and_float(types) -> bool:
|
|
243
|
+
"""Whether a list of these element types holds an integer beside a
|
|
244
|
+
float, itself or inside its lists. A SQL array of them is an array of
|
|
245
|
+
DOUBLE, so `[1, 2.5]` comes back `[1.0, 2.5]`: right where the list is
|
|
246
|
+
only tested (1 = 1.0), a different value where its elements are part
|
|
247
|
+
of the result."""
|
|
248
|
+
kinds = {t.kind for t in types}
|
|
249
|
+
if 'integer' in kinds and 'float' in kinds:
|
|
250
|
+
return True
|
|
251
|
+
inner = [t.element for t in types if t.kind == 'list' and t.element is not None]
|
|
252
|
+
return mixes_integer_and_float(inner) if inner else False
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def of_value(value: Any) -> CypherType:
|
|
256
|
+
"""The type of a Python value, as a literal or a column sample."""
|
|
257
|
+
if value is None or (isinstance(value, float) and math.isnan(value)):
|
|
258
|
+
return NULL
|
|
259
|
+
if isinstance(value, bool):
|
|
260
|
+
return BOOLEAN
|
|
261
|
+
if isinstance(value, int):
|
|
262
|
+
return INTEGER
|
|
263
|
+
if isinstance(value, float):
|
|
264
|
+
return FLOAT
|
|
265
|
+
if getattr(value, 'datatype', None) is not None and isinstance(value, str):
|
|
266
|
+
return UNKNOWN # a literal with no Python value (prograph.schema.xsd.Lexical)
|
|
267
|
+
if isinstance(value, str):
|
|
268
|
+
return STRING
|
|
269
|
+
if isinstance(value, datetime.datetime) or hasattr(value, 'to_pydatetime'):
|
|
270
|
+
return DATETIME
|
|
271
|
+
if isinstance(value, datetime.date):
|
|
272
|
+
return DATE
|
|
273
|
+
if isinstance(value, datetime.time):
|
|
274
|
+
return TIME
|
|
275
|
+
if isinstance(value, decimal.Decimal):
|
|
276
|
+
# An exact number, as a DECIMAL column's (a frame column of
|
|
277
|
+
# Decimals is one to DuckDB).
|
|
278
|
+
return DECIMAL
|
|
279
|
+
if isinstance(value, dict):
|
|
280
|
+
return MAP
|
|
281
|
+
if isinstance(value, (list, tuple)):
|
|
282
|
+
return list_of(common([of_value(v) for v in value]))
|
|
283
|
+
if hasattr(value, 'tolist'):
|
|
284
|
+
return of_value(value.tolist())
|
|
285
|
+
if hasattr(value, 'item'):
|
|
286
|
+
return of_value(value.item())
|
|
287
|
+
return UNKNOWN
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def differ(types, numbers_widen: bool = False) -> bool:
|
|
291
|
+
"""Whether values of these types have no one SQL type: two known types
|
|
292
|
+
that are not one (an integer beside a float included), or maps whose
|
|
293
|
+
one key has two such types, or lists whose elements do. A struct field
|
|
294
|
+
holds one SQL type, and the database casts one side to the other's (a
|
|
295
|
+
UNION of `{key: '1'}` and `{key: 1}` gave '1' twice) or refuses. Unknown types and nulls do not count.
|
|
296
|
+
|
|
297
|
+
`numbers_widen`: numbers of different kinds share the type `common`
|
|
298
|
+
gives them, a float or a decimal, and do not differ, however deep (a
|
|
299
|
+
list's elements, a map's fields): a Cypher union of an integer and a
|
|
300
|
+
float is a float, as the Python executor's answer is."""
|
|
301
|
+
known = [t for t in types if t.kind not in ('unknown', 'null')]
|
|
302
|
+
if numbers_widen and len(known) > 1 and all(t.is_number for t in known):
|
|
303
|
+
return False
|
|
304
|
+
if numbers_widen and len(known) > 1 and all(t.kind == 'list' for t in known):
|
|
305
|
+
return differ([t.element or NULL for t in known], numbers_widen)
|
|
306
|
+
if len(set(known)) > 1:
|
|
307
|
+
return True
|
|
308
|
+
if len(known) > 1 and known[0].kind == 'list':
|
|
309
|
+
return differ([t.element or NULL for t in known], numbers_widen)
|
|
310
|
+
maps = [t for t in known if t.kind == 'map' and t.fields is not None]
|
|
311
|
+
if len(maps) > 1:
|
|
312
|
+
keys = dict.fromkeys(k for m in maps for k, _ in m.fields)
|
|
313
|
+
return any(differ([m.field_type(k) for m in maps], numbers_widen) for k in keys)
|
|
314
|
+
return False
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def common(types) -> CypherType:
|
|
318
|
+
"""The one type several values share: nulls do not count, integer and
|
|
319
|
+
float give float, anything else mixed is UNKNOWN."""
|
|
320
|
+
kinds = [t for t in types if t.kind != 'null']
|
|
321
|
+
if not kinds:
|
|
322
|
+
return NULL
|
|
323
|
+
first = kinds[0]
|
|
324
|
+
if any(t.exact for t in kinds) and all(t.is_number for t in kinds):
|
|
325
|
+
# A decimal among integers and decimals is a decimal; beside a
|
|
326
|
+
# double it is a double (DECIMAL equals FLOAT, so it is read here).
|
|
327
|
+
return DECIMAL if all(t.exact for t in kinds if t.kind == 'float') else FLOAT
|
|
328
|
+
if all(t.kind == 'map' for t in kinds) and any(t.fields is not None for t in kinds):
|
|
329
|
+
# Maps compare equal whatever their fields: the fields' types are
|
|
330
|
+
# joined, and the result is no one struct's key set, so one map's
|
|
331
|
+
# fields never stand for another's.
|
|
332
|
+
return _join_maps(kinds)
|
|
333
|
+
if first.kind == 'string' and all(t.kind == 'string' for t in kinds):
|
|
334
|
+
# Texts: the digits each claims (`ExprTranslator.text_size`)
|
|
335
|
+
# joined, the widest whole part and scale; a text that claims none
|
|
336
|
+
# known (None) leaves none known.
|
|
337
|
+
claims = [t.digits for t in kinds]
|
|
338
|
+
if None in claims:
|
|
339
|
+
return STRING
|
|
340
|
+
from dataclasses import replace as _replace
|
|
341
|
+
return _replace(STRING, digits=(max(w for w, _ in claims), max(s for _, s in claims)))
|
|
342
|
+
if all(t == first for t in kinds):
|
|
343
|
+
return first
|
|
344
|
+
if all(t.is_number for t in kinds):
|
|
345
|
+
return FLOAT
|
|
346
|
+
if all(t.kind == 'list' for t in kinds):
|
|
347
|
+
# Lists share the type their elements share: an empty list and a
|
|
348
|
+
# list of nulls have no element type and do not count, as a null
|
|
349
|
+
# does not among values. One empty list in a column used to leave
|
|
350
|
+
# the whole column UNKNOWN.
|
|
351
|
+
element = common([t.element or NULL for t in kinds])
|
|
352
|
+
return list_of(element) if element != UNKNOWN else UNKNOWN
|
|
353
|
+
return UNKNOWN
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _precision(t: str) -> Optional[int]:
|
|
357
|
+
"""A date-time type name's fractional-second digits (`CypherType.precision`): TIMESTAMP(3),
|
|
358
|
+
DATETIME2(7), DuckDB's TIMESTAMP_S, _MS and _NS; None where the name states none."""
|
|
359
|
+
unit = {'TIMESTAMP_S': 0, 'TIMESTAMP_MS': 3, 'TIMESTAMP_NS': 9}.get(t)
|
|
360
|
+
if unit is not None:
|
|
361
|
+
return unit
|
|
362
|
+
found = re.match(r'(?:TIMESTAMP|DATETIME2?)\s*\(\s*(\d+)\s*\)', t)
|
|
363
|
+
return int(found.group(1)) if found else None
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
_INTEGER_FIELD = ('BIGINT', 'LONG', 'INTEGER', 'INT')
|
|
367
|
+
_TEMPORAL_STRUCTS = (
|
|
368
|
+
(('MONTHS', 'DAYS', 'SECONDS', 'NANOSECONDS'), DURATION),
|
|
369
|
+
(('EPOCH_SECONDS', 'NANOSECONDS', 'OFFSET_SECONDS', 'ZONE'), DATETIME_STRUCT),
|
|
370
|
+
(('LOCAL_SECONDS', 'NANOSECONDS'), LOCALDATETIME_STRUCT),
|
|
371
|
+
(('NANOSECONDS_OF_DAY', 'OFFSET_SECONDS'), TIME_STRUCT),
|
|
372
|
+
(('NANOSECONDS_OF_DAY',), LOCALTIME_STRUCT),
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def temporal_struct_fields(kind: CypherType) -> Tuple[str, ...]:
|
|
377
|
+
"""The fields, lower-cased and in order, of the struct `temporal_struct` reads as `kind`."""
|
|
378
|
+
return next(tuple(n.lower() for n in names) for names, k in _TEMPORAL_STRUCTS if k == kind)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def temporal_struct(t: str) -> Optional[CypherType]:
|
|
382
|
+
"""The Cypher temporal an (upper-cased) SQL struct type holds by its fields, exactly these in this order,
|
|
383
|
+
integers but a date-time's ZONE (text, null for an offset alone) (rule cypher-source-temporal):
|
|
384
|
+
months, days, seconds, nanoseconds a duration; epoch_seconds, nanoseconds, offset_seconds, zone a
|
|
385
|
+
date-time; local_seconds, nanoseconds a local date-time; nanoseconds_of_day, offset_seconds a time;
|
|
386
|
+
nanoseconds_of_day a local time. None for any other type. DuckDB writes `STRUCT("MONTHS" BIGINT, ...)`,
|
|
387
|
+
Spark `STRUCT<MONTHS:BIGINT,...>`."""
|
|
388
|
+
m = re.fullmatch(r'STRUCT\s*[(<](.*)[)>]', t.strip())
|
|
389
|
+
if m is None:
|
|
390
|
+
return None
|
|
391
|
+
fields = [re.split(r'[\s:]+', f.strip(), maxsplit=1) for f in m.group(1).split(',')]
|
|
392
|
+
if any(len(f) != 2 for f in fields):
|
|
393
|
+
return None
|
|
394
|
+
names = tuple(f[0].strip('"`') for f in fields)
|
|
395
|
+
for wanted, kind in _TEMPORAL_STRUCTS:
|
|
396
|
+
if names == wanted and all(f[1].strip() in (('VARCHAR', 'STRING', 'TEXT') if n == 'ZONE' else _INTEGER_FIELD)
|
|
397
|
+
for n, f in zip(names, fields)):
|
|
398
|
+
return kind
|
|
399
|
+
return None
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
# The database types known to have no natural XSD datatype (R2RML 10.2), by name as the catalogs write them: a
|
|
403
|
+
# column of one is a plain literal of the database's text. A name not here and not typed above stays UNKNOWN
|
|
404
|
+
# (a column nobody has typed is refused by name, never guessed to be text). PostgreSQL's catalog calls an
|
|
405
|
+
# enum, a composite and an extension's type USER-DEFINED; Spark's all-null column is VOID.
|
|
406
|
+
_TEXTUAL_NAMES = frozenset({
|
|
407
|
+
'JSON', 'JSONB', 'INET', 'CIDR', 'MACADDR', 'MACADDR8', 'MONEY', 'XML', 'XMLTYPE', 'TSVECTOR', 'TSQUERY',
|
|
408
|
+
'POINT', 'LINE', 'LSEG', 'BOX', 'PATH', 'POLYGON', 'CIRCLE', 'USER-DEFINED', 'VOID', 'VARIANT', 'GEOMETRY',
|
|
409
|
+
'GEOGRAPHY', 'HIERARCHYID', 'SQL_VARIANT', 'SET', 'INTERVAL', 'PG_LSN', 'TXID_SNAPSHOT', 'OID'})
|
|
410
|
+
_TEXTUAL_PREFIXES = ('INTERVAL', 'ENUM', 'UNION', 'STRUCT', 'MAP')
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
@home("no-natural-datatype-text", "A column of a type with no natural RDF datatype",
|
|
414
|
+
terms=("no natural datatype", "textual column", "jsonb", "inet", "interval", "struct column", "all-null column",
|
|
415
|
+
"CAST_TEXT"))
|
|
416
|
+
def of_sql_type(sql_type: str, timestamp_is_an_instant: bool = False) -> CypherType:
|
|
417
|
+
"""A column's type from a database type name (DuckDB, Spark, SQL).
|
|
418
|
+
|
|
419
|
+
`timestamp_is_an_instant`: the database's bare TIMESTAMP is an instant
|
|
420
|
+
shown in the session's zone (Spark and Databricks, whose type with no
|
|
421
|
+
zone is TIMESTAMP_NTZ), so it is read as zoned and its
|
|
422
|
+
text is its instant in UTC with `Z`. Elsewhere (DuckDB,
|
|
423
|
+
PostgreSQL) a bare TIMESTAMP has no zone."""
|
|
424
|
+
t = sql_type.upper().strip()
|
|
425
|
+
if t.endswith('[]'):
|
|
426
|
+
return list_of(of_sql_type(t[:-2], timestamp_is_an_instant))
|
|
427
|
+
if t.startswith('ARRAY<') and t.endswith('>'):
|
|
428
|
+
return list_of(of_sql_type(t[6:-1], timestamp_is_an_instant))
|
|
429
|
+
if t in ('MEDIUMINT', 'YEAR'):
|
|
430
|
+
return NARROW_INTEGER # MySQL's 24-bit integer and its year
|
|
431
|
+
# DuckDB's BIGNUM (VARINT) is not typed: its values reach the plan as the engine's binary form, which
|
|
432
|
+
# no integer reader decodes, so such a column is refused by name, never guessed.
|
|
433
|
+
if t in ('HUGEINT', 'UHUGEINT', 'UBIGINT'):
|
|
434
|
+
# Wider than 64 bits: a value can have 39 (or 20) digits.
|
|
435
|
+
return CypherType('integer', digits=(20 if t == 'UBIGINT' else 39, 0))
|
|
436
|
+
if t in ('TINYINT', 'SMALLINT', 'INTEGER', 'INT', 'SHORT', 'BYTE', 'UTINYINT',
|
|
437
|
+
'USMALLINT', 'UINTEGER', 'INT32'):
|
|
438
|
+
return NARROW_INTEGER
|
|
439
|
+
if t in ('BIGINT', 'LONG', 'INT64'):
|
|
440
|
+
return INTEGER
|
|
441
|
+
if t.startswith('DECIMAL') or t.startswith('NUMERIC') or t.startswith('NUMBER'):
|
|
442
|
+
sized = re.fullmatch(r'(?:DECIMAL|NUMERIC)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)', t)
|
|
443
|
+
if sized:
|
|
444
|
+
return CypherType('float', exact=True, digits=(int(sized.group(1)), int(sized.group(2))))
|
|
445
|
+
return DECIMAL
|
|
446
|
+
if t in ('BLOB', 'BYTEA', 'BINARY', 'VARBINARY', 'LONGBLOB', 'RAW') or t.startswith('VARBINARY') \
|
|
447
|
+
or t.startswith('BINARY(') or t.startswith('BLOB') \
|
|
448
|
+
or t in ('IMAGE', 'TINYBLOB', 'MEDIUMBLOB'):
|
|
449
|
+
return BINARY
|
|
450
|
+
if t in ('REAL', 'FLOAT4', 'FLOAT'):
|
|
451
|
+
# 32 bits on DuckDB, Spark, Databricks and PostgreSQL (REAL); SQL Server's FLOAT is 64 bits,
|
|
452
|
+
# and there the dialect reads the column as it is (FLOAT32_VALUE).
|
|
453
|
+
return replace(FLOAT, single=True)
|
|
454
|
+
if t in ('FLOAT', 'DOUBLE', 'REAL', 'FLOAT4', 'FLOAT8', 'DOUBLE PRECISION',
|
|
455
|
+
'BINARY_DOUBLE', 'BINARY_FLOAT'):
|
|
456
|
+
# Oracle's NUMBER with a scale; one that holds integers only is
|
|
457
|
+
# reported as INTEGER (oracle_graph.number_is_integer). Oracle's
|
|
458
|
+
# own float types, BINARY_DOUBLE and BINARY_FLOAT, were missing:
|
|
459
|
+
# a column of one had no type, and nothing that chooses a SQL form
|
|
460
|
+
# by a float's type did so on a connected Oracle.
|
|
461
|
+
return FLOAT
|
|
462
|
+
if t in ('CHAR', 'CHARACTER', 'BPCHAR', 'NCHAR') or re.fullmatch(r'N?CHAR(ACTER)?\s*\(\s*\d+\s*\)', t):
|
|
463
|
+
return replace(STRING, padded=True)
|
|
464
|
+
# A Databricks or Spark string under a collation (`string collate UTF8_LCASE_RTRIM`) is a string: its
|
|
465
|
+
# values are text whatever the collation. Whether it compares bytes is the collation's question
|
|
466
|
+
# (`collations.compares_bytes`, read from the same type text); it was left untyped, and a key
|
|
467
|
+
# column of one could not be built into an IRI at all.
|
|
468
|
+
collation = re.search(r'\s+COLLATE\s+([A-Za-z0-9_.]+)$', t)
|
|
469
|
+
t = re.sub(r'\s+COLLATE\s+[A-Za-z0-9_.]+$', '', t)
|
|
470
|
+
if t in ('TINYTEXT', 'MEDIUMTEXT', 'LONGTEXT', 'NTEXT', 'CLOB', 'NCLOB', 'LONG', 'ROWID', 'UROWID'):
|
|
471
|
+
return STRING # MySQL's, SQL Server's and Oracle's long text and Oracle's row addresses
|
|
472
|
+
if t in ('VARCHAR', 'STRING', 'TEXT') or t.startswith('VARCHAR') \
|
|
473
|
+
or t.startswith('CHAR') or t.startswith('NVARCHAR'):
|
|
474
|
+
if collation is not None and collation.group(1).upper() != 'UTF8_BINARY':
|
|
475
|
+
return replace(STRING, collated=True)
|
|
476
|
+
return STRING
|
|
477
|
+
if t in ('UUID', 'UNIQUEIDENTIFIER'):
|
|
478
|
+
# A uuid has no XSD datatype: a plain string of the database's own text (R2RML 10.2), lower case
|
|
479
|
+
# 8-4-4-4-12 on PostgreSQL and DuckDB, upper case on SQL Server (rule uuid-text).
|
|
480
|
+
return replace(STRING, uuid=True)
|
|
481
|
+
if t == 'CITEXT':
|
|
482
|
+
# PostgreSQL's case-insensitive text (the citext extension): its values are text, compared
|
|
483
|
+
# without case, so it is read as text under the binary collation (reader BINARY_TEXT).
|
|
484
|
+
return replace(STRING, collated=True)
|
|
485
|
+
bits = re.fullmatch(r'BIT\s*\(\s*(\d+)\s*\)', t)
|
|
486
|
+
if bits is not None or t in ('BIT VARYING', 'VARBIT') \
|
|
487
|
+
or t.startswith('BIT VARYING') or t.startswith('VARBIT'):
|
|
488
|
+
return TEXTUAL # a bit string is not a boolean: its database text
|
|
489
|
+
if t in ('BOOLEAN', 'BOOL', 'BIT'):
|
|
490
|
+
return BOOLEAN
|
|
491
|
+
if t == 'DATE':
|
|
492
|
+
return DATE
|
|
493
|
+
if t.startswith('TIMESTAMP') and (t.endswith('WITH TIME ZONE') or t == 'TIMESTAMPTZ'):
|
|
494
|
+
# DuckDB's TIMESTAMPTZ, PostgreSQL's and Oracle's TIMESTAMP WITH
|
|
495
|
+
# TIME ZONE: an instant.
|
|
496
|
+
return replace(ZONED_DATETIME, precision=_precision(t))
|
|
497
|
+
if t == 'TIMESTAMP_LTZ' or (timestamp_is_an_instant and t == 'TIMESTAMP'):
|
|
498
|
+
# Spark's and Databricks' TIMESTAMP (TIMESTAMP_LTZ): an instant;
|
|
499
|
+
# their TIMESTAMP_NTZ is a local date-time.
|
|
500
|
+
return ZONED_DATETIME
|
|
501
|
+
if t == 'SMALLDATETIME':
|
|
502
|
+
return DATETIME
|
|
503
|
+
if t == 'SMALLMONEY':
|
|
504
|
+
return CypherType('float', exact=True, digits=(10, 4))
|
|
505
|
+
if t.startswith('TIMESTAMP') or t.startswith('DATETIME'):
|
|
506
|
+
return replace(DATETIME, precision=_precision(t))
|
|
507
|
+
if t == 'TIMETZ' or (t.startswith('TIME ') and 'WITH TIME ZONE' in t and 'WITHOUT' not in t):
|
|
508
|
+
return ZONED_TIME
|
|
509
|
+
if t == 'TIME' or t.startswith('TIME '):
|
|
510
|
+
return TIME
|
|
511
|
+
held = temporal_struct(t)
|
|
512
|
+
if held is not None:
|
|
513
|
+
return held
|
|
514
|
+
if t.startswith('STRUCT') or t.startswith('MAP') or t == 'JSON' or t == 'JSONB':
|
|
515
|
+
return MAP
|
|
516
|
+
if t in _TEXTUAL_NAMES or t.startswith(_TEXTUAL_PREFIXES):
|
|
517
|
+
return TEXTUAL
|
|
518
|
+
return UNKNOWN
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def of_frame(frame) -> Dict[str, CypherType]:
|
|
522
|
+
"""Each column's type in a pandas DataFrame. An object column is typed
|
|
523
|
+
by its non-null values when they share one; an integer column with a
|
|
524
|
+
NULL, which pandas holds as float64, is still a float here (the values
|
|
525
|
+
alone cannot say otherwise). A float column with no value at all is
|
|
526
|
+
UNKNOWN: float64 is what pandas gives an empty column
|
|
527
|
+
(`pd.DataFrame({'id': []})`), so it says nothing about what the column
|
|
528
|
+
will hold, and a text key written to such a table was refused as "not a
|
|
529
|
+
float". A frame of no rows types such a column STRING: no value can
|
|
530
|
+
contradict it, and its maps give no term, so a query over every map (a
|
|
531
|
+
whole-graph pattern) is built rather than refused over a column it never
|
|
532
|
+
reads a value of. An object column whose every row is None is typed STRING
|
|
533
|
+
for the same reason: it holds no value, so its maps give no term (an
|
|
534
|
+
OPTIONAL over it is unbound, a join on it has no row)."""
|
|
535
|
+
out: Dict[str, CypherType] = {}
|
|
536
|
+
for column in frame.columns:
|
|
537
|
+
kind = frame[column].dtype.kind
|
|
538
|
+
arrow = getattr(frame[column].dtype, 'pyarrow_dtype', None)
|
|
539
|
+
if arrow is not None and str(arrow).startswith('date'):
|
|
540
|
+
# An Arrow date (date32[pyarrow], date64) reports kind 'M', as datetime64 does.
|
|
541
|
+
out[column] = DATE
|
|
542
|
+
continue
|
|
543
|
+
if kind in 'iu':
|
|
544
|
+
out[column] = NARROW_INTEGER if frame[column].dtype.itemsize < 8 else INTEGER
|
|
545
|
+
elif kind == 'f':
|
|
546
|
+
out[column] = FLOAT if frame[column].notna().any() else (STRING if len(frame) == 0 else UNKNOWN)
|
|
547
|
+
elif kind == 'b':
|
|
548
|
+
out[column] = BOOLEAN
|
|
549
|
+
elif kind == 'M':
|
|
550
|
+
zoned = getattr(frame[column].dtype, 'tz', None) is not None
|
|
551
|
+
out[column] = ZONED_DATETIME if zoned else DATETIME
|
|
552
|
+
elif kind in 'OSU':
|
|
553
|
+
sample = [of_value(v) for v in frame[column].dropna().head(200)]
|
|
554
|
+
out[column] = common(sample) if sample else STRING
|
|
555
|
+
else:
|
|
556
|
+
name = str(frame[column].dtype)
|
|
557
|
+
if name in ('Int8', 'Int16', 'Int32', 'UInt8', 'UInt16', 'UInt32'):
|
|
558
|
+
out[column] = NARROW_INTEGER
|
|
559
|
+
elif name in ('Int64', 'UInt64'):
|
|
560
|
+
out[column] = INTEGER
|
|
561
|
+
elif name in ('Float32', 'Float64'):
|
|
562
|
+
out[column] = FLOAT
|
|
563
|
+
elif name == 'boolean':
|
|
564
|
+
out[column] = BOOLEAN
|
|
565
|
+
elif name in ('string', 'str'):
|
|
566
|
+
out[column] = STRING
|
|
567
|
+
else:
|
|
568
|
+
out[column] = UNKNOWN
|
|
569
|
+
return out
|