openstatspec 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openstatspec-0.5.0 → openstatspec-0.6.0}/CHANGELOG.md +18 -1
- {openstatspec-0.5.0 → openstatspec-0.6.0}/PKG-INFO +1 -1
- {openstatspec-0.5.0 → openstatspec-0.6.0}/pyproject.toml +1 -1
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/binding.py +39 -7
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/syntax.py +15 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/inplace_transform.py +28 -39
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/profiles.py +28 -8
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/wide.py +6 -2
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/transform/validation.py +45 -3
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_inplace_transform.py +10 -5
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_sql_services.py +92 -2
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_transform_frontend.py +59 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/.github/verify_release_ref.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/.github/workflows/ci.yml +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/.github/workflows/release.yml +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/.gitignore +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/README.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/THIRD_PARTY_NOTICES.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/docs/architecture.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/docs/in-place-transformation.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/docs/pyspssio-spike.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/docs/release-readiness.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/docs/sav-profile.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/docs/sql-transformation-workflow.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/docs/transformations.md +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/__init__.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/api.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/cli.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/core/__init__.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/core/results.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/__init__.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/sas/.gitkeep +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/__init__.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/compiler.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/execution.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/frontends/stata/.gitkeep +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/spss/__init__.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/spss/dictionary.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/spss/raw_dictionary.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/spss/sav.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/__init__.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/capabilities.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/catalog_api.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/catalog_verification.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/database_urls.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/dolt_conformance.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/normative.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/sql/workflow.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/transform/__init__.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/transform/errors.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/transform/plan.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/src/openstatspec/transform/schema.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/conformance.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_atomic_import.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_attribute_catalog.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_catalog_lifecycle.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_catalog_persistence_review.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_cli.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_conditional_inplace_transform.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_document_round_trip.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_dolt_conformance.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_loss_reports.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_normative_catalog.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_official_conformance.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_pyspssio_catalog_authority.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_release_ref.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_review_transaction_boundary.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_sav_sqlite.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_spss_catalog_preflight.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_sql_profiles.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_sql_workflow.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_strict_catalog_review.py +0 -0
- {openstatspec-0.5.0 → openstatspec-0.6.0}/tests/test_vls_compatible_names.py +0 -0
|
@@ -4,7 +4,24 @@ All notable changes to this reference implementation are documented here.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
_No unreleased changes._
|
|
8
|
+
|
|
9
|
+
## 0.6.0 - 2026-08-05
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- Completed Python parity for bounded in-place SPSS schema transformations:
|
|
14
|
+
STRING, DELETE VARIABLES, string RECODE width validation, and durable
|
|
15
|
+
delete/recreate physical-column bindings.
|
|
16
|
+
- Added PostgreSQL physical attribute-slot preflight and real-service
|
|
17
|
+
regression coverage for delete/recreate at the column limit.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Schema-changing SPSS plans use the v0.3 contract and fail closed for
|
|
22
|
+
unsupported STRING/DELETE variable ranges.
|
|
23
|
+
|
|
24
|
+
## 0.5.0 - 2026-08-05
|
|
8
25
|
### Fixed
|
|
9
26
|
|
|
10
27
|
- Removed the temporary `*_catalog` compatibility schema and made SAV/ZSAV
|
|
@@ -216,6 +216,41 @@ def _result_type(
|
|
|
216
216
|
return result.value.type
|
|
217
217
|
|
|
218
218
|
|
|
219
|
+
def _validate_recode_string_width(
|
|
220
|
+
result: RecodeResult,
|
|
221
|
+
source: VariableDefinition,
|
|
222
|
+
target: VariableDefinition,
|
|
223
|
+
span: SourceSpan,
|
|
224
|
+
) -> None:
|
|
225
|
+
if target.storage_kind != "string" or target.declared_string_width is None:
|
|
226
|
+
return
|
|
227
|
+
if result.kind == "literal":
|
|
228
|
+
assert result.value is not None
|
|
229
|
+
if result.value.type != "string":
|
|
230
|
+
return
|
|
231
|
+
assert isinstance(result.value.value, str)
|
|
232
|
+
if len(result.value.value.encode("utf-8")) > target.declared_string_width:
|
|
233
|
+
raise frontend_error(
|
|
234
|
+
"string_width_exceeded",
|
|
235
|
+
"A RECODE string literal exceeds the target's declared string width.",
|
|
236
|
+
span=span,
|
|
237
|
+
variable=target.name,
|
|
238
|
+
declared_string_width=target.declared_string_width,
|
|
239
|
+
)
|
|
240
|
+
return
|
|
241
|
+
if result.kind == "copy" and source.storage_kind == "string":
|
|
242
|
+
source_width = source.declared_string_width
|
|
243
|
+
if source_width is None or source_width > target.declared_string_width:
|
|
244
|
+
raise frontend_error(
|
|
245
|
+
"string_width_exceeded",
|
|
246
|
+
"A RECODE COPY result can exceed the target's declared string width.",
|
|
247
|
+
span=span,
|
|
248
|
+
source=source.name,
|
|
249
|
+
variable=target.name,
|
|
250
|
+
declared_string_width=target.declared_string_width,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
219
254
|
def _bind_recode(
|
|
220
255
|
command: RecodeCommandSyntax, variables: list[VariableDefinition],
|
|
221
256
|
) -> tuple[list[RecodeOperation], list[SourceSpan]]:
|
|
@@ -252,6 +287,8 @@ def _bind_recode(
|
|
|
252
287
|
else_result: RecodeResult | None = None
|
|
253
288
|
for clause in command.clauses:
|
|
254
289
|
result = _result(clause.result, source)
|
|
290
|
+
if target_mode == "replace":
|
|
291
|
+
_validate_recode_string_width(result, source, source, clause.result.span)
|
|
255
292
|
if clause.match.kind == "else":
|
|
256
293
|
else_result = result
|
|
257
294
|
continue
|
|
@@ -259,6 +296,8 @@ def _bind_recode(
|
|
|
259
296
|
unmatched = else_result or RecodeResult(
|
|
260
297
|
"system_missing" if target_mode == "create" else "copy"
|
|
261
298
|
)
|
|
299
|
+
if target_mode == "replace" and else_result is None:
|
|
300
|
+
_validate_recode_string_width(unmatched, source, source, command.span)
|
|
262
301
|
result_types = {
|
|
263
302
|
_result_type(result, source)
|
|
264
303
|
for result in [*(rule.result for rule in rules), unmatched]
|
|
@@ -339,13 +378,6 @@ def bind_spss_syntax(
|
|
|
339
378
|
index, variable = _resolve(
|
|
340
379
|
variables, variable_token.text, variable_token.span,
|
|
341
380
|
)
|
|
342
|
-
if len(variables) == 1:
|
|
343
|
-
raise frontend_error(
|
|
344
|
-
"cannot_delete_last_variable",
|
|
345
|
-
"DELETE VARIABLES cannot remove the final dataset variable.",
|
|
346
|
-
span=variable_token.span,
|
|
347
|
-
variable=variable.name,
|
|
348
|
-
)
|
|
349
381
|
operations.append(DeleteVariableOperation(variable.name))
|
|
350
382
|
spans.append(command.span)
|
|
351
383
|
del variables[index]
|
|
@@ -563,10 +563,24 @@ class _Parser:
|
|
|
563
563
|
end = self.expects("period", "Expected '.' after EXECUTE.")
|
|
564
564
|
return ExecuteCommandSyntax(_joined_span(start.span, end.span))
|
|
565
565
|
|
|
566
|
+
@staticmethod
|
|
567
|
+
def reject_to_range(variables: tuple[Token, ...], command: str) -> None:
|
|
568
|
+
range_token = next(
|
|
569
|
+
(variable for variable in variables if variable.text.casefold() == "to"),
|
|
570
|
+
None,
|
|
571
|
+
)
|
|
572
|
+
if range_token is not None:
|
|
573
|
+
raise frontend_error(
|
|
574
|
+
"unsupported_spss_feature",
|
|
575
|
+
f"{command} variable ranges using TO are not supported.",
|
|
576
|
+
span=range_token.span,
|
|
577
|
+
)
|
|
578
|
+
|
|
566
579
|
def string(self, start: Token) -> StringCommandSyntax:
|
|
567
580
|
variables = self.variable_list(
|
|
568
581
|
stop_kinds=frozenset({"left_paren", "period", "eof"}),
|
|
569
582
|
)
|
|
583
|
+
self.reject_to_range(variables, "STRING")
|
|
570
584
|
self.expects("left_paren", "Expected '(' before a STRING width.")
|
|
571
585
|
width_token = self.expects(
|
|
572
586
|
"identifier", "STRING requires a width such as A20.",
|
|
@@ -595,6 +609,7 @@ class _Parser:
|
|
|
595
609
|
def delete_variables(self, start: Token) -> DeleteVariablesCommandSyntax:
|
|
596
610
|
self.expects_keyword("VARIABLES")
|
|
597
611
|
variables = self.variable_list(stop_kinds=frozenset({"period", "eof"}))
|
|
612
|
+
self.reject_to_range(variables, "DELETE VARIABLES")
|
|
598
613
|
end = self.expects("period", "Expected '.' after DELETE VARIABLES.")
|
|
599
614
|
return DeleteVariablesCommandSyntax(
|
|
600
615
|
variables, _joined_span(start.span, end.span),
|
|
@@ -389,22 +389,30 @@ def _delete_variable_metadata(
|
|
|
389
389
|
))
|
|
390
390
|
|
|
391
391
|
|
|
392
|
-
def
|
|
392
|
+
def _assert_postgresql_column_slot_available(
|
|
393
393
|
connection: Any,
|
|
394
394
|
*,
|
|
395
|
-
|
|
396
|
-
|
|
395
|
+
relation: Table,
|
|
396
|
+
target_profile: SqlProfile,
|
|
397
397
|
) -> None:
|
|
398
|
-
"""
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
398
|
+
"""Reject PostgreSQL creates when dropped columns exhaust attribute slots."""
|
|
399
|
+
if connection.dialect.name != "postgresql":
|
|
400
|
+
return
|
|
401
|
+
qualified_relation = connection.dialect.identifier_preparer.format_table(relation)
|
|
402
|
+
attribute_slots = connection.execute(text(
|
|
403
|
+
"SELECT COUNT(*) FROM pg_attribute "
|
|
404
|
+
"WHERE attrelid = to_regclass(:relation_name) AND attnum > 0"
|
|
405
|
+
), {"relation_name": qualified_relation}).scalar_one()
|
|
406
|
+
if not isinstance(attribute_slots, int) or attribute_slots < 1:
|
|
407
|
+
raise TransformationError(
|
|
408
|
+
"physical_table_missing",
|
|
409
|
+
"The target dataset's physical wide table does not exist.",
|
|
410
|
+
)
|
|
411
|
+
if attribute_slots >= target_profile.max_source_variables + 1:
|
|
412
|
+
raise TransformationError(
|
|
413
|
+
"source_variable_limit",
|
|
414
|
+
"PostgreSQL physical column slots are exhausted for this dataset table.",
|
|
406
415
|
)
|
|
407
|
-
variable["source_ordinal"] = source_ordinal
|
|
408
416
|
|
|
409
417
|
|
|
410
418
|
def _failure_boundary(_name: str) -> None:
|
|
@@ -555,6 +563,9 @@ def _apply_plan_on_connection(
|
|
|
555
563
|
)
|
|
556
564
|
by_name = {str(row["source_name"]).casefold(): row for row in variables}
|
|
557
565
|
used_physical = {str(row["physical_name"]).casefold() for row in variables}
|
|
566
|
+
next_source_ordinal = max(
|
|
567
|
+
(int(row["source_ordinal"]) for row in variables), default=0,
|
|
568
|
+
) + 1
|
|
558
569
|
quote = connection.dialect.identifier_preparer.quote
|
|
559
570
|
qualified_table = connection.dialect.identifier_preparer.format_table(relation)
|
|
560
571
|
numeric_type = (
|
|
@@ -603,6 +614,9 @@ def _apply_plan_on_connection(
|
|
|
603
614
|
)
|
|
604
615
|
)
|
|
605
616
|
if creates_target:
|
|
617
|
+
_assert_postgresql_column_slot_available(
|
|
618
|
+
connection, relation=relation, target_profile=target_profile,
|
|
619
|
+
)
|
|
606
620
|
target_name = (
|
|
607
621
|
operation.variable
|
|
608
622
|
if isinstance(operation, CreateVariableOperation)
|
|
@@ -631,7 +645,7 @@ def _apply_plan_on_connection(
|
|
|
631
645
|
created_target = {
|
|
632
646
|
"variable_id": str(uuid4()),
|
|
633
647
|
"dataset_id": dataset_id,
|
|
634
|
-
"source_ordinal":
|
|
648
|
+
"source_ordinal": next_source_ordinal,
|
|
635
649
|
"source_name": target_name,
|
|
636
650
|
"physical_name": target_physical,
|
|
637
651
|
"storage_kind": storage_kind,
|
|
@@ -648,6 +662,7 @@ def _apply_plan_on_connection(
|
|
|
648
662
|
)
|
|
649
663
|
connection.execute(insert(core.variable).values(**created_target))
|
|
650
664
|
variables.append(created_target)
|
|
665
|
+
next_source_ordinal += 1
|
|
651
666
|
by_name[str(created_target["source_name"]).casefold()] = created_target
|
|
652
667
|
_failure_boundary("catalog")
|
|
653
668
|
if isinstance(operation, CreateVariableOperation):
|
|
@@ -725,36 +740,10 @@ def _apply_plan_on_connection(
|
|
|
725
740
|
)
|
|
726
741
|
_delete_variable_metadata(connection, core=core, variable=variable)
|
|
727
742
|
by_name.pop(operation.variable.casefold(), None)
|
|
728
|
-
used_physical.discard(str(variable["physical_name"]).casefold())
|
|
729
743
|
variables = [
|
|
730
744
|
row for row in variables
|
|
731
745
|
if row["variable_id"] != variable["variable_id"]
|
|
732
746
|
]
|
|
733
|
-
_compact_variable_ordinals(
|
|
734
|
-
connection, core=core, variables=variables,
|
|
735
|
-
)
|
|
736
|
-
canonical_physical = {"__case_ordinal"}
|
|
737
|
-
for remaining_variable in variables:
|
|
738
|
-
expected_physical = physical_name(
|
|
739
|
-
str(remaining_variable["source_name"]), canonical_physical,
|
|
740
|
-
)
|
|
741
|
-
current_physical = str(remaining_variable["physical_name"])
|
|
742
|
-
if current_physical == expected_physical:
|
|
743
|
-
continue
|
|
744
|
-
connection.exec_driver_sql(
|
|
745
|
-
f"ALTER TABLE {qualified_table} RENAME COLUMN "
|
|
746
|
-
f"{quote(current_physical)} TO {quote(expected_physical)}"
|
|
747
|
-
)
|
|
748
|
-
connection.execute(
|
|
749
|
-
update(core.variable)
|
|
750
|
-
.where(
|
|
751
|
-
core.variable.c.variable_id
|
|
752
|
-
== remaining_variable["variable_id"]
|
|
753
|
-
)
|
|
754
|
-
.values(physical_name=expected_physical)
|
|
755
|
-
)
|
|
756
|
-
remaining_variable["physical_name"] = expected_physical
|
|
757
|
-
used_physical = canonical_physical
|
|
758
747
|
relation = Table(
|
|
759
748
|
table_name, MetaData(), schema=dataset.get("physical_table_schema"),
|
|
760
749
|
autoload_with=connection,
|
|
@@ -124,6 +124,7 @@ def preflight(
|
|
|
124
124
|
variables_or_count: int | Iterable[Mapping[str, Any]],
|
|
125
125
|
*,
|
|
126
126
|
rows: Iterable[Mapping[str, Any]] | None = None,
|
|
127
|
+
require_canonical_mapping: bool = True,
|
|
127
128
|
) -> None:
|
|
128
129
|
"""Validate strict target capabilities before any source dataset is created."""
|
|
129
130
|
variables = None if isinstance(variables_or_count, int) else list(variables_or_count)
|
|
@@ -143,6 +144,7 @@ def preflight(
|
|
|
143
144
|
|
|
144
145
|
used = {"__case_ordinal"}
|
|
145
146
|
source_names: set[str] = set()
|
|
147
|
+
physical_names = {"__case_ordinal"}
|
|
146
148
|
for expected_ordinal, variable in enumerate(variables, start=1):
|
|
147
149
|
source_name = variable.get("source_name")
|
|
148
150
|
if not isinstance(source_name, str) or not source_name or source_name in source_names:
|
|
@@ -151,18 +153,36 @@ def preflight(
|
|
|
151
153
|
source_name=source_name,
|
|
152
154
|
)
|
|
153
155
|
source_names.add(source_name)
|
|
154
|
-
expected_name = _physical_name(source_name, used)
|
|
155
156
|
actual_name = variable.get("physical_name")
|
|
156
|
-
if
|
|
157
|
+
if not isinstance(actual_name, str) or not actual_name:
|
|
157
158
|
raise _exceeded(
|
|
158
159
|
"physical_identifier_mapping_invalid",
|
|
159
|
-
f"{source_name!r}
|
|
160
|
-
source_name=source_name,
|
|
161
|
-
|
|
160
|
+
f"{source_name!r} has no physical variable identifier.",
|
|
161
|
+
source_name=source_name, actual_physical_name=actual_name,
|
|
162
|
+
)
|
|
163
|
+
if actual_name.casefold() in physical_names:
|
|
164
|
+
raise _exceeded(
|
|
165
|
+
"physical_identifier_collision",
|
|
166
|
+
"physical variable identifiers must be unique.",
|
|
167
|
+
source_name=source_name, physical_name=actual_name,
|
|
168
|
+
)
|
|
169
|
+
physical_names.add(actual_name.casefold())
|
|
170
|
+
if require_canonical_mapping:
|
|
171
|
+
expected_name = _physical_name(source_name, used)
|
|
172
|
+
if variable.get("ordinal") != expected_ordinal or actual_name != expected_name:
|
|
173
|
+
raise _exceeded(
|
|
174
|
+
"physical_identifier_mapping_invalid",
|
|
175
|
+
f"{source_name!r} must map deterministically to {expected_name!r} in source order.",
|
|
176
|
+
source_name=source_name, expected_physical_name=expected_name,
|
|
177
|
+
actual_physical_name=actual_name,
|
|
178
|
+
)
|
|
179
|
+
preflight_identifier(
|
|
180
|
+
profile, expected_name, role="physical variable identifier",
|
|
181
|
+
)
|
|
182
|
+
else:
|
|
183
|
+
preflight_identifier(
|
|
184
|
+
profile, actual_name, role="physical variable identifier",
|
|
162
185
|
)
|
|
163
|
-
preflight_identifier(
|
|
164
|
-
profile, expected_name, role="physical variable identifier",
|
|
165
|
-
)
|
|
166
186
|
if variable.get("storage_kind") == "string":
|
|
167
187
|
declared_width = variable.get("string_width")
|
|
168
188
|
if declared_width is not None and (
|
|
@@ -1239,7 +1239,9 @@ def read_wide_dataset(
|
|
|
1239
1239
|
)
|
|
1240
1240
|
for row in response_sets
|
|
1241
1241
|
}, ensure_ascii=False)
|
|
1242
|
-
preflight(
|
|
1242
|
+
preflight(
|
|
1243
|
+
profile, variables, rows=rows, require_canonical_mapping=False,
|
|
1244
|
+
)
|
|
1243
1245
|
return dataset, variables, rows
|
|
1244
1246
|
|
|
1245
1247
|
|
|
@@ -1705,7 +1707,9 @@ def validate_wide_dataset(
|
|
|
1705
1707
|
database_url=database_url, dataset_id=dataset_id, profile=profile,
|
|
1706
1708
|
dolt_conformance_source=dolt_conformance_source,
|
|
1707
1709
|
)
|
|
1708
|
-
preflight(
|
|
1710
|
+
preflight(
|
|
1711
|
+
profile, variables, rows=rows, require_canonical_mapping=False,
|
|
1712
|
+
)
|
|
1709
1713
|
validate_spss_catalog(
|
|
1710
1714
|
variables,
|
|
1711
1715
|
case_weight_variable=dataset.get("case_weight_variable"),
|
|
@@ -100,6 +100,38 @@ def _result_type(
|
|
|
100
100
|
return result.value.type
|
|
101
101
|
|
|
102
102
|
|
|
103
|
+
def _validate_recode_string_width(
|
|
104
|
+
result: RecodeResult,
|
|
105
|
+
source: VariableDefinition,
|
|
106
|
+
target: VariableDefinition,
|
|
107
|
+
) -> None:
|
|
108
|
+
if target.storage_kind != "string" or target.declared_string_width is None:
|
|
109
|
+
return
|
|
110
|
+
if result.kind == "literal":
|
|
111
|
+
assert result.value is not None
|
|
112
|
+
if result.value.type != "string":
|
|
113
|
+
return
|
|
114
|
+
assert isinstance(result.value.value, str)
|
|
115
|
+
if len(result.value.value.encode("utf-8")) > target.declared_string_width:
|
|
116
|
+
raise frontend_error(
|
|
117
|
+
"string_width_exceeded",
|
|
118
|
+
"A RECODE string literal exceeds the target's declared string width.",
|
|
119
|
+
variable=target.name,
|
|
120
|
+
declared_string_width=target.declared_string_width,
|
|
121
|
+
)
|
|
122
|
+
return
|
|
123
|
+
if result.kind == "copy" and source.storage_kind == "string":
|
|
124
|
+
source_width = source.declared_string_width
|
|
125
|
+
if source_width is None or source_width > target.declared_string_width:
|
|
126
|
+
raise frontend_error(
|
|
127
|
+
"string_width_exceeded",
|
|
128
|
+
"A RECODE COPY result can exceed the target's declared string width.",
|
|
129
|
+
source=source.name,
|
|
130
|
+
variable=target.name,
|
|
131
|
+
declared_string_width=target.declared_string_width,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
103
135
|
def _bind_recode(
|
|
104
136
|
operation: RecodeOperation, variables: list[VariableDefinition]
|
|
105
137
|
) -> None:
|
|
@@ -122,6 +154,10 @@ def _bind_recode(
|
|
|
122
154
|
)
|
|
123
155
|
for rule in operation.rules:
|
|
124
156
|
_validate_match(rule.match, source)
|
|
157
|
+
if operation.target_mode == "replace":
|
|
158
|
+
_validate_recode_string_width(rule.result, source, source)
|
|
159
|
+
if operation.target_mode == "replace":
|
|
160
|
+
_validate_recode_string_width(operation.unmatched, source, source)
|
|
125
161
|
result_types = {
|
|
126
162
|
_result_type(result, source)
|
|
127
163
|
for result in [
|
|
@@ -282,12 +318,16 @@ def bind_transformation_plan(
|
|
|
282
318
|
if not isinstance(schema, VariableSchema):
|
|
283
319
|
raise TypeError("schema must be a VariableSchema.")
|
|
284
320
|
variables = list(schema.variables)
|
|
285
|
-
for operation in plan.operations:
|
|
321
|
+
for operation_index, operation in enumerate(plan.operations):
|
|
322
|
+
later_create = any(
|
|
323
|
+
isinstance(later_operation, CreateVariableOperation)
|
|
324
|
+
for later_operation in plan.operations[operation_index + 1:]
|
|
325
|
+
)
|
|
286
326
|
if isinstance(operation, CreateVariableOperation):
|
|
287
327
|
_bind_create(operation, variables)
|
|
288
328
|
continue
|
|
289
329
|
if isinstance(operation, DeleteVariableOperation):
|
|
290
|
-
_bind_delete(operation, variables)
|
|
330
|
+
_bind_delete(operation, variables, allow_empty=later_create)
|
|
291
331
|
continue
|
|
292
332
|
if isinstance(operation, RecodeOperation):
|
|
293
333
|
_bind_recode(operation, variables)
|
|
@@ -367,9 +407,11 @@ def _bind_create(
|
|
|
367
407
|
def _bind_delete(
|
|
368
408
|
operation: DeleteVariableOperation,
|
|
369
409
|
variables: list[VariableDefinition],
|
|
410
|
+
*,
|
|
411
|
+
allow_empty: bool,
|
|
370
412
|
) -> None:
|
|
371
413
|
index, variable = _resolve(variables, operation.variable)
|
|
372
|
-
if len(variables) == 1:
|
|
414
|
+
if len(variables) == 1 and not allow_empty:
|
|
373
415
|
raise frontend_error(
|
|
374
416
|
"cannot_delete_last_variable",
|
|
375
417
|
"A dataset must retain at least one variable.",
|
|
@@ -191,7 +191,7 @@ def test_string_declaration_creates_column_and_catalog_variable(catalog) -> None
|
|
|
191
191
|
)["valid"] is True
|
|
192
192
|
|
|
193
193
|
|
|
194
|
-
def
|
|
194
|
+
def test_delete_preserves_surviving_physical_bindings_and_ordinals(tmp_path) -> None:
|
|
195
195
|
path = tmp_path / "collision-delete.sqlite"
|
|
196
196
|
url = f"sqlite:///{path}"
|
|
197
197
|
openstatspec.initialize_catalog(database_url=url)
|
|
@@ -242,9 +242,9 @@ def test_delete_recanonicalizes_surviving_collision_columns(tmp_path) -> None:
|
|
|
242
242
|
"select source_name, physical_name, source_ordinal from variable "
|
|
243
243
|
"where dataset_id = ? order by source_ordinal",
|
|
244
244
|
(dataset_id,),
|
|
245
|
-
).fetchall() == [("a_b", "
|
|
245
|
+
).fetchall() == [("a_b", "a_b_2", 2), ("keep", "keep", 3)]
|
|
246
246
|
assert connection.execute(
|
|
247
|
-
"select
|
|
247
|
+
"select a_b_2, keep from data_collision_source order by __case_ordinal"
|
|
248
248
|
).fetchall() == [(2.0, 3.0), (5.0, 6.0)]
|
|
249
249
|
connection.close()
|
|
250
250
|
assert validate_wide_dataset(
|
|
@@ -361,7 +361,7 @@ def test_delete_then_recreate_same_name_resolves_operations_in_order(catalog) ->
|
|
|
361
361
|
(dataset_id,),
|
|
362
362
|
).fetchall()
|
|
363
363
|
assert [(row[0], row[2]) for row in variables] == [
|
|
364
|
-
("other",
|
|
364
|
+
("other", 2), ("score", 3),
|
|
365
365
|
]
|
|
366
366
|
physical = {row[0]: row[1] for row in variables}
|
|
367
367
|
assert connection.execute(
|
|
@@ -414,8 +414,13 @@ def test_temporary_target_type_is_not_taken_from_same_name_recreation(catalog) -
|
|
|
414
414
|
"from variable where dataset_id = ? order by source_ordinal",
|
|
415
415
|
(dataset_id,),
|
|
416
416
|
).fetchall() == [("score", "numeric", None), ("tmp", "string", 4)]
|
|
417
|
+
physical = dict(connection.execute(
|
|
418
|
+
"select source_name, physical_name from variable where dataset_id = ?",
|
|
419
|
+
(dataset_id,),
|
|
420
|
+
).fetchall())
|
|
417
421
|
assert connection.execute(
|
|
418
|
-
f'SELECT score, tmp
|
|
422
|
+
f'SELECT "{physical["score"]}", "{physical["tmp"]}" '
|
|
423
|
+
f'FROM "{table_name}" ORDER BY __case_ordinal'
|
|
419
424
|
).fetchall() == [(1.0, ""), (2.0, ""), (3.0, "")]
|
|
420
425
|
connection.close()
|
|
421
426
|
assert validate_wide_dataset(
|
|
@@ -6,12 +6,17 @@ from uuid import uuid4
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
import pyspssio
|
|
8
8
|
import pytest
|
|
9
|
-
from sqlalchemy import create_engine, inspect as inspect_database, text
|
|
9
|
+
from sqlalchemy import MetaData, create_engine, inspect as inspect_database, text
|
|
10
10
|
from sqlalchemy.exc import DBAPIError
|
|
11
11
|
|
|
12
12
|
import openstatspec
|
|
13
13
|
from openstatspec.core import UnsupportedOperationError
|
|
14
14
|
from openstatspec.sql.dolt_conformance import DoltConformanceSource
|
|
15
|
+
from openstatspec.sql.normative import (
|
|
16
|
+
catalog as normative_catalog,
|
|
17
|
+
delete_dataset_representation,
|
|
18
|
+
)
|
|
19
|
+
from openstatspec.sql.wide import create_wide_dataset
|
|
15
20
|
from conformance import compare_sav_semantics, write_supported_semantics_fixture
|
|
16
21
|
|
|
17
22
|
|
|
@@ -206,4 +211,89 @@ def test_live_dolt_candidate_limit_probe_smoke() -> None:
|
|
|
206
211
|
connection.execute(text(
|
|
207
212
|
f"DROP TABLE IF EXISTS {quote(table_name)}"
|
|
208
213
|
))
|
|
209
|
-
engine.dispose()
|
|
214
|
+
engine.dispose()
|
|
215
|
+
|
|
216
|
+
def test_live_postgresql_in_place_create_rejects_exhausted_physical_slots() -> None:
|
|
217
|
+
database_url = os.environ.get("OPENSTATSPEC_POSTGRES_URL")
|
|
218
|
+
if not database_url:
|
|
219
|
+
pytest.skip("OPENSTATSPEC_POSTGRES_URL is not configured")
|
|
220
|
+
|
|
221
|
+
token = uuid4().hex[:12]
|
|
222
|
+
dataset_name = f"inplace_slots_{token}"
|
|
223
|
+
base_variable = {
|
|
224
|
+
"storage_kind": "numeric",
|
|
225
|
+
"string_width": None,
|
|
226
|
+
"label": "",
|
|
227
|
+
"format": "F8.0",
|
|
228
|
+
"print_format": "[5, 8, 0]",
|
|
229
|
+
"write_format": "[5, 8, 0]",
|
|
230
|
+
"measure": "scale",
|
|
231
|
+
"role": "input",
|
|
232
|
+
"alignment": "right",
|
|
233
|
+
"display_width": 8,
|
|
234
|
+
"attributes": "{}",
|
|
235
|
+
"compat_name": None,
|
|
236
|
+
"value_labels": "{}",
|
|
237
|
+
"missing_ranges": "[]",
|
|
238
|
+
}
|
|
239
|
+
variables = [
|
|
240
|
+
{
|
|
241
|
+
**base_variable,
|
|
242
|
+
"ordinal": ordinal,
|
|
243
|
+
"source_name": f"v{ordinal}",
|
|
244
|
+
"physical_name": f"v{ordinal}",
|
|
245
|
+
}
|
|
246
|
+
for ordinal in range(1, 1_600)
|
|
247
|
+
]
|
|
248
|
+
created = None
|
|
249
|
+
engine = create_engine(database_url)
|
|
250
|
+
try:
|
|
251
|
+
openstatspec.initialize_catalog(database_url=database_url)
|
|
252
|
+
created = create_wide_dataset(
|
|
253
|
+
database_url=database_url,
|
|
254
|
+
dataset_id=dataset_name,
|
|
255
|
+
source_name="slot-limit.sav",
|
|
256
|
+
source_format="SAV",
|
|
257
|
+
source_sha256="0" * 64,
|
|
258
|
+
rows=[],
|
|
259
|
+
variables=variables,
|
|
260
|
+
)
|
|
261
|
+
openstatspec.install_in_place_transformation_schema(
|
|
262
|
+
database_url=database_url,
|
|
263
|
+
)
|
|
264
|
+
plan = openstatspec.TransformationPlan(
|
|
265
|
+
(
|
|
266
|
+
openstatspec.DeleteVariableOperation("v1599"),
|
|
267
|
+
openstatspec.CreateVariableOperation("replacement", "numeric"),
|
|
268
|
+
),
|
|
269
|
+
contract="openstatspec-transformation-plan-v0.3",
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
with pytest.raises(openstatspec.TransformationError) as caught:
|
|
273
|
+
openstatspec.apply_transformation_plan_in_place(
|
|
274
|
+
database_url=database_url,
|
|
275
|
+
dataset_id=created["dataset_id"],
|
|
276
|
+
plan=plan,
|
|
277
|
+
actor="service-test",
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
assert caught.value.code == "source_variable_limit"
|
|
281
|
+
with engine.connect() as connection:
|
|
282
|
+
assert connection.execute(text(
|
|
283
|
+
"SELECT COUNT(*) FROM variable WHERE dataset_id = :dataset_id"
|
|
284
|
+
), {"dataset_id": created["dataset_id"]}).scalar_one() == 1_599
|
|
285
|
+
assert connection.execute(text(
|
|
286
|
+
"SELECT COUNT(*) FROM pg_attribute "
|
|
287
|
+
"WHERE attrelid = to_regclass(:relation_name) AND attnum > 0"
|
|
288
|
+
), {"relation_name": created["data_table"]}).scalar_one() == 1_600
|
|
289
|
+
finally:
|
|
290
|
+
if created is not None:
|
|
291
|
+
with engine.begin() as connection:
|
|
292
|
+
delete_dataset_representation(
|
|
293
|
+
connection, normative_catalog(MetaData()), created["dataset_id"],
|
|
294
|
+
)
|
|
295
|
+
quote = connection.dialect.identifier_preparer.quote
|
|
296
|
+
connection.execute(text(
|
|
297
|
+
f"DROP TABLE IF EXISTS {quote(created['data_table'])}"
|
|
298
|
+
))
|
|
299
|
+
engine.dispose()
|
|
@@ -15,6 +15,7 @@ from openstatspec.frontends.spss import (
|
|
|
15
15
|
)
|
|
16
16
|
from openstatspec.transform import (
|
|
17
17
|
CreateVariableOperation,
|
|
18
|
+
DeleteVariableOperation,
|
|
18
19
|
RecodeMatch,
|
|
19
20
|
RecodeOperation,
|
|
20
21
|
RecodeResult,
|
|
@@ -611,6 +612,64 @@ def test_custom_nonempty_input_alias_is_canonical() -> None:
|
|
|
611
612
|
assert plan.input_alias == "survey"
|
|
612
613
|
|
|
613
614
|
|
|
615
|
+
@pytest.mark.parametrize(
|
|
616
|
+
("source", "schema"),
|
|
617
|
+
[
|
|
618
|
+
(
|
|
619
|
+
"STRING first TO last (A8).",
|
|
620
|
+
_schema(
|
|
621
|
+
VariableDefinition("first", "numeric"),
|
|
622
|
+
VariableDefinition("last", "numeric"),
|
|
623
|
+
),
|
|
624
|
+
),
|
|
625
|
+
(
|
|
626
|
+
"DELETE VARIABLES first TO last.",
|
|
627
|
+
_schema(
|
|
628
|
+
VariableDefinition("first", "numeric"),
|
|
629
|
+
VariableDefinition("last", "numeric"),
|
|
630
|
+
),
|
|
631
|
+
),
|
|
632
|
+
],
|
|
633
|
+
)
|
|
634
|
+
def test_string_and_delete_variables_reject_unsupported_to_ranges(
|
|
635
|
+
source: str, schema: VariableSchema,
|
|
636
|
+
) -> None:
|
|
637
|
+
assert _error(source, schema).code == "unsupported_spss_feature"
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def test_delete_final_variable_requires_a_later_explicit_create() -> None:
|
|
641
|
+
schema = _schema(VariableDefinition("only", "numeric"))
|
|
642
|
+
|
|
643
|
+
assert _error("DELETE VARIABLES only.", schema).code == "cannot_delete_last_variable"
|
|
644
|
+
|
|
645
|
+
bound = _compile("DELETE VARIABLES only. STRING replacement (A3).", schema)
|
|
646
|
+
assert bound.plan.operations == (
|
|
647
|
+
DeleteVariableOperation("only"),
|
|
648
|
+
CreateVariableOperation("replacement", "string", 3),
|
|
649
|
+
)
|
|
650
|
+
assert bound.output_schema.variables == (
|
|
651
|
+
VariableDefinition("replacement", "string", declared_string_width=3),
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def test_recode_string_literals_respect_declared_width() -> None:
|
|
656
|
+
schema = _schema(VariableDefinition(
|
|
657
|
+
"note", "string", declared_string_width=3,
|
|
658
|
+
))
|
|
659
|
+
|
|
660
|
+
assert _error("RECODE note ('a' = 'abcd').", schema).code == "string_width_exceeded"
|
|
661
|
+
assert _compile("RECODE note ('a' = '\u00e4b').", schema).plan.operations
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def test_recode_copy_preserves_source_declared_width() -> None:
|
|
665
|
+
schema = _schema(VariableDefinition(
|
|
666
|
+
"note", "string", declared_string_width=3,
|
|
667
|
+
))
|
|
668
|
+
|
|
669
|
+
bound = _compile("RECODE note ('yes' = COPY).", schema)
|
|
670
|
+
assert bound.output_schema == schema
|
|
671
|
+
|
|
672
|
+
|
|
614
673
|
def test_generic_plan_binding_validates_sequential_schema_state() -> None:
|
|
615
674
|
plan = TransformationPlan((
|
|
616
675
|
RecodeOperation(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|