framework-py 0.1.0.dev27__py3-none-any.whl → 0.1.0.dev28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framework/data.py +109 -20
- {framework_py-0.1.0.dev27.dist-info → framework_py-0.1.0.dev28.dist-info}/METADATA +1 -1
- {framework_py-0.1.0.dev27.dist-info → framework_py-0.1.0.dev28.dist-info}/RECORD +5 -5
- {framework_py-0.1.0.dev27.dist-info → framework_py-0.1.0.dev28.dist-info}/WHEEL +0 -0
- {framework_py-0.1.0.dev27.dist-info → framework_py-0.1.0.dev28.dist-info}/licenses/LICENSE +0 -0
framework/data.py
CHANGED
|
@@ -436,13 +436,39 @@ def data_read(
|
|
|
436
436
|
return data
|
|
437
437
|
|
|
438
438
|
|
|
439
|
-
def
|
|
440
|
-
"""The
|
|
439
|
+
def _catalog_entry(name: str) -> DataInfo | None:
|
|
440
|
+
"""The catalog entry ``name`` already has, or None."""
|
|
441
|
+
try:
|
|
442
|
+
data = settings_read().get("data")
|
|
443
|
+
except FrameworkError:
|
|
444
|
+
return None
|
|
445
|
+
if catalog_locate(name, data) is None:
|
|
446
|
+
return None
|
|
447
|
+
try:
|
|
448
|
+
return data_info(name)
|
|
449
|
+
except DataNotFoundError:
|
|
450
|
+
# An entry without a path is saved like a new name
|
|
451
|
+
return None
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _delimiter_name(delimiter: str) -> str:
|
|
455
|
+
"""``comma``, ``tab``, ``semicolon``, or ``space`` for a delimiter or its character."""
|
|
456
|
+
_separator(delimiter)
|
|
457
|
+
for key, character in DELIMITERS.items():
|
|
458
|
+
if delimiter in (key, character):
|
|
459
|
+
return key
|
|
460
|
+
return delimiter
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None, DataInfo | None]:
|
|
464
|
+
"""The file to write, its type, the catalog name (None when ``path`` is a file path),
|
|
465
|
+
and the catalog entry the name already has (None for a new name or a file path).
|
|
441
466
|
|
|
442
467
|
A relative file path starts at the project root, as the catalog's paths do, so a
|
|
443
468
|
notebook in ``notebooks/`` writes where a script in the root would. A dot name that
|
|
444
469
|
ends in an extension, such as ``inputs.final.x.csv``, is the name ``inputs.final.x``
|
|
445
|
-
saved as that type.
|
|
470
|
+
saved as that type. A name already in the catalog is saved to its entry's file, in
|
|
471
|
+
its entry's type, as Stata's ``fw_save`` does.
|
|
446
472
|
"""
|
|
447
473
|
has_slash = "/" in path or "\\" in path
|
|
448
474
|
extension = re.search(r"\.(csv|parquet|rds)$", path, re.IGNORECASE)
|
|
@@ -458,7 +484,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
|
|
|
458
484
|
if not path.lower().endswith(f".{type}"):
|
|
459
485
|
path = f"{path}.{type}"
|
|
460
486
|
inform(f"Note: Added .{type} extension to path")
|
|
461
|
-
return in_root(path, root), type, None
|
|
487
|
+
return in_root(path, root), type, None, None
|
|
462
488
|
|
|
463
489
|
if extension:
|
|
464
490
|
given = extension.group(1).lower()
|
|
@@ -467,6 +493,18 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
|
|
|
467
493
|
type = given
|
|
468
494
|
path = path[: extension.start()]
|
|
469
495
|
|
|
496
|
+
entry = _catalog_entry(path)
|
|
497
|
+
if entry is not None:
|
|
498
|
+
# tsv is delimited text with tabs, which data_save() writes as csv
|
|
499
|
+
entry_type = "csv" if entry.type == "tsv" else entry.type
|
|
500
|
+
if type is not None and type.lower() != entry_type:
|
|
501
|
+
raise FrameworkError(
|
|
502
|
+
f"'{path}' is a {entry.type} file in the data catalog "
|
|
503
|
+
f"({relative_to_root(entry.path, root)}), but type is '{type}'.\n"
|
|
504
|
+
"Leave type out to save it as its entry says, or save it under another name."
|
|
505
|
+
)
|
|
506
|
+
return entry.path, entry_type, path, entry
|
|
507
|
+
|
|
470
508
|
parts = path.split(".")
|
|
471
509
|
if len(parts) == 1 or not all(parts):
|
|
472
510
|
suffix = type or "parquet"
|
|
@@ -497,7 +535,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
|
|
|
497
535
|
)
|
|
498
536
|
type = type or "parquet"
|
|
499
537
|
target = in_root(folder, root) / f"{'_'.join(parts[end:])}.{type}"
|
|
500
|
-
return target, type, path
|
|
538
|
+
return target, type, path, None
|
|
501
539
|
|
|
502
540
|
|
|
503
541
|
def _in_settings_folder(folder: Path) -> bool:
|
|
@@ -598,7 +636,7 @@ def data_save(
|
|
|
598
636
|
data: pd.DataFrame,
|
|
599
637
|
path: str,
|
|
600
638
|
type: str | None = None,
|
|
601
|
-
delimiter: str =
|
|
639
|
+
delimiter: str | None = None,
|
|
602
640
|
locked: bool | None = None,
|
|
603
641
|
force: bool = False,
|
|
604
642
|
unlock: bool = False,
|
|
@@ -617,14 +655,20 @@ def data_save(
|
|
|
617
655
|
file path with a ``/``, ending in ``.csv`` or ``.parquet``. A name's leading
|
|
618
656
|
parts select a folder from ``directories``: the longest run that
|
|
619
657
|
names a key, such as ``inputs_intermediate``, ``inputs_private_raw``, or a
|
|
620
|
-
course's ``data``. The rest becomes the file name. A name is also added
|
|
621
|
-
the data catalog, so ``data_read()`` finds it. A name
|
|
658
|
+
course's ``data``. The rest becomes the file name. A new name is also added
|
|
659
|
+
to the data catalog, so ``data_read()`` finds it. A name already in the
|
|
660
|
+
catalog is saved to its entry's file, in its entry's type and delimiter, and
|
|
661
|
+
the entry is left as it is (its description and other fields are kept). A
|
|
662
|
+
name may end in
|
|
622
663
|
``.csv`` or ``.parquet`` to choose the type: ``"inputs.final.x.csv"`` is
|
|
623
664
|
the name ``inputs.final.x`` saved as CSV. A relative file path starts at the
|
|
624
665
|
project root, wherever Python runs.
|
|
625
|
-
type: ``"parquet"`` or ``"csv"``. Default:
|
|
626
|
-
|
|
627
|
-
|
|
666
|
+
type: ``"parquet"`` or ``"csv"``. Default: the catalog entry's type, then the
|
|
667
|
+
file extension, then Parquet for a new catalog name. A type that differs
|
|
668
|
+
from the catalog entry's is an error.
|
|
669
|
+
delimiter: Delimiter for CSV files: comma, tab, semicolon, or space. Default:
|
|
670
|
+
the catalog entry's delimiter, else comma. A delimiter that differs from the
|
|
671
|
+
catalog entry's is an error.
|
|
628
672
|
locked: Whether later reads must find the file unchanged. Default: unlocked, as
|
|
629
673
|
in R, so a pipeline can save the data again. Pass ``locked=True`` to lock data
|
|
630
674
|
you mean to keep as it is. Left out, a save keeps a lock the data already
|
|
@@ -648,14 +692,24 @@ def data_save(
|
|
|
648
692
|
"""
|
|
649
693
|
if not isinstance(data, pd.DataFrame):
|
|
650
694
|
raise FrameworkError("data_save() needs a pandas DataFrame")
|
|
651
|
-
target, file_type, name = _save_target(path, type)
|
|
695
|
+
target, file_type, name, entry = _save_target(path, type)
|
|
696
|
+
if entry is not None and entry.delimiter is not None and file_type == "csv":
|
|
697
|
+
if delimiter is not None and _delimiter_name(delimiter) != entry.delimiter:
|
|
698
|
+
raise FrameworkError(
|
|
699
|
+
f"'{name}' is {entry.delimiter}-delimited in the data catalog, but "
|
|
700
|
+
f"delimiter is '{delimiter}'. Leave delimiter out to save it as its entry "
|
|
701
|
+
"says, or save it under another name."
|
|
702
|
+
)
|
|
703
|
+
delimiter = entry.delimiter
|
|
704
|
+
delimiter = _delimiter_name(delimiter or "comma")
|
|
652
705
|
if file_type == "rds":
|
|
653
706
|
raise UnsupportedFormatError(
|
|
654
707
|
"Python does not write .rds files. Save as 'parquet' or 'csv', which both "
|
|
655
708
|
"languages read."
|
|
656
709
|
)
|
|
657
710
|
if file_type not in _SAVE_TYPES:
|
|
658
|
-
|
|
711
|
+
where = f" ('{name}' in the data catalog)" if entry is not None else ""
|
|
712
|
+
raise UnsupportedFormatError(f"data_save() writes csv or parquet, not {file_type}{where}")
|
|
659
713
|
|
|
660
714
|
folder = target.parent
|
|
661
715
|
# A missing folder is created when it is a settings folder or inside one (git
|
|
@@ -734,7 +788,21 @@ def data_save(
|
|
|
734
788
|
|
|
735
789
|
ledger_push_quietly(key, digest, target)
|
|
736
790
|
|
|
737
|
-
if name is not None:
|
|
791
|
+
if name is not None and entry is not None:
|
|
792
|
+
# The entry already names this file. Only its lock can change, so the
|
|
793
|
+
# catalog's flag agrees with framework.db; everything else is left as it is.
|
|
794
|
+
if entry.locked != locked:
|
|
795
|
+
try:
|
|
796
|
+
_catalog_set(name, {"locked": True} if locked else {}, drop=("locked",))
|
|
797
|
+
inform(f"Data catalog entry {'locked' if locked else 'unlocked'}: {name}")
|
|
798
|
+
except FrameworkError as error:
|
|
799
|
+
warnings.warn(
|
|
800
|
+
f"File saved but could not update the lock in the data catalog for "
|
|
801
|
+
f"'{name}': {error}",
|
|
802
|
+
FrameworkWarning,
|
|
803
|
+
stacklevel=2,
|
|
804
|
+
)
|
|
805
|
+
elif name is not None:
|
|
738
806
|
try:
|
|
739
807
|
data_add(target, name=name, type=file_type, delimiter=delimiter, locked=locked)
|
|
740
808
|
except FrameworkError as error:
|
|
@@ -768,11 +836,18 @@ def _derive_name(file_path: Path, config: dict[str, Any], root: Path) -> str:
|
|
|
768
836
|
return file_path.stem
|
|
769
837
|
|
|
770
838
|
|
|
771
|
-
def _catalog_set(name: str, entry: dict[str, Any]) ->
|
|
839
|
+
def _catalog_set(name: str, entry: dict[str, Any], drop: Sequence[str] = ()) -> bool:
|
|
772
840
|
"""Write one catalog entry into settings.yml or the split file that holds ``data``.
|
|
773
841
|
|
|
774
|
-
|
|
842
|
+
An existing entry is updated, not replaced: the fields in ``entry`` are set, the
|
|
843
|
+
fields in ``drop`` that ``entry`` leaves out are removed, and every other field (a
|
|
844
|
+
description, notes, stata_names) is kept. The file is edited in place, so its
|
|
845
|
+
comments and other entries keep their form.
|
|
846
|
+
|
|
847
|
+
Returns:
|
|
848
|
+
Whether the entry already existed.
|
|
775
849
|
"""
|
|
850
|
+
existed = False
|
|
776
851
|
settings_file = find_settings_file()
|
|
777
852
|
if settings_file is None:
|
|
778
853
|
raise SettingsError("Configuration file 'settings.yml' (or legacy config.yml) not found")
|
|
@@ -782,9 +857,20 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
|
|
|
782
857
|
|
|
783
858
|
def set_entry(current: dict[str, Any]) -> None:
|
|
784
859
|
# An entry already written with dotted keys is updated where it is.
|
|
860
|
+
nonlocal existed
|
|
785
861
|
existing = catalog_locate(name, current)
|
|
786
862
|
if existing is not None:
|
|
787
|
-
|
|
863
|
+
existed = True
|
|
864
|
+
holder, key = existing
|
|
865
|
+
old = holder[key]
|
|
866
|
+
if not isinstance(old, dict):
|
|
867
|
+
# A bare path entry becomes a mapping
|
|
868
|
+
old = {"path": old} if isinstance(old, str) else {}
|
|
869
|
+
holder[key] = old
|
|
870
|
+
for field in drop:
|
|
871
|
+
if field not in entry:
|
|
872
|
+
old.pop(field, None)
|
|
873
|
+
old.update(entry)
|
|
788
874
|
return
|
|
789
875
|
node = current
|
|
790
876
|
parts = name.split(".")
|
|
@@ -810,7 +896,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
|
|
|
810
896
|
return {"data": content}
|
|
811
897
|
|
|
812
898
|
_yaml.update(in_root(source, settings_file.parent), in_split)
|
|
813
|
-
return
|
|
899
|
+
return existed
|
|
814
900
|
|
|
815
901
|
def in_settings(raw: Any) -> Any:
|
|
816
902
|
raw = raw if isinstance(raw, dict) else {}
|
|
@@ -821,6 +907,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
|
|
|
821
907
|
return raw
|
|
822
908
|
|
|
823
909
|
_yaml.update(settings_file, in_settings)
|
|
910
|
+
return existed
|
|
824
911
|
|
|
825
912
|
|
|
826
913
|
def data_add(
|
|
@@ -889,7 +976,9 @@ def data_add(
|
|
|
889
976
|
if locked:
|
|
890
977
|
entry["locked"] = True
|
|
891
978
|
if update_config:
|
|
892
|
-
|
|
893
|
-
|
|
979
|
+
# An existing entry keeps its other fields (description, notes); the
|
|
980
|
+
# delimiter and lock follow this file
|
|
981
|
+
existed = _catalog_set(name, entry, drop=("delimiter", "locked"))
|
|
982
|
+
inform(f"Data spec {'updated in' if existed else 'added to'} config: {name}")
|
|
894
983
|
inform(f'\nYou can now read this data with: data_read("{name}")')
|
|
895
984
|
return entry
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: framework-py
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev28
|
|
4
4
|
Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
|
|
5
5
|
Project-URL: Homepage, https://framework.pub
|
|
6
6
|
Project-URL: Source, https://github.com/frameworktools/framework-py
|
|
@@ -31,7 +31,7 @@ framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
|
|
|
31
31
|
framework/connections.py,sha256=99JqrNAi2zBpNYyxLV6SYj-NAJjh56pYh9jaMSYOHCU,32742
|
|
32
32
|
framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
|
|
33
33
|
framework/create.py,sha256=3ykMUhMslVsjiE__9Scx5v_6CU2Ye36wWqlp17j_vGw,53030
|
|
34
|
-
framework/data.py,sha256=
|
|
34
|
+
framework/data.py,sha256=HpiRl5qTk1o9G-XXNDgsWhNOgXlyYegv8-8KDz-HyPA,40853
|
|
35
35
|
framework/data_registry.py,sha256=2hbXsoY9IF5Zj6lkiTE2RO2c5j2Z2vXKm2iTN7FdsQk,22499
|
|
36
36
|
framework/db.py,sha256=UYEUCvk4vGytBkmdNTUxRpYPtoFSDHwTpgpfWFw31aE,8522
|
|
37
37
|
framework/display.py,sha256=0FuKUt67uf5h1WjD4bwzoruc2XAN4y_j3mz_f5FskhE,5334
|
|
@@ -141,7 +141,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
|
|
|
141
141
|
framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
|
|
142
142
|
framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
|
|
143
143
|
framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
|
|
144
|
-
framework_py-0.1.0.
|
|
145
|
-
framework_py-0.1.0.
|
|
146
|
-
framework_py-0.1.0.
|
|
147
|
-
framework_py-0.1.0.
|
|
144
|
+
framework_py-0.1.0.dev28.dist-info/METADATA,sha256=cTispVoaGZgGGcA8GR630W4VE96ZRIRwoxs1HjTeIdE,4682
|
|
145
|
+
framework_py-0.1.0.dev28.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
146
|
+
framework_py-0.1.0.dev28.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
|
|
147
|
+
framework_py-0.1.0.dev28.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|