framework-py 0.1.0.dev27__py3-none-any.whl → 0.1.0.dev28__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
framework/data.py CHANGED
@@ -436,13 +436,39 @@ def data_read(
436
436
  return data
437
437
 
438
438
 
439
- def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
440
- """The file to write, its type, and the catalog name (None when ``path`` is a file path).
439
+ def _catalog_entry(name: str) -> DataInfo | None:
440
+ """The catalog entry ``name`` already has, or None."""
441
+ try:
442
+ data = settings_read().get("data")
443
+ except FrameworkError:
444
+ return None
445
+ if catalog_locate(name, data) is None:
446
+ return None
447
+ try:
448
+ return data_info(name)
449
+ except DataNotFoundError:
450
+ # An entry without a path is saved like a new name
451
+ return None
452
+
453
+
454
+ def _delimiter_name(delimiter: str) -> str:
455
+ """``comma``, ``tab``, ``semicolon``, or ``space`` for a delimiter or its character."""
456
+ _separator(delimiter)
457
+ for key, character in DELIMITERS.items():
458
+ if delimiter in (key, character):
459
+ return key
460
+ return delimiter
461
+
462
+
463
+ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None, DataInfo | None]:
464
+ """The file to write, its type, the catalog name (None when ``path`` is a file path),
465
+ and the catalog entry the name already has (None for a new name or a file path).
441
466
 
442
467
  A relative file path starts at the project root, as the catalog's paths do, so a
443
468
  notebook in ``notebooks/`` writes where a script in the root would. A dot name that
444
469
  ends in an extension, such as ``inputs.final.x.csv``, is the name ``inputs.final.x``
445
- saved as that type.
470
+ saved as that type. A name already in the catalog is saved to its entry's file, in
471
+ its entry's type, as Stata's ``fw_save`` does.
446
472
  """
447
473
  has_slash = "/" in path or "\\" in path
448
474
  extension = re.search(r"\.(csv|parquet|rds)$", path, re.IGNORECASE)
@@ -458,7 +484,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
458
484
  if not path.lower().endswith(f".{type}"):
459
485
  path = f"{path}.{type}"
460
486
  inform(f"Note: Added .{type} extension to path")
461
- return in_root(path, root), type, None
487
+ return in_root(path, root), type, None, None
462
488
 
463
489
  if extension:
464
490
  given = extension.group(1).lower()
@@ -467,6 +493,18 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
467
493
  type = given
468
494
  path = path[: extension.start()]
469
495
 
496
+ entry = _catalog_entry(path)
497
+ if entry is not None:
498
+ # tsv is delimited text with tabs, which data_save() writes as csv
499
+ entry_type = "csv" if entry.type == "tsv" else entry.type
500
+ if type is not None and type.lower() != entry_type:
501
+ raise FrameworkError(
502
+ f"'{path}' is a {entry.type} file in the data catalog "
503
+ f"({relative_to_root(entry.path, root)}), but type is '{type}'.\n"
504
+ "Leave type out to save it as its entry says, or save it under another name."
505
+ )
506
+ return entry.path, entry_type, path, entry
507
+
470
508
  parts = path.split(".")
471
509
  if len(parts) == 1 or not all(parts):
472
510
  suffix = type or "parquet"
@@ -497,7 +535,7 @@ def _save_target(path: str, type: str | None) -> tuple[Path, str, str | None]:
497
535
  )
498
536
  type = type or "parquet"
499
537
  target = in_root(folder, root) / f"{'_'.join(parts[end:])}.{type}"
500
- return target, type, path
538
+ return target, type, path, None
501
539
 
502
540
 
503
541
  def _in_settings_folder(folder: Path) -> bool:
@@ -598,7 +636,7 @@ def data_save(
598
636
  data: pd.DataFrame,
599
637
  path: str,
600
638
  type: str | None = None,
601
- delimiter: str = "comma",
639
+ delimiter: str | None = None,
602
640
  locked: bool | None = None,
603
641
  force: bool = False,
604
642
  unlock: bool = False,
@@ -617,14 +655,20 @@ def data_save(
617
655
  file path with a ``/``, ending in ``.csv`` or ``.parquet``. A name's leading
618
656
  parts select a folder from ``directories``: the longest run that
619
657
  names a key, such as ``inputs_intermediate``, ``inputs_private_raw``, or a
620
- course's ``data``. The rest becomes the file name. A name is also added to
621
- the data catalog, so ``data_read()`` finds it. A name may end in
658
+ course's ``data``. The rest becomes the file name. A new name is also added
659
+ to the data catalog, so ``data_read()`` finds it. A name already in the
660
+ catalog is saved to its entry's file, in its entry's type and delimiter, and
661
+ the entry is left as it is (its description and other fields are kept). A
662
+ name may end in
622
663
  ``.csv`` or ``.parquet`` to choose the type: ``"inputs.final.x.csv"`` is
623
664
  the name ``inputs.final.x`` saved as CSV. A relative file path starts at the
624
665
  project root, wherever Python runs.
625
- type: ``"parquet"`` or ``"csv"``. Default: from the file extension,
626
- or Parquet for a catalog name.
627
- delimiter: Delimiter for CSV files: comma, tab, semicolon, or space.
666
+ type: ``"parquet"`` or ``"csv"``. Default: the catalog entry's type, then the
667
+ file extension, then Parquet for a new catalog name. A type that differs
668
+ from the catalog entry's is an error.
669
+ delimiter: Delimiter for CSV files: comma, tab, semicolon, or space. Default:
670
+ the catalog entry's delimiter, else comma. A delimiter that differs from the
671
+ catalog entry's is an error.
628
672
  locked: Whether later reads must find the file unchanged. Default: unlocked, as
629
673
  in R, so a pipeline can save the data again. Pass ``locked=True`` to lock data
630
674
  you mean to keep as it is. Left out, a save keeps a lock the data already
@@ -648,14 +692,24 @@ def data_save(
648
692
  """
649
693
  if not isinstance(data, pd.DataFrame):
650
694
  raise FrameworkError("data_save() needs a pandas DataFrame")
651
- target, file_type, name = _save_target(path, type)
695
+ target, file_type, name, entry = _save_target(path, type)
696
+ if entry is not None and entry.delimiter is not None and file_type == "csv":
697
+ if delimiter is not None and _delimiter_name(delimiter) != entry.delimiter:
698
+ raise FrameworkError(
699
+ f"'{name}' is {entry.delimiter}-delimited in the data catalog, but "
700
+ f"delimiter is '{delimiter}'. Leave delimiter out to save it as its entry "
701
+ "says, or save it under another name."
702
+ )
703
+ delimiter = entry.delimiter
704
+ delimiter = _delimiter_name(delimiter or "comma")
652
705
  if file_type == "rds":
653
706
  raise UnsupportedFormatError(
654
707
  "Python does not write .rds files. Save as 'parquet' or 'csv', which both "
655
708
  "languages read."
656
709
  )
657
710
  if file_type not in _SAVE_TYPES:
658
- raise UnsupportedFormatError(f"data_save() writes csv or parquet, not {file_type}")
711
+ where = f" ('{name}' in the data catalog)" if entry is not None else ""
712
+ raise UnsupportedFormatError(f"data_save() writes csv or parquet, not {file_type}{where}")
659
713
 
660
714
  folder = target.parent
661
715
  # A missing folder is created when it is a settings folder or inside one (git
@@ -734,7 +788,21 @@ def data_save(
734
788
 
735
789
  ledger_push_quietly(key, digest, target)
736
790
 
737
- if name is not None:
791
+ if name is not None and entry is not None:
792
+ # The entry already names this file. Only its lock can change, so the
793
+ # catalog's flag agrees with framework.db; everything else is left as it is.
794
+ if entry.locked != locked:
795
+ try:
796
+ _catalog_set(name, {"locked": True} if locked else {}, drop=("locked",))
797
+ inform(f"Data catalog entry {'locked' if locked else 'unlocked'}: {name}")
798
+ except FrameworkError as error:
799
+ warnings.warn(
800
+ f"File saved but could not update the lock in the data catalog for "
801
+ f"'{name}': {error}",
802
+ FrameworkWarning,
803
+ stacklevel=2,
804
+ )
805
+ elif name is not None:
738
806
  try:
739
807
  data_add(target, name=name, type=file_type, delimiter=delimiter, locked=locked)
740
808
  except FrameworkError as error:
@@ -768,11 +836,18 @@ def _derive_name(file_path: Path, config: dict[str, Any], root: Path) -> str:
768
836
  return file_path.stem
769
837
 
770
838
 
771
- def _catalog_set(name: str, entry: dict[str, Any]) -> None:
839
+ def _catalog_set(name: str, entry: dict[str, Any], drop: Sequence[str] = ()) -> bool:
772
840
  """Write one catalog entry into settings.yml or the split file that holds ``data``.
773
841
 
774
- The file is edited in place, so its comments and other entries keep their form.
842
+ An existing entry is updated, not replaced: the fields in ``entry`` are set, the
843
+ fields in ``drop`` that ``entry`` leaves out are removed, and every other field (a
844
+ description, notes, stata_names) is kept. The file is edited in place, so its
845
+ comments and other entries keep their form.
846
+
847
+ Returns:
848
+ Whether the entry already existed.
775
849
  """
850
+ existed = False
776
851
  settings_file = find_settings_file()
777
852
  if settings_file is None:
778
853
  raise SettingsError("Configuration file 'settings.yml' (or legacy config.yml) not found")
@@ -782,9 +857,20 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
782
857
 
783
858
  def set_entry(current: dict[str, Any]) -> None:
784
859
  # An entry already written with dotted keys is updated where it is.
860
+ nonlocal existed
785
861
  existing = catalog_locate(name, current)
786
862
  if existing is not None:
787
- existing[0][existing[1]] = entry
863
+ existed = True
864
+ holder, key = existing
865
+ old = holder[key]
866
+ if not isinstance(old, dict):
867
+ # A bare path entry becomes a mapping
868
+ old = {"path": old} if isinstance(old, str) else {}
869
+ holder[key] = old
870
+ for field in drop:
871
+ if field not in entry:
872
+ old.pop(field, None)
873
+ old.update(entry)
788
874
  return
789
875
  node = current
790
876
  parts = name.split(".")
@@ -810,7 +896,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
810
896
  return {"data": content}
811
897
 
812
898
  _yaml.update(in_root(source, settings_file.parent), in_split)
813
- return
899
+ return existed
814
900
 
815
901
  def in_settings(raw: Any) -> Any:
816
902
  raw = raw if isinstance(raw, dict) else {}
@@ -821,6 +907,7 @@ def _catalog_set(name: str, entry: dict[str, Any]) -> None:
821
907
  return raw
822
908
 
823
909
  _yaml.update(settings_file, in_settings)
910
+ return existed
824
911
 
825
912
 
826
913
  def data_add(
@@ -889,7 +976,9 @@ def data_add(
889
976
  if locked:
890
977
  entry["locked"] = True
891
978
  if update_config:
892
- _catalog_set(name, entry)
893
- inform(f"Data spec added to config: {name}")
979
+ # An existing entry keeps its other fields (description, notes); the
980
+ # delimiter and lock follow this file
981
+ existed = _catalog_set(name, entry, drop=("delimiter", "locked"))
982
+ inform(f"Data spec {'updated in' if existed else 'added to'} config: {name}")
894
983
  inform(f'\nYou can now read this data with: data_read("{name}")')
895
984
  return entry
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: framework-py
3
- Version: 0.1.0.dev27
3
+ Version: 0.1.0.dev28
4
4
  Summary: Structured data science projects: a data catalog, integrity tracking, caching, outputs, and AI guardrails
5
5
  Project-URL: Homepage, https://framework.pub
6
6
  Project-URL: Source, https://github.com/frameworktools/framework-py
@@ -31,7 +31,7 @@ framework/config_home.py,sha256=iSb0VuENgNOSnsZ_v53xFqjRiGgrB7OzZbHbyDOqKzs,8265
31
31
  framework/connections.py,sha256=99JqrNAi2zBpNYyxLV6SYj-NAJjh56pYh9jaMSYOHCU,32742
32
32
  framework/console.py,sha256=zQ0LuFDcxe1W1kuc5E1bJ03h8d3cvwFoqDT8dAJQyN0,1267
33
33
  framework/create.py,sha256=3ykMUhMslVsjiE__9Scx5v_6CU2Ye36wWqlp17j_vGw,53030
34
- framework/data.py,sha256=cAOxwmF_nowXIZPl6vW5-ZUyI_z16H1bRJM60gJgeoo,36522
34
+ framework/data.py,sha256=HpiRl5qTk1o9G-XXNDgsWhNOgXlyYegv8-8KDz-HyPA,40853
35
35
  framework/data_registry.py,sha256=2hbXsoY9IF5Zj6lkiTE2RO2c5j2Z2vXKm2iTN7FdsQk,22499
36
36
  framework/db.py,sha256=UYEUCvk4vGytBkmdNTUxRpYPtoFSDHwTpgpfWFw31aE,8522
37
37
  framework/display.py,sha256=0FuKUt67uf5h1WjD4bwzoruc2XAN4y_j3mz_f5FskhE,5334
@@ -141,7 +141,7 @@ framework/templates/stubs/notebook-default.qmd,sha256=8Js1S2BcHYeQWmJS-kt31gBvIj
141
141
  framework/templates/stubs/notebook-minimal.qmd,sha256=2U2k8judffhrh_6m6r6OAbQBCYEtViAjeKRBqBYCJTM,178
142
142
  framework/templates/stubs/notebook-revealjs.qmd,sha256=_ztT_gHoudpB5DIjlbvtB5zQ8163DeV8xbWcT-HgJrI,409
143
143
  framework/templates/stubs/script-default.py,sha256=vbWqgW0_rHot17mQC3tYb92g_9L-cbTOeL3y99BRrSk,216
144
- framework_py-0.1.0.dev27.dist-info/METADATA,sha256=dUHbI-_zRBu2n8F4xgWwa9oUNAZ4Jjp7UAAcO6cCSsU,4682
145
- framework_py-0.1.0.dev27.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
146
- framework_py-0.1.0.dev27.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
147
- framework_py-0.1.0.dev27.dist-info/RECORD,,
144
+ framework_py-0.1.0.dev28.dist-info/METADATA,sha256=cTispVoaGZgGGcA8GR630W4VE96ZRIRwoxs1HjTeIdE,4682
145
+ framework_py-0.1.0.dev28.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
146
+ framework_py-0.1.0.dev28.dist-info/licenses/LICENSE,sha256=giOOgSwALkZimC7uhLZQK8Untez641bLW1-2-Cjvmq0,1072
147
+ framework_py-0.1.0.dev28.dist-info/RECORD,,