pipelime-python 2.2.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. pipelime/__init__.py +1 -1
  2. pipelime/choixe/ast/nodes.py +41 -3
  3. pipelime/choixe/utils/common.py +35 -1
  4. pipelime/choixe/visitors/decoder.py +5 -2
  5. pipelime/choixe/visitors/inspector.py +8 -5
  6. pipelime/choixe/visitors/processor.py +19 -8
  7. pipelime/choixe/xconfig.py +3 -3
  8. pipelime/cli/main.py +16 -8
  9. pipelime/cli/pretty_print.py +72 -77
  10. pipelime/cli/tui/tui.py +3 -2
  11. pipelime/cli/tui/utils.py +27 -24
  12. pipelime/cli/utils.py +64 -37
  13. pipelime/commands/general.py +56 -53
  14. pipelime/commands/interfaces.py +92 -106
  15. pipelime/commands/piper.py +13 -19
  16. pipelime/commands/resume.py +9 -7
  17. pipelime/commands/shell.py +1 -3
  18. pipelime/commands/split_ops.py +15 -28
  19. pipelime/commands/tempman.py +2 -2
  20. pipelime/commands/toy_dataset.py +1 -3
  21. pipelime/piper/__init__.py +1 -0
  22. pipelime/piper/checkpoint.py +5 -3
  23. pipelime/piper/model.py +105 -80
  24. pipelime/piper/parsers/choixe_parser.py +1 -1
  25. pipelime/piper/progress/listener/receiver/zmq.py +1 -1
  26. pipelime/piper/progress/model.py +3 -3
  27. pipelime/piper/progress/tracker/zmq.py +1 -1
  28. pipelime/sequences/grabber.py +3 -2
  29. pipelime/sequences/pipes/base.py +2 -3
  30. pipelime/sequences/pipes/mapping.py +4 -3
  31. pipelime/sequences/pipes/operations.py +7 -14
  32. pipelime/sequences/pipes/validation.py +3 -5
  33. pipelime/sequences/pipes/writers.py +9 -7
  34. pipelime/sequences/sample.py +6 -2
  35. pipelime/sequences/samples_sequence.py +16 -17
  36. pipelime/sequences/sources/from_callable.py +1 -1
  37. pipelime/sequences/sources/raw.py +1 -1
  38. pipelime/sequences/sources/readers.py +22 -13
  39. pipelime/sequences/sources/toy_dataset.py +3 -2
  40. pipelime/sequences/utils.py +3 -8
  41. pipelime/stages/augmentations.py +30 -36
  42. pipelime/stages/base.py +33 -42
  43. pipelime/stages/entities.py +167 -99
  44. pipelime/stages/item_info.py +3 -2
  45. pipelime/stages/item_replacement.py +7 -8
  46. pipelime/stages/item_sources.py +1 -1
  47. pipelime/stages/key_transformations.py +3 -2
  48. pipelime/utils/pydantic_compat.py +1064 -0
  49. pipelime/utils/pydantic_types.py +186 -220
  50. {pipelime_python-2.2.0.dist-info → pipelime_python-3.0.0.dist-info}/METADATA +9 -4
  51. {pipelime_python-2.2.0.dist-info → pipelime_python-3.0.0.dist-info}/RECORD +54 -53
  52. {pipelime_python-2.2.0.dist-info → pipelime_python-3.0.0.dist-info}/WHEEL +1 -1
  53. {pipelime_python-2.2.0.dist-info → pipelime_python-3.0.0.dist-info}/entry_points.txt +0 -0
  54. {pipelime_python-2.2.0.dist-info → pipelime_python-3.0.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,14 +1,15 @@
1
1
  import typing as t
2
2
 
3
- import pydantic.v1 as pyd
3
+ import pydantic as pyd
4
4
 
5
5
  import pipelime.commands.interfaces as pl_interfaces
6
6
  import pipelime.utils.pydantic_types as pl_types
7
- from pipelime.piper import PipelimeCommand, PiperPortType
7
+ from pipelime.piper import Field, PipelimeCommand, PiperPortType
8
8
  from pipelime.stages import StageInput
9
+ from pipelime.utils.pydantic_compat import PipelimeModel, model_title
9
10
 
10
11
 
11
- class OutputTime(pyd.BaseModel):
12
+ class OutputTime(PipelimeModel):
12
13
  nanosec: int
13
14
 
14
15
  def __repr__(self) -> str:
@@ -20,7 +21,7 @@ class OutputTime(pyd.BaseModel):
20
21
  return time_to_str(self.nanosec)
21
22
 
22
23
 
23
- class OutputStageTime(pyd.BaseModel):
24
+ class OutputStageTime(PipelimeModel):
24
25
  stages: t.Mapping[str, OutputTime]
25
26
 
26
27
  def __repr__(self) -> str:
@@ -55,7 +56,7 @@ class TimeItCommand(PipelimeCommand, title="timeit"):
55
56
  )
56
57
  )
57
58
 
58
- operations: t.Optional[pl_types.YamlInput] = pyd.Field(
59
+ operations: t.Optional[pl_types.YamlInput] = Field(
59
60
  None,
60
61
  alias="op",
61
62
  description=(
@@ -68,29 +69,29 @@ class TimeItCommand(PipelimeCommand, title="timeit"):
68
69
  ),
69
70
  )
70
71
 
71
- skip_first: pyd.NonNegativeInt = pyd.Field(
72
+ skip_first: pyd.NonNegativeInt = Field(
72
73
  1, alias="s", description="Skip the first n samples, then start the timer."
73
74
  )
74
- max_samples: t.Optional[pyd.PositiveInt] = pyd.Field(
75
+ max_samples: t.Optional[pyd.PositiveInt] = Field(
75
76
  None,
76
77
  alias="m",
77
78
  description="Grab at most `max_samples` and take the average time.",
78
79
  )
79
- repeat: pyd.PositiveInt = pyd.Field(
80
+ repeat: pyd.PositiveInt = Field(
80
81
  1, alias="r", description="Repeat the measurement `repeat` times."
81
82
  )
82
- process: bool = pyd.Field(
83
+ process: bool = Field(
83
84
  False,
84
85
  alias="p",
85
86
  description=(
86
87
  "Measure process time instead of using a performance counter clock."
87
88
  ),
88
89
  )
89
- clear_output_folder: bool = pyd.Field(
90
+ clear_output_folder: bool = Field(
90
91
  True, alias="c", description="Remove the output folder before each run."
91
92
  )
92
93
 
93
- average_time: t.Optional[OutputTime] = pyd.Field(
94
+ average_time: t.Optional[OutputTime] = Field(
94
95
  None,
95
96
  description="The average time to get a sample from the sequence.",
96
97
  exclude=True,
@@ -150,7 +151,7 @@ class TimeItCommand(PipelimeCommand, title="timeit"):
150
151
  class StageTimingCommand(PipelimeCommand, title="stage-time"):
151
152
  """Measures the average time to get a sample through some stages."""
152
153
 
153
- stages: t.Union[StageInput, t.Sequence[StageInput]] = pyd.Field(
154
+ stages: t.Union[StageInput, t.Sequence[StageInput]] = Field(
154
155
  ...,
155
156
  alias="s",
156
157
  description=(
@@ -170,14 +171,14 @@ class StageTimingCommand(PipelimeCommand, title="stage-time"):
170
171
  alias="g"
171
172
  )
172
173
 
173
- skip_first: pyd.NonNegativeInt = pyd.Field(
174
+ skip_first: pyd.NonNegativeInt = Field(
174
175
  1,
175
176
  alias="s",
176
177
  description=(
177
178
  "Skip the first n samples in EACH EXECUTION PROCESS, then start the timer."
178
179
  ),
179
180
  )
180
- max_samples: t.Optional[pyd.PositiveInt] = pyd.Field(
181
+ max_samples: t.Optional[pyd.PositiveInt] = Field(
181
182
  None,
182
183
  alias="m",
183
184
  description=(
@@ -185,10 +186,10 @@ class StageTimingCommand(PipelimeCommand, title="stage-time"):
185
186
  "and take the average time."
186
187
  ),
187
188
  )
188
- repeat: pyd.PositiveInt = pyd.Field(
189
+ repeat: pyd.PositiveInt = Field(
189
190
  1, alias="r", description="Repeat the measurement `repeat` times."
190
191
  )
191
- process: bool = pyd.Field(
192
+ process: bool = Field(
192
193
  False,
193
194
  alias="p",
194
195
  description=(
@@ -196,7 +197,7 @@ class StageTimingCommand(PipelimeCommand, title="stage-time"):
196
197
  ),
197
198
  )
198
199
 
199
- average_time: t.Optional[OutputStageTime] = pyd.Field(
200
+ average_time: t.Optional[OutputStageTime] = Field(
200
201
  None,
201
202
  description="The average time to get a sample through the stages.",
202
203
  exclude=True,
@@ -211,12 +212,8 @@ class StageTimingCommand(PipelimeCommand, title="stage-time"):
211
212
  stages = self.stages if isinstance(self.stages, t.Sequence) else [self.stages]
212
213
  names = []
213
214
  for st in stages:
214
- stage_cls = st.__root__.__class__
215
- stage_name = (
216
- stage_cls.__config__.title
217
- if stage_cls.__config__.title
218
- else stage_cls.__name__
219
- )
215
+ stage_cls = type(st.root)
216
+ stage_name = model_title(stage_cls)
220
217
  names.append(stage_name)
221
218
  for i, n in enumerate(names[::-1]):
222
219
  names[-i - 1] = f"{n}-{names.count(n)}"
@@ -274,7 +271,7 @@ class StageTimingCommand(PipelimeCommand, title="stage-time"):
274
271
  class PipeCommand(PipelimeCommand, title="pipe"):
275
272
  """A general-purpose command to build up linear pipelines."""
276
273
 
277
- operations: pl_types.YamlInput = pyd.Field(
274
+ operations: pl_types.YamlInput = Field(
278
275
  ...,
279
276
  alias="op",
280
277
  description=(
@@ -309,7 +306,8 @@ class PipeCommand(PipelimeCommand, title="pipe"):
309
306
  alias="g"
310
307
  )
311
308
 
312
- @pyd.validator("operations")
309
+ @pyd.field_validator("operations")
310
+ @classmethod
313
311
  def _validate_operations(cls, v: pl_types.YamlInput) -> pl_types.YamlInput:
314
312
  if not v.value or not isinstance(v.value, (t.Mapping, t.Sequence)):
315
313
  raise ValueError(f"Invalid pipeline: {v.value}")
@@ -382,7 +380,7 @@ class ConcatCommand(PipelimeCommand, title="cat"):
382
380
  alias="g"
383
381
  )
384
382
 
385
- interleave: bool = pyd.Field(False, description="If TRUE, interleaves samples.")
383
+ interleave: bool = Field(False, description="If TRUE, interleaves samples.")
386
384
 
387
385
  def run(self):
388
386
  inputs = self.inputs if isinstance(self.inputs, t.Sequence) else [self.inputs]
@@ -424,7 +422,7 @@ class ZipCommand(PipelimeCommand, title="zip"):
424
422
  )
425
423
  )
426
424
 
427
- key_format: t.Union[str, t.Sequence[str]] = pyd.Field(
425
+ key_format: t.Union[str, t.Sequence[str]] = Field(
428
426
  "*",
429
427
  description=(
430
428
  "The zipped samples' key format FOR EACH INPUT SEQUENCE EXCEPT THE FIRST "
@@ -469,7 +467,7 @@ class ZipCommand(PipelimeCommand, title="zip"):
469
467
  class ValidateCommand(PipelimeCommand, title="validate"):
470
468
  """Outputs a minimal schema which will validate the given input."""
471
469
 
472
- class OutputSchemaDefinition(pyd.BaseModel):
470
+ class OutputSchemaDefinition(PipelimeModel):
473
471
  schema_def: t.Any
474
472
 
475
473
  def __repr__(self) -> str:
@@ -480,7 +478,7 @@ class ValidateCommand(PipelimeCommand, title="validate"):
480
478
 
481
479
  return yaml.safe_dump(self.schema_def, sort_keys=False)
482
480
 
483
- class OutputCmdLineSchema(pyd.BaseModel):
481
+ class OutputCmdLineSchema(PipelimeModel):
484
482
  schema_def: t.Any
485
483
 
486
484
  def __repr__(self) -> str:
@@ -537,7 +535,7 @@ class ValidateCommand(PipelimeCommand, title="validate"):
537
535
  )
538
536
  )
539
537
 
540
- max_samples: int = pyd.Field(
538
+ max_samples: int = Field(
541
539
  0,
542
540
  alias="m",
543
541
  description=(
@@ -545,7 +543,7 @@ class ValidateCommand(PipelimeCommand, title="validate"):
545
543
  "Negative values count from the end, while if 0 all samples are checked."
546
544
  ),
547
545
  )
548
- root_key_path: str = pyd.Field(
546
+ root_key_path: str = Field(
549
547
  "input.schema", alias="r", description="Root key path for the output schema."
550
548
  )
551
549
 
@@ -553,14 +551,14 @@ class ValidateCommand(PipelimeCommand, title="validate"):
553
551
  alias="g"
554
552
  )
555
553
 
556
- output_schema_def: t.Optional[OutputSchemaDefinition] = pyd.Field(
554
+ output_schema_def: t.Optional[OutputSchemaDefinition] = Field(
557
555
  None,
558
556
  description="yaml schema definition",
559
557
  exclude=True,
560
558
  repr=False,
561
559
  piper_port=PiperPortType.OUTPUT,
562
560
  )
563
- output_cmd_line_schema: t.Optional[OutputCmdLineSchema] = pyd.Field(
561
+ output_cmd_line_schema: t.Optional[OutputCmdLineSchema] = Field(
564
562
  None,
565
563
  description="Schema definition on command line",
566
564
  exclude=True,
@@ -598,13 +596,15 @@ class ValidateCommand(PipelimeCommand, title="validate"):
598
596
  ignore_extra_keys=False,
599
597
  lazy=(self.max_samples == 0),
600
598
  max_samples=self.max_samples,
601
- ).dict(by_alias=True)
599
+ ).model_dump(by_alias=True)
602
600
 
603
601
  if self.root_key_path: # pragma: no branch
604
602
  import pydash as py_
605
603
 
604
+ from pipelime.choixe.utils.common import pydash_path
605
+
606
606
  tmp_dict = {}
607
- py_.set_(tmp_dict, self.root_key_path, sample_validation)
607
+ py_.set_(tmp_dict, pydash_path(self.root_key_path), sample_validation)
608
608
  sample_validation = tmp_dict
609
609
 
610
610
  self.output_schema_def = ValidateCommand.OutputSchemaDefinition(
@@ -618,7 +618,7 @@ class ValidateCommand(PipelimeCommand, title="validate"):
618
618
  class MapCommand(PipelimeCommand, title="map"):
619
619
  """Apply a stage on a dataset."""
620
620
 
621
- stage: StageInput = pyd.Field(
621
+ stage: StageInput = Field(
622
622
  ...,
623
623
  alias="s",
624
624
  description=(
@@ -659,7 +659,7 @@ class MapCommand(PipelimeCommand, title="map"):
659
659
  class MapIfCommand(PipelimeCommand, title="map-if"):
660
660
  """Apply a stage on a dataset if a condition is met."""
661
661
 
662
- stage: StageInput = pyd.Field(
662
+ stage: StageInput = Field(
663
663
  ...,
664
664
  alias="s",
665
665
  description=(
@@ -685,7 +685,7 @@ class MapIfCommand(PipelimeCommand, title="map-if"):
685
685
  alias="g"
686
686
  )
687
687
 
688
- condition: pl_types.CallableDef = pyd.Field(
688
+ condition: pl_types.CallableDef = Field(
689
689
  ...,
690
690
  alias="c",
691
691
  description=(
@@ -714,7 +714,7 @@ class MapIfCommand(PipelimeCommand, title="map-if"):
714
714
  class SortCommand(PipelimeCommand, title="sort"):
715
715
  """Sort a dataset by metadata values or according to a custom sorting function."""
716
716
 
717
- sort_key: t.Optional[str] = pyd.Field(
717
+ sort_key: t.Optional[str] = Field(
718
718
  None,
719
719
  alias="k",
720
720
  description=(
@@ -723,7 +723,7 @@ class SortCommand(PipelimeCommand, title="sort"):
723
723
  "Use `\\` to escape the `.` character."
724
724
  ),
725
725
  )
726
- sort_fn: t.Optional[pl_types.CallableDef] = pyd.Field(
726
+ sort_fn: t.Optional[pl_types.CallableDef] = Field(
727
727
  None,
728
728
  alias="f",
729
729
  description=(
@@ -777,14 +777,15 @@ class SortCommand(PipelimeCommand, title="sort"):
777
777
  class FilterCommand(PipelimeCommand, title="filter"):
778
778
  """Filter samples by metadata values or according to a custom sorting function."""
779
779
 
780
- filter_query: t.Optional[str] = pyd.Field(
780
+ filter_query: t.Optional[str] = Field(
781
781
  None,
782
782
  alias="q",
783
783
  description=("A dictquery (cfr. https://github.com/cyberlis/dictquery)."),
784
784
  )
785
- filter_fn: t.Optional[pl_types.CallableDef] = pyd.Field(
785
+ filter_fn: t.Optional[pl_types.CallableDef] = Field(
786
786
  None,
787
787
  alias="f",
788
+ validate_default=True,
788
789
  description=(
789
790
  "A `class.path.func`, `file.py:func`, `lambda...` or `func:::def func...` "
790
791
  "of a callable `(Sample) -> bool` returning True for any valid sample."
@@ -807,11 +808,12 @@ class FilterCommand(PipelimeCommand, title="filter"):
807
808
  alias="g"
808
809
  )
809
810
 
810
- @pyd.validator("filter_fn", always=True)
811
+ @pyd.field_validator("filter_fn")
812
+ @classmethod
811
813
  def _check_filters(
812
- cls, v: t.Optional[pl_types.CallableDef], values: t.Mapping[str, t.Any]
814
+ cls, v: t.Optional[pl_types.CallableDef], info: pyd.ValidationInfo
813
815
  ) -> t.Optional[pl_types.CallableDef]:
814
- fquery = values.get("filter_query", None)
816
+ fquery = info.data.get("filter_query", None)
815
817
  if (v is None) == (fquery is None):
816
818
  raise ValueError("You should define either `filter_query` or `filter_fn`")
817
819
  return v
@@ -872,7 +874,7 @@ class SliceCommand(PipelimeCommand, title="slice"):
872
874
  alias="s"
873
875
  )
874
876
 
875
- shuffle: t.Union[bool, pyd.PositiveInt] = pyd.Field(
877
+ shuffle: t.Union[bool, pyd.PositiveInt] = Field(
876
878
  False,
877
879
  alias="shf",
878
880
  description=(
@@ -909,9 +911,10 @@ class SetMetadataCommand(FilterCommand, title="set-meta"):
909
911
  The metadata is set only on samples selected by a dictquery or a filter function.
910
912
  """
911
913
 
912
- filter_fn: t.Optional[pl_types.CallableDef] = pyd.Field(
914
+ filter_fn: t.Optional[pl_types.CallableDef] = Field(
913
915
  None,
914
916
  alias="f",
917
+ validate_default=True,
915
918
  description=(
916
919
  "A `class.path.func`, `file.py:func`, `lambda...` or `func:::def func...` "
917
920
  "of a callable returning True for any valid sample.\n"
@@ -923,10 +926,10 @@ class SetMetadataCommand(FilterCommand, title="set-meta"):
923
926
  ),
924
927
  )
925
928
 
926
- key_path: str = pyd.Field(
929
+ key_path: str = Field(
927
930
  ..., alias="k", description="The metadata key in pydash dot notation."
928
931
  )
929
- value: pl_types.YamlInput = pyd.Field(
932
+ value: pl_types.YamlInput = Field(
930
933
  None, alias="v", description="The value to set, ie, any valid yaml/json value."
931
934
  )
932
935
 
@@ -968,7 +971,7 @@ class FilterDuplicatesCommand(PipelimeCommand, title="filter-duplicates"):
968
971
  )
969
972
  )
970
973
 
971
- algorithm: str = pyd.Field(
974
+ algorithm: str = Field(
972
975
  "sha256",
973
976
  description=(
974
977
  "The hashing algorithm from `hashlib` to use. Only algorithms that"
@@ -976,7 +979,7 @@ class FilterDuplicatesCommand(PipelimeCommand, title="filter-duplicates"):
976
979
  ),
977
980
  )
978
981
 
979
- keys: t.Union[str, t.Sequence[str]] = pyd.Field(
982
+ keys: t.Union[str, t.Sequence[str]] = Field(
980
983
  ...,
981
984
  alias="k",
982
985
  description=(
@@ -1078,12 +1081,12 @@ class CopySharedItemsCommand(PipelimeCommand, title="copy-shared-items"):
1078
1081
  grabber: pl_interfaces.GrabberInterface = pl_interfaces.GrabberInterface.pyd_field(
1079
1082
  alias="g"
1080
1083
  )
1081
- key_list: t.Sequence[str] = pyd.Field(
1084
+ key_list: t.Sequence[str] = Field(
1082
1085
  ...,
1083
1086
  alias="k",
1084
1087
  description="The keys to copy. Must be present in source dataset.",
1085
1088
  )
1086
- force_shared: bool = pyd.Field(
1089
+ force_shared: bool = Field(
1087
1090
  False,
1088
1091
  alias="f",
1089
1092
  description="If True, the items will be copied as shared items",