fameio 3.2.0__py3-none-any.whl → 3.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. fameio/cli/convert_results.py +4 -6
  2. fameio/cli/make_config.py +3 -5
  3. fameio/cli/options.py +6 -4
  4. fameio/cli/parser.py +53 -29
  5. fameio/cli/reformat.py +58 -0
  6. fameio/input/__init__.py +4 -4
  7. fameio/input/loader/__init__.py +4 -6
  8. fameio/input/loader/controller.py +11 -16
  9. fameio/input/loader/loader.py +11 -9
  10. fameio/input/metadata.py +26 -29
  11. fameio/input/resolver.py +4 -6
  12. fameio/input/scenario/agent.py +18 -16
  13. fameio/input/scenario/attribute.py +85 -31
  14. fameio/input/scenario/contract.py +78 -38
  15. fameio/input/scenario/exception.py +3 -6
  16. fameio/input/scenario/fameiofactory.py +7 -12
  17. fameio/input/scenario/generalproperties.py +7 -8
  18. fameio/input/scenario/scenario.py +14 -18
  19. fameio/input/scenario/stringset.py +5 -6
  20. fameio/input/schema/agenttype.py +8 -10
  21. fameio/input/schema/attribute.py +30 -36
  22. fameio/input/schema/java_packages.py +6 -7
  23. fameio/input/schema/schema.py +9 -11
  24. fameio/input/validator.py +178 -41
  25. fameio/input/writer.py +20 -29
  26. fameio/logs.py +28 -19
  27. fameio/output/agent_type.py +14 -16
  28. fameio/output/conversion.py +9 -12
  29. fameio/output/csv_writer.py +33 -23
  30. fameio/output/data_transformer.py +11 -11
  31. fameio/output/execution_dao.py +170 -0
  32. fameio/output/input_dao.py +16 -19
  33. fameio/output/output_dao.py +7 -7
  34. fameio/output/reader.py +8 -10
  35. fameio/output/yaml_writer.py +2 -3
  36. fameio/scripts/__init__.py +15 -4
  37. fameio/scripts/convert_results.py +18 -17
  38. fameio/scripts/exception.py +1 -1
  39. fameio/scripts/make_config.py +3 -4
  40. fameio/scripts/reformat.py +71 -0
  41. fameio/scripts/reformat.py.license +3 -0
  42. fameio/series.py +78 -47
  43. fameio/time.py +56 -18
  44. fameio/tools.py +42 -4
  45. {fameio-3.2.0.dist-info → fameio-3.4.0.dist-info}/METADATA +64 -40
  46. fameio-3.4.0.dist-info/RECORD +60 -0
  47. {fameio-3.2.0.dist-info → fameio-3.4.0.dist-info}/entry_points.txt +1 -0
  48. fameio-3.2.0.dist-info/RECORD +0 -56
  49. {fameio-3.2.0.dist-info → fameio-3.4.0.dist-info}/LICENSE.txt +0 -0
  50. {fameio-3.2.0.dist-info → fameio-3.4.0.dist-info}/LICENSES/Apache-2.0.txt +0 -0
  51. {fameio-3.2.0.dist-info → fameio-3.4.0.dist-info}/LICENSES/CC-BY-4.0.txt +0 -0
  52. {fameio-3.2.0.dist-info → fameio-3.4.0.dist-info}/LICENSES/CC0-1.0.txt +0 -0
  53. {fameio-3.2.0.dist-info → fameio-3.4.0.dist-info}/WHEEL +0 -0
fameio/time.py CHANGED
@@ -17,14 +17,15 @@ START_IN_REAL_TIME = "2000-01-01_00:00:00"
17
17
  DATE_FORMAT = "%Y-%m-%d_%H:%M:%S"
18
18
  DATE_REGEX = re.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}:[0-9]{2}:[0-9]{2}")
19
19
  FAME_FIRST_DATETIME = dt.datetime.strptime(START_IN_REAL_TIME, DATE_FORMAT)
20
+ TIME_SPAN_REGEX = re.compile(r"^[0-9]+\s*[A-z]+$")
20
21
 
21
22
 
22
23
  class ConversionError(InputError, OutputError):
23
- """Indicates that time stamp conversion failed"""
24
+ """Indicates that time stamp conversion failed."""
24
25
 
25
26
 
26
27
  class TimeUnit(Enum):
27
- """Time units defined in FAME"""
28
+ """Time units defined in FAME."""
28
29
 
29
30
  SECONDS = auto()
30
31
  MINUTES = auto()
@@ -36,7 +37,7 @@ class TimeUnit(Enum):
36
37
 
37
38
 
38
39
  class Constants:
39
- """Time steps in FAME simulations associated with corresponding TimeUnits"""
40
+ """Time steps in FAME simulations associated with corresponding TimeUnits."""
40
41
 
41
42
  FIRST_YEAR = 2000
42
43
  STEPS_PER_SECOND = 1
@@ -63,7 +64,7 @@ class Constants:
63
64
 
64
65
 
65
66
  class FameTime:
66
- """Handles conversion of TimeSteps and TimeDurations into TimeStamps and vice versa"""
67
+ """Handles conversion of TimeSteps and TimeDurations into TimeStamps and vice versa."""
67
68
 
68
69
  _TIME_UNIT_UNKNOWN = "TimeUnit conversion of '{}' not implemented."
69
70
  _FORMAT_INVALID = "'{}' is not recognised as time stamp string - check its format."
@@ -71,11 +72,12 @@ class FameTime:
71
72
  _INVALID_TOO_LARGE = "Cannot convert time stamp string '{}' - last day of leap year is Dec 30th!"
72
73
  _NO_TIMESTAMP = "Time value expected, but '{}' is neither a time stamp string nor an integer."
73
74
  _INVALID_DATE_FORMAT = "Received invalid date format '{}'."
75
+ _INVALID_SPAN_FORMAT = "Time span must be provided in the format '<positive integer> <TimeUnit>' but was: '{}'"
76
+ _INVALID_TIME_UNIT = f"Time span unit '{{}}' unknown, must be one of: {[u.name for u in TimeUnit]}"
74
77
 
75
78
  @staticmethod
76
79
  def convert_datetime_to_fame_time_step(datetime_string: str) -> int:
77
- """
78
- Converts Datetime string to FAME time step
80
+ """Converts Datetime string to FAME time step.
79
81
 
80
82
  Args:
81
83
  datetime_string: a datetime in FAME formatting
@@ -101,8 +103,7 @@ class FameTime:
101
103
 
102
104
  @staticmethod
103
105
  def _convert_to_datetime(datetime_string: str) -> dt.datetime:
104
- """
105
- Converts given `datetime_string` in FAME formatting to real-world datetime
106
+ """Converts given `datetime_string` in FAME formatting to real-world datetime.
106
107
 
107
108
  Args:
108
109
  datetime_string: to be converted to datetime
@@ -120,8 +121,7 @@ class FameTime:
120
121
 
121
122
  @staticmethod
122
123
  def convert_fame_time_step_to_datetime(fame_time_steps: int, date_format: str = DATE_FORMAT) -> str:
123
- """
124
- Converts given `fame_time_steps` to corresponding real-world datetime string in `date_format`
124
+ """Converts given `fame_time_steps` to corresponding real-world datetime string in `date_format`.
125
125
 
126
126
  Args:
127
127
  fame_time_steps: an integer representing time in FAME's internal format
@@ -146,8 +146,7 @@ class FameTime:
146
146
 
147
147
  @staticmethod
148
148
  def convert_time_span_to_fame_time_steps(value: int, unit: TimeUnit) -> int:
149
- """
150
- Converts value of `TimeUnit.UNIT` to FAME time steps
149
+ """Converts value of `TimeUnit.UNIT` to FAME time steps.
151
150
 
152
151
  Args:
153
152
  value: amount of the units to be converted
@@ -164,16 +163,42 @@ class FameTime:
164
163
  return steps * value
165
164
  raise log_error(ConversionError(FameTime._TIME_UNIT_UNKNOWN.format(unit)))
166
165
 
166
+ @staticmethod
167
+ def convert_text_to_time_span(string: str) -> int:
168
+ """Converts given string in form of "<positive integer> <TimeUnit>" to a time span in FAME time steps.
169
+
170
+ Args:
171
+ string: to convert to a time span from
172
+
173
+ Returns:
174
+ FAME time steps equivalent of `value x unit`
175
+
176
+ Raises:
177
+ ConversionError: if an unknown time unit or an unsupported format is used, logged with level "ERROR"
178
+ """
179
+ string = string.strip()
180
+ if TIME_SPAN_REGEX.fullmatch(string) is None:
181
+ raise log_error(ConversionError(FameTime._INVALID_SPAN_FORMAT.format(string)))
182
+ multiple, unit_name = string.split()
183
+ unit_name = unit_name.upper()
184
+ if not unit_name.endswith("S"):
185
+ unit_name += "S"
186
+ try:
187
+ unit = TimeUnit[unit_name]
188
+ except KeyError as e:
189
+ raise log_error(ConversionError(FameTime._INVALID_TIME_UNIT.format(unit_name))) from e
190
+ return FameTime.convert_time_span_to_fame_time_steps(int(multiple), unit)
191
+
167
192
  @staticmethod
168
193
  def is_datetime(string: Any) -> bool:
169
- """Returns `True` if given `string` matches Datetime string format and can be converted to FAME time step"""
194
+ """Returns `True` if given `string` matches Datetime string format and can be converted to FAME time step."""
170
195
  if isinstance(string, str):
171
196
  return DATE_REGEX.fullmatch(string.strip()) is not None
172
197
  return False
173
198
 
174
199
  @staticmethod
175
200
  def is_fame_time_compatible(value: int | str) -> bool:
176
- """Returns `True` if given int or string `value` can be converted to a FAME time step"""
201
+ """Returns `True` if given int or string `value` can be converted to a FAME time step."""
177
202
  if isinstance(value, int):
178
203
  return True
179
204
  if isinstance(value, str):
@@ -182,7 +207,7 @@ class FameTime:
182
207
 
183
208
  @staticmethod
184
209
  def _is_integer(string: str) -> bool:
185
- """Returns `True` if given string can be interpreted as integer"""
210
+ """Returns `True` if given string can be interpreted as integer."""
186
211
  try:
187
212
  int(string)
188
213
  except ValueError:
@@ -191,9 +216,10 @@ class FameTime:
191
216
 
192
217
  @staticmethod
193
218
  def convert_string_if_is_datetime(value: int | str) -> int:
194
- """
195
- Returns FAME time steps of given `value`. If it is a valid FAME datetime string it is converted to
196
- FAME time steps, or, if given `value` is an integer, it is returned without modification.
219
+ """Returns FAME time steps of given `value`.
220
+
221
+ If it is a valid FAME datetime string it is converted to FAME time steps.
222
+ Else if given `value` is an integer, it is returned without modification.
197
223
 
198
224
  Args:
199
225
  value: to be converted
@@ -211,3 +237,15 @@ class FameTime:
211
237
  return int(value)
212
238
  except ValueError as e:
213
239
  raise log_error(ConversionError(FameTime._NO_TIMESTAMP.format(value))) from e
240
+
241
+ @staticmethod
242
+ def get_first_fame_time_step_of_year(year: int) -> int:
243
+ """Returns FAME time in integer format of first time step of given year.
244
+
245
+ Args:
246
+ year: to get the first time step for
247
+
248
+ Returns:
249
+ first time step in the requested year in integer format
250
+ """
251
+ return (year - FAME_FIRST_DATETIME.year) * Constants.STEPS_PER_YEAR
fameio/tools.py CHANGED
@@ -6,25 +6,63 @@ from __future__ import annotations
6
6
  from pathlib import Path
7
7
  from typing import Any
8
8
 
9
+ from fameio.logs import log_error
10
+
11
+ CSV_FILE_SUFFIX = ".csv"
12
+
13
+ _ERR_INVALID_PATTERN = "Pattern '{}' cannot be used here due to: '{}'"
14
+
9
15
 
10
16
  def keys_to_lower(dictionary: dict[str, Any]) -> dict[str, Any]:
11
- """Returns new dictionary content of given `dictionary` but its top-level `keys` in lower case"""
17
+ """Returns new dictionary content of given `dictionary` but its top-level `keys` in lower case."""
12
18
  return {keys.lower(): value for keys, value in dictionary.items()}
13
19
 
14
20
 
15
21
  def ensure_is_list(value: Any) -> list:
16
- """Returns a list: Either the provided `value` if it is a list, or a new list containing the provided value"""
22
+ """Returns a list: Either the provided `value` if it is a list, or a new list containing the provided value."""
17
23
  if isinstance(value, list):
18
24
  return value
19
25
  return [value]
20
26
 
21
27
 
22
28
  def ensure_path_exists(path: Path | str):
23
- """Creates a specified path if not already existent"""
29
+ """Creates a specified path if not already existent."""
24
30
  Path(path).mkdir(parents=True, exist_ok=True)
25
31
 
26
32
 
27
33
  def clean_up_file_name(name: str) -> str:
28
- """Returns given `name` replacing spaces and colons with underscore, and slashed with a dash"""
34
+ """Returns given `name` replacing spaces and colons with underscore, and slashed with a dash."""
29
35
  translation_table = str.maketrans({" ": "_", ":": "_", "/": "-"})
30
36
  return name.translate(translation_table)
37
+
38
+
39
+ def get_csv_files_with_pattern(base_path: Path, pattern: str) -> list[Path]:
40
+ """Find all csv files matching the given `pattern` based on the given `base_path`.
41
+
42
+ Args:
43
+ base_path: to start the search from
44
+ pattern: to match the files against that are to be found
45
+
46
+ Returns:
47
+ Full file paths for files ending with ".csv" and matching the given pattern
48
+
49
+ Raises:
50
+ ValueError: if pattern cannot be used to search path, logged with level "ERROR"
51
+ """
52
+ try:
53
+ return [file for file in base_path.glob(pattern) if file.suffix.lower() == CSV_FILE_SUFFIX]
54
+ except NotImplementedError as e:
55
+ raise log_error(ValueError(_ERR_INVALID_PATTERN.format(pattern, e))) from e
56
+
57
+
58
+ def extend_file_name(original_file: Path, appendix: str) -> Path:
59
+ """Return original file path, but appending `FILE_NAME_APPENDIX` before the suffix.
60
+
61
+ Args:
62
+ original_file: from which to derive the new path
63
+ appendix: to be added to the end of the file name before the suffix
64
+
65
+ Returns:
66
+ new file path including the appendix in the file name
67
+ """
68
+ return Path(original_file.parent, original_file.stem + appendix + original_file.suffix)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: fameio
3
- Version: 3.2.0
3
+ Version: 3.4.0
4
4
  Summary: Tools for input preparation and output digestion of FAME models
5
5
  License: Apache-2.0
6
6
  Keywords: FAME,fameio,agent-based modelling,energy systems
@@ -47,8 +47,7 @@ SPDX-License-Identifier: Apache-2.0 -->
47
47
  *Tools for input preparation and output digestion of FAME models*
48
48
 
49
49
  FAME-Io compiles input for FAME models in protobuf format and extracts model outputs to human-readable files.
50
- Please visit the [FAME-Wiki](https://gitlab.com/fame-framework/wiki/-/wikis/home) to get an explanation of FAME and its
51
- components.
50
+ Please visit the [FAME-Wiki](https://gitlab.com/fame-framework/wiki/-/wikis/home) to get an explanation of FAME and its components.
52
51
 
53
52
  # Installation
54
53
 
@@ -56,30 +55,26 @@ We recommend installing `fameio` using PyPI:
56
55
 
57
56
  pip install fameio
58
57
 
59
- You may also use `pipx`. For detailed information please refer to the
60
- official `pipx` [documentation](https://github.com/pypa/pipx).
58
+ You may also use `pipx`. For detailed information please refer to the official `pipx` [documentation](https://github.com/pypa/pipx).
61
59
 
62
60
  pipx install fameio
63
61
 
64
- `fameio` is currently developed and tested for Python 3.8 or higher.
62
+ `fameio` is currently developed and tested for Python 3.9 or higher.
65
63
  See the `pyproject.toml` for a complete listing of dependencies.
66
64
 
67
65
  # Usage
68
66
 
69
- FAME-Io currently offers two main scripts `makeFameRunConfig` and `convertFameResults`.
70
- Both are automatically installed with the package.
67
+ FAME-Io currently offers two main scripts `makeFameRunConfig` and `convertFameResults`, plus a helper script `reformatTimeSeries`
68
+ All are automatically installed with the package.
71
69
  The first one creates a protobuf file for FAME applications using YAML definition files and CSV files.
72
- The latter one reads output files from FAME applications in protobuf format and converts them to CSV files.
70
+ The second one reads output files from FAME applications in protobuf format and converts them to CSV files.
71
+ The third script reformats time series CSV files to FAME format.
73
72
 
74
- You may use the [example data](https://gitlab.com/dlr-ve/esy/amiris/examples) provided for
75
- the [AMIRIS](https://gitlab.com/dlr-ve/esy/amiris/amiris) model which can be used to simulate electricity markets
76
- in [Germany](https://gitlab.com/dlr-ve/esy/amiris/examples/-/tree/main/Germany2019), [Austria](https://gitlab.com/dlr-ve/esy/amiris/examples/-/tree/main/Austria2019),
77
- and a simple [proof-of-concept model](https://gitlab.com/dlr-ve/esy/amiris/examples/-/tree/main/Simple).
73
+ You may use the [example data](https://gitlab.com/dlr-ve/esy/amiris/examples) provided for the [AMIRIS](https://gitlab.com/dlr-ve/esy/amiris/amiris) model which can be used to simulate electricity markets in [Germany](https://gitlab.com/dlr-ve/esy/amiris/examples/-/tree/main/Germany2019), [Austria](https://gitlab.com/dlr-ve/esy/amiris/examples/-/tree/main/Austria2019), and a simple [proof-of-concept model](https://gitlab.com/dlr-ve/esy/amiris/examples/-/tree/main/Simple).
78
74
 
79
75
  ## Make a FAME run configuration
80
76
 
81
- Digests configuration files in YAML format, combines them with CSV data files and creates a single input file for FAME
82
- applications in protobuf format.
77
+ Digests configuration files in YAML format, combines them with CSV data files and creates a single input file for FAME applications in protobuf format.
83
78
  Call structure:
84
79
 
85
80
  makeFameRunConfig -f <path/to/scenario.yaml>
@@ -512,7 +507,7 @@ Contracts:
512
507
  ReceiverId: 2
513
508
  ProductName: ProductOfAgent_1
514
509
  FirstDeliveryTime: -25
515
- DeliveryIntervalInSteps: 3600
510
+ Every: 3600
516
511
  Metadata:
517
512
  Some: "additional information can go here"
518
513
 
@@ -520,7 +515,7 @@ Contracts:
520
515
  ReceiverId: 1
521
516
  ProductName: ProductOfAgent_2
522
517
  FirstDeliveryTime: -22
523
- DeliveryIntervalInSteps: 3600
518
+ Every: 1 hour
524
519
  Attributes:
525
520
  ProductAppendix: value
526
521
  TimeOffset: 42
@@ -532,7 +527,8 @@ Contract Parameters:
532
527
  * `ReceiverId` unique ID of agent receiving the product
533
528
  * `ProductName` name of the product to be sent
534
529
  * `FirstDeliveryTime` first time of delivery in the format "seconds after the January 1st 2000, 00:00:00"
535
- * `DeliveryIntervalInSteps` delay time in between deliveries in seconds
530
+ * `Every` delay time in between deliveries; either an integer value in seconds, or a qualified time span in the format "<integer> <TimeUnit>(s)", where TimeUnit is one of "second", "minute", "hour", "day", "week", "month", "year" - with an options "s" at the end; mind that week, month, and year refer to fixed-length intervals of 168, 730, and 8760 hours.
531
+ * `DeliveryIntervalInSteps` deprecated; delay time in between deliveries in seconds (use instead of `Every`)
536
532
  * `Metadata` can be assigned to add further helpful information about a Contract
537
533
  * `Attributes` can be set to include additional information as `int`, `float`, `enum`, or `dict` data types
538
534
 
@@ -546,59 +542,72 @@ example:
546
542
  ```yaml
547
543
  Contracts:
548
544
  # effectively 3 similar contracts (0 -> 11), (0 -> 12), (0 -> 13)
549
- # with otherwise identical ProductName, FirstDeliveryTime & DeliveryIntervalInSteps
545
+ # with otherwise identical ProductName, FirstDeliveryTime, and Every
550
546
  - SenderId: 0
551
547
  ReceiverId: [ 11, 12, 13 ]
552
548
  ProductName: MyOtherProduct
553
549
  FirstDeliveryTime: 100
554
- DeliveryIntervalInSteps: 3600
550
+ Every: 1 hour
555
551
 
556
552
  # effectively 3 similar contracts (1 -> 10), (2 -> 10), (3 -> 10)
557
- # with otherwise identical ProductName, FirstDeliveryTime & DeliveryIntervalInSteps
553
+ # with otherwise identical ProductName, FirstDeliveryTime, and Every
558
554
  - SenderId: [ 1, 2, 3 ]
559
555
  ReceiverId: 10
560
556
  ProductName: MyProduct
561
557
  FirstDeliveryTime: 100
562
- DeliveryIntervalInSteps: 3600
558
+ Every: 1 hour
563
559
 
564
560
  # effectively 3 similar contracts (1 -> 11), (2 -> 12), (3 -> 13)
565
- # with otherwise identical ProductName, FirstDeliveryTime & DeliveryIntervalInSteps
561
+ # with otherwise identical ProductName, FirstDeliveryTime, and Every
566
562
  - SenderId: [ 1, 2, 3 ]
567
563
  ReceiverId: [ 11, 12, 13 ]
568
564
  ProductName: MyThirdProduct
569
565
  FirstDeliveryTime: 100
570
- DeliveryIntervalInSteps: 3600
566
+ Every: 1 hour
571
567
  ```
572
568
 
573
- Combined with YAML anchors complex contract chains can be easily reduced to a minimum of required configuration.
574
- The following example is equivalent to the previous one and allows a quick extension of contracts to a new couple of
575
- agents e.g. (4;14):
569
+ When combined with YAML anchors, the complexity of extensive contract chains can be reduced.
570
+ The following example is equivalent to the previous one, enabling contracts to be quickly extended to a new group of agents:
576
571
 
577
572
  ```yaml
578
573
  Groups:
579
- - &agentList1: [ 1,2,3 ]
580
- - &agentList2: [ 11,12,13 ]
574
+ - &agentList1: [ 1, 2, 3 ]
575
+ - &agentList2: [ 11, 12, 13 ]
581
576
 
582
577
  Contracts:
583
578
  - SenderId: 0
584
579
  ReceiverId: *agentList2
585
580
  ProductName: MyOtherProduct
586
581
  FirstDeliveryTime: 100
587
- DeliveryIntervalInSteps: 3600
582
+ Every: 1 hour
588
583
 
589
584
  - SenderId: *agentList1
590
585
  ReceiverId: 10
591
586
  ProductName: MyProduct
592
587
  FirstDeliveryTime: 100
593
- DeliveryIntervalInSteps: 3600
588
+ Every: 1 hour
594
589
 
595
590
  - SenderId: *agentList1
596
591
  ReceiverId: *agentList2
597
592
  ProductName: MyThirdProduct
598
593
  FirstDeliveryTime: 100
599
- DeliveryIntervalInSteps: 3600
594
+ Every: 1 hour
600
595
  ```
601
596
 
597
+ Lists can be nested in senders and receivers as follows:
598
+
599
+ ```
600
+ Groups:
601
+ - SenderId: 42
602
+ ReceiverId: [1, [2, 3], [4, [5]]]
603
+ ProductName: AnImportantProduct
604
+ FirstDeliveryTime: 101
605
+ Every: 30 minutes
606
+ ```
607
+
608
+ This feature should not be overused, as nested lists can become messy and difficult to read.
609
+ Special care should be taken when using it with an N-to-N mapping, as it will be difficult to check whether senders and receivers are matched correctly.
610
+
602
611
  #### StringSets
603
612
 
604
613
  This optional section defines values of type `string_set`.
@@ -644,10 +653,8 @@ TIME_SERIES inputs are not directly fed into the Scenario YAML file.
644
653
  Instead, TIME_SERIES reference a CSV file that can be stored some place else.
645
654
  These CSV files follow a specific structure:
646
655
 
647
- * They should contain exactly two columns - any other columns are ignored.
648
- A warning is raised if more than two non-empty columns are detected.
649
- * The first column must be a time stamp in form `YYYY-MM-DD_hh:mm:ss` or
650
- a [FAME-Timestamp](https://gitlab.com/fame-framework/wiki/-/wikis/architecture/decisions/TimeStamp) integer value.
656
+ * They should contain exactly two columns - any other columns are ignored. A warning is raised if more than two non-empty columns are detected.
657
+ * The first column must be a time stamp in form `YYYY-MM-DD_hh:mm:ss` or a [FAME-Timestamp](https://gitlab.com/fame-framework/wiki/-/wikis/architecture/decisions/TimeStamp) integer value.
651
658
  * The second column must be a numerical value (either integer or floating-point)
652
659
  * The separator of the two columns is a semicolon
653
660
  * The data must **not** have headers, except for comments marked with `#`
@@ -663,11 +670,10 @@ Exemplary content of a valid CSV file:
663
670
  2016-01-01_00:00:00;42 # optional comment on this particular data point
664
671
  2017-01-01_00:00:00;0.1
665
672
 
666
- Please refer also to the detailed article about `TimeStamps` in
667
- the [FAME-Wiki](https://gitlab.com/fame-framework/wiki/-/wikis/TimeStamp).
668
- For large CSV files (with more than 20,000 rows) we recommend using the integer representation of FAME-Timestamps in the
669
- first column (instead of text representation) to improve conversion speed.
673
+ Please refer also to the detailed article about `TimeStamps` in the [FAME-Wiki](https://gitlab.com/fame-framework/wiki/-/wikis/TimeStamp).
674
+ For large CSV files (with more than 20,000 rows) we recommend using the integer representation of FAME-Timestamps in the first column (instead of text representation) to improve conversion speed.
670
675
  A warning will be raised for very large files (exceeding 50,000 rows) that require time stamp conversion.
676
+ Use `reformatTimeSeries` to convert one or multiple timeseries CSV files into FAME format to improve conversion speed and avoid this warning.
671
677
 
672
678
  ### Split and join multiple YAML files
673
679
 
@@ -936,6 +942,24 @@ run_config = handle_args(my_arg_string, my_defaults)
936
942
  convert_results(run_config)
937
943
  ```
938
944
 
945
+ ## Reformat time series
946
+
947
+ Takes CSV time series files and reformats them into FAME time format.
948
+ This improves speed of run configuration creation but also reduces readability of the CSV files' content.
949
+ Thus, we recommend to apply this reformatting only for CSV time series files with more than 20,000 lines.
950
+
951
+ Call structure:
952
+
953
+ reformatTimeSeries -fp <file_or_file_pattern*.csv>
954
+
955
+ You may also specify any of the following arguments:
956
+
957
+ | Command | Action |
958
+ |-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
959
+ | `-l` or `--log` <option> | Sets the logging level. Default is `WARNING`. Options are `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. |
960
+ | `-lf` or `--logfile` <file> | Sets the logging file. Default is `None`. If `None` is provided, all logs get only printed to the console. |
961
+ | `--replace` or `--no-replace` | If `--replace` is specified, existing csv files are replaced with the new content. Otherwise and by default, new files are created extending the original file name by "_reformatted". |
962
+
939
963
  ## Cite FAME-Io
940
964
 
941
965
  If you use FAME-Io for academic work, please cite as follows.
@@ -0,0 +1,60 @@
1
+ fameio/__init__.py,sha256=LiE7kRXW0pMIB4hTPC0T_ppGz9O0swd0Ca1-b99hOMc,229
2
+ fameio/cli/__init__.py,sha256=YAxIBl8azJDFaKr0iGaba94UH3Xy-KhQtxrGtwz3FpM,179
3
+ fameio/cli/convert_results.py,sha256=hSdxWFsF7Q6Z9sWRRJso-xFqxBgAyToTy32aJJnOSH4,3451
4
+ fameio/cli/make_config.py,sha256=vfxREWB4NUf_eZior0L4cu-B3DQZAtUq5splAnN36m0,2720
5
+ fameio/cli/options.py,sha256=WG07XKzRTat4Pk57DkdEDv0iTM7SzijrSLr7HXwRKkY,1322
6
+ fameio/cli/parser.py,sha256=yV5YNve0Rv_cVDpZhRK-wmnXyxCJ0_587aR2c3wVCuU,10441
7
+ fameio/cli/reformat.py,sha256=4UyxaDbT7f-nDs7crgt38s9RYXfM--bNLL4jiCtdJEw,2181
8
+ fameio/input/__init__.py,sha256=WNyeYBuu0aQ363JX_ULUhI6OQtC123y_3k5zl1TUJS0,508
9
+ fameio/input/loader/__init__.py,sha256=VtDJtTItjHTZ04uB30t70kRn4-ElWr2C-hj6y1_cnLs,2753
10
+ fameio/input/loader/controller.py,sha256=lpzcn8iGgWV6wAMduE-1P7unNFvVFESX5iMj6boKI0E,7714
11
+ fameio/input/loader/loader.py,sha256=zvsmFuK5vNOhBFwFPR2wQ-vcl-Mwip-NgTJUXpfTwYw,5321
12
+ fameio/input/metadata.py,sha256=oV11xeI9fXyY1Qcl79mV1UyA3K4eH1751qHKAMa8cYE,6568
13
+ fameio/input/resolver.py,sha256=NakBjnCCWRMz-8gTC_Ggx-2tXq-u6OPjfBOua0Rd2nA,1902
14
+ fameio/input/scenario/__init__.py,sha256=Pb8O9rVOTwEo48WIgiq1kBnpovpc4D_syC6EjTFGHew,404
15
+ fameio/input/scenario/agent.py,sha256=n0H8nHwQFfAeTwdJceJpi9VV1muYjJu_PjmzC_vrP84,5526
16
+ fameio/input/scenario/attribute.py,sha256=pp9cquxfBUKNFwV3bTDBkDXv1k8ThBiAL6Eh-ag4kQk,11386
17
+ fameio/input/scenario/contract.py,sha256=fv4XRMMF8X9Q1LMNJsWcPfQx-EY40gDBWsEMJ7rsaBQ,15324
18
+ fameio/input/scenario/exception.py,sha256=o64hd7FQrkTF6Ze075Cbt4TM3OkcyJFVSi4qexLuMoU,1939
19
+ fameio/input/scenario/fameiofactory.py,sha256=HgLHVQGKsTPEFy8K1ILB7F_lJtHoMhu89inOgDWYP5k,2800
20
+ fameio/input/scenario/generalproperties.py,sha256=C3ND-PLb1FrCRheIBIyxXHKsZVMW8GZVHUidyInqrOw,3749
21
+ fameio/input/scenario/scenario.py,sha256=GeS-IDYB-Q6-26QPeKQgA1-FVW90tBHHTBcfrNiwYJA,8376
22
+ fameio/input/scenario/stringset.py,sha256=a1PehtkMVaVd1S_HCu9-3DxEWCXflzM32HDdqFZSCDY,2529
23
+ fameio/input/schema/__init__.py,sha256=oIsJVC0hUhwJf_XIBpd1NNTR8sOWEthAy98m6uR7gKc,327
24
+ fameio/input/schema/agenttype.py,sha256=pOXRJXpiOWDfIq7el52CcPqbnnt8lkiXtIcT_i4F_xc,5684
25
+ fameio/input/schema/attribute.py,sha256=Q5pn2BK2QP6L0t8xI7QN5SVqylWYIfyBJoTvIoiTHkw,14901
26
+ fameio/input/schema/java_packages.py,sha256=WlU7Vjesscqiiv8i24J1AhEwsJwG4QviC6k6QOddG24,2944
27
+ fameio/input/schema/schema.py,sha256=5KXd1d9KxQ6JCMYqI81yOEBUX4s_uEPXM8twXzDEjaA,3851
28
+ fameio/input/validator.py,sha256=VMgj1u-qedarKuMrD3fkBZhdYVWvYha768iEdI58Fvc,24040
29
+ fameio/input/writer.py,sha256=XE70AysjcJgmIjepzdzYV7S3lmb4F8V0Yvs8krRzXRo,15867
30
+ fameio/logs.py,sha256=kRaGxHjvmB2WjxKSHr-G9IoVCexkWaALzE4Mtky7Kik,4519
31
+ fameio/output/__init__.py,sha256=ixupBH6NNsC36gL4egdS6jg1fcpH1nFqJGeAUKDrS3E,211
32
+ fameio/output/agent_type.py,sha256=qyt7sVOo6KY1NSbe384Io9dSjXxw0KJQ-RS7TZtcgPM,6551
33
+ fameio/output/conversion.py,sha256=qUC5Xfe8p72TKP5_PuxLnGUo2pPIevumV2PH-GB5aRA,4692
34
+ fameio/output/csv_writer.py,sha256=WIILdfS2gk4z9LIE8Ds58_ZLA3PfglMGJrdEsk0_71k,8103
35
+ fameio/output/data_transformer.py,sha256=mlyCrOdvVM9s-A6ROekequPif56qOaRAKvQjK86dXY0,5251
36
+ fameio/output/execution_dao.py,sha256=w7_6a462RAF1D4URnzMGOT4aKqV9oNGAZMHZR2yLb4A,6803
37
+ fameio/output/input_dao.py,sha256=NC9LMj9I93Ugt1BfFJH1j8sfpuO7cYXw0Ypoqa3LS7c,8460
38
+ fameio/output/output_dao.py,sha256=1j39qH_utLEE_AYRw2aoWLAomEUgScFECDIphwYysUA,4888
39
+ fameio/output/reader.py,sha256=uuhZnNeT1plV5iO_qkoI2L1k87WaU8uhmSUFpFnO0hI,6569
40
+ fameio/output/yaml_writer.py,sha256=vxbJTeqBGVMao7P6_ODpv20ANtswlPYPIob9T7tn5GI,1091
41
+ fameio/scripts/__init__.py,sha256=obt5YXd1mRl9VLswVQnLJzkkU0rbv5TbxoQoPdro3qM,1278
42
+ fameio/scripts/__init__.py.license,sha256=2-OqCNxP4504xY2XQqseYypJi1_Qx4xJSzO3t7c3ACM,107
43
+ fameio/scripts/convert_results.py,sha256=ML7v7XxyDW3bjCHRfG07rnudjEafpyfJSyXjfL9Bycc,7994
44
+ fameio/scripts/convert_results.py.license,sha256=EXKiZn-aoR7nO3auGMNGk9upHbobPLHAIBYUO0S6LUg,107
45
+ fameio/scripts/exception.py,sha256=q0tewaxn_xM1QgqSYVw_rjth2e9DPaTgWWCjDtYxjLg,236
46
+ fameio/scripts/make_config.py,sha256=O6GghQ1M1Y6bxlaS1c9joB066SCaCWvG_V-WTFqFreU,2016
47
+ fameio/scripts/make_config.py.license,sha256=EXKiZn-aoR7nO3auGMNGk9upHbobPLHAIBYUO0S6LUg,107
48
+ fameio/scripts/reformat.py,sha256=jYJsl0UkXtZyn2GyA-QVAARilkHa_ZBWa5CGNIGNDuo,2850
49
+ fameio/scripts/reformat.py.license,sha256=EXKiZn-aoR7nO3auGMNGk9upHbobPLHAIBYUO0S6LUg,107
50
+ fameio/series.py,sha256=ipjDsDmgVlVzaYvXwXcWM8fpCy5w8hTNWTi4hPmvOuM,13654
51
+ fameio/time.py,sha256=S8TagfGA7B44JPkj8YfC3ftkxODNTC6ZXELNQyHC1VA,10201
52
+ fameio/tools.py,sha256=metmgKuZ0lubmTIPY3w_ertDxSLQtHIa6OlpcapyIk4,2478
53
+ fameio-3.4.0.dist-info/entry_points.txt,sha256=IUbTceB_CLFOHulubEf9jgiCFsV2TchlzCssmjbiOKI,176
54
+ fameio-3.4.0.dist-info/LICENSE.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
55
+ fameio-3.4.0.dist-info/LICENSES/Apache-2.0.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
56
+ fameio-3.4.0.dist-info/LICENSES/CC-BY-4.0.txt,sha256=y9WvMYKGt0ZW8UXf9QkZB8wj1tjJrQngKR7CSXeSukE,19051
57
+ fameio-3.4.0.dist-info/LICENSES/CC0-1.0.txt,sha256=9Ofzc7m5lpUDN-jUGkopOcLZC3cl6brz1QhKInF60yg,7169
58
+ fameio-3.4.0.dist-info/METADATA,sha256=qRURN8lvabRE9CmzKRuwsoi_1rKyv71TGkkdLJUAydE,42374
59
+ fameio-3.4.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
60
+ fameio-3.4.0.dist-info/RECORD,,
@@ -1,4 +1,5 @@
1
1
  [console_scripts]
2
2
  convertFameResults=fameio.scripts:convertFameResults
3
3
  makeFameRunConfig=fameio.scripts:makeFameRunConfig
4
+ reformatTimeSeries=fameio.scripts:reformatTimeSeries
4
5
 
@@ -1,56 +0,0 @@
1
- fameio/__init__.py,sha256=LiE7kRXW0pMIB4hTPC0T_ppGz9O0swd0Ca1-b99hOMc,229
2
- fameio/cli/__init__.py,sha256=YAxIBl8azJDFaKr0iGaba94UH3Xy-KhQtxrGtwz3FpM,179
3
- fameio/cli/convert_results.py,sha256=ix5pcmci8GG6q5JTteA9m-na1YmyJTqOuqfEs5CSKGE,3460
4
- fameio/cli/make_config.py,sha256=-kw6-I4mFU5NRi2MjKhnRzU9WxEqGNlrzjX6lHr9Wpo,2734
5
- fameio/cli/options.py,sha256=k_bVfVo8GEoqJzLLfzVpfrol51IA9AWjIGASQsATruY,1269
6
- fameio/cli/parser.py,sha256=8T3FRpeiBm-6VkILTd_XVatFYpopiZjFRGNSs_l35Vo,8909
7
- fameio/input/__init__.py,sha256=73a3vchTU9c--1uJcJTTaB8CGajnTD-ufraQjNtzh8k,504
8
- fameio/input/loader/__init__.py,sha256=yo-sEnI7i0puMpKHENw9zaNq6VfYDLVyw6T0GXiOu6Q,2715
9
- fameio/input/loader/controller.py,sha256=yb7lLc3KDBzdZ7myZ6Q6FPBwX3DkSfHU7f8ArtAbcuI,7769
10
- fameio/input/loader/loader.py,sha256=uxIt6lwPuo6gpme-BA-W6eLayVic9cH5m6M7ZnJ-xGk,5324
11
- fameio/input/metadata.py,sha256=vaDSKmk433XMvZhehAU60LdASEhrs29mp1zpQJ0NffY,6544
12
- fameio/input/resolver.py,sha256=RK6X4mSv2OPEO_ZXc6VJWN0pfvA0okkSse7fHuCOVP0,1920
13
- fameio/input/scenario/__init__.py,sha256=Pb8O9rVOTwEo48WIgiq1kBnpovpc4D_syC6EjTFGHew,404
14
- fameio/input/scenario/agent.py,sha256=D2rt94jNqoCpvVDVa2TUXrxw2D_L9dCzOOGXleDVQE8,5451
15
- fameio/input/scenario/attribute.py,sha256=T7eTDU80HFAVZqpjkh1W7-PSnE9epQZ_gdLTMHdeEv4,9720
16
- fameio/input/scenario/contract.py,sha256=H6XWbVUmRJzpAfWMFW7R4f8upuYAWtEumUOy-Rqx-Uk,13174
17
- fameio/input/scenario/exception.py,sha256=uZSiudjgQ0cwIDJPJoowsTRJxm6MmcobEOpzSDDZfUI,1944
18
- fameio/input/scenario/fameiofactory.py,sha256=-LHVHirqa_TXA0RvABRQSV_tM--VTYtDpWlt4K9kmIw,2847
19
- fameio/input/scenario/generalproperties.py,sha256=gEiOPN0vRDy0ChsN4aG7D5GyKVfTTEAl6G7ZIoVLfgc,3751
20
- fameio/input/scenario/scenario.py,sha256=9jdJ37MFSUoklAyVi2q_4719o8vcHXtAQCig2B3mkiY,8396
21
- fameio/input/scenario/stringset.py,sha256=qm2C8HTWVcZIf20gj456FpTfkD1fVu6icoc4NpkMJ5w,2534
22
- fameio/input/schema/__init__.py,sha256=oIsJVC0hUhwJf_XIBpd1NNTR8sOWEthAy98m6uR7gKc,327
23
- fameio/input/schema/agenttype.py,sha256=vouiTB_p63y1ceXZXpsUaGdh6QLx_TroFA7peoTcgIE,5699
24
- fameio/input/schema/attribute.py,sha256=XpCfbZybuftLv5iQlsoJQ99MJEdNt3sGu38gxuJ_V8s,14916
25
- fameio/input/schema/java_packages.py,sha256=nbvOcj1P0GPyKDQFwK7fO0rS16ElOd7YPEKt5qteYF0,2948
26
- fameio/input/schema/schema.py,sha256=D6mkCm1CjNxe4xPATkaFf9hqfX9fbBRtbZp3gb8lqLE,3844
27
- fameio/input/validator.py,sha256=gDAW51la4p3kzjoWbL1G_Hjoab1d5hYFD88rMOfUKSs,19213
28
- fameio/input/writer.py,sha256=QUd1m3a-xMlgoWoO1JTrFG2IVWst-gVh9ZSa4XAIhtA,15937
29
- fameio/logs.py,sha256=idBSshztqzXQM92yZWDIB9pNAUMQIsY4JyRh2fqdXz0,4252
30
- fameio/output/__init__.py,sha256=ixupBH6NNsC36gL4egdS6jg1fcpH1nFqJGeAUKDrS3E,211
31
- fameio/output/agent_type.py,sha256=2GGDzyMRNIn7B8fzwwx_6TSwRkDTEA6w5eHs67ck3jM,6462
32
- fameio/output/conversion.py,sha256=Zps-U1SMvRkYGtTm3T2EYbjmmpKYvV5XoI_sxiYto0I,4676
33
- fameio/output/csv_writer.py,sha256=Ye26DZ0A_xDvOTi0zPQp7MqtfIibQqWYrNaypI67c0o,7602
34
- fameio/output/data_transformer.py,sha256=haQFtuiplfndfG5uAZjiEf39zeOhRXVV8ZXH0-PLEfU,5250
35
- fameio/output/input_dao.py,sha256=PZuZO6FpcVsTvOhr4JLiC0oHW2lSkongtUDmfmr2nCg,8432
36
- fameio/output/output_dao.py,sha256=slKQJM--dlyr48ZaHp6fsmsyxaokPGSizQETbM7S2as,4890
37
- fameio/output/reader.py,sha256=fiitnvTGLv4uEz4GT2-2UdCZRlj_AOUrQqQA_8irOaU,6599
38
- fameio/output/yaml_writer.py,sha256=vuXqbCsO5yvKJreoEbI7qiMg1rNEkhGzK9XYZiMmDTs,1095
39
- fameio/scripts/__init__.py,sha256=9HlariFV2QOJ69XIJtLSIyKBBGmJN2Vgih3r1K7Ln5Y,937
40
- fameio/scripts/__init__.py.license,sha256=2-OqCNxP4504xY2XQqseYypJi1_Qx4xJSzO3t7c3ACM,107
41
- fameio/scripts/convert_results.py,sha256=2k9cYh1rzA5ukfQEbrGH8GIK86v7z8PyS0FNRHx-sVY,7694
42
- fameio/scripts/convert_results.py.license,sha256=EXKiZn-aoR7nO3auGMNGk9upHbobPLHAIBYUO0S6LUg,107
43
- fameio/scripts/exception.py,sha256=sgGGTIVKUzflp0J7H4lEsPBgALHqZ898BILeCVrJRzs,235
44
- fameio/scripts/make_config.py,sha256=htGUKVUJtTL6i8kF1PJPGZEGJCqPvNKoW-dMg0BLhrE,2021
45
- fameio/scripts/make_config.py.license,sha256=EXKiZn-aoR7nO3auGMNGk9upHbobPLHAIBYUO0S6LUg,107
46
- fameio/series.py,sha256=P5DvPOwzZxN5h4b0LFSqT0xw8WSqHXtb9-bp3j_povA,12192
47
- fameio/time.py,sha256=-rF7-XEDgbvDmIL3nGSpOtJwtjfa05jfCysnyepQ2Wo,8458
48
- fameio/tools.py,sha256=SsPWTeqc2vS4_F4bBkLXY2LX5fp6ITe0f2FTxkejmlY,1104
49
- fameio-3.2.0.dist-info/entry_points.txt,sha256=jvQVfwJjZXPWQjJlhj1Dt6PTeblryTc1GxjKeK90twI,123
50
- fameio-3.2.0.dist-info/LICENSE.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
51
- fameio-3.2.0.dist-info/LICENSES/Apache-2.0.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
52
- fameio-3.2.0.dist-info/LICENSES/CC-BY-4.0.txt,sha256=y9WvMYKGt0ZW8UXf9QkZB8wj1tjJrQngKR7CSXeSukE,19051
53
- fameio-3.2.0.dist-info/LICENSES/CC0-1.0.txt,sha256=9Ofzc7m5lpUDN-jUGkopOcLZC3cl6brz1QhKInF60yg,7169
54
- fameio-3.2.0.dist-info/METADATA,sha256=2k0dbtDQhG3ep0HyXlt8UQf0Xy1b4kc-ysGL--Etp6I,39946
55
- fameio-3.2.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
56
- fameio-3.2.0.dist-info/RECORD,,
File without changes