testomaton 0.2.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {testomaton-0.2.2/src/testomaton.egg-info → testomaton-0.4.0}/PKG-INFO +92 -21
  2. testomaton-0.2.2/PKG-INFO → testomaton-0.4.0/README.md +73 -41
  3. {testomaton-0.2.2 → testomaton-0.4.0}/setup.py +23 -10
  4. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/beaver.py +2 -2
  5. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/constraint.py +133 -91
  6. testomaton-0.4.0/src/testomaton/errors.py +85 -0
  7. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/generator.py +214 -37
  8. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/jigsaw.py +2 -2
  9. testomaton-0.4.0/src/testomaton/model.py +872 -0
  10. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/solver.py +132 -83
  11. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/tomato.py +177 -53
  12. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/validator.py +2 -6
  13. testomaton-0.2.2/README.md → testomaton-0.4.0/src/testomaton.egg-info/PKG-INFO +112 -16
  14. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton.egg-info/SOURCES.txt +3 -2
  15. testomaton-0.4.0/src/testomaton.egg-info/entry_points.txt +4 -0
  16. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton.egg-info/top_level.txt +3 -0
  17. testomaton-0.4.0/test/test_parsing_errors.py +440 -0
  18. testomaton-0.2.2/src/testomaton/exit_codes.py +0 -20
  19. testomaton-0.2.2/src/testomaton/model.py +0 -605
  20. testomaton-0.2.2/src/testomaton.egg-info/entry_points.txt +0 -4
  21. {testomaton-0.2.2 → testomaton-0.4.0}/LICENSE +0 -0
  22. {testomaton-0.2.2 → testomaton-0.4.0}/pyproject.toml +0 -0
  23. {testomaton-0.2.2 → testomaton-0.4.0}/setup.cfg +0 -0
  24. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton/__init__.py +0 -0
  25. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton.egg-info/dependency_links.txt +0 -0
  26. {testomaton-0.2.2 → testomaton-0.4.0}/src/testomaton.egg-info/requires.txt +0 -0
  27. {testomaton-0.2.2 → testomaton-0.4.0}/test/test_model.py +0 -0
@@ -1,11 +1,12 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: testomaton
3
- Version: 0.2.2
3
+ Version: 0.4.0
4
4
  Summary: Model based combinatorial test data generator
5
5
  Home-page: https://bitbucket.org/testify-no/tomato
6
- Author: Patryk Chamuczyński, Testify AS
7
- Author-email: p.chamuczynski@testify.no
6
+ Author: Testify AS
7
+ Author-email: tomato@testify.no
8
8
  License: GNU Affero General Public License v3 or later (AGPLv3+)
9
+ Project-URL: Changelog, https://bitbucket.org/testify-no/tomato/src/develop/CHANGELOG.md
9
10
  Keywords: testing pairwise test_generation
10
11
  Classifier: Development Status :: 4 - Beta
11
12
  Classifier: Programming Language :: Python :: 3
@@ -15,13 +16,26 @@ Classifier: Environment :: Console
15
16
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
17
  Classifier: Topic :: Software Development :: Testing
17
18
  Classifier: Programming Language :: Python :: 3
18
- Requires-Python: >=3.6
19
+ Requires-Python: >=3.7
19
20
  Description-Content-Type: text/markdown
20
21
  License-File: LICENSE
21
22
  Requires-Dist: python-sat
22
23
  Requires-Dist: pyparsing
23
24
  Requires-Dist: pyaml
24
25
  Requires-Dist: regex
26
+ Dynamic: author
27
+ Dynamic: author-email
28
+ Dynamic: classifier
29
+ Dynamic: description
30
+ Dynamic: description-content-type
31
+ Dynamic: home-page
32
+ Dynamic: keywords
33
+ Dynamic: license
34
+ Dynamic: license-file
35
+ Dynamic: project-url
36
+ Dynamic: requires-dist
37
+ Dynamic: requires-python
38
+ Dynamic: summary
25
39
 
26
40
  # Introduction
27
41
  Testomaton is a suite of tools for combinatoric testing. It consists of __tomato__ - a combinatoric test generator and two tools used for postprocessing tomato's output - __beaver__ and __jigsaw__.
@@ -62,6 +76,7 @@ Testomaton is a suite of tools for combinatoric testing. It consists of __tomato
62
76
  - [Constraints manipulation](#constraints-manipulation)
63
77
  - [Filtering parsed elements](#filtering-parsed-elements)
64
78
  - [Format of the output](#format-of-the-output)
79
+ - [Validating models](#validating-models)
65
80
  - [Validating tests](#validating-tests)
66
81
  - [Using tomato in test code](#using-tomato-in-test-code)
67
82
  - [Test postprocessing](#test-postprocessing)
@@ -119,10 +134,10 @@ Jigsaw is a simple .csv manipulation util that can be used to add, remove, repla
119
134
  Tomato is the core of the testomaton suite. Simplifying, it reads a model of a test function and generates rows of tests.
120
135
  The model is defined in a yaml format and provides the description of what values can be assigned to individual parameters of the function.
121
136
 
122
- ┌───────────────┐
123
- │ test function │
124
- └───────┬───────┘
125
- ┌───────────────────┬───────────┴──┬────────────────┐
137
+ ┌───────────────┐
138
+ │ test function │
139
+ └───────┬───────┘
140
+ ┌───────────────────┬───────┴──────┬────────────────┐
126
141
  X1 X2 [...] Xn
127
142
  ┌────┬──┴─┬────┐ ┌────┬──┴─┬────┐ ┌────┬──┴─┬────┐
128
143
  x11 x12 [...] x1m x21 x22 [...] x2m xn1 xn2 [...] xnm
@@ -379,7 +394,7 @@ A linked parameter is a parameter that is a copy of a global parameter. Linked p
379
394
  Linked parameters may link to a structure or to a leaf parameter, but must always link to a parameter that is directly defined as a global parameter.
380
395
  It means that a linked parameter may not link to a subparameter of a global structure.
381
396
  Optionally, linked parameters may have `constraints whitelist` or `constraints blacklist` element that defines what constraints of the link shall be considered in the linked parameter.
382
- Obviously, these fields are mutually exclusive (a parameter cannot have both a whitelist and a blacklist). Elements of the lists are names of the constraints that should be filtered, separated by a comma (`,`).
397
+ Obviously, these fields are mutually exclusive (a parameter cannot have both a whitelist and a blacklist). Elements of the lists are names or labels of the constraints that should be filtered, separated by a comma (`,`).
383
398
  Note that although a comma is allowed when naming or labelling a constraint (or anything else), using it may have unexpected consequences and make the model not work as intended. Using quotes when defining white and blacklists may help here.
384
399
  As mentioned above, parts of structures may also be defined as linked parameters.
385
400
 
@@ -543,7 +558,7 @@ functions:
543
558
  ```
544
559
  In this example, whenever the value of `bad guy` is `Jadis`, the location will always be `White Castle`, unlike like in the previous example when `Jadis` and `Cair Paravel` could coexist.
545
560
 
546
- Invariants and implications are in fact different forms of the same logic semantics. Using implications is introduced for convenient notation, but in the end all implications may be reduced to invariants, because any expression `"IF A THEN B"` can also be noted as `"NOT A OR B"`
561
+ Invariants and implications are in fact different forms of the same logic semantics. Using implications is introduced for convenient notation, but in the end all implications may be reduced to invariants, because any expression `"IF A THEN B"` can also be noted as `"NOT A OR B"`.
547
562
 
548
563
  ### Assignments
549
564
  Assignments are defining values of output parameters in the tests. The syntax of an assignment is `"IF CONDITION THEN ASSIGNMENTS_LIST"` (or alternatively `"CONDITION => ASSIGNMENT_LIST"`),
@@ -573,14 +588,23 @@ functions:
573
588
  It is technically allowed to define two different assignments for the same condition, but the result of such operation are undefined. Tomato will not warn if that happens.
574
589
 
575
590
  ### Aliases
576
- Aliases are macros that allow defining short names for long statements used in constraints and assignments. Aliases may be then used by their names in other statements.
591
+ Aliases are macros that allow defining short names for long statements used in constraints and assignments. Aliases may be then used by their names in other statements of all logic elements (aliases, constraints or assignments).
577
592
 
578
593
  ```yaml
579
594
  functions:
580
595
  - function: duel
581
596
  parameters:
582
597
  - parameter: Good guy
583
- choices: [Peter, Susan, Edmund, Lucy]
598
+ parameters:
599
+ - parameter: name
600
+ choices: [Peter, Susan, Edmund, Lucy]
601
+ - parameter: weapon
602
+ choices: [sword, bow, dagger]
603
+ logic:
604
+ - alias: female
605
+ expression: "'name' IN ['Susan', 'Lucy']"
606
+ - constraint: sword
607
+ expression: "IF 'female' THEN 'weapon' IS NOT 'sword'"
584
608
  - parameter: Bad guy
585
609
  choices: [Jadis, Maugrim]
586
610
  - parameter: location
@@ -588,14 +612,18 @@ functions:
588
612
  - output parameter: duration
589
613
  default value: 5 minutes
590
614
  logic:
591
- - alias: Lucy against Jadis
592
- expression: "'Good guy' IS 'Lucy' AND 'Bad guy' IS 'Jadis'"
593
- - constraint: Lucy against Jadis in Cair Paravel
594
- expression: "IF 'Lucy against Jadis' THEN 'location' IS 'Cair Paravel'"
595
- - assignment: Lucy against Jadis duel duration
596
- expression: "IF 'Lucy against Jadis' THEN 'duration'='10 minutes'"
615
+ - alias: Cair Paravel
616
+ expression: "'location' IS 'Cair Paravel'"
617
+ - alias: Girls against Jadis
618
+ expression: "'Good guy::female' AND 'Bad guy' IS 'Jadis'"
619
+ - constraint: Girls against Jadis in Cair Paravel
620
+ expression: "IF 'Girls against Jadis' THEN 'location' IS 'Cair Paravel'"
621
+ - assignment: Cair Paravel
622
+ expression: "IF 'Cair Paravel' THEN 'duration'='10 minutes'"
597
623
  ```
598
624
 
625
+ An alias may be accessed from higher levels of the hierarchy. In the example above the alias `female` in the structure `Good guy` is accessed from the constraint `sword` within the same structure as well as from the alias `Girls against Jadis` on the function level.
626
+
599
627
  ### Statements
600
628
  A _statement_ is a core concept in the model logic. Statements define invariants, coditions and results of implications and conditions of assignments.
601
629
  Statements may also be assigned to aliases. Statements are built from _primitive statements_ using logical operations like `AND`, `OR`, `NOT` and grouping those in parentheses.
@@ -611,7 +639,11 @@ Both `PARAMETER` and `CHOICE` may refer to lower level in hierarchy if a nested
611
639
  `'PARAMETER' IN/NOT IN ['CHOICE 1', 'CHOICE 2', ...]`
612
640
  This statement defines a situation when a value of a parameter is defined by one of the choices on the list (or not belong to the list, if using `NOT IN`). It is equivalent to expression `'PARAMETER' IS 'CHOICE 1' OR 'PARAMETER IS 'CHOICE 2' OR...`
613
641
 
614
- The names of all elements (parameters and chocies) from the model mentioned in primitive statements must always be surrounded by single quotes.
642
+ `'PARAMETER' LABEL IN/NOT IN ['label 1', 'label 2', ...]`, `'PARAMETER' LABEL IS/IS NOT 'label'`
643
+ Choice labels can also be used in constraints to only allow specific values/choices which have the allowed labels. The statement building works the same as regular primitive statements, but with the added `LABEL` section to specify that we are restricting based
644
+ on labels and not choice values.
645
+
646
+ The names of all elements (parameters, choices and labels) from the model mentioned in primitive statements must always be surrounded by single quotes.
615
647
 
616
648
  ```yaml
617
649
  global parameters:
@@ -693,6 +725,22 @@ If your function has white characters in the name, use quotes (e.g. `$ tomato -f
693
725
  There are three main types of generation algorithms used by tomato: _cartesian_, _random_ and _nwise_ that may be optionally customized with some additional options.
694
726
  By default, the nwise algorithm is used with the parameter N set to 2, which means that the tool will generate tests with pairwise coverage.
695
727
 
728
+ ### Demo and progress
729
+ In order to have more info during the data generation, the user may use demo levels (1-3) or progress.
730
+
731
+ The demo functionality will print how many tuples/test cases there are left before generation is complete. Using demo level 1
732
+ will only print this to stderr and not change the generation speed or such. On the other hand, using demo levels 2 and 3 will print the info to
733
+ stderr and also slow down the generation so that the user can more easily see what is happening and follow along.
734
+ The different demo levels are used as such:
735
+ `--demo-level 1`, `--demo-level 2`, `--demo-level 3`
736
+
737
+ The progress functionality is an experimental feature that may be subject to change in the future. It allows the user to see how many tuples/test cases
738
+ have been generated and how many are left. This allows the user to see exactly how many tests they will have and also helps to follow along so that they can
739
+ know when the generation will finish. The progress is formatted as (completed tuples or test cases / total tuples or test cases).
740
+ The progress functionality is used as such:
741
+ `--show-progress`
742
+
743
+
696
744
  ### Cartesian generator
697
745
  The cartesian generator is the simplest generator that will output all possible combinations of parameter values that are valid according to defined constraints.
698
746
  This parameter does not take any additional options.
@@ -745,7 +793,27 @@ The parameter `--coverage` defines percentage of tuples that must be covered.
745
793
 
746
794
  The way the nwise algorithm works is by building a set of all possible n-tuples that validate the constraints and building tests for individual tuples. It tries to build such a test that covers as many tuples in the set as possible. After the test is constructed, covered tuples are removed from the set. The algorithm repeats until the set of uncovered tuples is empty.
747
795
 
748
- The working of the algorithm is demonstrated if tomato is used with `--demo-level` parameter 1, 2 or 3.
796
+ The parameter `-T|--tuples-from` allows to define a set of parameters that will be covered by the n-tuples. For the example above, if we generate tests by the command:
797
+
798
+ ```bash
799
+ $ tomato --tuples-from 'good guy','bad guy'
800
+ ```
801
+
802
+ we will get something like this:
803
+ ```
804
+ good guy,weapon,bad guy
805
+ Peter,sword,Jadis
806
+ Susan,bow,Jadis
807
+ Peter,dagger,Maugrim
808
+ Lucy,sword,Jadis
809
+ Edmund,bow,Jadis
810
+ Lucy,sword,Maugrim
811
+ Susan,bow,Maugrim
812
+ Edmund,bow,Maugrim
813
+ ```
814
+ In the generated tests, all the combinations of `good guy` and `bad guy` parameters are covered, not necessarily covering the interactions with the remaining parameters.
815
+
816
+ The working of the n-wise algorithm is demonstrated if tomato is used with `--demo-level` parameter 1, 2 or 3.
749
817
  The higher the value is, the more intermediate info is printed (on the error output) and the longer the algorithm waits on each step.
750
818
 
751
819
  ### Constraints manipulation
@@ -833,6 +901,9 @@ Then, the following rows will be filled with tests consisting values of choices
833
901
  - `--use-choice-names` - choice names will be used instead of values. Nested choices will have `::` between hierarchy levels,
834
902
  - `-s|--separator SEPARATOR` - use `SEPARATOR` instead of `,`. May be useful if some of the choices contain `,`.
835
903
 
904
+ ## Validating models
905
+ If tomato is called to generate tests for an invalid model, it will print the encountered errors and exit with an error code. It is also possible to call tomato only in order to validate the model. This is done with `-v|--validate-model` option. Tomato will check the correctness of the model and quit, without generating tests. If the model is correct, it will print a confirmation message on the error output and quit with `0` exit code. Otherwise it will print all the error messages and quit with a positive exit code.
906
+
836
907
  ## Validating tests
837
908
  Tomato may also be used to validate tests using `-V|--validate-tests [TEST_FILE]` option. This is a useful feature in situations when one wants to define a model having some sample tests.
838
909
  Tomato will parse the model, read the tests and print the tests that could be generated from the model unaltered on the standard output.
@@ -1,28 +1,3 @@
1
- Metadata-Version: 2.1
2
- Name: testomaton
3
- Version: 0.2.2
4
- Summary: Model based combinatorial test data generator
5
- Home-page: https://bitbucket.org/testify-no/tomato
6
- Author: Patryk Chamuczyński, Testify AS
7
- Author-email: p.chamuczynski@testify.no
8
- License: GNU Affero General Public License v3 or later (AGPLv3+)
9
- Keywords: testing pairwise test_generation
10
- Classifier: Development Status :: 4 - Beta
11
- Classifier: Programming Language :: Python :: 3
12
- Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
13
- Classifier: Operating System :: OS Independent
14
- Classifier: Environment :: Console
15
- Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
- Classifier: Topic :: Software Development :: Testing
17
- Classifier: Programming Language :: Python :: 3
18
- Requires-Python: >=3.6
19
- Description-Content-Type: text/markdown
20
- License-File: LICENSE
21
- Requires-Dist: python-sat
22
- Requires-Dist: pyparsing
23
- Requires-Dist: pyaml
24
- Requires-Dist: regex
25
-
26
1
  # Introduction
27
2
  Testomaton is a suite of tools for combinatoric testing. It consists of __tomato__ - a combinatoric test generator and two tools used for postprocessing tomato's output - __beaver__ and __jigsaw__.
28
3
 
@@ -62,6 +37,7 @@ Testomaton is a suite of tools for combinatoric testing. It consists of __tomato
62
37
  - [Constraints manipulation](#constraints-manipulation)
63
38
  - [Filtering parsed elements](#filtering-parsed-elements)
64
39
  - [Format of the output](#format-of-the-output)
40
+ - [Validating models](#validating-models)
65
41
  - [Validating tests](#validating-tests)
66
42
  - [Using tomato in test code](#using-tomato-in-test-code)
67
43
  - [Test postprocessing](#test-postprocessing)
@@ -119,10 +95,10 @@ Jigsaw is a simple .csv manipulation util that can be used to add, remove, repla
119
95
  Tomato is the core of the testomaton suite. Simplifying, it reads a model of a test function and generates rows of tests.
120
96
  The model is defined in a yaml format and provides the description of what values can be assigned to individual parameters of the function.
121
97
 
122
- ┌───────────────┐
123
- │ test function │
124
- └───────┬───────┘
125
- ┌───────────────────┬───────────┴──┬────────────────┐
98
+ ┌───────────────┐
99
+ │ test function │
100
+ └───────┬───────┘
101
+ ┌───────────────────┬───────┴──────┬────────────────┐
126
102
  X1 X2 [...] Xn
127
103
  ┌────┬──┴─┬────┐ ┌────┬──┴─┬────┐ ┌────┬──┴─┬────┐
128
104
  x11 x12 [...] x1m x21 x22 [...] x2m xn1 xn2 [...] xnm
@@ -379,7 +355,7 @@ A linked parameter is a parameter that is a copy of a global parameter. Linked p
379
355
  Linked parameters may link to a structure or to a leaf parameter, but must always link to a parameter that is directly defined as a global parameter.
380
356
  It means that a linked parameter may not link to a subparameter of a global structure.
381
357
  Optionally, linked parameters may have `constraints whitelist` or `constraints blacklist` element that defines what constraints of the link shall be considered in the linked parameter.
382
- Obviously, these fields are mutually exclusive (a parameter cannot have both a whitelist and a blacklist). Elements of the lists are names of the constraints that should be filtered, separated by a comma (`,`).
358
+ Obviously, these fields are mutually exclusive (a parameter cannot have both a whitelist and a blacklist). Elements of the lists are names or labels of the constraints that should be filtered, separated by a comma (`,`).
383
359
  Note that although a comma is allowed when naming or labelling a constraint (or anything else), using it may have unexpected consequences and make the model not work as intended. Using quotes when defining white and blacklists may help here.
384
360
  As mentioned above, parts of structures may also be defined as linked parameters.
385
361
 
@@ -543,7 +519,7 @@ functions:
543
519
  ```
544
520
  In this example, whenever the value of `bad guy` is `Jadis`, the location will always be `White Castle`, unlike like in the previous example when `Jadis` and `Cair Paravel` could coexist.
545
521
 
546
- Invariants and implications are in fact different forms of the same logic semantics. Using implications is introduced for convenient notation, but in the end all implications may be reduced to invariants, because any expression `"IF A THEN B"` can also be noted as `"NOT A OR B"`
522
+ Invariants and implications are in fact different forms of the same logic semantics. Using implications is introduced for convenient notation, but in the end all implications may be reduced to invariants, because any expression `"IF A THEN B"` can also be noted as `"NOT A OR B"`.
547
523
 
548
524
  ### Assignments
549
525
  Assignments are defining values of output parameters in the tests. The syntax of an assignment is `"IF CONDITION THEN ASSIGNMENTS_LIST"` (or alternatively `"CONDITION => ASSIGNMENT_LIST"`),
@@ -573,14 +549,23 @@ functions:
573
549
  It is technically allowed to define two different assignments for the same condition, but the result of such operation are undefined. Tomato will not warn if that happens.
574
550
 
575
551
  ### Aliases
576
- Aliases are macros that allow defining short names for long statements used in constraints and assignments. Aliases may be then used by their names in other statements.
552
+ Aliases are macros that allow defining short names for long statements used in constraints and assignments. Aliases may be then used by their names in other statements of all logic elements (aliases, constraints or assignments).
577
553
 
578
554
  ```yaml
579
555
  functions:
580
556
  - function: duel
581
557
  parameters:
582
558
  - parameter: Good guy
583
- choices: [Peter, Susan, Edmund, Lucy]
559
+ parameters:
560
+ - parameter: name
561
+ choices: [Peter, Susan, Edmund, Lucy]
562
+ - parameter: weapon
563
+ choices: [sword, bow, dagger]
564
+ logic:
565
+ - alias: female
566
+ expression: "'name' IN ['Susan', 'Lucy']"
567
+ - constraint: sword
568
+ expression: "IF 'female' THEN 'weapon' IS NOT 'sword'"
584
569
  - parameter: Bad guy
585
570
  choices: [Jadis, Maugrim]
586
571
  - parameter: location
@@ -588,14 +573,18 @@ functions:
588
573
  - output parameter: duration
589
574
  default value: 5 minutes
590
575
  logic:
591
- - alias: Lucy against Jadis
592
- expression: "'Good guy' IS 'Lucy' AND 'Bad guy' IS 'Jadis'"
593
- - constraint: Lucy against Jadis in Cair Paravel
594
- expression: "IF 'Lucy against Jadis' THEN 'location' IS 'Cair Paravel'"
595
- - assignment: Lucy against Jadis duel duration
596
- expression: "IF 'Lucy against Jadis' THEN 'duration'='10 minutes'"
576
+ - alias: Cair Paravel
577
+ expression: "'location' IS 'Cair Paravel'"
578
+ - alias: Girls against Jadis
579
+ expression: "'Good guy::female' AND 'Bad guy' IS 'Jadis'"
580
+ - constraint: Girls against Jadis in Cair Paravel
581
+ expression: "IF 'Girls against Jadis' THEN 'location' IS 'Cair Paravel'"
582
+ - assignment: Cair Paravel
583
+ expression: "IF 'Cair Paravel' THEN 'duration'='10 minutes'"
597
584
  ```
598
585
 
586
+ An alias may be accessed from higher levels of the hierarchy. In the example above the alias `female` in the structure `Good guy` is accessed from the constraint `sword` within the same structure as well as from the alias `Girls against Jadis` on the function level.
587
+
599
588
  ### Statements
600
589
  A _statement_ is a core concept in the model logic. Statements define invariants, coditions and results of implications and conditions of assignments.
601
590
  Statements may also be assigned to aliases. Statements are built from _primitive statements_ using logical operations like `AND`, `OR`, `NOT` and grouping those in parentheses.
@@ -611,7 +600,11 @@ Both `PARAMETER` and `CHOICE` may refer to lower level in hierarchy if a nested
611
600
  `'PARAMETER' IN/NOT IN ['CHOICE 1', 'CHOICE 2', ...]`
612
601
  This statement defines a situation when a value of a parameter is defined by one of the choices on the list (or not belong to the list, if using `NOT IN`). It is equivalent to expression `'PARAMETER' IS 'CHOICE 1' OR 'PARAMETER IS 'CHOICE 2' OR...`
613
602
 
614
- The names of all elements (parameters and chocies) from the model mentioned in primitive statements must always be surrounded by single quotes.
603
+ `'PARAMETER' LABEL IN/NOT IN ['label 1', 'label 2', ...]`, `'PARAMETER' LABEL IS/IS NOT 'label'`
604
+ Choice labels can also be used in constraints to only allow specific values/choices which have the allowed labels. The statement building works the same as regular primitive statements, but with the added `LABEL` section to specify that we are restricting based
605
+ on labels and not choice values.
606
+
607
+ The names of all elements (parameters, choices and labels) from the model mentioned in primitive statements must always be surrounded by single quotes.
615
608
 
616
609
  ```yaml
617
610
  global parameters:
@@ -693,6 +686,22 @@ If your function has white characters in the name, use quotes (e.g. `$ tomato -f
693
686
  There are three main types of generation algorithms used by tomato: _cartesian_, _random_ and _nwise_ that may be optionally customized with some additional options.
694
687
  By default, the nwise algorithm is used with the parameter N set to 2, which means that the tool will generate tests with pairwise coverage.
695
688
 
689
+ ### Demo and progress
690
+ In order to have more info during the data generation, the user may use demo levels (1-3) or progress.
691
+
692
+ The demo functionality will print how many tuples/test cases there are left before generation is complete. Using demo level 1
693
+ will only print this to stderr and not change the generation speed or such. On the other hand, using demo levels 2 and 3 will print the info to
694
+ stderr and also slow down the generation so that the user can more easily see what is happening and follow along.
695
+ The different demo levels are used as such:
696
+ `--demo-level 1`, `--demo-level 2`, `--demo-level 3`
697
+
698
+ The progress functionality is an experimental feature that may be subject to change in the future. It allows the user to see how many tuples/test cases
699
+ have been generated and how many are left. This allows the user to see exactly how many tests they will have and also helps to follow along so that they can
700
+ know when the generation will finish. The progress is formatted as (completed tuples or test cases / total tuples or test cases).
701
+ The progress functionality is used as such:
702
+ `--show-progress`
703
+
704
+
696
705
  ### Cartesian generator
697
706
  The cartesian generator is the simplest generator that will output all possible combinations of parameter values that are valid according to defined constraints.
698
707
  This parameter does not take any additional options.
@@ -745,7 +754,27 @@ The parameter `--coverage` defines percentage of tuples that must be covered.
745
754
 
746
755
  The way the nwise algorithm works is by building a set of all possible n-tuples that validate the constraints and building tests for individual tuples. It tries to build such a test that covers as many tuples in the set as possible. After the test is constructed, covered tuples are removed from the set. The algorithm repeats until the set of uncovered tuples is empty.
747
756
 
748
- The working of the algorithm is demonstrated if tomato is used with `--demo-level` parameter 1, 2 or 3.
757
+ The parameter `-T|--tuples-from` allows to define a set of parameters that will be covered by the n-tuples. For the example above, if we generate tests by the command:
758
+
759
+ ```bash
760
+ $ tomato --tuples-from 'good guy','bad guy'
761
+ ```
762
+
763
+ we will get something like this:
764
+ ```
765
+ good guy,weapon,bad guy
766
+ Peter,sword,Jadis
767
+ Susan,bow,Jadis
768
+ Peter,dagger,Maugrim
769
+ Lucy,sword,Jadis
770
+ Edmund,bow,Jadis
771
+ Lucy,sword,Maugrim
772
+ Susan,bow,Maugrim
773
+ Edmund,bow,Maugrim
774
+ ```
775
+ In the generated tests, all the combinations of `good guy` and `bad guy` parameters are covered, not necessarily covering the interactions with the remaining parameters.
776
+
777
+ The working of the n-wise algorithm is demonstrated if tomato is used with `--demo-level` parameter 1, 2 or 3.
749
778
  The higher the value is, the more intermediate info is printed (on the error output) and the longer the algorithm waits on each step.
750
779
 
751
780
  ### Constraints manipulation
@@ -833,6 +862,9 @@ Then, the following rows will be filled with tests consisting values of choices
833
862
  - `--use-choice-names` - choice names will be used instead of values. Nested choices will have `::` between hierarchy levels,
834
863
  - `-s|--separator SEPARATOR` - use `SEPARATOR` instead of `,`. May be useful if some of the choices contain `,`.
835
864
 
865
+ ## Validating models
866
+ If tomato is called to generate tests for an invalid model, it will print the encountered errors and exit with an error code. It is also possible to call tomato only in order to validate the model. This is done with `-v|--validate-model` option. Tomato will check the correctness of the model and quit, without generating tests. If the model is correct, it will print a confirmation message on the error output and quit with `0` exit code. Otherwise it will print all the error messages and quit with a positive exit code.
867
+
836
868
  ## Validating tests
837
869
  Tomato may also be used to validate tests using `-V|--validate-tests [TEST_FILE]` option. This is a useful feature in situations when one wants to define a model having some sample tests.
838
870
  Tomato will parse the model, read the tests and print the tests that could be generated from the model unaltered on the standard output.
@@ -1,13 +1,15 @@
1
1
  import setuptools
2
2
 
3
- with open("README.md", "r") as fh:
4
- long_description = fh.read()
3
+ with open("README.md", "r", encoding="utf8") as fh:
4
+ readme = fh.read()
5
+
6
+ long_description = readme
5
7
 
6
8
  setuptools.setup(
7
9
  name="testomaton",
8
- version="0.2.2",
9
- author="Patryk Chamuczyński, Testify AS",
10
- author_email="p.chamuczynski@testify.no",
10
+ version="0.4.0",
11
+ author="Testify AS",
12
+ author_email="tomato@testify.no",
11
13
  description="Model based combinatorial test data generator",
12
14
  long_description_content_type="text/markdown",
13
15
  long_description=long_description,
@@ -23,20 +25,31 @@ setuptools.setup(
23
25
  "Topic :: Software Development :: Testing",
24
26
  "Programming Language :: Python :: 3"
25
27
  ],
26
- python_requires='>=3.6',
28
+ python_requires='>=3.7',
27
29
  keywords = 'testing pairwise test_generation',
28
30
  package_dir = {"": "src",},
31
+ project_urls = {
32
+ 'Changelog': 'https://bitbucket.org/testify-no/tomato/src/develop/CHANGELOG.md',
33
+ },
29
34
  packages=setuptools.find_packages(where="src"),
30
- py_modules=['tomato', 'generator', 'model', 'constraint', 'solver', 'beaver'],
35
+ py_modules=['tomato',
36
+ 'generator',
37
+ 'model',
38
+ 'constraint',
39
+ 'solver',
40
+ 'beaver',
41
+ 'errors',
42
+ 'jigsaw',
43
+ 'validator',],
31
44
  install_requires=['python-sat',
32
45
  'pyparsing',
33
46
  'pyaml',
34
47
  'regex'],
35
48
  entry_points={
36
49
  'console_scripts':[
37
- 'tomato=testomaton.tomato:main',
38
- 'beaver=testomaton.beaver:main',
39
- 'jigsaw=testomaton.jigsaw:main',
50
+ 'tomato=testomaton.tomato:tomato',
51
+ 'beaver=testomaton.beaver:beaver',
52
+ 'jigsaw=testomaton.jigsaw:jigsaw',
40
53
  ]
41
54
  },
42
55
  )
@@ -151,7 +151,7 @@ def evaluate_token(row_id, token, values):
151
151
  else:
152
152
  return token
153
153
 
154
- def main():
154
+ def beaver():
155
155
  args = parse_args()
156
156
 
157
157
  input_columns = next(read_csv(args.input_file, separator=args.input_separator))
@@ -177,4 +177,4 @@ def main():
177
177
  csv_writer.writerow(output)
178
178
 
179
179
  if __name__ == '__main__':
180
- main()
180
+ beaver()