pbia 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pbia-1.1.0/LICENSE +21 -0
- pbia-1.1.0/PKG-INFO +612 -0
- pbia-1.1.0/README.md +571 -0
- pbia-1.1.0/pbia/__init__.py +36 -0
- pbia-1.1.0/pbia/__main__.py +6 -0
- pbia-1.1.0/pbia/capture.py +2039 -0
- pbia-1.1.0/pbia/cli.py +309 -0
- pbia-1.1.0/pbia/constant.py +142 -0
- pbia-1.1.0/pbia/dax.py +82 -0
- pbia-1.1.0/pbia/plugin.py +587 -0
- pbia-1.1.0/pbia/results.py +323 -0
- pbia-1.1.0/pbia/runner.py +95 -0
- pbia-1.1.0/pbia/sql.py +208 -0
- pbia-1.1.0/pbia/templates/config.example.json +52 -0
- pbia-1.1.0/pbia/templates/gitignore.txt +20 -0
- pbia-1.1.0/pbia/templates/main.py +91 -0
- pbia-1.1.0/pbia/templates/pipelines/azure-pipelines.yml +95 -0
- pbia-1.1.0/pbia/templates/pipelines/github-actions.yml +81 -0
- pbia-1.1.0/pbia/templates/pytest.ini +10 -0
- pbia-1.1.0/pbia/templates/test_template.py +295 -0
- pbia-1.1.0/pbia/testkit.py +1416 -0
- pbia-1.1.0/pbia/utils/__init__.py +0 -0
- pbia-1.1.0/pbia/utils/browser.py +434 -0
- pbia-1.1.0/pbia/utils/common.py +413 -0
- pbia-1.1.0/pbia/utils/compare.py +314 -0
- pbia-1.1.0/pbia/utils/db.py +513 -0
- pbia-1.1.0/pbia/utils/excel.py +461 -0
- pbia-1.1.0/pbia/utils/logs.py +136 -0
- pbia-1.1.0/pbia/utils/powerbi.py +1238 -0
- pbia-1.1.0/pbia/utils/workspace.py +349 -0
- pbia-1.1.0/pbia.egg-info/PKG-INFO +612 -0
- pbia-1.1.0/pbia.egg-info/SOURCES.txt +36 -0
- pbia-1.1.0/pbia.egg-info/dependency_links.txt +1 -0
- pbia-1.1.0/pbia.egg-info/entry_points.txt +2 -0
- pbia-1.1.0/pbia.egg-info/requires.txt +23 -0
- pbia-1.1.0/pbia.egg-info/top_level.txt +1 -0
- pbia-1.1.0/pyproject.toml +50 -0
- pbia-1.1.0/setup.cfg +4 -0
pbia-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 pbia contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pbia-1.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pbia
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Power BI DAX vs SQL QA automation: capture visual queries, validate DAX against developer SQL, pytest test cases
|
|
5
|
+
Author: HP
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: power bi,dax,sql,qa,testing,pytest,data validation
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Framework :: Pytest
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Classifier: Topic :: Database
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: openpyxl>=3.1
|
|
23
|
+
Requires-Dist: playwright>=1.40
|
|
24
|
+
Requires-Dist: pytest<9,>=7.4
|
|
25
|
+
Requires-Dist: pytest-xdist>=3.3
|
|
26
|
+
Provides-Extra: mssql
|
|
27
|
+
Requires-Dist: pyodbc>=5.0; extra == "mssql"
|
|
28
|
+
Provides-Extra: postgres
|
|
29
|
+
Requires-Dist: sqlalchemy>=2.0; extra == "postgres"
|
|
30
|
+
Requires-Dist: psycopg2-binary; extra == "postgres"
|
|
31
|
+
Provides-Extra: mysql
|
|
32
|
+
Requires-Dist: sqlalchemy>=2.0; extra == "mysql"
|
|
33
|
+
Requires-Dist: pymysql; extra == "mysql"
|
|
34
|
+
Provides-Extra: oracle
|
|
35
|
+
Requires-Dist: sqlalchemy>=2.0; extra == "oracle"
|
|
36
|
+
Requires-Dist: oracledb; extra == "oracle"
|
|
37
|
+
Provides-Extra: snowflake
|
|
38
|
+
Requires-Dist: sqlalchemy>=2.0; extra == "snowflake"
|
|
39
|
+
Requires-Dist: snowflake-sqlalchemy; extra == "snowflake"
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
|
|
42
|
+
# pbia - Power BI DAX vs SQL Validation
|
|
43
|
+
|
|
44
|
+
Checks that the numbers in Power BI reports match the source database, query by query, and that every visual
|
|
45
|
+
actually loads on the page. Several reports, each with several pages, are handled from **one command: `pbia`**
|
|
46
|
+
(installed from the wheel; in this source folder `python run.py` does the same).
|
|
47
|
+
|
|
48
|
+
Works with any Power BI report. Databases: SQL Server / Azure SQL, PostgreSQL, MySQL, Oracle, Snowflake, SQLite.
|
|
49
|
+
Python 3.10+.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Flow of execution
|
|
54
|
+
|
|
55
|
+
```text
|
|
56
|
+
config.json ("reports": [...])
|
|
57
|
+
|
|
|
58
|
+
v
|
|
59
|
+
1. pbia --capture browser: capture every visual's query, page by page
|
|
60
|
+
| -> reports/<Report>/<Page>/QA_<Page>.xlsx (new rows; your edits are kept)
|
|
61
|
+
| -> reports/<Report>/<Page>/PAGE_<Page>.jsonc (reference, from the workbook)
|
|
62
|
+
v
|
|
63
|
+
2. you edit QA_<Page>.xlsx paste Dev SQL, fix DAX that does not work, add your own rows (C1 ...),
|
|
64
|
+
| set SKIP / INTENTIONAL
|
|
65
|
+
v
|
|
66
|
+
3. pbia --validate --dax run the workbook DAX -> DAX Status / Result -> JSONC
|
|
67
|
+
4. pbia --validate --sql run DAX + SQL and compare -> SQL Status / Match -> JSONC
|
|
68
|
+
|
|
|
69
|
+
v
|
|
70
|
+
5. pbia --test your own pytest checks -> results/<date>/test.log + report_<name>_<date>.xlsx
|
|
71
|
+
| + results/final_report.jsonc
|
|
72
|
+
v
|
|
73
|
+
6. python main.py step 5 + your own code on the final report (what a pipeline runs)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
**The workbook is the source of truth.** Every command updates `QA_<Page>.xlsx` first and then rewrites
|
|
77
|
+
`PAGE_<Page>.jsonc` from it. The JSONC is for reference: it holds the page and report details, every query as it is
|
|
78
|
+
in the workbook plus the DAX that was originally captured and the last results, and the page filters.
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Install and start a project
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install "pbia[mssql]" # from PyPI; [mssql] = SQL Server / Azure SQL driver (pyodbc)
|
|
86
|
+
playwright install chromium # once, for --capture / qa.page / qa.check_ui
|
|
87
|
+
mkdir my-pbi-qa && cd my-pbi-qa
|
|
88
|
+
pbia --init # config.json, pytest.ini, tests/_template.py, .gitignore,
|
|
89
|
+
# main.py, pipelines/
|
|
90
|
+
az login # Power BI and Azure SQL tokens come from the Azure CLI
|
|
91
|
+
pbia --capture
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
One install serves every Power BI project: each project is just a folder with its own `config.json` (`pbia --init`
|
|
95
|
+
creates it; existing files are never overwritten). `pbia` works in the nearest folder with a `config.json`, from
|
|
96
|
+
the current folder upwards (or the folder in the `PBIA_HOME` environment variable), so it can be run from
|
|
97
|
+
`tests/` too. Database drivers are extras: `[mssql]`, `[postgres]`, `[mysql]`, `[oracle]`, `[snowflake]`.
|
|
98
|
+
|
|
99
|
+
`pbia --init` also writes this guide into the project (`README.md`, or `PBIA_README.md` when the project already
|
|
100
|
+
has a README).
|
|
101
|
+
|
|
102
|
+
**New release** (maintainers): set the same version in `pyproject.toml` and `pbia/__init__.py`, commit, then
|
|
103
|
+
`git tag v1.2.0 && git push origin v1.2.0` - the *Build and release* GitHub workflow builds, checks and publishes
|
|
104
|
+
it to PyPI (Trusted Publishing). Local build: `pip install build` then `python -m build` -> `dist/`.
|
|
105
|
+
|
|
106
|
+
**From the source folder without installing:** `pip install -r requirements.txt`, then `python run.py ...`
|
|
107
|
+
(same commands as `pbia ...`).
|
|
108
|
+
|
|
109
|
+
`config.json` and `.temp/` (tokens, browser session, capture cache) are git-ignored. Put the database password in
|
|
110
|
+
the `QA_DB_PASSWORD` environment variable instead of `config.json`.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## Commands - `pbia`
|
|
115
|
+
|
|
116
|
+
`pbia --help-all` prints the full guide; `pbia --version` the version. `python -m pbia ...` and, in the source
|
|
117
|
+
folder, `python run.py ...` are the same command.
|
|
118
|
+
|
|
119
|
+
| Command | What it does |
|
|
120
|
+
| :--- | :--- |
|
|
121
|
+
| `pbia --init` | new project in the current folder: `config.json`, `pytest.ini`, `tests/_template.py`, `.gitignore` |
|
|
122
|
+
| `pbia --list` | configured reports, pages on disk, query counts |
|
|
123
|
+
| `pbia --capture` | guided capture: each page opens, you click the filters you need, **ENTER** = next page, **q** + ENTER = stop |
|
|
124
|
+
| `pbia --capture --auto-filters` | unattended: waits `page_load_wait` s (30), then clicks every filter for `filter_click_time` s (60) per page |
|
|
125
|
+
| `pbia --capture --test-filters` | only tests the filters (does each option re-query the visuals?) - updates `filters` in the JSONC only |
|
|
126
|
+
| `pbia --capture --refresh-dax --report "R/P"` | opens just that page and takes its **current** DAX: a changed DAX updates its existing row (marked *DAX updated - update the Dev SQL*), new visuals are added |
|
|
127
|
+
| `pbia --capture --regenerate` | **no browser**: rebuilds the workbooks / JSONC from the raw capture cache (`.temp/cache/<Report>.json`) |
|
|
128
|
+
| `pbia --generate` | builds / refreshes `QA_<Page>.xlsx` from `PAGE_<Page>.jsonc` (restores missing rows, never overwrites what is typed) |
|
|
129
|
+
| `pbia --validate --dax` | runs the DAX of every workbook row against Power BI |
|
|
130
|
+
| `pbia --validate --sql` | runs DAX + developer SQL (read-only) and compares them |
|
|
131
|
+
| `pbia --test [pytest options]` | runs `tests/test_*.py` (everything after `--test` goes to pytest) |
|
|
132
|
+
| `pbia --test --case TC_001` | runs only these test case IDs (repeatable, commas, `*` wildcard) |
|
|
133
|
+
| `pbia --test --run-env DEV` | runs as environment DEV (else the `RUN_ENV` variable, else LOC): test cases with `run_env` run only there |
|
|
134
|
+
| `pbia --test --workers 4` | 4 tests at the same time (one log, one Excel report per name, one final report) |
|
|
135
|
+
| `python main.py [options]` | `pbia --test` + `results/final_report.jsonc` + your own code (`after_run`) - what a pipeline runs |
|
|
136
|
+
|
|
137
|
+
Options for every command:
|
|
138
|
+
|
|
139
|
+
| Option | Use |
|
|
140
|
+
| :--- | :--- |
|
|
141
|
+
| `--report "Report"` | only this report (folder or name, case-insensitive, partial) |
|
|
142
|
+
| `--report "Report/Page"` | only this page; repeat `--report` (or use commas) for more |
|
|
143
|
+
| `--workers N` | `--validate`: run N DAX queries at the same time (default 1) |
|
|
144
|
+
| `--config path` | another config file |
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
pbia --capture --report "Sales Report"
|
|
148
|
+
pbia --capture --regenerate --report "Sales Report/Sales Overview"
|
|
149
|
+
pbia --validate --dax --report "sales/overview"
|
|
150
|
+
pbia --validate --sql --report "Sales Report" --report "Finance/Budget" --workers 4
|
|
151
|
+
pbia --test tests/test_sales_overview.py -s
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
**Logs:** every command also writes what it prints to `results/<YYYY-MM-DD>/<command>.log` (`capture.log`,
|
|
155
|
+
`generate.log`, `validate_dax.log`, `validate_sql.log`), appended per run with a header, the duration and the exit
|
|
156
|
+
code. `--test` writes the detailed `test.log` (see the pytest section).
|
|
157
|
+
|
|
158
|
+
Exit code: `0` all passed, `1` a failure (DAX error, SQL mismatch / error, connection problem), `2` bad command - so
|
|
159
|
+
any command can gate a CI pipeline. The old `capture.py`, `dax.py` and `sql.py` are no longer run directly.
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Configuration - `config.json`
|
|
164
|
+
|
|
165
|
+
```json
|
|
166
|
+
{
|
|
167
|
+
"powerbi": {
|
|
168
|
+
"group_id": "me", // default workspace (GUID, or "me") for reports without their own
|
|
169
|
+
"token": "", // optional: 'az login' is used otherwise
|
|
170
|
+
"output_dir": "reports",
|
|
171
|
+
"cache_dir": ".temp/cache", // raw captured queries (--regenerate)
|
|
172
|
+
"page_load_wait": 30, // --auto-filters / --test-filters
|
|
173
|
+
"query_timeout": 120, // seconds one DAX query may take ("The read operation timed out")
|
|
174
|
+
"filter_click_time": 60, // --auto-filters
|
|
175
|
+
"filter_options_per_slicer": 10 // --test-filters (0 = all)
|
|
176
|
+
},
|
|
177
|
+
"reports": [
|
|
178
|
+
{
|
|
179
|
+
"name": "Sales Report", // empty = taken from Power BI on the first capture (and saved here)
|
|
180
|
+
"group_id": "", // empty = powerbi.group_id
|
|
181
|
+
"report_id": "<guid after /reports/ in the report URL>",
|
|
182
|
+
"dataset_id": "", // empty = looked up from the report (and saved here)
|
|
183
|
+
"pages": [] // [] = all pages (found on the first capture and saved here)
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
"name": "Finance",
|
|
187
|
+
"group_id": "<another workspace guid>",
|
|
188
|
+
"report_id": "<guid>",
|
|
189
|
+
"database": {"database": "finance_db"} // optional: overrides parts of "database" for this report
|
|
190
|
+
}
|
|
191
|
+
],
|
|
192
|
+
"database": {
|
|
193
|
+
"db_type": "mssql", // mssql, postgres, mysql, oracle, snowflake, sqlite
|
|
194
|
+
"host": "your-server.database.windows.net",
|
|
195
|
+
"port": 1433,
|
|
196
|
+
"database": "your_db",
|
|
197
|
+
"user": "you@company.com",
|
|
198
|
+
"auth_mode": "entra_mfa", // entra_mfa, ssms_interactive, sql_login, windows
|
|
199
|
+
"schema": "dbo",
|
|
200
|
+
"read_only": true
|
|
201
|
+
},
|
|
202
|
+
"compare": {"decimals": 2, "abs_tolerance": 0.01, "rel_tolerance": 0, "duration_tolerance": 0, "blank_as_zero": false}
|
|
203
|
+
}
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
(`config.example.json` has every setting; the comments above are for reading only - JSON files cannot contain them.)
|
|
207
|
+
An older single-report `config.json` (`powerbi.report_id` / `page_ids`) is converted to this layout automatically
|
|
208
|
+
the first time `pbia` runs; the old file is copied to `.temp/backup/`.
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## The workbook - `reports/<Report>/<Page>/QA_<Page>.xlsx`
|
|
213
|
+
|
|
214
|
+
One row per query. **Green columns are yours**; the others are written by the commands.
|
|
215
|
+
|
|
216
|
+
| Column | Use |
|
|
217
|
+
| :--- | :--- |
|
|
218
|
+
| **Query ID** | captured rows get numbers; give your own rows an ID like `C1`, `C2` |
|
|
219
|
+
| **Visual Name / Visual Type / Active Filters** | filled by capture when empty; free text |
|
|
220
|
+
| **DAX Query (EDIT / PASTE HERE)** | the DAX that is run. Starts as the captured DAX - fix it here when it does not work |
|
|
221
|
+
| **Dev SQL Query (EDIT / PASTE HERE)** | SQL that returns the same data as the DAX (same columns, same order) |
|
|
222
|
+
| **Skip / Intentional** | `SKIP` - not validated. `INTENTIONAL` - an error / mismatch is expected and is not a failure |
|
|
223
|
+
| **Developer Notes** | free text |
|
|
224
|
+
| DAX Change | written by a DAX refresh: *DAX updated ... - update the Dev SQL* (then *Dev SQL updated since* once you edit it) |
|
|
225
|
+
| DAX Status, DAX Result, SQL Status, SQL Result, DAX vs SQL Match?, Error Message, Last Run | written by `--validate` |
|
|
226
|
+
|
|
227
|
+
- **Captured again = same row.** A re-capture recognises a query it captured before (by its captured DAX) and keeps
|
|
228
|
+
that row's Query ID, DAX, SQL, flag and notes. New queries are added with the next free Query ID.
|
|
229
|
+
- **Queries capture missed:** add a row with your own Query ID, the DAX and the SQL - it is validated like the rest
|
|
230
|
+
and kept by every command.
|
|
231
|
+
- **The report's DAX changed?** `pbia --capture --refresh-dax --report "Report/Page"` (or `qa.refresh_dax()`
|
|
232
|
+
in a test) opens only that page and takes its current DAX. A query whose DAX changed is matched to its existing row
|
|
233
|
+
(same visual: type + measure + dimensions, else visual name): its DAX is replaced (the old one goes to
|
|
234
|
+
`dax_history` in the JSONC) and the **DAX Change** column says *DAX updated ... - update the Dev SQL*. While that
|
|
235
|
+
Dev SQL is unchanged, a failing check of the row is **`SQL_OUTDATED`** ("DAX was updated, SQL wasn't"); editing the
|
|
236
|
+
SQL clears it. New visuals become new rows; rows not seen on the page are listed and kept.
|
|
237
|
+
Rows marked **SKIP** are never refreshed: their DAX stays as it is (and their query is not added again as a new row).
|
|
238
|
+
- **The DAX cell is never changed by a command** (except a refresh, as above). Before running, the DAX is cleaned up (names / values escaped,
|
|
239
|
+
filter values converted to the column's data type, known Power BI errors repaired). If that version fails, the
|
|
240
|
+
DAX is run exactly as typed. The DAX that actually ran is stored as `executed_dax` in the JSONC.
|
|
241
|
+
- Extra columns you add to the sheet are kept. Close the workbook in Excel before running a command.
|
|
242
|
+
- The file name prefix is `WORKBOOK_PREFIX = "QA_"` in `pbia/utils/common.py`.
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## `PAGE_<Page>.jsonc` (reference)
|
|
247
|
+
|
|
248
|
+
```text
|
|
249
|
+
report_name, report_id, group_id, dataset_id, page_name, page_id, workbook, generated_at, summary
|
|
250
|
+
queries: [ {query_id, visual_name, visual_type, source (captured / custom), dax, sql, skip, intentional,
|
|
251
|
+
notes, filters, captured_dax, dax_result {status, rows_count, executed_dax, rows_preview ...},
|
|
252
|
+
sql_result {status, comparison, rows_preview ...}} ... ]
|
|
253
|
+
filters, filter_tests
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Do not edit it - it is rewritten from the workbook by every command.
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
## SQL validation rules - `--validate --sql`
|
|
261
|
+
|
|
262
|
+
| Status | Meaning |
|
|
263
|
+
| :--- | :--- |
|
|
264
|
+
| `SUCCESS` | SQL result matches the DAX result |
|
|
265
|
+
| `MISMATCH` | SQL returned different data (differing rows are printed) |
|
|
266
|
+
| `ERROR` | the SQL failed |
|
|
267
|
+
| `DAX_ERROR` | the DAX failed, so nothing could be compared |
|
|
268
|
+
| `PENDING_DEV_SQL` | no SQL in the workbook yet |
|
|
269
|
+
| `SKIPPED` | marked `SKIP` |
|
|
270
|
+
| `INTENTIONAL` | marked `INTENTIONAL` and the error / mismatch happened |
|
|
271
|
+
| `SQL_OUTDATED` | the DAX was refreshed from the report but the Dev SQL was not updated since, and the check failed - update the Dev SQL |
|
|
272
|
+
|
|
273
|
+
**Comparison:** columns are matched by position and row order is ignored. Text is trimmed and case-insensitive;
|
|
274
|
+
blank and NULL are equal; a midnight datetime equals the date. Numbers:
|
|
275
|
+
|
|
276
|
+
- `decimals`: round both sides first (half-up: 12.345 -> 12.35); `null` = no rounding
|
|
277
|
+
- `abs_tolerance` / `rel_tolerance`: allowed difference after rounding (0.001 relative = 0.1%)
|
|
278
|
+
- `duration_tolerance`: seconds allowed between `h:m:s` durations (`0:2:31` vs `0:2:30` passes with 1); durations are compared as seconds, so `0:2:30` = `00:02:30`. Numbers are not affected
|
|
279
|
+
- `blank_as_zero`: `true` makes a blank cell (null / None / empty / `"null"` / `"(Blank)"`) count as 0, so a DAX blank
|
|
280
|
+
equals SQL `0`, `0.00` or `NULL`. Default `false`: blank only equals blank
|
|
281
|
+
- formatted numbers are read as numbers: `1,234.50`, `$1,234`, `1 234`, `(1,234)` = -1234, `45%` = 0.45,
|
|
282
|
+
`1.234,56` = 1234.56; numbers inside text are normalised the same way (`1,123/12` equals `1123/12`)
|
|
283
|
+
|
|
284
|
+
**Production safety:** only a single `SELECT` / `WITH` / `EXPLAIN` statement is allowed (`INSERT`,
|
|
285
|
+
`SELECT ... INTO`, `UPDATE`, `DELETE`, `DROP`, `EXEC`, ... are rejected), SQL Server connections use
|
|
286
|
+
`ApplicationIntent=ReadOnly` with `READ UNCOMMITTED` and low deadlock priority, PostgreSQL / MySQL sessions are
|
|
287
|
+
`READ ONLY`, SQLite files are opened read-only, and every query is rolled back.
|
|
288
|
+
|
|
289
|
+
---
|
|
290
|
+
|
|
291
|
+
## Test cases with pytest - `pbia --test`
|
|
292
|
+
|
|
293
|
+
Write your own DAX vs SQL checks as pytest files. They use the same code as `--validate`.
|
|
294
|
+
|
|
295
|
+
```bash
|
|
296
|
+
cp tests/_template.py tests/test_sales_overview.py # then set TEST_FOLDER and keep the tests you need
|
|
297
|
+
pbia --test # all tests/test_*.py
|
|
298
|
+
pbia --test tests/test_sales_overview.py -k revenue -s
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
**Module-level settings in a test file**
|
|
302
|
+
|
|
303
|
+
| Setting | Use |
|
|
304
|
+
| :--- | :--- |
|
|
305
|
+
| `TEST_FOLDER` | **required**: `"Report/Page"` (folder or name), a page folder, or its `QA_<Page>.xlsx` |
|
|
306
|
+
| `DECIMALS`, `ABS_TOLERANCE`, `REL_TOLERANCE`, `DURATION_TOLERANCE` | rounding / tolerance for the file (otherwise `config.json` `compare`); `qa.check(..., decimals=0, duration_tol=1)` per check |
|
|
307
|
+
| `BLANK_AS_ZERO` | `True`: blank / null counts as 0 (DAX blank = SQL `0` / `0.00` / `NULL` / empty); per check `qa.check(id, blank_as_zero=True)` |
|
|
308
|
+
| `STOP_ON_FAIL` | `False` (default): a failed check is recorded and the test continues (its other checks run); the test is marked FAILED at the end with every failure. `True`: stop at the first failure |
|
|
309
|
+
| `REPORT_NAME` | report file name (default: the file name without `test_`) |
|
|
310
|
+
| `REPORT_COLUMNS` | report layout: a list of record keys, or `{"Header": key or function(record)}` |
|
|
311
|
+
| `build_report_row(record)` | function: build each report row yourself |
|
|
312
|
+
| `transform_dax(rows, query_id)`, `transform_sql(rows, query_id)` | functions: change every result before it is compared |
|
|
313
|
+
|
|
314
|
+
**In a test, the `qa` fixture gives you**
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
qa.check(3) # workbook DAX + SQL, compare, assert (SKIP / INTENTIONAL honoured)
|
|
318
|
+
qa.check("C1") # your own workbook row
|
|
319
|
+
dax = qa.run_dax(4) # or qa.run_dax(dax="EVALUATE ...") -> QueryResult
|
|
320
|
+
sql = qa.run_sql(4) # or qa.run_sql(sql="SELECT ...")
|
|
321
|
+
dax = dax.select("Product", "Revenue Won").round(2) # select/drop/rename/where/map/apply/sort ...
|
|
322
|
+
sql.rows[0]["revenue"] = 0 # or edit the rows (list of dicts) directly
|
|
323
|
+
qa.compare(dax, sql, decimals=2, name="by product") # compare, add a report row, assert
|
|
324
|
+
qa.note(owner="data-team") # extra report columns
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
`@pytest.mark.parametrize("query_id", query_ids(TEST_FOLDER))` gives one test per workbook row.
|
|
328
|
+
|
|
329
|
+
**Refresh the DAX from the report, read values shown on the page, use other pages**
|
|
330
|
+
|
|
331
|
+
```python
|
|
332
|
+
out = qa.refresh_dax() # open the page, take its current DAX into the workbook
|
|
333
|
+
out["updated"] # rows whose DAX changed -> their Dev SQL needs updating
|
|
334
|
+
qa.refresh_dax(3) # only row 3 (or several: qa.refresh_dax(3, 7))
|
|
335
|
+
qa.refresh_dax(query_id) # in a parametrized test: each test refreshes its own row
|
|
336
|
+
|
|
337
|
+
# qa.page = the report page open in the browser, a Playwright Page: anything Playwright can do works on it
|
|
338
|
+
qa.page.get_by_text("2024", exact=True).first.click() # pick a slicer value, click, fill, evaluate ...
|
|
339
|
+
qa.page.screenshot(path="results/page.png")
|
|
340
|
+
shown = qa.page.value(title="Revenue Won") # what the card SHOWS: "1.23M" -> 1230000.0
|
|
341
|
+
qa.compare(qa.run_dax(3), shown, rel_tol=0.005) # page vs DAX (the shown value is rounded)
|
|
342
|
+
qa.page.value(pattern=r"Target:\s*([\d.,]+[KMB]?)") # a hardcoded text on the page (regex, group 1)
|
|
343
|
+
qa.page.value(selector="div.card .value") # any element (CSS selector)
|
|
344
|
+
qa.open("Finance/Budget").value(title="Revenue by Product") # another report / page, same as qa.page
|
|
345
|
+
qa.page.playwright # the Playwright Page itself (e.g. for playwright's expect())
|
|
346
|
+
|
|
347
|
+
qa.workbook # TEST_FOLDER's workbook rows: query_ids, visual(id), dev_input(id)
|
|
348
|
+
qa.check(5, page="Finance/Budget") # a row of another page's workbook
|
|
349
|
+
qa.run_dax(2, page="Finance/Budget"); qa.run_sql(2, page="Finance/Budget")
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
**UI level - did the visual load on the page?** `qa.check_ui` follows the page load in the browser the way the
|
|
353
|
+
report itself works: the query each visual sends (with the `VisualId` that sent it), Power BI's answer (HTTP status,
|
|
354
|
+
time, an error inside the answer) and what the visual then shows (error text such as *Can't display this visual*,
|
|
355
|
+
a spinner that never stopped, nothing drawn):
|
|
356
|
+
|
|
357
|
+
```python
|
|
358
|
+
qa.check_ui(query_id) # a row that is not LOADED fails the test (soft, like qa.check)
|
|
359
|
+
qa.check_ui() # every row of the page
|
|
360
|
+
st = qa.ui_status(5)["5"] # the same without failing: status, message, http_status, ms,
|
|
361
|
+
# error, visual_id, ui_state, ui_title
|
|
362
|
+
qa.page.visuals() # every visual's state now: loaded / error / loading / empty
|
|
363
|
+
```
|
|
364
|
+
|
|
365
|
+
| UI status | meaning |
|
|
366
|
+
|---|---|
|
|
367
|
+
| `LOADED` | the page sent the row's query, got data back, and its visual shows no error |
|
|
368
|
+
| `NOT_LOADED` | the page never sent this row's query (visual missing / hidden / not drawn - or its DAX changed: `qa.refresh_dax`) |
|
|
369
|
+
| `QUERY_ERROR` | Power BI answered with an error (HTTP error or an error inside the result), or never answered |
|
|
370
|
+
| `UI_ERROR` | the visual shows an error on the page |
|
|
371
|
+
| `UI_LOADING` | the visual was still loading when the page went quiet |
|
|
372
|
+
|
|
373
|
+
The page load is watched **once per run** and reused by every `check_ui` / `ui_status` (and by `qa.refresh_dax`);
|
|
374
|
+
`fresh=True` loads it again. The visual on the page is found by the row's *Visual Name / UI Title*; when it is not
|
|
375
|
+
found by name, the status comes from the query and its answer only. Rows marked SKIP are skipped. The report gets
|
|
376
|
+
the columns `ui_state`, `ui_title`, `http_status`, `ui_ms`, `visual_id`; `test.log` lists every visual's state.
|
|
377
|
+
|
|
378
|
+
- `qa.compare(a, b)` compares any two results: DAX, SQL or page values.
|
|
379
|
+
- **A page is captured once per test run.** The first `qa.refresh_dax(...)` on a page opens it and records its
|
|
380
|
+
queries (about 10-30 s); every later call in the same run - other tests, other rows - reuses that capture and only
|
|
381
|
+
updates the workbook. `qa.refresh_dax(fresh=True)` captures again. In a parametrized test pass the row:
|
|
382
|
+
`qa.refresh_dax(query_id)`.
|
|
383
|
+
- `qa.page` opens on first use and keeps the same tab, so clicks and filters you apply stay for the next
|
|
384
|
+
`qa.page.value(...)`; `qa.page.reload_report()` resets it. Tests that never touch it never start a browser.
|
|
385
|
+
- One browser is used for the whole run, signed in with the session saved by the first `pbia --capture`. If the
|
|
386
|
+
sign-in expired, sign in in its window (the run waits `powerbi.login_wait` = 180 s). `powerbi.headless: true`
|
|
387
|
+
hides the window once the session works.
|
|
388
|
+
- After a refresh, `qa.check(...)` on a refreshed row runs the new DAX; if its old Dev SQL then fails, the result is
|
|
389
|
+
`SQL_OUTDATED` with the message *DAX was updated from the report ... but the Dev SQL was not - update the Dev SQL*.
|
|
390
|
+
|
|
391
|
+
**Test case IDs - `@test_case`**
|
|
392
|
+
|
|
393
|
+
```python
|
|
394
|
+
from pbia import query_ids, test_case
|
|
395
|
+
|
|
396
|
+
@test_case("TC_SO_001", title="Revenue Won card matches SQL", owner="data-team", priority="P1")
|
|
397
|
+
def revenue_won_card(qa): # no "test_" needed: every @test_case function is a test case
|
|
398
|
+
qa.check(3)
|
|
399
|
+
|
|
400
|
+
@test_case("TC_SO_002")
|
|
401
|
+
def missed_kpi(qa):
|
|
402
|
+
"""Missed KPI (own workbook row C1) matches SQL""" # first docstring line = title when none is given
|
|
403
|
+
qa.check("C1")
|
|
404
|
+
|
|
405
|
+
@test_case("SO_V{query_id}", title="Visual {query_id}: DAX = SQL") # one ID per parameter: SO_V1, SO_V2 ...
|
|
406
|
+
@pytest.mark.parametrize("query_id", query_ids(TEST_FOLDER))
|
|
407
|
+
def test_all_visuals(qa, query_id):
|
|
408
|
+
qa.check(query_id)
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
- The ID (and title) is shown in the live log and `test.log` (`TEST START [TC_SO_001] ...`), and **Test Case ID** /
|
|
412
|
+
**Test Case** are the first columns of the report; extra details (`owner=`, `priority=`, ...) become report
|
|
413
|
+
columns. The Summary sheet counts the test cases.
|
|
414
|
+
- Every ID must belong to one test function - a duplicate stops the run and names both functions.
|
|
415
|
+
- `pbia --test --case TC_SO_001 --case "SO_V*"` runs only those test cases (case-insensitive).
|
|
416
|
+
- The ID is also written to `--junitxml` output as the `test_case_id` property.
|
|
417
|
+
|
|
418
|
+
**Results**: everything a test run produces goes to `results/<YYYY-MM-DD>/`:
|
|
419
|
+
|
|
420
|
+
| File | Content |
|
|
421
|
+
| :--- | :--- |
|
|
422
|
+
| `test.log` | detailed log, appended per run: run settings (config, reports, database, rounding), every test's start and outcome with duration, every DAX / SQL query with its full text, the DAX actually run when it was cleaned up, status, row count, timing, column names and the first rows, transforms applied, comparison result and differing rows, the full failure text, and a run summary |
|
|
423
|
+
| `report_<REPORT_NAME>_<YYYY-MM-DD>.xlsx` | *Results* sheet (one row per comparison, or per test when nothing was compared) and *Summary* sheet. If the file is open in Excel, it is saved with a time suffix instead |
|
|
424
|
+
|
|
425
|
+
While the tests run, a **compact live view** is printed in the console - one line per event, cut to the window
|
|
426
|
+
width, outcomes in colour, the file name once per file:
|
|
427
|
+
|
|
428
|
+
```text
|
|
429
|
+
tests\test_sales_overview.py
|
|
430
|
+
|
|
431
|
+
13:29:35 START [SO_V3] test_all_visuals[3]
|
|
432
|
+
13:29:36 DAX #3 SUCCESS 1 row(s) 1338 ms
|
|
433
|
+
13:29:41 SQL #3 SUCCESS 1 row(s) 5146 ms
|
|
434
|
+
13:29:41 CHECK #3 M Visit 12 M -> SUCCESS 1 row(s) match
|
|
435
|
+
13:29:41 P PASSED [SO_V3] test_all_visuals[3] (6.49s)
|
|
436
|
+
|
|
437
|
+
13:29:41 START [SO_V4] test_all_visuals[4]
|
|
438
|
+
13:29:41 S SKIPPED [SO_V4] test_all_visuals[4] (0.0s) Query 4 is marked SKIP in the workbook
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
Every outcome line starts with a letter: **P** passed, **F** failed, **S** skipped, **X** intentional (xfail), **E** error.
|
|
442
|
+
|
|
443
|
+
A failed check does not stop its test (unless `STOP_ON_FAIL = True`): the test's other checks still run and the test is marked **F** at the end with all its failures ("2 checks failed - ...").
|
|
444
|
+
|
|
445
|
+
Query texts, row previews and full failure texts are in `test.log` only (a failed test also shows its full log under
|
|
446
|
+
"Captured log call"). Run without `-v`: with it, pytest adds its own per-test lines on top of these.
|
|
447
|
+
|
|
448
|
+
---
|
|
449
|
+
|
|
450
|
+
## Test case details and environments - `@test_case(..., run_env=...)`
|
|
451
|
+
|
|
452
|
+
```python
|
|
453
|
+
@test_case("TC_SO_001", title="Revenue Won card matches SQL",
|
|
454
|
+
description="Revenue Won card vs the finance SQL, whole numbers",
|
|
455
|
+
run_env=["DEV", "PRD"], owner="data-team", priority="P1")
|
|
456
|
+
def revenue_won_card(qa):
|
|
457
|
+
qa.check(3, decimals=0)
|
|
458
|
+
```
|
|
459
|
+
|
|
460
|
+
- `title` is the test case name and `description` what it checks (defaults: the first line of the docstring / the
|
|
461
|
+
rest of it). Both are columns of the Excel report and fields of the final report.
|
|
462
|
+
- `run_env` - the environments the test case runs in: `"DEV"`, `"DEV,PRD"` or `["DEV", "PRD"]`. No `run_env` =
|
|
463
|
+
every environment. `@pytest.mark.run_env("DEV")` on a test, a class or a module (`pytestmark`) does the same.
|
|
464
|
+
- The environment of a run comes from the **`RUN_ENV` environment variable** - never from `config.json` -
|
|
465
|
+
`PRD`, `DEV` or `LOC` (default `LOC`; `prod` / `production` / `local` ... are accepted). `--run-env DEV` sets it
|
|
466
|
+
for one run. Test cases for other environments are not run; they are listed under `not_run` in the final report.
|
|
467
|
+
An unknown name (in `RUN_ENV` or in a decorator) stops the run with an error instead of silently running nothing.
|
|
468
|
+
- In a test: `from pbia.constant import RUN_ENV` -> `"PRD"` / `"DEV"` / `"LOC"`. All environment settings are
|
|
469
|
+
read in `pbia/constant.py`.
|
|
470
|
+
|
|
471
|
+
```bash
|
|
472
|
+
RUN_ENV=DEV pbia --test # or: pbia --test --run-env DEV
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
---
|
|
476
|
+
|
|
477
|
+
## Final report - `results/final_report.jsonc` and `main.py`
|
|
478
|
+
|
|
479
|
+
Every test run (`pbia --test`, `python main.py`, plain `pytest`, serial or parallel) ends with
|
|
480
|
+
**`results/final_report.jsonc`**: one file with every test case, at a fixed path for pipelines and your own code
|
|
481
|
+
(a copy is kept as `results/<date>/final_report_<HHMMSS>.jsonc`; `--final-report PATH` or `PBIA_FINAL_REPORT`
|
|
482
|
+
write it elsewhere).
|
|
483
|
+
|
|
484
|
+
| Part | Holds |
|
|
485
|
+
| :--- | :--- |
|
|
486
|
+
| `run` | `run_id`, `env`, `status` (PASSED / FAILED / NO_TESTS / ERROR), `exit_code`, `started`, `finished`, `duration_s`, `workers`, `host`, versions, `args`, `case_filter`, `ci` (pipeline build id / number, branch, commit, url) |
|
|
487
|
+
| `summary` | `total`, `passed`, `failed`, `error`, `skipped`, `xfail`, `xpass`, `pass_rate`, `not_run_in_env`, `checks` {total, passed, failed, other} |
|
|
488
|
+
| `test_cases[]` | `test_case_id`, `test_case_name`, `description`, `status` (PASSED / FAILED / ERROR / SKIPPED / XFAIL / XPASS), `remarks`, `run_env`, `test`, `test_function`, `test_file`, `report_name`, `pbi_reports`, `pages`, `started`, `finished`, `duration_s`, `worker`, `checks`, `details[]` (one per check: query id, visual, status, message, row counts, differences, errors, timings, settings, queries), `extra` (`owner=` ... and `qa.note()` fields) |
|
|
489
|
+
| `not_run[]` | test cases left out because of `RUN_ENV`, with the reason |
|
|
490
|
+
| `files` | `log`, `excel_reports`, `final_report`, `archive`, `junit` |
|
|
491
|
+
|
|
492
|
+
`remarks` says in one line why a test case ended as it did, e.g. *"2 of 5 check(s) failed - #5 M Visit:
|
|
493
|
+
MISMATCH - 403 of 830 row(s) differ ..."*, *"Skipped: Query 4 is marked SKIP in the workbook"*,
|
|
494
|
+
*"Error: Could not connect to the database ..."*, *"All 3 check(s) passed"*.
|
|
495
|
+
|
|
496
|
+
**`main.py`** (in the project root; `pbia --init` creates it) runs every test, writes the final report and then
|
|
497
|
+
calls **`after_run(report)`** - your code: reshape the report and send it wherever you want (examples inside: your
|
|
498
|
+
own JSON, a CSV, an API call). Its exit code is the tests' exit code, so a pipeline step fails when a test fails
|
|
499
|
+
(`FAIL_ON_TEST_FAILURE`, `FAIL_WHEN_NO_TESTS` change that).
|
|
500
|
+
|
|
501
|
+
```bash
|
|
502
|
+
python main.py # all test cases
|
|
503
|
+
python main.py --run-env DEV --workers 4 # any pbia --test / pytest option works too
|
|
504
|
+
```
|
|
505
|
+
|
|
506
|
+
From any other Python code:
|
|
507
|
+
|
|
508
|
+
```python
|
|
509
|
+
from pbia import run_tests, load_final_report, flatten
|
|
510
|
+
result = run_tests(["--run-env", "DEV"]) # result.exit_code, result.report (dict), result.final_report (path)
|
|
511
|
+
report = load_final_report() # the last run's report
|
|
512
|
+
rows = flatten(report) # one flat dict per test case (details=True: one per check)
|
|
513
|
+
```
|
|
514
|
+
|
|
515
|
+
---
|
|
516
|
+
|
|
517
|
+
## Parallel runs - `--workers N`
|
|
518
|
+
|
|
519
|
+
```bash
|
|
520
|
+
pbia --test --workers 4 # or: python main.py --workers 4, or PBIA_WORKERS=4
|
|
521
|
+
```
|
|
522
|
+
|
|
523
|
+
Runs 4 tests at the same time (pytest-xdist, installed with pbia). Each worker is its own process with its own
|
|
524
|
+
Power BI / database connections (and browser, for UI tests); a free worker takes the next test. The console still
|
|
525
|
+
shows one line per outcome (with the worker, `[gw2]`), and the main process writes **one** `test.log` (each test's
|
|
526
|
+
lines kept together), **one** Excel report per `REPORT_NAME` and **one** final report - the same files as a serial
|
|
527
|
+
run. `qa.refresh_dax` writes to a workbook under a lock, so two workers never write the same workbook at once.
|
|
528
|
+
`--dist loadfile` keeps each test file on one worker (one browser page load per page for UI tests); any `-n` /
|
|
529
|
+
`--dist` option you pass is used as given.
|
|
530
|
+
|
|
531
|
+
Most of a run is waiting for Power BI and the database, so 4-8 workers usually cut the time to a fraction. Keep
|
|
532
|
+
Power BI's limits in mind (executeQueries allows about 120 queries per minute per user).
|
|
533
|
+
|
|
534
|
+
---
|
|
535
|
+
|
|
536
|
+
## Pipelines (GitHub Actions / Azure DevOps)
|
|
537
|
+
|
|
538
|
+
`pbia --init` puts two ready examples in `pipelines/`: `github-actions.yml` (copy to `.github/workflows/`) and
|
|
539
|
+
`azure-pipelines.yml`. Both: install the wheel, write `config.json` from a secret when it is not committed, sign in
|
|
540
|
+
to Azure with a **service principal**, run `python main.py --junitxml=results/junit.xml`, and keep `results/` as
|
|
541
|
+
the run's artifact (Azure DevOps also shows the JUnit results in its Tests tab). `RUN_ENV` and the number of
|
|
542
|
+
workers are pipeline parameters.
|
|
543
|
+
|
|
544
|
+
What a pipeline needs:
|
|
545
|
+
|
|
546
|
+
- **The workbooks**: `reports/<Report>/<Page>/QA_*.xlsx` are the test data - commit them (take `reports/` out of
|
|
547
|
+
`.gitignore`).
|
|
548
|
+
- **Power BI**: the tenant settings *Service principals can use Fabric/Power BI APIs* and *Dataset Execute Queries
|
|
549
|
+
REST API* on, and the service principal is a member (Viewer or more) of each report's workspace. The token comes
|
|
550
|
+
from the pipeline's `az login` (or set `PBIA_POWERBI_TOKEN`).
|
|
551
|
+
- **Azure SQL**: `CREATE USER [<app name>] FROM EXTERNAL PROVIDER` + `db_datareader`, and `auth_mode`
|
|
552
|
+
`entra_mfa` (the token comes from `az login`, or `PBIA_SQL_TOKEN`). With `sql_login`, `QA_DB_PASSWORD` from a
|
|
553
|
+
secret.
|
|
554
|
+
- **UI tests** (`qa.page`, `qa.check_ui`, `qa.refresh_dax`) open Power BI in a browser, which needs an interactive
|
|
555
|
+
sign-in: give them `run_env="LOC"` so they run only on your machine.
|
|
556
|
+
|
|
557
|
+
In a pipeline (`CI`, `TF_BUILD`, `GITHUB_ACTIONS`, `GITLAB_CI`, `JENKINS_URL` ... set) pbia never waits for a
|
|
558
|
+
sign-in: the browser is headless, a Power BI sign-in page or an interactive database login fails at once with the
|
|
559
|
+
reason, and the pipeline's build id / branch / commit / url go into the final report.
|
|
560
|
+
|
|
561
|
+
**Environment variables** (all read in `pbia/constant.py`):
|
|
562
|
+
|
|
563
|
+
| Variable | Use |
|
|
564
|
+
| :--- | :--- |
|
|
565
|
+
| `RUN_ENV` | `PRD` / `DEV` / `LOC` (default `LOC`): which test cases run |
|
|
566
|
+
| `PBIA_WORKERS` | tests at the same time (default 1; `--workers` wins) |
|
|
567
|
+
| `PBIA_FINAL_REPORT` | final report path (default `results/final_report.jsonc`) |
|
|
568
|
+
| `PBIA_HEADLESS` | `1` / `0`: hide the test browser (default: hidden in a pipeline) |
|
|
569
|
+
| `PBIA_POWERBI_TOKEN`, `PBIA_SQL_TOKEN` | access tokens instead of the Azure CLI |
|
|
570
|
+
| `QA_DB_PASSWORD` | database password (`sql_login`) |
|
|
571
|
+
| `PBIA_HOME` | the project folder (default: nearest folder with a `config.json`) |
|
|
572
|
+
|
|
573
|
+
---
|
|
574
|
+
|
|
575
|
+
## Project layout
|
|
576
|
+
|
|
577
|
+
```text
|
|
578
|
+
pyproject.toml package definition: python -m build --wheel -> dist/pbia-<version>-py3-none-any.whl
|
|
579
|
+
run.py source folder only: same as the pbia command
|
|
580
|
+
conftest.py source folder only: loads the pbia pytest plugin
|
|
581
|
+
main.py runs every test, writes results/final_report.jsonc, then your own code (after_run)
|
|
582
|
+
pipelines/ GitHub Actions / Azure DevOps examples
|
|
583
|
+
config.json reports, database, comparison rules
|
|
584
|
+
reports/<Report>/<Page>/
|
|
585
|
+
QA_<Page>.xlsx source of truth per page
|
|
586
|
+
PAGE_<Page>.jsonc reference copy
|
|
587
|
+
pbia/ the package
|
|
588
|
+
cli.py the pbia command (all commands, --init)
|
|
589
|
+
capture.py --capture: browser capture, query translation, filter testing
|
|
590
|
+
dax.py --validate --dax
|
|
591
|
+
sql.py --validate --sql
|
|
592
|
+
testkit.py pytest toolkit: qa fixture API, QueryResult, @test_case, Excel report writer
|
|
593
|
+
plugin.py pytest plugin (-p pbia.plugin): fixtures qa / qa_session, run_env / --case selection, live
|
|
594
|
+
log, parallel workers, reports
|
|
595
|
+
results.py final report: collected per test (also from parallel workers), written, load_final_report
|
|
596
|
+
runner.py run_tests(): what pbia --test and main.py run (--workers -> pytest-xdist)
|
|
597
|
+
constant.py settings from environment variables: RUN_ENV, PBIA_WORKERS, CI detection ...
|
|
598
|
+
templates/ what pbia --init copies (config.example.json, pytest.ini, test template, .gitignore,
|
|
599
|
+
main.py, pipelines/)
|
|
600
|
+
utils/
|
|
601
|
+
common.py config (reports), project folder, file names, JSONC, Azure CLI tokens
|
|
602
|
+
workspace.py report / page discovery, --report targets, PAGE_*.jsonc written from the workbook
|
|
603
|
+
excel.py the workbook: read, merge captured queries, write results
|
|
604
|
+
powerbi.py DAX clean-up, type alignment, auto-repair, executeQueries client, parallel runner
|
|
605
|
+
browser.py test browser: page capture, page values, UI load watching (qa.check_ui)
|
|
606
|
+
db.py read-only database connections and query guard
|
|
607
|
+
compare.py DAX vs SQL result comparison
|
|
608
|
+
tests/
|
|
609
|
+
_template.py copy to tests/test_<name>.py
|
|
610
|
+
results/ final_report.jsonc (last test run)
|
|
611
|
+
results/<date>/ test.log, Excel reports, final report copies and the logs of every pbia command
|
|
612
|
+
```
|