gitdata-lib 0.0.12__tar.gz → 0.0.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gitdata_lib-0.0.17/CHANGELOG.md +69 -0
- gitdata_lib-0.0.17/PKG-INFO +258 -0
- gitdata_lib-0.0.17/README.md +214 -0
- gitdata_lib-0.0.17/gitdata/__version__.py +1 -0
- gitdata_lib-0.0.17/gitdata/cli/__init__.py +114 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/cli/gitdata_get.py +5 -3
- gitdata_lib-0.0.17/gitdata/cli/gitdata_init.py +22 -0
- gitdata_lib-0.0.17/gitdata/cli/gitdata_ls.py +84 -0
- gitdata_lib-0.0.17/gitdata/cli/gitdata_scan.py +150 -0
- gitdata_lib-0.0.17/gitdata/cli/gitdata_secrets.py +129 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/config.py +7 -2
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/common.py +35 -12
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/http.py +24 -8
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/local.py +5 -5
- gitdata_lib-0.0.17/gitdata/connectors/tabular.py +629 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/__init__.py +13 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/common.py +3 -2
- gitdata_lib-0.0.17/gitdata/database/oracle.py +99 -0
- gitdata_lib-0.0.17/gitdata/digester.py +170 -0
- gitdata_lib-0.0.17/gitdata/encryption.py +25 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/graphs.py +10 -2
- gitdata_lib-0.0.17/gitdata/inspection.py +166 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/queues.py +14 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/repositories.py +33 -9
- gitdata_lib-0.0.17/gitdata/secrets.py +359 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/facts.py +12 -26
- gitdata_lib-0.0.17/gitdata_lib.egg-info/PKG-INFO +258 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/SOURCES.txt +10 -1
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/requires.txt +4 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/requirements.txt +1 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/setup.py +5 -3
- gitdata_lib-0.0.17/tests/test_blob.py +42 -0
- gitdata_lib-0.0.12/CHANGELOG.md +0 -0
- gitdata_lib-0.0.12/PKG-INFO +0 -36
- gitdata_lib-0.0.12/README.md +0 -5
- gitdata_lib-0.0.12/gitdata/__version__.py +0 -1
- gitdata_lib-0.0.12/gitdata/cli/__init__.py +0 -70
- gitdata_lib-0.0.12/gitdata/cli/gitdata_scan.py +0 -94
- gitdata_lib-0.0.12/gitdata/digester.py +0 -94
- gitdata_lib-0.0.12/gitdata_lib.egg-info/PKG-INFO +0 -36
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/LICENSE +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/MANIFEST.in +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/__init__.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/assets/README.md +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/buckets.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/__init__.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/fake.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/gitlab.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/mysql.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/mysql_setup.sql +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/mysql_setup_test_database.sql +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/postgresql.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/sqlite3.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/sqlite3_setup.sql +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/sqlite3_setup_test_data.sql +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/ext/__init__.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/ext/connectors/__init__.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/json.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/solutions.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/sql.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/__init__.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/common.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/entities.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/tables.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/utils.py +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/dependency_links.txt +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/entry_points.txt +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/top_level.txt +0 -0
- {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/setup.cfg +0 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
All notable changes to this project are documented in this file.
|
|
3
|
+
|
|
4
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
5
|
+
and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [v0.0.17] - 2026-09-28
|
|
10
|
+
- add direct `gitdata ls` and streaming `gitdata scan` for CSV, SQLite, and MariaDB refs without digestion or storage
|
|
11
|
+
- allow `gitdata ls` on a MariaDB server ref to list databases visible to the user
|
|
12
|
+
- accept canonical `database.table` selection in MariaDB/MySQL refs while retaining `#table` compatibility
|
|
13
|
+
- use `database-file/table.column` SQLite refs without fragment selectors
|
|
14
|
+
- eagerly count database rows before sampling unless `gitdata scan --no-count` is supplied
|
|
15
|
+
- allow SQLite column refs to scan one column and behave as leaves under `gitdata ls`
|
|
16
|
+
- format `gitdata scan` as a terminal-width-aware table with aligned numeric columns and bounded values
|
|
17
|
+
- default omitted MariaDB usernames to the current OS user and attempt passwordless authentication when no password environment variable is specified
|
|
18
|
+
- add reusable CSV, SQLite, and `mysqldump` examples for manual inspection
|
|
19
|
+
- exit cleanly without a traceback when a command is interrupted with Ctrl+C
|
|
20
|
+
- announce `gitdata scan` after ref validation and connection, immediately before sampling
|
|
21
|
+
- add `gitdata ls -l` database/table statistics, column schema, and index details
|
|
22
|
+
- hide database-specific system objects from `gitdata ls` unless `-a` is supplied
|
|
23
|
+
- distinguish node relationships from literal values with structural predicates
|
|
24
|
+
- preserve repeated predicate values when reading graph nodes
|
|
25
|
+
- escape colliding source predicates and omit factless values without dangling relationships
|
|
26
|
+
- verify multiple source graphs retain independent roots in memory and SQLite repositories
|
|
27
|
+
- require re-ingestion of older graph facts whose unprefixed relationships cannot be distinguished from literal IDs
|
|
28
|
+
|
|
29
|
+
## [v0.0.16] - 2026-09-19
|
|
30
|
+
- add optional Oracle Database support with `python-oracledb` Thin mode and live CI integration tests
|
|
31
|
+
- add initial encryption module and unit tests
|
|
32
|
+
- refactor secrets module into a backend-agnostic core with pluggable storage
|
|
33
|
+
- add `GITDATA_ENCRYPTION_KEY` and docker `/run/secrets` key resolution for secrets
|
|
34
|
+
- mask secret values by default in secrets listing APIs
|
|
35
|
+
- persist secrets in the local `.gitdata` repository via EntityStore
|
|
36
|
+
- add `gitdata secret` command to list, get, set, and delete secrets
|
|
37
|
+
- add secrets resolve API that hard-fails listing missing names
|
|
38
|
+
- add `gitdata secret status`, `resolve`, and `clear` for share handoff
|
|
39
|
+
- keep secret values out of logs, HTTP facts, and routine CLI output
|
|
40
|
+
- rename CLI command to `gitdata secret` (`secrets` remains an alias)
|
|
41
|
+
- add `gitdata init` command to initialize repositories
|
|
42
|
+
- switch local repository storage to a single `.gitdata` sqlite file
|
|
43
|
+
|
|
44
|
+
## [v0.0.15] - 2025-08-03
|
|
45
|
+
- add secrets module stub
|
|
46
|
+
|
|
47
|
+
## [v0.0.14] - 2025-08-03
|
|
48
|
+
- add support for environment variable style config keys
|
|
49
|
+
|
|
50
|
+
## [v0.0.13] - 2025-08-03
|
|
51
|
+
- make config filename a constant
|
|
52
|
+
- improve CLI help and `get` output readability
|
|
53
|
+
- expand queue tests and database-specific test configuration
|
|
54
|
+
|
|
55
|
+
## [v0.0.12] - 2024-05-01
|
|
56
|
+
- add `scan` command and HTTP connector support
|
|
57
|
+
- add SQL module and related tests
|
|
58
|
+
- improve CLI behavior and test/CI setup
|
|
59
|
+
|
|
60
|
+
## [v0.0.11] - 2021-12-31
|
|
61
|
+
- refactor CLI and connector internals
|
|
62
|
+
- add fake and local connector support
|
|
63
|
+
- continue datastore and graph/facts foundation work
|
|
64
|
+
|
|
65
|
+
## [v0.0.3] - 2021-10-10
|
|
66
|
+
- add CLI entrypoint and related command wiring
|
|
67
|
+
|
|
68
|
+
## [v0.0.1] - 2021-10-09
|
|
69
|
+
- initial packaged release with config, database, stores, queues, and connectors
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gitdata-lib
|
|
3
|
+
Version: 0.0.17
|
|
4
|
+
Summary: Data extraction and analysis library
|
|
5
|
+
Home-page: https://github.com/gitdata/gitdata-lib
|
|
6
|
+
Author: DSI Labs
|
|
7
|
+
Author-email: support@gitdata.com
|
|
8
|
+
Classifier: Development Status :: 1 - Planning
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
19
|
+
Requires-Python: >=3.9,<3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: python-decouple>=3.4
|
|
23
|
+
Requires-Dist: Unipath>=1.1
|
|
24
|
+
Requires-Dist: PyMySQL==0.10.1
|
|
25
|
+
Requires-Dist: psycopg2-binary==2.9.1
|
|
26
|
+
Requires-Dist: python-dotenv
|
|
27
|
+
Requires-Dist: requests
|
|
28
|
+
Requires-Dist: docopt
|
|
29
|
+
Requires-Dist: faker
|
|
30
|
+
Requires-Dist: cryptography
|
|
31
|
+
Provides-Extra: oracle
|
|
32
|
+
Requires-Dist: oracledb<4,>=1.3; extra == "oracle"
|
|
33
|
+
Dynamic: author
|
|
34
|
+
Dynamic: author-email
|
|
35
|
+
Dynamic: classifier
|
|
36
|
+
Dynamic: description
|
|
37
|
+
Dynamic: description-content-type
|
|
38
|
+
Dynamic: home-page
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
Dynamic: provides-extra
|
|
41
|
+
Dynamic: requires-dist
|
|
42
|
+
Dynamic: requires-python
|
|
43
|
+
Dynamic: summary
|
|
44
|
+
|
|
45
|
+
# GitData Lib
|
|
46
|
+
Data Wrangling for Everyone.
|
|
47
|
+
|
|
48
|
+
GitData is a fast, scalable, distributed data exploration system
|
|
49
|
+
with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.
|
|
50
|
+
|
|
51
|
+
## Development installation
|
|
52
|
+
|
|
53
|
+
Install the checkout into a Python 3.9 virtual environment so the `gitdata`
|
|
54
|
+
command works without setting `PYTHONPATH`:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
python3.9 -m venv .venv
|
|
58
|
+
. .venv/bin/activate
|
|
59
|
+
python -m pip install -e .
|
|
60
|
+
gitdata --help
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The editable installation continues to use code from the checkout, so source
|
|
64
|
+
changes do not require reinstalling it. Activate `.venv` in each new shell.
|
|
65
|
+
Alternatively, if `pipx` is installed, put the editable command on your normal
|
|
66
|
+
`PATH` without activating a project environment:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pipx install --python python3.9 --editable .
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Direct inspection
|
|
73
|
+
|
|
74
|
+
`ls` and `scan` inspect live Connector output directly. They do not initialize a
|
|
75
|
+
repository, digest the data into facts, or write to a store.
|
|
76
|
+
|
|
77
|
+
The repository includes ready-to-use examples. Run these commands from the
|
|
78
|
+
repository root:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
# List tables, list one table's columns, scan the table, and scan one column.
|
|
82
|
+
gitdata ls examples/inspection.sqlite
|
|
83
|
+
# Add -l for table row/column counts and estimated sizes.
|
|
84
|
+
gitdata ls -l examples/inspection.sqlite
|
|
85
|
+
gitdata ls examples/inspection.sqlite/customers
|
|
86
|
+
# On a table, -l shows schema details and indexes.
|
|
87
|
+
gitdata ls -l examples/inspection.sqlite/customers
|
|
88
|
+
gitdata scan --limit 2 examples/inspection.sqlite/customers
|
|
89
|
+
gitdata scan --limit 2 examples/inspection.sqlite/customers.email
|
|
90
|
+
|
|
91
|
+
# A headered CSV can be inspected in the same way.
|
|
92
|
+
gitdata ls examples/locations-headered.csv
|
|
93
|
+
gitdata scan --limit 10 examples/locations-headered.csv
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Database scans issue `COUNT(*)` before sampling so the report normally includes
|
|
97
|
+
an exact table row count. Use `--no-count` to skip that query when counting would
|
|
98
|
+
be expensive; if the sample does not exhaust the rows, the report then shows a
|
|
99
|
+
lower bound. Row sampling remains lazy and never reads beyond `--limit`. Once a
|
|
100
|
+
ref has been validated and connected, the command writes `Scanning...` to standard
|
|
101
|
+
error before sampling begins. Invalid and server-only refs are rejected without
|
|
102
|
+
first claiming that a scan has started. Pressing Ctrl+C stops any command cleanly
|
|
103
|
+
without a Python traceback.
|
|
104
|
+
|
|
105
|
+
`gitdata ls -l` follows the detailed listings from `datascan`: server refs show
|
|
106
|
+
database table/row/size statistics, database refs show table column/row/size
|
|
107
|
+
statistics, and table refs show column schema and index details. Plain `ls`
|
|
108
|
+
continues to print names only. Database-specific system objects are hidden by
|
|
109
|
+
default: MariaDB/MySQL omits `information_schema`, `mysql`,
|
|
110
|
+
`performance_schema`, and `sys`, while SQLite omits `sqlite_*` tables. Add
|
|
111
|
+
`-a`/`--all` to include them; `-la` combines system objects with the long
|
|
112
|
+
listing. SQLite sizes are estimated from up to 100 rows; MariaDB/MySQL uses
|
|
113
|
+
`information_schema` statistics.
|
|
114
|
+
|
|
115
|
+
SQLite refs form a hierarchy beneath the database filename:
|
|
116
|
+
|
|
117
|
+
```text
|
|
118
|
+
path/to/database.sqlite database
|
|
119
|
+
path/to/database.sqlite/table table
|
|
120
|
+
path/to/database.sqlite/table.column column
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`ls` lists immediate children: tables below a database and columns below a table.
|
|
124
|
+
A column is a leaf, so `ls` on a column produces no entries. `scan` on a column
|
|
125
|
+
profiles only that column while still counting rows in its table by default.
|
|
126
|
+
Literal dots in table or column identifiers must be percent-encoded as `%2E`.
|
|
127
|
+
|
|
128
|
+
### MariaDB example
|
|
129
|
+
|
|
130
|
+
`examples/mariadb.sql` is a `mysqldump` of a `gitdata_example` database with
|
|
131
|
+
`customers`, `orders`, and `products` tables. Load it into a disposable MariaDB
|
|
132
|
+
container:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
docker run --name gitdata-example-mariadb \
|
|
136
|
+
-e MARIADB_ROOT_PASSWORD=example \
|
|
137
|
+
-p 3307:3306 \
|
|
138
|
+
-d mariadb:10.7@sha256:9a48ac9f196f3d4fd6fea2cab59a49df9e7ca459bf14b2f7b85a0e38a5454571
|
|
139
|
+
|
|
140
|
+
until docker exec gitdata-example-mariadb \
|
|
141
|
+
mariadb-admin ping -h 127.0.0.1 -uroot -pexample --silent
|
|
142
|
+
do
|
|
143
|
+
sleep 1
|
|
144
|
+
done
|
|
145
|
+
|
|
146
|
+
docker exec -i gitdata-example-mariadb \
|
|
147
|
+
mariadb -uroot -pexample < examples/mariadb.sql
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Keep the password out of the ref by naming an environment variable:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
export GITDATA_EXAMPLE_MARIADB_PASSWORD=example
|
|
154
|
+
MARIADB_SERVER_REF='mysql://root@127.0.0.1:3307?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
|
|
155
|
+
MARIADB_REF='mysql://root@127.0.0.1:3307/gitdata_example?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
|
|
156
|
+
MARIADB_TABLE_REF='mysql://root@127.0.0.1:3307/gitdata_example.customers?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
|
|
157
|
+
|
|
158
|
+
gitdata ls "$MARIADB_SERVER_REF" # databases visible to root
|
|
159
|
+
gitdata ls -l "$MARIADB_SERVER_REF" # database statistics
|
|
160
|
+
gitdata ls "$MARIADB_REF" # tables in gitdata_example
|
|
161
|
+
gitdata ls -l "$MARIADB_REF" # table statistics
|
|
162
|
+
gitdata ls -l "$MARIADB_TABLE_REF" # column schema and indexes
|
|
163
|
+
gitdata scan --limit 2 "$MARIADB_TABLE_REF"
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
If the username is omitted, the connector uses the current operating-system
|
|
167
|
+
username. If `password_env` is omitted, it attempts a passwordless connection.
|
|
168
|
+
For example:
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
gitdata ls 'mysql://zeno'
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Inline passwords remain unsupported; use `password_env` when a password is
|
|
175
|
+
required.
|
|
176
|
+
|
|
177
|
+
MariaDB/MySQL refs follow the SQL qualification users already expect:
|
|
178
|
+
|
|
179
|
+
```text
|
|
180
|
+
mysql://host server
|
|
181
|
+
mysql://host/database database
|
|
182
|
+
mysql://host/database.table table
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
The earlier `database#table` form used a URI fragment as a generic nested-resource
|
|
186
|
+
selector. It remains accepted for MariaDB/MySQL compatibility, but
|
|
187
|
+
`database.table` is canonical. A literal dot inside an identifier must be
|
|
188
|
+
percent-encoded as `%2E`. SQLite does not use fragments; its `/table.column`
|
|
189
|
+
hierarchy follows the database filename.
|
|
190
|
+
|
|
191
|
+
Remove the disposable database when finished:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
docker rm -f gitdata-example-mariadb
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## Running the complete test suite locally
|
|
198
|
+
|
|
199
|
+
Docker is the only host dependency for the complete Python 3.9 test matrix:
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
bash tests/run_with_docker.sh
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
The runner builds a disposable test image, runs the general and SQLite suites,
|
|
206
|
+
and then starts MariaDB, PostgreSQL, and Oracle Free Lite one at a time. Each
|
|
207
|
+
database container and network is removed before the next service starts and is
|
|
208
|
+
also cleaned up if a test fails.
|
|
209
|
+
|
|
210
|
+
The service images are pinned by digest. They can be replaced without editing
|
|
211
|
+
the script by setting `GITDATA_TEST_MARIADB_IMAGE`,
|
|
212
|
+
`GITDATA_TEST_POSTGRES_IMAGE`, or `GITDATA_TEST_ORACLE_IMAGE`.
|
|
213
|
+
|
|
214
|
+
## Oracle Database
|
|
215
|
+
|
|
216
|
+
Oracle support uses the Python DB-API driver `python-oracledb`, which defaults
|
|
217
|
+
to Thin mode. The adapter does not initialize Thick mode, so Oracle Instant
|
|
218
|
+
Client is not required. The driver is optional and is loaded only when an
|
|
219
|
+
Oracle connection is requested:
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
pip install 'gitdata-lib[oracle]'
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Connect with an Oracle service name:
|
|
226
|
+
|
|
227
|
+
```python
|
|
228
|
+
import os
|
|
229
|
+
|
|
230
|
+
from gitdata.database import connect
|
|
231
|
+
|
|
232
|
+
db = connect(
|
|
233
|
+
'oracle',
|
|
234
|
+
host='127.0.0.1',
|
|
235
|
+
port=1521,
|
|
236
|
+
service_name='FREEPDB1',
|
|
237
|
+
user='app',
|
|
238
|
+
password=os.environ['ORACLE_PASSWORD'],
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
rows = list(db('SELECT 1 FROM dual'))
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
A complete `dsn` can be supplied instead of `host`, `port`, and
|
|
245
|
+
`service_name`. Installing `gitdata-lib` without the `oracle` extra does not
|
|
246
|
+
install or import `python-oracledb`; requesting an Oracle connection in that
|
|
247
|
+
case reports the extra that must be installed.
|
|
248
|
+
|
|
249
|
+
The Oracle integration tests use these environment variables:
|
|
250
|
+
|
|
251
|
+
- `GITDATA_TEST_ORACLE_USER`
|
|
252
|
+
- `GITDATA_TEST_ORACLE_PASSWORD`
|
|
253
|
+
- `GITDATA_TEST_ORACLE_SERVICE_NAME`
|
|
254
|
+
- `GITDATA_TEST_ORACLE_HOST` (defaults to `127.0.0.1`)
|
|
255
|
+
- `GITDATA_TEST_ORACLE_PORT` (defaults to `1521`)
|
|
256
|
+
|
|
257
|
+
They have been exercised against Oracle Free using the
|
|
258
|
+
`container-registry.oracle.com/database/free:latest` container image.
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# GitData Lib
|
|
2
|
+
Data Wrangling for Everyone.
|
|
3
|
+
|
|
4
|
+
GitData is a fast, scalable, distributed data exploration system
|
|
5
|
+
with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.
|
|
6
|
+
|
|
7
|
+
## Development installation
|
|
8
|
+
|
|
9
|
+
Install the checkout into a Python 3.9 virtual environment so the `gitdata`
|
|
10
|
+
command works without setting `PYTHONPATH`:
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
python3.9 -m venv .venv
|
|
14
|
+
. .venv/bin/activate
|
|
15
|
+
python -m pip install -e .
|
|
16
|
+
gitdata --help
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The editable installation continues to use code from the checkout, so source
|
|
20
|
+
changes do not require reinstalling it. Activate `.venv` in each new shell.
|
|
21
|
+
Alternatively, if `pipx` is installed, put the editable command on your normal
|
|
22
|
+
`PATH` without activating a project environment:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pipx install --python python3.9 --editable .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Direct inspection
|
|
29
|
+
|
|
30
|
+
`ls` and `scan` inspect live Connector output directly. They do not initialize a
|
|
31
|
+
repository, digest the data into facts, or write to a store.
|
|
32
|
+
|
|
33
|
+
The repository includes ready-to-use examples. Run these commands from the
|
|
34
|
+
repository root:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
# List tables, list one table's columns, scan the table, and scan one column.
|
|
38
|
+
gitdata ls examples/inspection.sqlite
|
|
39
|
+
# Add -l for table row/column counts and estimated sizes.
|
|
40
|
+
gitdata ls -l examples/inspection.sqlite
|
|
41
|
+
gitdata ls examples/inspection.sqlite/customers
|
|
42
|
+
# On a table, -l shows schema details and indexes.
|
|
43
|
+
gitdata ls -l examples/inspection.sqlite/customers
|
|
44
|
+
gitdata scan --limit 2 examples/inspection.sqlite/customers
|
|
45
|
+
gitdata scan --limit 2 examples/inspection.sqlite/customers.email
|
|
46
|
+
|
|
47
|
+
# A headered CSV can be inspected in the same way.
|
|
48
|
+
gitdata ls examples/locations-headered.csv
|
|
49
|
+
gitdata scan --limit 10 examples/locations-headered.csv
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Database scans issue `COUNT(*)` before sampling so the report normally includes
|
|
53
|
+
an exact table row count. Use `--no-count` to skip that query when counting would
|
|
54
|
+
be expensive; if the sample does not exhaust the rows, the report then shows a
|
|
55
|
+
lower bound. Row sampling remains lazy and never reads beyond `--limit`. Once a
|
|
56
|
+
ref has been validated and connected, the command writes `Scanning...` to standard
|
|
57
|
+
error before sampling begins. Invalid and server-only refs are rejected without
|
|
58
|
+
first claiming that a scan has started. Pressing Ctrl+C stops any command cleanly
|
|
59
|
+
without a Python traceback.
|
|
60
|
+
|
|
61
|
+
`gitdata ls -l` follows the detailed listings from `datascan`: server refs show
|
|
62
|
+
database table/row/size statistics, database refs show table column/row/size
|
|
63
|
+
statistics, and table refs show column schema and index details. Plain `ls`
|
|
64
|
+
continues to print names only. Database-specific system objects are hidden by
|
|
65
|
+
default: MariaDB/MySQL omits `information_schema`, `mysql`,
|
|
66
|
+
`performance_schema`, and `sys`, while SQLite omits `sqlite_*` tables. Add
|
|
67
|
+
`-a`/`--all` to include them; `-la` combines system objects with the long
|
|
68
|
+
listing. SQLite sizes are estimated from up to 100 rows; MariaDB/MySQL uses
|
|
69
|
+
`information_schema` statistics.
|
|
70
|
+
|
|
71
|
+
SQLite refs form a hierarchy beneath the database filename:
|
|
72
|
+
|
|
73
|
+
```text
|
|
74
|
+
path/to/database.sqlite database
|
|
75
|
+
path/to/database.sqlite/table table
|
|
76
|
+
path/to/database.sqlite/table.column column
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`ls` lists immediate children: tables below a database and columns below a table.
|
|
80
|
+
A column is a leaf, so `ls` on a column produces no entries. `scan` on a column
|
|
81
|
+
profiles only that column while still counting rows in its table by default.
|
|
82
|
+
Literal dots in table or column identifiers must be percent-encoded as `%2E`.
|
|
83
|
+
|
|
84
|
+
### MariaDB example
|
|
85
|
+
|
|
86
|
+
`examples/mariadb.sql` is a `mysqldump` of a `gitdata_example` database with
|
|
87
|
+
`customers`, `orders`, and `products` tables. Load it into a disposable MariaDB
|
|
88
|
+
container:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
docker run --name gitdata-example-mariadb \
|
|
92
|
+
-e MARIADB_ROOT_PASSWORD=example \
|
|
93
|
+
-p 3307:3306 \
|
|
94
|
+
-d mariadb:10.7@sha256:9a48ac9f196f3d4fd6fea2cab59a49df9e7ca459bf14b2f7b85a0e38a5454571
|
|
95
|
+
|
|
96
|
+
until docker exec gitdata-example-mariadb \
|
|
97
|
+
mariadb-admin ping -h 127.0.0.1 -uroot -pexample --silent
|
|
98
|
+
do
|
|
99
|
+
sleep 1
|
|
100
|
+
done
|
|
101
|
+
|
|
102
|
+
docker exec -i gitdata-example-mariadb \
|
|
103
|
+
mariadb -uroot -pexample < examples/mariadb.sql
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Keep the password out of the ref by naming an environment variable:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
export GITDATA_EXAMPLE_MARIADB_PASSWORD=example
|
|
110
|
+
MARIADB_SERVER_REF='mysql://root@127.0.0.1:3307?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
|
|
111
|
+
MARIADB_REF='mysql://root@127.0.0.1:3307/gitdata_example?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
|
|
112
|
+
MARIADB_TABLE_REF='mysql://root@127.0.0.1:3307/gitdata_example.customers?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
|
|
113
|
+
|
|
114
|
+
gitdata ls "$MARIADB_SERVER_REF" # databases visible to root
|
|
115
|
+
gitdata ls -l "$MARIADB_SERVER_REF" # database statistics
|
|
116
|
+
gitdata ls "$MARIADB_REF" # tables in gitdata_example
|
|
117
|
+
gitdata ls -l "$MARIADB_REF" # table statistics
|
|
118
|
+
gitdata ls -l "$MARIADB_TABLE_REF" # column schema and indexes
|
|
119
|
+
gitdata scan --limit 2 "$MARIADB_TABLE_REF"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
If the username is omitted, the connector uses the current operating-system
|
|
123
|
+
username. If `password_env` is omitted, it attempts a passwordless connection.
|
|
124
|
+
For example:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
gitdata ls 'mysql://zeno'
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Inline passwords remain unsupported; use `password_env` when a password is
|
|
131
|
+
required.
|
|
132
|
+
|
|
133
|
+
MariaDB/MySQL refs follow the SQL qualification users already expect:
|
|
134
|
+
|
|
135
|
+
```text
|
|
136
|
+
mysql://host server
|
|
137
|
+
mysql://host/database database
|
|
138
|
+
mysql://host/database.table table
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The earlier `database#table` form used a URI fragment as a generic nested-resource
|
|
142
|
+
selector. It remains accepted for MariaDB/MySQL compatibility, but
|
|
143
|
+
`database.table` is canonical. A literal dot inside an identifier must be
|
|
144
|
+
percent-encoded as `%2E`. SQLite does not use fragments; its `/table.column`
|
|
145
|
+
hierarchy follows the database filename.
|
|
146
|
+
|
|
147
|
+
Remove the disposable database when finished:
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
docker rm -f gitdata-example-mariadb
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Running the complete test suite locally
|
|
154
|
+
|
|
155
|
+
Docker is the only host dependency for the complete Python 3.9 test matrix:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
bash tests/run_with_docker.sh
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The runner builds a disposable test image, runs the general and SQLite suites,
|
|
162
|
+
and then starts MariaDB, PostgreSQL, and Oracle Free Lite one at a time. Each
|
|
163
|
+
database container and network is removed before the next service starts and is
|
|
164
|
+
also cleaned up if a test fails.
|
|
165
|
+
|
|
166
|
+
The service images are pinned by digest. They can be replaced without editing
|
|
167
|
+
the script by setting `GITDATA_TEST_MARIADB_IMAGE`,
|
|
168
|
+
`GITDATA_TEST_POSTGRES_IMAGE`, or `GITDATA_TEST_ORACLE_IMAGE`.
|
|
169
|
+
|
|
170
|
+
## Oracle Database
|
|
171
|
+
|
|
172
|
+
Oracle support uses the Python DB-API driver `python-oracledb`, which defaults
|
|
173
|
+
to Thin mode. The adapter does not initialize Thick mode, so Oracle Instant
|
|
174
|
+
Client is not required. The driver is optional and is loaded only when an
|
|
175
|
+
Oracle connection is requested:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
pip install 'gitdata-lib[oracle]'
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Connect with an Oracle service name:
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
import os
|
|
185
|
+
|
|
186
|
+
from gitdata.database import connect
|
|
187
|
+
|
|
188
|
+
db = connect(
|
|
189
|
+
'oracle',
|
|
190
|
+
host='127.0.0.1',
|
|
191
|
+
port=1521,
|
|
192
|
+
service_name='FREEPDB1',
|
|
193
|
+
user='app',
|
|
194
|
+
password=os.environ['ORACLE_PASSWORD'],
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
rows = list(db('SELECT 1 FROM dual'))
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
A complete `dsn` can be supplied instead of `host`, `port`, and
|
|
201
|
+
`service_name`. Installing `gitdata-lib` without the `oracle` extra does not
|
|
202
|
+
install or import `python-oracledb`; requesting an Oracle connection in that
|
|
203
|
+
case reports the extra that must be installed.
|
|
204
|
+
|
|
205
|
+
The Oracle integration tests use these environment variables:
|
|
206
|
+
|
|
207
|
+
- `GITDATA_TEST_ORACLE_USER`
|
|
208
|
+
- `GITDATA_TEST_ORACLE_PASSWORD`
|
|
209
|
+
- `GITDATA_TEST_ORACLE_SERVICE_NAME`
|
|
210
|
+
- `GITDATA_TEST_ORACLE_HOST` (defaults to `127.0.0.1`)
|
|
211
|
+
- `GITDATA_TEST_ORACLE_PORT` (defaults to `1521`)
|
|
212
|
+
|
|
213
|
+
They have been exercised against Oracle Free using the
|
|
214
|
+
`container-registry.oracle.com/database/free:latest` container image.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '0.0.17'
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""
|
|
2
|
+
usage: gitdata [options] <command> [<args>...]
|
|
3
|
+
|
|
4
|
+
options:
|
|
5
|
+
-h, --help show help
|
|
6
|
+
-V, --version print version
|
|
7
|
+
-d, --debug debug
|
|
8
|
+
|
|
9
|
+
The most commonly used gitdata commands are:
|
|
10
|
+
init initialize a local gitdata repository
|
|
11
|
+
fetch fetch data to the local reposotiry
|
|
12
|
+
get get data
|
|
13
|
+
ls list immediate children of a ref
|
|
14
|
+
scan scan data
|
|
15
|
+
secret manage repository secrets
|
|
16
|
+
|
|
17
|
+
See 'gitdata help <command>' for more information on a specific command.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import importlib
|
|
21
|
+
import logging
|
|
22
|
+
import sys
|
|
23
|
+
|
|
24
|
+
from docopt import docopt
|
|
25
|
+
|
|
26
|
+
import gitdata
|
|
27
|
+
from gitdata.utils import trim
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
root_logger = logging.getLogger()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def print_help(doc):
|
|
34
|
+
"""Print help text"""
|
|
35
|
+
print(trim(doc))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_module_doc(name):
|
|
39
|
+
if name == 'secret':
|
|
40
|
+
name = 'secrets'
|
|
41
|
+
module_name = 'gitdata.cli.gitdata_' + name
|
|
42
|
+
try:
|
|
43
|
+
result = importlib.import_module(module_name).__doc__
|
|
44
|
+
except ModuleNotFoundError:
|
|
45
|
+
result = f'no help on topic {name!r}'
|
|
46
|
+
return result
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main():
|
|
50
|
+
"""CLI entry point with clean interrupt handling."""
|
|
51
|
+
try:
|
|
52
|
+
_main()
|
|
53
|
+
except KeyboardInterrupt:
|
|
54
|
+
print('\nInterrupted.', file=sys.stderr)
|
|
55
|
+
raise SystemExit(130)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _main():
|
|
59
|
+
"""Parse and dispatch a CLI command."""
|
|
60
|
+
|
|
61
|
+
if len(sys.argv) == 1:
|
|
62
|
+
print_help(__doc__)
|
|
63
|
+
sys.exit()
|
|
64
|
+
|
|
65
|
+
args = docopt(
|
|
66
|
+
__doc__,
|
|
67
|
+
version='gitdata version {}'.format(gitdata.__version__),
|
|
68
|
+
options_first=True
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
if args['--debug']:
|
|
72
|
+
print(args)
|
|
73
|
+
root_logger.setLevel(logging.DEBUG)
|
|
74
|
+
|
|
75
|
+
argv = [args['<command>']] + args['<args>']
|
|
76
|
+
command = args['<command>']
|
|
77
|
+
|
|
78
|
+
if command == 'help':
|
|
79
|
+
if args['<args>']:
|
|
80
|
+
topic = args['<args>'][0]
|
|
81
|
+
doc = get_module_doc(topic)
|
|
82
|
+
else:
|
|
83
|
+
doc = __doc__
|
|
84
|
+
print_help(doc)
|
|
85
|
+
sys.exit()
|
|
86
|
+
|
|
87
|
+
elif command == 'init':
|
|
88
|
+
from gitdata.cli.gitdata_init import init, __doc__ as doc
|
|
89
|
+
args = docopt(doc, argv=argv)
|
|
90
|
+
init(args)
|
|
91
|
+
|
|
92
|
+
elif command == 'get':
|
|
93
|
+
from gitdata.cli.gitdata_get import get, __doc__ as doc
|
|
94
|
+
args = docopt(doc, argv=argv)
|
|
95
|
+
get(args)
|
|
96
|
+
|
|
97
|
+
elif command == 'ls':
|
|
98
|
+
from gitdata.cli.gitdata_ls import ls_to_console, __doc__ as doc
|
|
99
|
+
args = docopt(doc, argv=argv)
|
|
100
|
+
ls_to_console(args)
|
|
101
|
+
|
|
102
|
+
elif command == 'scan':
|
|
103
|
+
from gitdata.cli.gitdata_scan import scan_to_console, __doc__ as doc
|
|
104
|
+
args = docopt(doc, argv=argv)
|
|
105
|
+
scan_to_console(args)
|
|
106
|
+
|
|
107
|
+
elif command in ('secret', 'secrets'):
|
|
108
|
+
from gitdata.cli.gitdata_secrets import secrets, __doc__ as doc
|
|
109
|
+
argv[0] = 'secret'
|
|
110
|
+
args = docopt(doc, argv=argv)
|
|
111
|
+
secrets(args)
|
|
112
|
+
|
|
113
|
+
else:
|
|
114
|
+
exit("%r is not a gitdata command. See 'gitdata help'." % args['<command>'])
|
|
@@ -14,9 +14,11 @@ def get(args):
|
|
|
14
14
|
if args['<ref>']:
|
|
15
15
|
for ref in args['<ref>']:
|
|
16
16
|
print('getting', ref)
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
result = gitdata.connectors.common.get(ref)
|
|
18
|
+
if isinstance(result, dict):
|
|
19
|
+
max_len = max(map(len, result.keys())) + 3
|
|
20
|
+
for k, v in result.items():
|
|
21
|
+
print('%s%s: %s' % (k, '.' * (max_len - len(k)), v))
|
|
20
22
|
else:
|
|
21
23
|
print(__doc__)
|
|
22
24
|
print(args)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""
|
|
2
|
+
usage: gitdata init [<path>]
|
|
3
|
+
|
|
4
|
+
options:
|
|
5
|
+
-h, --help
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import gitdata.repositories
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def init(args):
|
|
12
|
+
"""Initialize a gitdata repository."""
|
|
13
|
+
path = args['<path>'] or '.'
|
|
14
|
+
try:
|
|
15
|
+
repository_path, created = gitdata.repositories.init_repository(path)
|
|
16
|
+
except ValueError as error:
|
|
17
|
+
raise SystemExit('fatal: {}'.format(error))
|
|
18
|
+
|
|
19
|
+
if created:
|
|
20
|
+
print('Initialized empty GitData repository in {}'.format(repository_path))
|
|
21
|
+
else:
|
|
22
|
+
print('GitData repository already initialized in {}'.format(repository_path))
|