gitdata-lib 0.0.12__tar.gz → 0.0.17__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. gitdata_lib-0.0.17/CHANGELOG.md +69 -0
  2. gitdata_lib-0.0.17/PKG-INFO +258 -0
  3. gitdata_lib-0.0.17/README.md +214 -0
  4. gitdata_lib-0.0.17/gitdata/__version__.py +1 -0
  5. gitdata_lib-0.0.17/gitdata/cli/__init__.py +114 -0
  6. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/cli/gitdata_get.py +5 -3
  7. gitdata_lib-0.0.17/gitdata/cli/gitdata_init.py +22 -0
  8. gitdata_lib-0.0.17/gitdata/cli/gitdata_ls.py +84 -0
  9. gitdata_lib-0.0.17/gitdata/cli/gitdata_scan.py +150 -0
  10. gitdata_lib-0.0.17/gitdata/cli/gitdata_secrets.py +129 -0
  11. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/config.py +7 -2
  12. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/common.py +35 -12
  13. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/http.py +24 -8
  14. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/local.py +5 -5
  15. gitdata_lib-0.0.17/gitdata/connectors/tabular.py +629 -0
  16. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/__init__.py +13 -0
  17. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/common.py +3 -2
  18. gitdata_lib-0.0.17/gitdata/database/oracle.py +99 -0
  19. gitdata_lib-0.0.17/gitdata/digester.py +170 -0
  20. gitdata_lib-0.0.17/gitdata/encryption.py +25 -0
  21. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/graphs.py +10 -2
  22. gitdata_lib-0.0.17/gitdata/inspection.py +166 -0
  23. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/queues.py +14 -0
  24. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/repositories.py +33 -9
  25. gitdata_lib-0.0.17/gitdata/secrets.py +359 -0
  26. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/facts.py +12 -26
  27. gitdata_lib-0.0.17/gitdata_lib.egg-info/PKG-INFO +258 -0
  28. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/SOURCES.txt +10 -1
  29. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/requires.txt +4 -0
  30. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/requirements.txt +1 -0
  31. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/setup.py +5 -3
  32. gitdata_lib-0.0.17/tests/test_blob.py +42 -0
  33. gitdata_lib-0.0.12/CHANGELOG.md +0 -0
  34. gitdata_lib-0.0.12/PKG-INFO +0 -36
  35. gitdata_lib-0.0.12/README.md +0 -5
  36. gitdata_lib-0.0.12/gitdata/__version__.py +0 -1
  37. gitdata_lib-0.0.12/gitdata/cli/__init__.py +0 -70
  38. gitdata_lib-0.0.12/gitdata/cli/gitdata_scan.py +0 -94
  39. gitdata_lib-0.0.12/gitdata/digester.py +0 -94
  40. gitdata_lib-0.0.12/gitdata_lib.egg-info/PKG-INFO +0 -36
  41. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/LICENSE +0 -0
  42. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/MANIFEST.in +0 -0
  43. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/__init__.py +0 -0
  44. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/assets/README.md +0 -0
  45. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/buckets.py +0 -0
  46. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/__init__.py +0 -0
  47. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/fake.py +0 -0
  48. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/connectors/gitlab.py +0 -0
  49. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/mysql.py +0 -0
  50. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/mysql_setup.sql +0 -0
  51. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/mysql_setup_test_database.sql +0 -0
  52. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/postgresql.py +0 -0
  53. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/sqlite3.py +0 -0
  54. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/sqlite3_setup.sql +0 -0
  55. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/database/sqlite3_setup_test_data.sql +0 -0
  56. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/ext/__init__.py +0 -0
  57. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/ext/connectors/__init__.py +0 -0
  58. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/json.py +0 -0
  59. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/solutions.py +0 -0
  60. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/sql.py +0 -0
  61. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/__init__.py +0 -0
  62. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/common.py +0 -0
  63. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/entities.py +0 -0
  64. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/stores/tables.py +0 -0
  65. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata/utils.py +0 -0
  66. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/dependency_links.txt +0 -0
  67. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/entry_points.txt +0 -0
  68. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/gitdata_lib.egg-info/top_level.txt +0 -0
  69. {gitdata_lib-0.0.12 → gitdata_lib-0.0.17}/setup.cfg +0 -0
@@ -0,0 +1,69 @@
1
+ # Changelog
2
+ All notable changes to this project are documented in this file.
3
+
4
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5
+ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [Unreleased]
8
+
9
+ ## [v0.0.17] - 2026-09-28
10
+ - add direct `gitdata ls` and streaming `gitdata scan` for CSV, SQLite, and MariaDB refs without digestion or storage
11
+ - allow `gitdata ls` on a MariaDB server ref to list databases visible to the user
12
+ - accept canonical `database.table` selection in MariaDB/MySQL refs while retaining `#table` compatibility
13
+ - use `database-file/table.column` SQLite refs without fragment selectors
14
+ - eagerly count database rows before sampling unless `gitdata scan --no-count` is supplied
15
+ - allow SQLite column refs to scan one column and behave as leaves under `gitdata ls`
16
+ - format `gitdata scan` as a terminal-width-aware table with aligned numeric columns and bounded values
17
+ - default omitted MariaDB usernames to the current OS user and attempt passwordless authentication when no password environment variable is specified
18
+ - add reusable CSV, SQLite, and `mysqldump` examples for manual inspection
19
+ - exit cleanly without a traceback when a command is interrupted with Ctrl+C
20
+ - announce `gitdata scan` after ref validation and connection, immediately before sampling
21
+ - add `gitdata ls -l` database/table statistics, column schema, and index details
22
+ - hide database-specific system objects from `gitdata ls` unless `-a` is supplied
23
+ - distinguish node relationships from literal values with structural predicates
24
+ - preserve repeated predicate values when reading graph nodes
25
+ - escape colliding source predicates and omit factless values without dangling relationships
26
+ - verify multiple source graphs retain independent roots in memory and SQLite repositories
27
+ - require re-ingestion of older graph facts whose unprefixed relationships cannot be distinguished from literal IDs
28
+
29
+ ## [v0.0.16] - 2026-09-19
30
+ - add optional Oracle Database support with `python-oracledb` Thin mode and live CI integration tests
31
+ - add initial encryption module and unit tests
32
+ - refactor secrets module into a backend-agnostic core with pluggable storage
33
+ - add `GITDATA_ENCRYPTION_KEY` and docker `/run/secrets` key resolution for secrets
34
+ - mask secret values by default in secrets listing APIs
35
+ - persist secrets in the local `.gitdata` repository via EntityStore
36
+ - add `gitdata secret` command to list, get, set, and delete secrets
37
+ - add secrets resolve API that hard-fails listing missing names
38
+ - add `gitdata secret status`, `resolve`, and `clear` for share handoff
39
+ - keep secret values out of logs, HTTP facts, and routine CLI output
40
+ - rename CLI command to `gitdata secret` (`secrets` remains an alias)
41
+ - add `gitdata init` command to initialize repositories
42
+ - switch local repository storage to a single `.gitdata` sqlite file
43
+
44
+ ## [v0.0.15] - 2025-08-03
45
+ - add secrets module stub
46
+
47
+ ## [v0.0.14] - 2025-08-03
48
+ - add support for environment variable style config keys
49
+
50
+ ## [v0.0.13] - 2025-08-03
51
+ - make config filename a constant
52
+ - improve CLI help and `get` output readability
53
+ - expand queue tests and database-specific test configuration
54
+
55
+ ## [v0.0.12] - 2024-05-01
56
+ - add `scan` command and HTTP connector support
57
+ - add SQL module and related tests
58
+ - improve CLI behavior and test/CI setup
59
+
60
+ ## [v0.0.11] - 2021-12-31
61
+ - refactor CLI and connector internals
62
+ - add fake and local connector support
63
+ - continue datastore and graph/facts foundation work
64
+
65
+ ## [v0.0.3] - 2021-10-10
66
+ - add CLI entrypoint and related command wiring
67
+
68
+ ## [v0.0.1] - 2021-10-09
69
+ - initial packaged release with config, database, stores, queues, and connectors
@@ -0,0 +1,258 @@
1
+ Metadata-Version: 2.4
2
+ Name: gitdata-lib
3
+ Version: 0.0.17
4
+ Summary: Data extraction and analysis library
5
+ Home-page: https://github.com/gitdata/gitdata-lib
6
+ Author: DSI Labs
7
+ Author-email: support@gitdata.com
8
+ Classifier: Development Status :: 1 - Planning
9
+ Classifier: Environment :: Console
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Topic :: Database :: Front-Ends
19
+ Requires-Python: >=3.9,<3.10
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: python-decouple>=3.4
23
+ Requires-Dist: Unipath>=1.1
24
+ Requires-Dist: PyMySQL==0.10.1
25
+ Requires-Dist: psycopg2-binary==2.9.1
26
+ Requires-Dist: python-dotenv
27
+ Requires-Dist: requests
28
+ Requires-Dist: docopt
29
+ Requires-Dist: faker
30
+ Requires-Dist: cryptography
31
+ Provides-Extra: oracle
32
+ Requires-Dist: oracledb<4,>=1.3; extra == "oracle"
33
+ Dynamic: author
34
+ Dynamic: author-email
35
+ Dynamic: classifier
36
+ Dynamic: description
37
+ Dynamic: description-content-type
38
+ Dynamic: home-page
39
+ Dynamic: license-file
40
+ Dynamic: provides-extra
41
+ Dynamic: requires-dist
42
+ Dynamic: requires-python
43
+ Dynamic: summary
44
+
45
+ # GitData Lib
46
+ Data Wrangling for Everyone.
47
+
48
+ GitData is a fast, scalable, distributed data exploration system
49
+ with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.
50
+
51
+ ## Development installation
52
+
53
+ Install the checkout into a Python 3.9 virtual environment so the `gitdata`
54
+ command works without setting `PYTHONPATH`:
55
+
56
+ ```bash
57
+ python3.9 -m venv .venv
58
+ . .venv/bin/activate
59
+ python -m pip install -e .
60
+ gitdata --help
61
+ ```
62
+
63
+ The editable installation continues to use code from the checkout, so source
64
+ changes do not require reinstalling it. Activate `.venv` in each new shell.
65
+ Alternatively, if `pipx` is installed, put the editable command on your normal
66
+ `PATH` without activating a project environment:
67
+
68
+ ```bash
69
+ pipx install --python python3.9 --editable .
70
+ ```
71
+
72
+ ## Direct inspection
73
+
74
+ `ls` and `scan` inspect live Connector output directly. They do not initialize a
75
+ repository, digest the data into facts, or write to a store.
76
+
77
+ The repository includes ready-to-use examples. Run these commands from the
78
+ repository root:
79
+
80
+ ```bash
81
+ # List tables, list one table's columns, scan the table, and scan one column.
82
+ gitdata ls examples/inspection.sqlite
83
+ # Add -l for table row/column counts and estimated sizes.
84
+ gitdata ls -l examples/inspection.sqlite
85
+ gitdata ls examples/inspection.sqlite/customers
86
+ # On a table, -l shows schema details and indexes.
87
+ gitdata ls -l examples/inspection.sqlite/customers
88
+ gitdata scan --limit 2 examples/inspection.sqlite/customers
89
+ gitdata scan --limit 2 examples/inspection.sqlite/customers.email
90
+
91
+ # A headered CSV can be inspected in the same way.
92
+ gitdata ls examples/locations-headered.csv
93
+ gitdata scan --limit 10 examples/locations-headered.csv
94
+ ```
95
+
96
+ Database scans issue `COUNT(*)` before sampling so the report normally includes
97
+ an exact table row count. Use `--no-count` to skip that query when counting would
98
+ be expensive; if the sample does not exhaust the rows, the report then shows a
99
+ lower bound. Row sampling remains lazy and never reads beyond `--limit`. Once a
100
+ ref has been validated and connected, the command writes `Scanning...` to standard
101
+ error before sampling begins. Invalid and server-only refs are rejected without
102
+ first claiming that a scan has started. Pressing Ctrl+C stops any command cleanly
103
+ without a Python traceback.
104
+
105
+ `gitdata ls -l` follows the detailed listings from `datascan`: server refs show
106
+ database table/row/size statistics, database refs show table column/row/size
107
+ statistics, and table refs show column schema and index details. Plain `ls`
108
+ continues to print names only. Database-specific system objects are hidden by
109
+ default: MariaDB/MySQL omits `information_schema`, `mysql`,
110
+ `performance_schema`, and `sys`, while SQLite omits `sqlite_*` tables. Add
111
+ `-a`/`--all` to include them; `-la` combines system objects with the long
112
+ listing. SQLite sizes are estimated from up to 100 rows; MariaDB/MySQL uses
113
+ `information_schema` statistics.
114
+
115
+ SQLite refs form a hierarchy beneath the database filename:
116
+
117
+ ```text
118
+ path/to/database.sqlite database
119
+ path/to/database.sqlite/table table
120
+ path/to/database.sqlite/table.column column
121
+ ```
122
+
123
+ `ls` lists immediate children: tables below a database and columns below a table.
124
+ A column is a leaf, so `ls` on a column produces no entries. `scan` on a column
125
+ profiles only that column while still counting rows in its table by default.
126
+ Literal dots in table or column identifiers must be percent-encoded as `%2E`.
127
+
128
+ ### MariaDB example
129
+
130
+ `examples/mariadb.sql` is a `mysqldump` of a `gitdata_example` database with
131
+ `customers`, `orders`, and `products` tables. Load it into a disposable MariaDB
132
+ container:
133
+
134
+ ```bash
135
+ docker run --name gitdata-example-mariadb \
136
+ -e MARIADB_ROOT_PASSWORD=example \
137
+ -p 3307:3306 \
138
+ -d mariadb:10.7@sha256:9a48ac9f196f3d4fd6fea2cab59a49df9e7ca459bf14b2f7b85a0e38a5454571
139
+
140
+ until docker exec gitdata-example-mariadb \
141
+ mariadb-admin ping -h 127.0.0.1 -uroot -pexample --silent
142
+ do
143
+ sleep 1
144
+ done
145
+
146
+ docker exec -i gitdata-example-mariadb \
147
+ mariadb -uroot -pexample < examples/mariadb.sql
148
+ ```
149
+
150
+ Keep the password out of the ref by naming an environment variable:
151
+
152
+ ```bash
153
+ export GITDATA_EXAMPLE_MARIADB_PASSWORD=example
154
+ MARIADB_SERVER_REF='mysql://root@127.0.0.1:3307?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
155
+ MARIADB_REF='mysql://root@127.0.0.1:3307/gitdata_example?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
156
+ MARIADB_TABLE_REF='mysql://root@127.0.0.1:3307/gitdata_example.customers?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
157
+
158
+ gitdata ls "$MARIADB_SERVER_REF" # databases visible to root
159
+ gitdata ls -l "$MARIADB_SERVER_REF" # database statistics
160
+ gitdata ls "$MARIADB_REF" # tables in gitdata_example
161
+ gitdata ls -l "$MARIADB_REF" # table statistics
162
+ gitdata ls -l "$MARIADB_TABLE_REF" # column schema and indexes
163
+ gitdata scan --limit 2 "$MARIADB_TABLE_REF"
164
+ ```
165
+
166
+ If the username is omitted, the connector uses the current operating-system
167
+ username. If `password_env` is omitted, it attempts a passwordless connection.
168
+ For example:
169
+
170
+ ```bash
171
+ gitdata ls 'mysql://zeno'
172
+ ```
173
+
174
+ Inline passwords remain unsupported; use `password_env` when a password is
175
+ required.
176
+
177
+ MariaDB/MySQL refs follow the SQL qualification users already expect:
178
+
179
+ ```text
180
+ mysql://host server
181
+ mysql://host/database database
182
+ mysql://host/database.table table
183
+ ```
184
+
185
+ The earlier `database#table` form used a URI fragment as a generic nested-resource
186
+ selector. It remains accepted for MariaDB/MySQL compatibility, but
187
+ `database.table` is canonical. A literal dot inside an identifier must be
188
+ percent-encoded as `%2E`. SQLite does not use fragments; its `/table.column`
189
+ hierarchy follows the database filename.
190
+
191
+ Remove the disposable database when finished:
192
+
193
+ ```bash
194
+ docker rm -f gitdata-example-mariadb
195
+ ```
196
+
197
+ ## Running the complete test suite locally
198
+
199
+ Docker is the only host dependency for the complete Python 3.9 test matrix:
200
+
201
+ ```bash
202
+ bash tests/run_with_docker.sh
203
+ ```
204
+
205
+ The runner builds a disposable test image, runs the general and SQLite suites,
206
+ and then starts MariaDB, PostgreSQL, and Oracle Free Lite one at a time. Each
207
+ database container and network is removed before the next service starts and is
208
+ also cleaned up if a test fails.
209
+
210
+ The service images are pinned by digest. They can be replaced without editing
211
+ the script by setting `GITDATA_TEST_MARIADB_IMAGE`,
212
+ `GITDATA_TEST_POSTGRES_IMAGE`, or `GITDATA_TEST_ORACLE_IMAGE`.
213
+
214
+ ## Oracle Database
215
+
216
+ Oracle support uses the Python DB-API driver `python-oracledb`, which defaults
217
+ to Thin mode. The adapter does not initialize Thick mode, so Oracle Instant
218
+ Client is not required. The driver is optional and is loaded only when an
219
+ Oracle connection is requested:
220
+
221
+ ```bash
222
+ pip install 'gitdata-lib[oracle]'
223
+ ```
224
+
225
+ Connect with an Oracle service name:
226
+
227
+ ```python
228
+ import os
229
+
230
+ from gitdata.database import connect
231
+
232
+ db = connect(
233
+ 'oracle',
234
+ host='127.0.0.1',
235
+ port=1521,
236
+ service_name='FREEPDB1',
237
+ user='app',
238
+ password=os.environ['ORACLE_PASSWORD'],
239
+ )
240
+
241
+ rows = list(db('SELECT 1 FROM dual'))
242
+ ```
243
+
244
+ A complete `dsn` can be supplied instead of `host`, `port`, and
245
+ `service_name`. Installing `gitdata-lib` without the `oracle` extra does not
246
+ install or import `python-oracledb`; requesting an Oracle connection in that
247
+ case reports the extra that must be installed.
248
+
249
+ The Oracle integration tests use these environment variables:
250
+
251
+ - `GITDATA_TEST_ORACLE_USER`
252
+ - `GITDATA_TEST_ORACLE_PASSWORD`
253
+ - `GITDATA_TEST_ORACLE_SERVICE_NAME`
254
+ - `GITDATA_TEST_ORACLE_HOST` (defaults to `127.0.0.1`)
255
+ - `GITDATA_TEST_ORACLE_PORT` (defaults to `1521`)
256
+
257
+ They have been exercised against Oracle Free using the
258
+ `container-registry.oracle.com/database/free:latest` container image.
@@ -0,0 +1,214 @@
1
+ # GitData Lib
2
+ Data Wrangling for Everyone.
3
+
4
+ GitData is a fast, scalable, distributed data exploration system
5
+ with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.
6
+
7
+ ## Development installation
8
+
9
+ Install the checkout into a Python 3.9 virtual environment so the `gitdata`
10
+ command works without setting `PYTHONPATH`:
11
+
12
+ ```bash
13
+ python3.9 -m venv .venv
14
+ . .venv/bin/activate
15
+ python -m pip install -e .
16
+ gitdata --help
17
+ ```
18
+
19
+ The editable installation continues to use code from the checkout, so source
20
+ changes do not require reinstalling it. Activate `.venv` in each new shell.
21
+ Alternatively, if `pipx` is installed, put the editable command on your normal
22
+ `PATH` without activating a project environment:
23
+
24
+ ```bash
25
+ pipx install --python python3.9 --editable .
26
+ ```
27
+
28
+ ## Direct inspection
29
+
30
+ `ls` and `scan` inspect live Connector output directly. They do not initialize a
31
+ repository, digest the data into facts, or write to a store.
32
+
33
+ The repository includes ready-to-use examples. Run these commands from the
34
+ repository root:
35
+
36
+ ```bash
37
+ # List tables, list one table's columns, scan the table, and scan one column.
38
+ gitdata ls examples/inspection.sqlite
39
+ # Add -l for table row/column counts and estimated sizes.
40
+ gitdata ls -l examples/inspection.sqlite
41
+ gitdata ls examples/inspection.sqlite/customers
42
+ # On a table, -l shows schema details and indexes.
43
+ gitdata ls -l examples/inspection.sqlite/customers
44
+ gitdata scan --limit 2 examples/inspection.sqlite/customers
45
+ gitdata scan --limit 2 examples/inspection.sqlite/customers.email
46
+
47
+ # A headered CSV can be inspected in the same way.
48
+ gitdata ls examples/locations-headered.csv
49
+ gitdata scan --limit 10 examples/locations-headered.csv
50
+ ```
51
+
52
+ Database scans issue `COUNT(*)` before sampling so the report normally includes
53
+ an exact table row count. Use `--no-count` to skip that query when counting would
54
+ be expensive; if the sample does not exhaust the rows, the report then shows a
55
+ lower bound. Row sampling remains lazy and never reads beyond `--limit`. Once a
56
+ ref has been validated and connected, the command writes `Scanning...` to standard
57
+ error before sampling begins. Invalid and server-only refs are rejected without
58
+ first claiming that a scan has started. Pressing Ctrl+C stops any command cleanly
59
+ without a Python traceback.
60
+
61
+ `gitdata ls -l` follows the detailed listings from `datascan`: server refs show
62
+ database table/row/size statistics, database refs show table column/row/size
63
+ statistics, and table refs show column schema and index details. Plain `ls`
64
+ continues to print names only. Database-specific system objects are hidden by
65
+ default: MariaDB/MySQL omits `information_schema`, `mysql`,
66
+ `performance_schema`, and `sys`, while SQLite omits `sqlite_*` tables. Add
67
+ `-a`/`--all` to include them; `-la` combines system objects with the long
68
+ listing. SQLite sizes are estimated from up to 100 rows; MariaDB/MySQL uses
69
+ `information_schema` statistics.
70
+
71
+ SQLite refs form a hierarchy beneath the database filename:
72
+
73
+ ```text
74
+ path/to/database.sqlite database
75
+ path/to/database.sqlite/table table
76
+ path/to/database.sqlite/table.column column
77
+ ```
78
+
79
+ `ls` lists immediate children: tables below a database and columns below a table.
80
+ A column is a leaf, so `ls` on a column produces no entries. `scan` on a column
81
+ profiles only that column while still counting rows in its table by default.
82
+ Literal dots in table or column identifiers must be percent-encoded as `%2E`.
83
+
84
+ ### MariaDB example
85
+
86
+ `examples/mariadb.sql` is a `mysqldump` of a `gitdata_example` database with
87
+ `customers`, `orders`, and `products` tables. Load it into a disposable MariaDB
88
+ container:
89
+
90
+ ```bash
91
+ docker run --name gitdata-example-mariadb \
92
+ -e MARIADB_ROOT_PASSWORD=example \
93
+ -p 3307:3306 \
94
+ -d mariadb:10.7@sha256:9a48ac9f196f3d4fd6fea2cab59a49df9e7ca459bf14b2f7b85a0e38a5454571
95
+
96
+ until docker exec gitdata-example-mariadb \
97
+ mariadb-admin ping -h 127.0.0.1 -uroot -pexample --silent
98
+ do
99
+ sleep 1
100
+ done
101
+
102
+ docker exec -i gitdata-example-mariadb \
103
+ mariadb -uroot -pexample < examples/mariadb.sql
104
+ ```
105
+
106
+ Keep the password out of the ref by naming an environment variable:
107
+
108
+ ```bash
109
+ export GITDATA_EXAMPLE_MARIADB_PASSWORD=example
110
+ MARIADB_SERVER_REF='mysql://root@127.0.0.1:3307?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
111
+ MARIADB_REF='mysql://root@127.0.0.1:3307/gitdata_example?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
112
+ MARIADB_TABLE_REF='mysql://root@127.0.0.1:3307/gitdata_example.customers?password_env=GITDATA_EXAMPLE_MARIADB_PASSWORD'
113
+
114
+ gitdata ls "$MARIADB_SERVER_REF" # databases visible to root
115
+ gitdata ls -l "$MARIADB_SERVER_REF" # database statistics
116
+ gitdata ls "$MARIADB_REF" # tables in gitdata_example
117
+ gitdata ls -l "$MARIADB_REF" # table statistics
118
+ gitdata ls -l "$MARIADB_TABLE_REF" # column schema and indexes
119
+ gitdata scan --limit 2 "$MARIADB_TABLE_REF"
120
+ ```
121
+
122
+ If the username is omitted, the connector uses the current operating-system
123
+ username. If `password_env` is omitted, it attempts a passwordless connection.
124
+ For example:
125
+
126
+ ```bash
127
+ gitdata ls 'mysql://zeno'
128
+ ```
129
+
130
+ Inline passwords remain unsupported; use `password_env` when a password is
131
+ required.
132
+
133
+ MariaDB/MySQL refs follow the SQL qualification users already expect:
134
+
135
+ ```text
136
+ mysql://host server
137
+ mysql://host/database database
138
+ mysql://host/database.table table
139
+ ```
140
+
141
+ The earlier `database#table` form used a URI fragment as a generic nested-resource
142
+ selector. It remains accepted for MariaDB/MySQL compatibility, but
143
+ `database.table` is canonical. A literal dot inside an identifier must be
144
+ percent-encoded as `%2E`. SQLite does not use fragments; its `/table.column`
145
+ hierarchy follows the database filename.
146
+
147
+ Remove the disposable database when finished:
148
+
149
+ ```bash
150
+ docker rm -f gitdata-example-mariadb
151
+ ```
152
+
153
+ ## Running the complete test suite locally
154
+
155
+ Docker is the only host dependency for the complete Python 3.9 test matrix:
156
+
157
+ ```bash
158
+ bash tests/run_with_docker.sh
159
+ ```
160
+
161
+ The runner builds a disposable test image, runs the general and SQLite suites,
162
+ and then starts MariaDB, PostgreSQL, and Oracle Free Lite one at a time. Each
163
+ database container and network is removed before the next service starts and is
164
+ also cleaned up if a test fails.
165
+
166
+ The service images are pinned by digest. They can be replaced without editing
167
+ the script by setting `GITDATA_TEST_MARIADB_IMAGE`,
168
+ `GITDATA_TEST_POSTGRES_IMAGE`, or `GITDATA_TEST_ORACLE_IMAGE`.
169
+
170
+ ## Oracle Database
171
+
172
+ Oracle support uses the Python DB-API driver `python-oracledb`, which defaults
173
+ to Thin mode. The adapter does not initialize Thick mode, so Oracle Instant
174
+ Client is not required. The driver is optional and is loaded only when an
175
+ Oracle connection is requested:
176
+
177
+ ```bash
178
+ pip install 'gitdata-lib[oracle]'
179
+ ```
180
+
181
+ Connect with an Oracle service name:
182
+
183
+ ```python
184
+ import os
185
+
186
+ from gitdata.database import connect
187
+
188
+ db = connect(
189
+ 'oracle',
190
+ host='127.0.0.1',
191
+ port=1521,
192
+ service_name='FREEPDB1',
193
+ user='app',
194
+ password=os.environ['ORACLE_PASSWORD'],
195
+ )
196
+
197
+ rows = list(db('SELECT 1 FROM dual'))
198
+ ```
199
+
200
+ A complete `dsn` can be supplied instead of `host`, `port`, and
201
+ `service_name`. Installing `gitdata-lib` without the `oracle` extra does not
202
+ install or import `python-oracledb`; requesting an Oracle connection in that
203
+ case reports the extra that must be installed.
204
+
205
+ The Oracle integration tests use these environment variables:
206
+
207
+ - `GITDATA_TEST_ORACLE_USER`
208
+ - `GITDATA_TEST_ORACLE_PASSWORD`
209
+ - `GITDATA_TEST_ORACLE_SERVICE_NAME`
210
+ - `GITDATA_TEST_ORACLE_HOST` (defaults to `127.0.0.1`)
211
+ - `GITDATA_TEST_ORACLE_PORT` (defaults to `1521`)
212
+
213
+ They have been exercised against Oracle Free using the
214
+ `container-registry.oracle.com/database/free:latest` container image.
@@ -0,0 +1 @@
1
+ __version__ = '0.0.17'
@@ -0,0 +1,114 @@
1
+ """
2
+ usage: gitdata [options] <command> [<args>...]
3
+
4
+ options:
5
+ -h, --help show help
6
+ -V, --version print version
7
+ -d, --debug debug
8
+
9
+ The most commonly used gitdata commands are:
10
+ init initialize a local gitdata repository
11
+ fetch fetch data to the local reposotiry
12
+ get get data
13
+ ls list immediate children of a ref
14
+ scan scan data
15
+ secret manage repository secrets
16
+
17
+ See 'gitdata help <command>' for more information on a specific command.
18
+ """
19
+
20
+ import importlib
21
+ import logging
22
+ import sys
23
+
24
+ from docopt import docopt
25
+
26
+ import gitdata
27
+ from gitdata.utils import trim
28
+
29
+
30
+ root_logger = logging.getLogger()
31
+
32
+
33
+ def print_help(doc):
34
+ """Print help text"""
35
+ print(trim(doc))
36
+
37
+
38
+ def get_module_doc(name):
39
+ if name == 'secret':
40
+ name = 'secrets'
41
+ module_name = 'gitdata.cli.gitdata_' + name
42
+ try:
43
+ result = importlib.import_module(module_name).__doc__
44
+ except ModuleNotFoundError:
45
+ result = f'no help on topic {name!r}'
46
+ return result
47
+
48
+
49
+ def main():
50
+ """CLI entry point with clean interrupt handling."""
51
+ try:
52
+ _main()
53
+ except KeyboardInterrupt:
54
+ print('\nInterrupted.', file=sys.stderr)
55
+ raise SystemExit(130)
56
+
57
+
58
+ def _main():
59
+ """Parse and dispatch a CLI command."""
60
+
61
+ if len(sys.argv) == 1:
62
+ print_help(__doc__)
63
+ sys.exit()
64
+
65
+ args = docopt(
66
+ __doc__,
67
+ version='gitdata version {}'.format(gitdata.__version__),
68
+ options_first=True
69
+ )
70
+
71
+ if args['--debug']:
72
+ print(args)
73
+ root_logger.setLevel(logging.DEBUG)
74
+
75
+ argv = [args['<command>']] + args['<args>']
76
+ command = args['<command>']
77
+
78
+ if command == 'help':
79
+ if args['<args>']:
80
+ topic = args['<args>'][0]
81
+ doc = get_module_doc(topic)
82
+ else:
83
+ doc = __doc__
84
+ print_help(doc)
85
+ sys.exit()
86
+
87
+ elif command == 'init':
88
+ from gitdata.cli.gitdata_init import init, __doc__ as doc
89
+ args = docopt(doc, argv=argv)
90
+ init(args)
91
+
92
+ elif command == 'get':
93
+ from gitdata.cli.gitdata_get import get, __doc__ as doc
94
+ args = docopt(doc, argv=argv)
95
+ get(args)
96
+
97
+ elif command == 'ls':
98
+ from gitdata.cli.gitdata_ls import ls_to_console, __doc__ as doc
99
+ args = docopt(doc, argv=argv)
100
+ ls_to_console(args)
101
+
102
+ elif command == 'scan':
103
+ from gitdata.cli.gitdata_scan import scan_to_console, __doc__ as doc
104
+ args = docopt(doc, argv=argv)
105
+ scan_to_console(args)
106
+
107
+ elif command in ('secret', 'secrets'):
108
+ from gitdata.cli.gitdata_secrets import secrets, __doc__ as doc
109
+ argv[0] = 'secret'
110
+ args = docopt(doc, argv=argv)
111
+ secrets(args)
112
+
113
+ else:
114
+ exit("%r is not a gitdata command. See 'gitdata help'." % args['<command>'])
@@ -14,9 +14,11 @@ def get(args):
14
14
  if args['<ref>']:
15
15
  for ref in args['<ref>']:
16
16
  print('getting', ref)
17
- pprint(
18
- gitdata.connectors.common.get(ref),
19
- )
17
+ result = gitdata.connectors.common.get(ref)
18
+ if isinstance(result, dict):
19
+ max_len = max(map(len, result.keys())) + 3
20
+ for k, v in result.items():
21
+ print('%s%s: %s' % (k, '.' * (max_len - len(k)), v))
20
22
  else:
21
23
  print(__doc__)
22
24
  print(args)
@@ -0,0 +1,22 @@
1
+ """
2
+ usage: gitdata init [<path>]
3
+
4
+ options:
5
+ -h, --help
6
+ """
7
+
8
+ import gitdata.repositories
9
+
10
+
11
+ def init(args):
12
+ """Initialize a gitdata repository."""
13
+ path = args['<path>'] or '.'
14
+ try:
15
+ repository_path, created = gitdata.repositories.init_repository(path)
16
+ except ValueError as error:
17
+ raise SystemExit('fatal: {}'.format(error))
18
+
19
+ if created:
20
+ print('Initialized empty GitData repository in {}'.format(repository_path))
21
+ else:
22
+ print('GitData repository already initialized in {}'.format(repository_path))