gitdata-lib 0.0.12__tar.gz → 0.0.18__tar.gz

gitdata_lib-0.0.18/CHANGELOG.md

@@ -0,0 +1,76 @@
+# Changelog
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [v0.0.18] - 2026-09-28
+- support and test Python 3.9 through 3.12
+- replace removed `collections` aliases with `collections.abc`
+- update PostgreSQL and CLI dependencies for Python 3.12
+- register explicit SQLite date and timestamp adapters and converters
+- run pytest-based CI and local test workflows across supported Python versions and database integrations
+
+## [v0.0.17] - 2026-09-28
+- add direct `gitdata ls` and streaming `gitdata scan` for CSV, SQLite, and MariaDB refs without digestion or storage
+- allow `gitdata ls` on a MariaDB server ref to list databases visible to the user
+- accept canonical `database.table` selection in MariaDB/MySQL refs while retaining `#table` compatibility
+- use `database-file/table.column` SQLite refs without fragment selectors
+- eagerly count database rows before sampling unless `gitdata scan --no-count` is supplied
+- allow SQLite column refs to scan one column and behave as leaves under `gitdata ls`
+- format `gitdata scan` as a terminal-width-aware table with aligned numeric columns and bounded values
+- default omitted MariaDB usernames to the current OS user and attempt passwordless authentication when no password environment variable is specified
+- add reusable CSV, SQLite, and `mysqldump` examples for manual inspection
+- exit cleanly without a traceback when a command is interrupted with Ctrl+C
+- announce `gitdata scan` after ref validation and connection, immediately before sampling
+- add `gitdata ls -l` database/table statistics, column schema, and index details
+- hide database-specific system objects from `gitdata ls` unless `-a` is supplied
+- distinguish node relationships from literal values with structural predicates
+- preserve repeated predicate values when reading graph nodes
+- escape colliding source predicates and omit factless values without dangling relationships
+- verify multiple source graphs retain independent roots in memory and SQLite repositories
+- require re-ingestion of older graph facts whose unprefixed relationships cannot be distinguished from literal IDs
+
+## [v0.0.16] - 2026-09-19
+- add optional Oracle Database support with `python-oracledb` Thin mode and live CI integration tests
+- add initial encryption module and unit tests
+- refactor secrets module into a backend-agnostic core with pluggable storage
+- add `GITDATA_ENCRYPTION_KEY` and docker `/run/secrets` key resolution for secrets
+- mask secret values by default in secrets listing APIs
+- persist secrets in the local `.gitdata` repository via EntityStore
+- add `gitdata secret` command to list, get, set, and delete secrets
+- add secrets resolve API that hard-fails listing missing names
+- add `gitdata secret status`, `resolve`, and `clear` for share handoff
+- keep secret values out of logs, HTTP facts, and routine CLI output
+- rename CLI command to `gitdata secret` (`secrets` remains an alias)
+- add `gitdata init` command to initialize repositories
+- switch local repository storage to a single `.gitdata` sqlite file
+
+## [v0.0.15] - 2025-08-03
+- add secrets module stub
+
+## [v0.0.14] - 2025-08-03
+- add support for environment variable style config keys
+
+## [v0.0.13] - 2025-08-03
+- make config filename a constant
+- improve CLI help and `get` output readability
+- expand queue tests and database-specific test configuration
+
+## [v0.0.12] - 2024-05-01
+- add `scan` command and HTTP connector support
+- add SQL module and related tests
+- improve CLI behavior and test/CI setup
+
+## [v0.0.11] - 2021-12-31
+- refactor CLI and connector internals
+- add fake and local connector support
+- continue datastore and graph/facts foundation work
+
+## [v0.0.3] - 2021-10-10
+- add CLI entrypoint and related command wiring
+
+## [v0.0.1] - 2021-10-09
+- initial packaged release with config, database, stores, queues, and connectors

gitdata_lib-0.0.18/PKG-INFO

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: gitdata-lib
+Version: 0.0.18
+Summary: Data extraction and analysis library
+Home-page: https://github.com/gitdata/gitdata-lib
+Author: DSI Labs
+Author-email: support@gitdata.com
+Classifier: Development Status :: 1 - Planning
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Database :: Front-Ends
+Requires-Python: >=3.9,<3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: python-decouple>=3.4
+Requires-Dist: Unipath>=1.1
+Requires-Dist: PyMySQL==0.10.1
+Requires-Dist: psycopg2-binary==2.9.10
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Requires-Dist: docopt-ng<1,>=0.9
+Requires-Dist: faker
+Requires-Dist: cryptography
+Provides-Extra: oracle
+Requires-Dist: oracledb<4,>=1.3; extra == "oracle"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# GitData
+
+GitData gives you one set of commands for exploring files and databases.
+List what is inside a source with `gitdata ls`, then preview its data with
+`gitdata scan`.
+
+## Install
+
+GitData supports Python 3.9 through 3.12.
+
+```bash
+python3 -m pip install gitdata-lib
+```
+
+Oracle support is optional:
+
+```bash
+python3 -m pip install 'gitdata-lib[oracle]'
+```
+
+## Explore a data source
+
+The same two commands work across CSV files, SQLite databases, and
+MariaDB/MySQL databases:
+
+```bash
+gitdata ls <source>
+gitdata scan <source>
+```
+
+Use `-l` for a detailed listing, `-a` to include system objects, and `--limit`
+to choose how many rows to preview.
+
+## CSV
+
+List the columns in a CSV file:
+
+```bash
+gitdata ls examples/locations-headered.csv
+```
+
+Preview its rows:
+
+```bash
+gitdata scan --limit 10 examples/locations-headered.csv
+```
+
+## SQLite
+
+Start with the database file, then move into a table or column:
+
+```bash
+# List tables
+gitdata ls examples/inspection.sqlite
+
+# Show table details
+gitdata ls -l examples/inspection.sqlite/customers
+
+# Preview rows
+gitdata scan --limit 5 examples/inspection.sqlite/customers
+
+# Examine one column
+gitdata scan --limit 5 examples/inspection.sqlite/customers.email
+```
+
+SQLite references follow this shape:
+
+```text
+path/to/database.sqlite
+path/to/database.sqlite/table
+path/to/database.sqlite/table.column
+```
+
+## MariaDB and MySQL
+
+Keep passwords in environment variables instead of putting them in database
+references:
+
+```bash
+export GITDATA_DB_PASSWORD='your-password'
+
+SERVER='mysql://user@db.example.com?password_env=GITDATA_DB_PASSWORD'
+DATABASE='mysql://user@db.example.com/sales?password_env=GITDATA_DB_PASSWORD'
+TABLE='mysql://user@db.example.com/sales.orders?password_env=GITDATA_DB_PASSWORD'
+```
+
+Explore the server, database, and table with the same commands:
+
+```bash
+# List databases
+gitdata ls "$SERVER"
+
+# List tables
+gitdata ls "$DATABASE"
+
+# Show columns and indexes
+gitdata ls -l "$TABLE"
+
+# Preview rows
+gitdata scan --limit 10 "$TABLE"
+```
+
+MariaDB/MySQL references follow this shape:
+
+```text
+mysql://user@host
+mysql://user@host/database
+mysql://user@host/database.table
+```
+
+If the username is omitted, GitData uses the current operating-system username.
+Use `gitdata ls -a` when you also want to see system databases or tables.
+
+### Try the included MariaDB example
+
+The repository includes `examples/mariadb.sql`. Load it into a disposable
+MariaDB container:
+
+```bash
+docker run --name gitdata-example-mariadb \
+  -e MARIADB_ROOT_PASSWORD=example \
+  -p 3307:3306 \
+  -d mariadb:10.7
+
+until docker exec gitdata-example-mariadb \
+  mariadb-admin ping -h 127.0.0.1 -uroot -pexample --silent
+do
+  sleep 1
+done
+
+docker exec -i gitdata-example-mariadb \
+  mariadb -uroot -pexample < examples/mariadb.sql
+
+export GITDATA_EXAMPLE_PASSWORD=example
+gitdata ls 'mysql://root@127.0.0.1:3307/gitdata_example?password_env=GITDATA_EXAMPLE_PASSWORD'
+gitdata scan --limit 5 'mysql://root@127.0.0.1:3307/gitdata_example.customers?password_env=GITDATA_EXAMPLE_PASSWORD'
+```
+
+Remove the example when finished:
+
+```bash
+docker rm -f gitdata-example-mariadb
+```
+
+## Command reference
+
+```text
+gitdata ls <source>          list databases, tables, or columns
+gitdata ls -l <source>       show a detailed listing
+gitdata ls -a <source>       include system objects
+gitdata scan <source>        preview and profile data
+gitdata scan --limit N ...   preview at most N rows
+```
+
+Run a command with `--help` for all options:
+
+```bash
+gitdata ls --help
+gitdata scan --help
+```
+
+## Oracle from Python
+
+Use the optional Oracle driver through the Python API:
+
+```python
+import os
+
+from gitdata.database import connect
+
+db = connect(
+    'oracle',
+    host='127.0.0.1',
+    port=1521,
+    service_name='FREEPDB1',
+    user='app',
+    password=os.environ['ORACLE_PASSWORD'],
+)
+
+rows = list(db('SELECT 1 FROM dual'))
+```

gitdata_lib-0.0.18/README.md

@@ -0,0 +1,181 @@
+# GitData
+
+GitData gives you one set of commands for exploring files and databases.
+List what is inside a source with `gitdata ls`, then preview its data with
+`gitdata scan`.
+
+## Install
+
+GitData supports Python 3.9 through 3.12.
+
+```bash
+python3 -m pip install gitdata-lib
+```
+
+Oracle support is optional:
+
+```bash
+python3 -m pip install 'gitdata-lib[oracle]'
+```
+
+## Explore a data source
+
+The same two commands work across CSV files, SQLite databases, and
+MariaDB/MySQL databases:
+
+```bash
+gitdata ls <source>
+gitdata scan <source>
+```
+
+Use `-l` for a detailed listing, `-a` to include system objects, and `--limit`
+to choose how many rows to preview.
+
+## CSV
+
+List the columns in a CSV file:
+
+```bash
+gitdata ls examples/locations-headered.csv
+```
+
+Preview its rows:
+
+```bash
+gitdata scan --limit 10 examples/locations-headered.csv
+```
+
+## SQLite
+
+Start with the database file, then move into a table or column:
+
+```bash
+# List tables
+gitdata ls examples/inspection.sqlite
+
+# Show table details
+gitdata ls -l examples/inspection.sqlite/customers
+
+# Preview rows
+gitdata scan --limit 5 examples/inspection.sqlite/customers
+
+# Examine one column
+gitdata scan --limit 5 examples/inspection.sqlite/customers.email
+```
+
+SQLite references follow this shape:
+
+```text
+path/to/database.sqlite
+path/to/database.sqlite/table
+path/to/database.sqlite/table.column
+```
+
+## MariaDB and MySQL
+
+Keep passwords in environment variables instead of putting them in database
+references:
+
+```bash
+export GITDATA_DB_PASSWORD='your-password'
+
+SERVER='mysql://user@db.example.com?password_env=GITDATA_DB_PASSWORD'
+DATABASE='mysql://user@db.example.com/sales?password_env=GITDATA_DB_PASSWORD'
+TABLE='mysql://user@db.example.com/sales.orders?password_env=GITDATA_DB_PASSWORD'
+```
+
+Explore the server, database, and table with the same commands:
+
+```bash
+# List databases
+gitdata ls "$SERVER"
+
+# List tables
+gitdata ls "$DATABASE"
+
+# Show columns and indexes
+gitdata ls -l "$TABLE"
+
+# Preview rows
+gitdata scan --limit 10 "$TABLE"
+```
+
+MariaDB/MySQL references follow this shape:
+
+```text
+mysql://user@host
+mysql://user@host/database
+mysql://user@host/database.table
+```
+
+If the username is omitted, GitData uses the current operating-system username.
+Use `gitdata ls -a` when you also want to see system databases or tables.
+
+### Try the included MariaDB example
+
+The repository includes `examples/mariadb.sql`. Load it into a disposable
+MariaDB container:
+
+```bash
+docker run --name gitdata-example-mariadb \
+  -e MARIADB_ROOT_PASSWORD=example \
+  -p 3307:3306 \
+  -d mariadb:10.7
+
+until docker exec gitdata-example-mariadb \
+  mariadb-admin ping -h 127.0.0.1 -uroot -pexample --silent
+do
+  sleep 1
+done
+
+docker exec -i gitdata-example-mariadb \
+  mariadb -uroot -pexample < examples/mariadb.sql
+
+export GITDATA_EXAMPLE_PASSWORD=example
+gitdata ls 'mysql://root@127.0.0.1:3307/gitdata_example?password_env=GITDATA_EXAMPLE_PASSWORD'
+gitdata scan --limit 5 'mysql://root@127.0.0.1:3307/gitdata_example.customers?password_env=GITDATA_EXAMPLE_PASSWORD'
+```
+
+Remove the example when finished:
+
+```bash
+docker rm -f gitdata-example-mariadb
+```
+
+## Command reference
+
+```text
+gitdata ls <source>          list databases, tables, or columns
+gitdata ls -l <source>       show a detailed listing
+gitdata ls -a <source>       include system objects
+gitdata scan <source>        preview and profile data
+gitdata scan --limit N ...   preview at most N rows
+```
+
+Run a command with `--help` for all options:
+
+```bash
+gitdata ls --help
+gitdata scan --help
+```
+
+## Oracle from Python
+
+Use the optional Oracle driver through the Python API:
+
+```python
+import os
+
+from gitdata.database import connect
+
+db = connect(
+    'oracle',
+    host='127.0.0.1',
+    port=1521,
+    service_name='FREEPDB1',
+    user='app',
+    password=os.environ['ORACLE_PASSWORD'],
+)
+
+rows = list(db('SELECT 1 FROM dual'))
+```

{gitdata_lib-0.0.12 → gitdata_lib-0.0.18}/gitdata/__init__.py

@@ -1,4 +1,4 @@
-"""
+r"""
    ____ _ _   ____        _
   / ___(_) |_|  _ \  __ _| |_ __ _
  | |  _| | __| | | |/ _` | __/ _` |

gitdata_lib-0.0.18/gitdata/__version__.py

@@ -0,0 +1 @@
+__version__ = '0.0.18'

gitdata_lib-0.0.18/gitdata/cli/__init__.py

@@ -0,0 +1,114 @@
+"""
+usage: gitdata [options] <command> [<args>...]
+
+options:
+    -h, --help      show help
+    -V, --version   print version
+    -d, --debug     debug
+
+The most commonly used gitdata commands are:
+    init        initialize a local gitdata repository
+    fetch       fetch data to the local reposotiry
+    get         get data
+    ls          list immediate children of a ref
+    scan        scan data
+    secret      manage repository secrets
+
+See 'gitdata help <command>' for more information on a specific command.
+"""
+
+import importlib
+import logging
+import sys
+
+from docopt import docopt
+
+import gitdata
+from gitdata.utils import trim
+
+
+root_logger = logging.getLogger()
+
+
+def print_help(doc):
+    """Print help text"""
+    print(trim(doc))
+
+
+def get_module_doc(name):
+    if name == 'secret':
+        name = 'secrets'
+    module_name = 'gitdata.cli.gitdata_' + name
+    try:
+        result = importlib.import_module(module_name).__doc__
+    except ModuleNotFoundError:
+        result = f'no help on topic {name!r}'
+    return result
+
+
+def main():
+    """CLI entry point with clean interrupt handling."""
+    try:
+        _main()
+    except KeyboardInterrupt:
+        print('\nInterrupted.', file=sys.stderr)
+        raise SystemExit(130)
+
+
+def _main():
+    """Parse and dispatch a CLI command."""
+
+    if len(sys.argv) == 1:
+        print_help(__doc__)
+        sys.exit()
+
+    args = docopt(
+        __doc__,
+        version='gitdata version {}'.format(gitdata.__version__),
+        options_first=True
+    )
+
+    if args['--debug']:
+        print(args)
+        root_logger.setLevel(logging.DEBUG)
+
+    argv = [args['<command>']] + args['<args>']
+    command = args['<command>']
+
+    if command == 'help':
+        if args['<args>']:
+            topic = args['<args>'][0]
+            doc = get_module_doc(topic)
+        else:
+            doc = __doc__
+        print_help(doc)
+        sys.exit()
+
+    elif command == 'init':
+        from gitdata.cli.gitdata_init import init, __doc__ as doc
+        args = docopt(doc, argv=argv)
+        init(args)
+
+    elif command == 'get':
+        from gitdata.cli.gitdata_get import get, __doc__ as doc
+        args = docopt(doc, argv=argv)
+        get(args)
+
+    elif command == 'ls':
+        from gitdata.cli.gitdata_ls import ls_to_console, __doc__ as doc
+        args = docopt(doc, argv=argv)
+        ls_to_console(args)
+
+    elif command == 'scan':
+        from gitdata.cli.gitdata_scan import scan_to_console, __doc__ as doc
+        args = docopt(doc, argv=argv)
+        scan_to_console(args)
+
+    elif command in ('secret', 'secrets'):
+        from gitdata.cli.gitdata_secrets import secrets, __doc__ as doc
+        argv[0] = 'secret'
+        args = docopt(doc, argv=argv)
+        secrets(args)
+
+    else:
+        exit("%r is not a gitdata command. See 'gitdata help'." % args['<command>'])

{gitdata_lib-0.0.12 → gitdata_lib-0.0.18}/gitdata/cli/gitdata_get.py

@@ -14,9 +14,11 @@ def get(args):
     if args['<ref>']:
         for ref in args['<ref>']:
             print('getting', ref)
-            pprint(
-                gitdata.connectors.common.get(ref),
-            )
+            result = gitdata.connectors.common.get(ref)
+            if isinstance(result, dict):
+                max_len = max(map(len, result.keys())) + 3
+                for k, v in result.items():
+                    print('%s%s: %s' % (k, '.' * (max_len - len(k)), v))
     else:
         print(__doc__)
         print(args)

gitdata_lib-0.0.18/gitdata/cli/gitdata_init.py

@@ -0,0 +1,22 @@
+"""
+usage: gitdata init [<path>]
+
+options:
+    -h, --help
+"""
+
+import gitdata.repositories
+
+
+def init(args):
+    """Initialize a gitdata repository."""
+    path = args['<path>'] or '.'
+    try:
+        repository_path, created = gitdata.repositories.init_repository(path)
+    except ValueError as error:
+        raise SystemExit('fatal: {}'.format(error))
+
+    if created:
+        print('Initialized empty GitData repository in {}'.format(repository_path))
+    else:
+        print('GitData repository already initialized in {}'.format(repository_path))