gitdata-lib 0.0.10__tar.gz → 0.0.12__tar.gz

{gitdata-lib-0.0.10 → gitdata_lib-0.0.12}/MANIFEST.in

@@ -1,2 +1,4 @@
 include README.md LICENSE CHANGELOG.md requirements.txt
-recursive-include gitdata *.py *.html *.pug *.sass *.sql *.ini *.js *.css *.gitinclude *.md *.json *.wsgi *.eot *.svg *.ttf *.woff *.woff2 *.png *.ico *.map *.gif LICENSE
+recursive-include gitdata *.py *.html *.pug *.sass *.sql *.ini *.js *.css *.gitinclude *.md *.json *.wsgi *.eot *.svg *.ttf *.woff *.woff2 *.png *.ico *.map *.gif LICENSE
+recursive-exclude gitdata/ext *.py
+recursive-include gitdata/ext __init__.py

{gitdata-lib-0.0.10/gitdata_lib.egg-info → gitdata_lib-0.0.12}/PKG-INFO

@@ -1,12 +1,10 @@
 Metadata-Version: 2.1
 Name: gitdata-lib
-Version: 0.0.10
+Version: 0.0.12
 Summary: Data extraction and analysis library
 Home-page: https://github.com/gitdata/gitdata-lib
 Author: DSI Labs
 Author-email: support@gitdata.com
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: Development Status :: 1 - Planning
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -22,12 +20,17 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Database :: Front-Ends
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: python-decouple>=3.4
+Requires-Dist: Unipath>=1.1
+Requires-Dist: PyMySQL==0.10.1
+Requires-Dist: psycopg2-binary==2.9.1
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Requires-Dist: docopt
+Requires-Dist: faker
 
 # GitData Lib
 Data Wrangling for Everyone.
 
-GitData is an easy to use, fast, scalable, distributed data extraction system
-with a rich set of commands that provide ways to gather, manage and query data in
-an unusually straightforward way.
-
-
+GitData is a fast, scalable, distributed data exploration system
+with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.

gitdata_lib-0.0.12/README.md

@@ -0,0 +1,5 @@
+# GitData Lib
+Data Wrangling for Everyone.
+
+GitData is a fast, scalable, distributed data exploration system
+with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.

{gitdata-lib-0.0.10 → gitdata_lib-0.0.12}/gitdata/__init__.py

@@ -8,14 +8,7 @@
 """
 
 from .utils import Record
-from .stores.records import table_of
+from .stores.tables import table_of
 from .stores.entities import store_of
 from .__version__ import __version__
-from .connectors.http import HttpConnector
-
-def fetch(ref):
-    """Fetch data
-
-    Stub for fetch data function.
-    """
-    return Record(HttpConnector().get(ref))
+from .connectors import fetch

gitdata_lib-0.0.12/gitdata/__version__.py

@@ -0,0 +1 @@
+__version__ = '0.0.12'

{gitdata-lib-0.0.10 → gitdata_lib-0.0.12}/gitdata/buckets.py

@@ -4,9 +4,11 @@
     stores blobs of data
 """
 
+import io
 import os
-import uuid
+import shutil
 
+import gitdata
 
 
 def mkdir_p(path):
@@ -19,10 +21,6 @@ def mkdir_p(path):
         os.makedirs(path)
 
 
-def new_id():
-    """Generate a random unique id"""
-    return uuid.uuid4().hex
-
 class BaseBucket:
     """BaseBucket"""
 
@@ -30,7 +28,7 @@ class BaseBucket:
 class FileBucket(BaseBucket):
     """File Bucket
 
-    FileBucket can store and retreive blobs of binary data.  This is
+    FileBucket can store and retrieve blobs of binary data.  This is
     useful for storing images or files as attachments where you want
     to put the binary data somewhere and be able to retreive it later
     if asked.  The id returned is generally stored in a database or
@@ -46,8 +44,7 @@ class FileBucket(BaseBucket):
     in the site.ini configuration file.
 
     >>> import gitdata, tempfile
-    >>> dir = tempfile.TemporaryDirectory()
-    >>> path = str(dir)
+    >>> path = tempfile.TemporaryDirectory().name
     >>> ids = ['id_%04d' % (9-i) for i in range(10)]
     >>> new_id = ids.pop
 
@@ -83,9 +80,22 @@ class FileBucket(BaseBucket):
     >>> for item_id in bucket.keys():
     ...     bucket.delete(item_id)
 
+    >>> bucket = FileBucket(path, new_id)
+    >>> item_id = bucket.puts(io.BytesIO(b'some data'))
+    >>> item_id
+    'id_0004'
+    >>> s = bucket.gets(item_id)
+    >>> s.read()
+    b'some data'
+    >>> s.close()
+    >>> bucket.exists(item_id)
+    True
+    >>> bucket.delete(item_id)
+    >>> bucket.exists(item_id)
+    False
     """
 
-    def __init__(self, path, id_factory=new_id):
+    def __init__(self, path, id_factory=gitdata.utils.new_uid):
         self.path = os.path.realpath(path)
         self.id_factory = id_factory
         mkdir_p(self.path)
@@ -95,7 +105,7 @@ class FileBucket(BaseBucket):
         pathname = os.path.join(self.path, item_id)
         if os.path.exists(pathname):
             raise Exception('duplicate item')
-        f = open(os.path.join(pathname), 'wb')
+        f = open(pathname, 'wb')
         try:
             f.write(item)
         finally:
@@ -109,6 +119,26 @@ class FileBucket(BaseBucket):
         with open(os.path.join(pathname), 'rb') as f:
             return f.read()
 
+    def puts(self, item):
+        item_id = self.id_factory()
+        pathname = os.path.join(self.path, item_id)
+        if os.path.exists(pathname):
+            raise Exception('duplicate item')
+        f = open(pathname, 'wb')
+        try:
+            shutil.copyfileobj(item, f)
+        finally:
+            f.close()
+        return item_id
+
+    def gets(self, item_id, default=None):
+        if not isinstance(item_id, str):
+            return default
+        pathname = os.path.join(self.path, item_id)
+        if not os.path.exists(pathname) and default is not None:
+            return default
+        return open(os.path.join(pathname), 'rb')
+
     def delete(self, item_id):
         pathname = os.path.join(self.path, item_id)
         os.remove(pathname)
@@ -120,5 +150,55 @@ class FileBucket(BaseBucket):
     def keys(self):
         return os.listdir(self.path)
 
+    def clear(self):
+        for key in self.keys():
+            self.delete(key)
+
+
+class MemoryBucket(BaseBucket):
+    """Memory Bucket
+
+    Memory based bucket.
+    """
+
+    def __init__(self, id_factory=gitdata.utils.new_uid):
+        self.id_factory = id_factory
+        self.items = {}
+
+    def put(self, item):
+        item_id = self.id_factory()
+        if item_id in self.items:
+            raise Exception('duplicate item')
+        self.items[item_id] = item
+        return item_id
+
+    def get(self, item_id, default=None):
+        value = self.items.get(item_id, default)
+        return value
+
+    def puts(self, item):
+        item_id = self.id_factory()
+        if item_id in self.items:
+            raise Exception('duplicate item')
+        self.items[item_id] = item.read()
+        return item_id
+
+    def gets(self, item_id, default=None):
+        value = self.items.get(item_id)
+        return io.BytesIO(value) if value else default
+        # return io.BytesIO(self.items.get(item_id, default))
+
+    def delete(self, item_id):
+        self.items.pop(item_id)
+
+    def exists(self, item_id):
+        return item_id in self.items
+
+    def keys(self):
+        return list(self.items.keys())
+
+    def clear(self):
+        self.items.clear()
+
 
-Bucket = FileBucket
+Bucket = FileBucket

gitdata_lib-0.0.12/gitdata/cli/__init__.py

@@ -0,0 +1,70 @@
+"""
+usage: gitdata [options] <command> [<args>...]
+
+options:
+    -h, --help      show help
+    -V, --version   print version
+    -d, --debug     debug
+
+The most commonly used gitdata commands are:
+    fetch       fetch data to the local reposotiry
+    get         get data
+    scan        scan data
+
+See 'gitdata help <command>' for more information on a specific command.
+"""
+
+import logging
+
+from docopt import docopt
+import sys
+
+import gitdata
+from gitdata.utils import trim
+
+
+root_logger = logging.getLogger()
+
+
+def print_help(doc):
+    """Print help text"""
+    print(trim(doc))
+
+
+def main():
+    """CLI main"""
+
+    if len(sys.argv) == 1:
+        print_help(__doc__)
+        sys.exit()
+
+    args = docopt(
+        __doc__,
+        version='gitdata version {}'.format(gitdata.__version__),
+        options_first=True
+    )
+
+    if args['--debug']:
+        print(args)
+        root_logger.setLevel(logging.DEBUG)
+
+    argv = [args['<command>']] + args['<args>']
+
+    if args['<command>'] == 'get':
+        from gitdata.cli.gitdata_get import get, __doc__ as doc
+        args = docopt(doc, argv=argv)
+        get(args)
+
+    elif args['<command>'] == 'scan':
+        from gitdata.cli.gitdata_scan import scan_to_console, __doc__ as doc
+        args = docopt(doc, argv=argv)
+        scan_to_console(args)
+
+    elif args['<command>'] in ['help', None]:
+        if args['<args>'] == ['get']:
+            from gitdata.cli.gitdata_get import __doc__ as doc
+            exit(doc)
+        exit(__doc__)
+
+    else:
+        exit("%r is not a gitdata command. See 'gitdata help'." % args['<command>'])

gitdata_lib-0.0.12/gitdata/cli/gitdata_get.py

@@ -0,0 +1,22 @@
+"""
+usage: gitdata get [options] [<ref>...]
+
+options:
+    -h, --help
+"""
+
+from pprint import pprint
+
+import gitdata.connectors.common
+
+
+def get(args):
+    if args['<ref>']:
+        for ref in args['<ref>']:
+            print('getting', ref)
+            pprint(
+                gitdata.connectors.common.get(ref),
+            )
+    else:
+        print(__doc__)
+        print(args)

gitdata_lib-0.0.12/gitdata/cli/gitdata_scan.py

@@ -0,0 +1,94 @@
+"""
+usage: gitdata scan [options] [<ref>...]
+
+options:
+    --limit <n>, -n <n>     limit number of records displayed
+    --name-columns          provide synthetic column names (handy for csv files that do not provide a header)
+    -h, --help
+"""
+
+import csv
+import os
+
+import gitdata
+from gitdata.connectors.common import load, connect
+from gitdata.connectors.http import HttpConnector
+
+
+def as_blobs(ref):
+    if os.path.isfile(ref):
+        with open(ref, 'rb') as f:
+            return [f.read()]
+
+    if ref.startswith('http'):
+        http_connector = HttpConnector()
+        result = http_connector.get(ref)
+        if result:
+            return [result['blob'].read()]
+
+    return []
+
+
+def as_tables(ref, args):
+    for blob in as_blobs(ref):
+        rows = []
+        try:
+            reader = csv.reader(blob.decode('utf8').splitlines())
+        except UnicodeDecodeError:
+            reader = csv.reader(blob.decode('Latin1').splitlines())
+        for row in reader:
+            rows.append(row)
+        yield gitdata.Record(
+            name=ref,
+            filename=ref,
+            rows=rows,
+            columns=[gitdata.Record(name=f'{chr(n+65)}', type='str') for n in range(len(rows[0]))]
+        )
+
+
+def scan(ref, args):
+
+    def scan_table(table):
+        header = gitdata.Record(
+            name=table.name,
+            row_count=len(table.rows),
+            column_count=len(table.rows[0])
+        )
+
+        limit = args['--limit'] and int(args['--limit']) or 5
+
+        if args['--name-columns']:
+            rows = [[t.name for t in table.columns]] + table.rows[:limit]
+        else:
+            rows = table.rows[:limit]
+
+        sample = gitdata.utils.ItemList(rows)
+        return str(header) + '\n' + str(sample) + str(list())
+
+    return ''.join(scan_table(table) for table in as_tables(ref, args))
+
+
+def console(output):
+    if output:
+        print(output)
+        print()
+
+
+def scan_to_console(args):
+    if args['<ref>']:
+        for ref in args['<ref>']:
+            console(scan(ref, args))
+            # connection = connect(ref)
+            # if connection:
+            #     print(connection)
+            #     for table in connection.get_tables():
+            #         print(f'scanning {table.name}')
+            # else:
+            #     print(f'unable to connect to {ref}')
+
+            # g = load(ref)
+            # print('scanning', ref)
+            # print(repr(g))
+            # print(g)
+    else:
+        print(__doc__)

{gitdata-lib-0.0.10 → gitdata_lib-0.0.12}/gitdata/config.py

@@ -152,7 +152,6 @@ def locate_config_file(filename='gitdata.ini', start='.'):
             return pathname
 
     for path in OrderedSet(parents(join(expanduser('~')))) - ancestors:
-        print(path)
         pathname = join(path, filename)
         if exists(pathname):
             return pathname

gitdata_lib-0.0.12/gitdata/connectors/__init__.py

@@ -0,0 +1,5 @@
+"""
+    connectors
+"""
+
+from .common import explore, connect, fetch, extract

{gitdata-lib-0.0.10 → gitdata_lib-0.0.12}/gitdata/connectors/common.py

@@ -2,20 +2,25 @@
     common connector
 """
 
+import datetime
+import getpass
 import inspect
 import importlib
 import logging
 import pkgutil
+import platform
 
 import gitdata.connectors
+import gitdata.ext.connectors
+import gitdata.graphs
 import gitdata.solutions
 
-
 logger = logging.getLogger(__name__)
 
 
 def get_connectors():
     """generate connectors"""
+
     path = gitdata.connectors.__path__
     for _, name, _ in pkgutil.iter_modules(path):
         if name != 'common':
@@ -30,6 +35,19 @@ def get_connectors():
                 if found:
                     yield obj
 
+    path = gitdata.ext.connectors.__path__
+    for _, name, _ in pkgutil.iter_modules(path):
+        module = importlib.import_module('gitdata.ext.connectors.' + name)
+        for _, obj in inspect.getmembers(module):
+            if obj == BaseConnector:
+                continue
+            try:
+                found = issubclass(obj, BaseConnector)
+            except TypeError:
+                found = False
+            if found:
+                yield obj
+
 
 def get_connector_graph():
     """generate connector graph"""
@@ -103,6 +121,15 @@ class BaseConnector:
     def connect(self, ref):
         """Connect to a ref"""
 
+    def extract(self, ref):
+        """Extract facts from a ref"""
+
+    def get_tables(self):
+        return []
+
+    def get_blobs(self):
+        return []
+
     # def explore(self, data):
     #     """Explore a graph
     #     """
@@ -113,16 +140,112 @@ def connect(ref):
     """Return a connection to a ref"""
 
     for connector in get_connectors():
-        if connector.connects(ref):
-            connection = connector()
-            connection.connect(ref)
+        connection = connector().connect(ref)
+        if connection:
             return connection
 
 
     # connectors = {
     #     connector.__name__: connector for connector in get_connectors()}
 
-
-
     # if 'System' in connectors:
     #     return connectors['System']()
+
+def add_connection_metadata(facts):
+    facts.update(
+        dict(
+            collected=datetime.datetime.now(),
+            node=platform.node(),
+            user=getpass.getuser()
+        )
+    )
+
+
+def fetch(ref):
+    """Fetch data
+
+    Fetch the facts in a datasource and store them.
+    """
+    connectors = get_connectors()
+    for connector in connectors:
+        if hasattr(connector, 'get'):
+            facts = connector().get(ref)
+            if facts:
+                add_connection_metadata(facts)
+                return facts
+
+def get(ref):
+    """Get Data
+
+    Get the facts from a datasource.
+    """
+    connectors = get_connectors()
+    for connector in connectors:
+        if hasattr(connector, 'get'):
+            facts = connector().get(ref)
+            if facts:
+                add_connection_metadata(facts)
+                return facts
+
+class Bunch:
+    """a handy bunch of variables"""
+    tables = []
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+    def __str__(self):
+        return f'{self.__class__.__name__}\n' + '\n  '.join(
+            f'{k:{(20-len(k))*"."}}: {v!r}'
+            for k, v in self.__dict__.items()
+        )
+
+class State(dict):
+    """State of Extraction"""
+
+    def find(self, kind):
+        """Return things of a kind"""
+        cache = self.setdefault('_cache', [])
+        for thing in self.setdefault(kind, []):
+            thing_id = id(thing)
+            if thing_id not in cache:
+                cache.append(thing_id)
+                yield thing
+
+    def add(self, kind, value):
+        """Add a value of a certain kind"""
+        self.setdefault(kind, []).append(value)
+
+
+def extract(ref):
+    """Extract a reference
+
+    Extract data from a reference.
+    """
+
+    connectors = get_connectors()
+
+    # ref = State(ref=refs)
+
+    extracting = True
+    while extracting:
+        extracting = any(
+            connector().extract(ref)
+            for connector in connectors
+        )
+        # print(extracting)
+
+    return Bunch(
+        # name=ref.get('name', ref['ref']),
+        # ref=ref['ref'],
+        # tables=ref['tables'],
+        # text=ref['text'],
+    )
+
+
+def load(ref):
+    """Get a Graph
+    """
+    graph = gitdata.graphs.Graph()
+    graph.add(get(ref))
+    return graph

gitdata_lib-0.0.12/gitdata/connectors/fake.py

@@ -0,0 +1,68 @@
+"""
+    mock data
+"""
+
+import faker
+
+from gitdata.connectors.common import BaseConnector
+
+
+class FakeConnector(BaseConnector):
+
+    @property
+    def person(self):
+        fake = faker.Faker()
+        person = fake.profile()
+        first_name, last_name = person['name'].split(maxsplit=1)
+        return dict(
+            first_name=first_name,
+            last_name=last_name,
+            birthdate=person['birthdate'],
+            sex=person['sex']
+        )
+
+    @property
+    def people(self):
+        while True:
+            yield self.person
+
+    @property
+    def address(self):
+        fake = faker.Faker()
+        return dict(
+            street=fake.street_address(),
+            city=fake.city(),
+            state=fake.state(),
+            postal_code=fake.postcode(),
+        )
+
+    @property
+    def addresses(self):
+        while True:
+            yield self.address
+
+    @property
+    def location(self):
+        fake = faker.Faker()
+        lat, lng, name, country_code, timezone = fake.location_on_land()
+        return dict(
+            latitude=lat,
+            longitude=lng,
+            name=name,
+            country_code=country_code,
+            timezone=timezone,
+        )
+
+    @property
+    def locations(self):
+        while True:
+            yield self.location
+
+    def get(self, ref):
+        if ref.startswith('fake'):
+            print('ref is', ref)
+            return dict(
+                people=self.people,
+                addresses=self.addresses,
+                locations=self.locations,
+            )