gitdata-lib 0.0.11__tar.gz → 0.0.12__tar.gz

{gitdata-lib-0.0.11/gitdata_lib.egg-info → gitdata_lib-0.0.12}/PKG-INFO

@@ -1,12 +1,10 @@
 Metadata-Version: 2.1
 Name: gitdata-lib
-Version: 0.0.11
+Version: 0.0.12
 Summary: Data extraction and analysis library
 Home-page: https://github.com/gitdata/gitdata-lib
 Author: DSI Labs
 Author-email: support@gitdata.com
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: Development Status :: 1 - Planning
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -22,11 +20,17 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Database :: Front-Ends
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: python-decouple>=3.4
+Requires-Dist: Unipath>=1.1
+Requires-Dist: PyMySQL==0.10.1
+Requires-Dist: psycopg2-binary==2.9.1
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Requires-Dist: docopt
+Requires-Dist: faker
 
 # GitData Lib
 Data Wrangling for Everyone.
 
 GitData is a fast, scalable, distributed data exploration system
 with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.
-
-

gitdata_lib-0.0.12/gitdata/__version__.py

@@ -0,0 +1 @@
+__version__ = '0.0.12'

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata/cli/__init__.py

@@ -9,6 +9,7 @@ options:
 The most commonly used gitdata commands are:
     fetch       fetch data to the local reposotiry
     get         get data
+    scan        scan data
 
 See 'gitdata help <command>' for more information on a specific command.
 """
@@ -16,6 +17,7 @@ See 'gitdata help <command>' for more information on a specific command.
 import logging
 
 from docopt import docopt
+import sys
 
 import gitdata
 from gitdata.utils import trim
@@ -32,6 +34,10 @@ def print_help(doc):
 def main():
     """CLI main"""
 
+    if len(sys.argv) == 1:
+        print_help(__doc__)
+        sys.exit()
+
     args = docopt(
         __doc__,
         version='gitdata version {}'.format(gitdata.__version__),
@@ -49,6 +55,11 @@ def main():
         args = docopt(doc, argv=argv)
         get(args)
 
+    elif args['<command>'] == 'scan':
+        from gitdata.cli.gitdata_scan import scan_to_console, __doc__ as doc
+        args = docopt(doc, argv=argv)
+        scan_to_console(args)
+
     elif args['<command>'] in ['help', None]:
         if args['<args>'] == ['get']:
             from gitdata.cli.gitdata_get import __doc__ as doc
@@ -56,4 +67,4 @@ def main():
         exit(__doc__)
 
     else:
-        exit("%r is not a git.py command. See 'git help'." % args['<command>'])
+        exit("%r is not a gitdata command. See 'gitdata help'." % args['<command>'])

gitdata_lib-0.0.12/gitdata/cli/gitdata_scan.py

@@ -0,0 +1,94 @@
+"""
+usage: gitdata scan [options] [<ref>...]
+
+options:
+    --limit <n>, -n <n>     limit number of records displayed
+    --name-columns          provide synthetic column names (handy for csv files that do not provide a header)
+    -h, --help
+"""
+
+import csv
+import os
+
+import gitdata
+from gitdata.connectors.common import load, connect
+from gitdata.connectors.http import HttpConnector
+
+
+def as_blobs(ref):
+    if os.path.isfile(ref):
+        with open(ref, 'rb') as f:
+            return [f.read()]
+
+    if ref.startswith('http'):
+        http_connector = HttpConnector()
+        result = http_connector.get(ref)
+        if result:
+            return [result['blob'].read()]
+
+    return []
+
+
+def as_tables(ref, args):
+    for blob in as_blobs(ref):
+        rows = []
+        try:
+            reader = csv.reader(blob.decode('utf8').splitlines())
+        except UnicodeDecodeError:
+            reader = csv.reader(blob.decode('Latin1').splitlines())
+        for row in reader:
+            rows.append(row)
+        yield gitdata.Record(
+            name=ref,
+            filename=ref,
+            rows=rows,
+            columns=[gitdata.Record(name=f'{chr(n+65)}', type='str') for n in range(len(rows[0]))]
+        )
+
+
+def scan(ref, args):
+
+    def scan_table(table):
+        header = gitdata.Record(
+            name=table.name,
+            row_count=len(table.rows),
+            column_count=len(table.rows[0])
+        )
+
+        limit = args['--limit'] and int(args['--limit']) or 5
+
+        if args['--name-columns']:
+            rows = [[t.name for t in table.columns]] + table.rows[:limit]
+        else:
+            rows = table.rows[:limit]
+
+        sample = gitdata.utils.ItemList(rows)
+        return str(header) + '\n' + str(sample) + str(list())
+
+    return ''.join(scan_table(table) for table in as_tables(ref, args))
+
+
+def console(output):
+    if output:
+        print(output)
+        print()
+
+
+def scan_to_console(args):
+    if args['<ref>']:
+        for ref in args['<ref>']:
+            console(scan(ref, args))
+            # connection = connect(ref)
+            # if connection:
+            #     print(connection)
+            #     for table in connection.get_tables():
+            #         print(f'scanning {table.name}')
+            # else:
+            #     print(f'unable to connect to {ref}')
+
+            # g = load(ref)
+            # print('scanning', ref)
+            # print(repr(g))
+            # print(g)
+    else:
+        print(__doc__)

gitdata_lib-0.0.12/gitdata/connectors/__init__.py

@@ -0,0 +1,5 @@
+"""
+    connectors
+"""
+
+from .common import explore, connect, fetch, extract

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata/connectors/common.py

@@ -12,16 +12,16 @@ import platform
 
 import gitdata.connectors
 import gitdata.ext.connectors
+import gitdata.graphs
 import gitdata.solutions
 
-
 logger = logging.getLogger(__name__)
 
 
 def get_connectors():
     """generate connectors"""
-    path = gitdata.connectors.__path__
 
+    path = gitdata.connectors.__path__
     for _, name, _ in pkgutil.iter_modules(path):
         if name != 'common':
             module = importlib.import_module('gitdata.connectors.' + name)
@@ -121,6 +121,15 @@ class BaseConnector:
     def connect(self, ref):
         """Connect to a ref"""
 
+    def extract(self, ref):
+        """Extract facts from a ref"""
+
+    def get_tables(self):
+        return []
+
+    def get_blobs(self):
+        return []
+
     # def explore(self, data):
     #     """Explore a graph
     #     """
@@ -131,17 +140,14 @@ def connect(ref):
     """Return a connection to a ref"""
 
     for connector in get_connectors():
-        if connector.connects(ref):
-            connection = connector()
-            connection.connect(ref)
+        connection = connector().connect(ref)
+        if connection:
             return connection
 
 
     # connectors = {
     #     connector.__name__: connector for connector in get_connectors()}
 
-
-
     # if 'System' in connectors:
     #     return connectors['System']()
 
@@ -180,3 +186,66 @@ def get(ref):
             if facts:
                 add_connection_metadata(facts)
                 return facts
+
+class Bunch:
+    """a handy bunch of variables"""
+    tables = []
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+    def __str__(self):
+        return f'{self.__class__.__name__}\n' + '\n  '.join(
+            f'{k:{(20-len(k))*"."}}: {v!r}'
+            for k, v in self.__dict__.items()
+        )
+
+class State(dict):
+    """State of Extraction"""
+
+    def find(self, kind):
+        """Return things of a kind"""
+        cache = self.setdefault('_cache', [])
+        for thing in self.setdefault(kind, []):
+            thing_id = id(thing)
+            if thing_id not in cache:
+                cache.append(thing_id)
+                yield thing
+
+    def add(self, kind, value):
+        """Add a value of a certain kind"""
+        self.setdefault(kind, []).append(value)
+
+
+def extract(ref):
+    """Extract a reference
+
+    Extract data from a reference.
+    """
+
+    connectors = get_connectors()
+
+    # ref = State(ref=refs)
+
+    extracting = True
+    while extracting:
+        extracting = any(
+            connector().extract(ref)
+            for connector in connectors
+        )
+        # print(extracting)
+
+    return Bunch(
+        # name=ref.get('name', ref['ref']),
+        # ref=ref['ref'],
+        # tables=ref['tables'],
+        # text=ref['text'],
+    )
+
+
+def load(ref):
+    """Get a Graph
+    """
+    graph = gitdata.graphs.Graph()
+    graph.add(get(ref))
+    return graph

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata/connectors/fake.py

@@ -41,9 +41,28 @@ class FakeConnector(BaseConnector):
         while True:
             yield self.address
 
+    @property
+    def location(self):
+        fake = faker.Faker()
+        lat, lng, name, country_code, timezone = fake.location_on_land()
+        return dict(
+            latitude=lat,
+            longitude=lng,
+            name=name,
+            country_code=country_code,
+            timezone=timezone,
+        )
+
+    @property
+    def locations(self):
+        while True:
+            yield self.location
+
     def get(self, ref):
         if ref.startswith('fake'):
+            print('ref is', ref)
             return dict(
                 people=self.people,
                 addresses=self.addresses,
+                locations=self.locations,
             )

gitdata_lib-0.0.12/gitdata/connectors/http.py

@@ -0,0 +1,59 @@
+"""
+    http connector
+"""
+
+import io
+import logging
+import urllib
+import urllib.parse
+
+import requests
+
+from gitdata.connectors.common import BaseConnector
+
+
+logger = logging.getLogger(__name__)
+
+
+class HttpConnector(BaseConnector):
+
+    def get(self, ref):
+        """Get Data"""
+        if ref.startswith('http://') or ref.startswith('https://'):
+            logger.debug(
+                '%s get %r',
+                self.__class__.__name__,
+                ref
+            )
+
+            u = urllib.parse.urlparse(ref)
+            endpoint=urllib.parse.urldefrag(ref)[0]
+            facts = dict(
+                url=ref,
+                endpoint=endpoint,
+                scheme=u.scheme,
+                netloc=u.netloc,
+                username=u.username,
+                password=u.password,
+                hostname=u.hostname,
+                port=u.port,
+                path=u.path,
+                lpath=str(endpoint).lower(),
+                query=u.query,
+                fragment=u.fragment,
+                name=u.fragment or str(u.path).split('/')[-1],
+            )
+
+            r = requests.get(ref)
+            if r.status_code == 200:
+                return dict(
+                    facts,
+                    blob=io.BytesIO(r.content)
+                )
+            else:
+                logger.error(
+                    'status %s - %s get %r',
+                    r.status_code,
+                    self.__class__.__name__,
+                    ref
+                )

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata/connectors/local.py

@@ -16,6 +16,17 @@ class FileConnector(BaseConnector):
     reads = ['location']
     writes = ['text', 'blob', 'stdout']
 
+    # def connects(self, ref):
+    #     return os.path.isfile(ref)
+
+    def connect(self, ref):
+        self.ref = ref
+        return os.path.isfile(ref)
+
+    def get_blobs(self):
+        with open(self.ref, 'rb') as f:
+            return [io.BytesIO(f.read())]
+
     def get(self, ref):
         if os.path.isfile(ref):
             stat = os.stat(ref)
@@ -26,6 +37,28 @@ class FileConnector(BaseConnector):
                 content = io.BytesIO(f.read())
 
             return dict(
+                ref=ref,
+                filename=filename,
+                size=stat.st_size,
+                modified=fromtimestamp(stat.st_mtime),
+                created=fromtimestamp(stat.st_ctime),
+                path=path,
+                blob=content
+            )
+
+    def extract(self, ref):
+        """extract data from a ref"""
+        print('extracting', ref)
+        if os.path.isfile(ref):
+            stat = os.stat(ref)
+            pathname = os.path.realpath(ref)
+            path, filename = os.path.split(pathname)
+
+            with open(ref, 'rb') as f:
+                content = io.BytesIO(f.read())
+
+            return dict(
+                ref=ref,
                 filename=filename,
                 size=stat.st_size,
                 modified=fromtimestamp(stat.st_mtime),

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata/database/common.py

@@ -344,7 +344,6 @@ class Database:
     def get_table_names(self):
         """get list of database tables"""
 
-
     def get_schema_names(self):
         """Return schema names"""
         return []

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata/graphs.py

@@ -253,8 +253,9 @@ class Graph:
         for k, v in kwargs.items():
             query.append(('?subject', k, v))
         subjects = set(record['subject'] for record in self.query(query))
-        result = self.get(subjects) or []
-        return result
+        if subjects:
+            return self.get(subjects) or []
+        return []
 
     def first(self, *args, **kwargs):
         """Find first node"""
@@ -272,3 +273,6 @@ class Graph:
 
     def __len__(self):
         return len(self.facts)
+
+    def __repr__(self):
+        return 'Graph({})'.format(len(self))

gitdata_lib-0.0.12/gitdata/sql.py

@@ -0,0 +1,235 @@
+"""
+    sql utilities
+"""
+
+import inspect
+import os
+
+import gitdata
+from gitdata.stores.tables import table_of
+from gitdata import Record
+
+
+from gitdata.utils import (
+    RecordList, load
+)
+
+
+def get_result_iterator(rows, kind, store=None):
+    """returns an iterator that iterates over the rows and zips the names onto
+    the items being iterated so they come back as dicts"""
+    names = [d[0] == 'id' and '_id' or d[0] for d in rows.cursor.description]
+    set_store = dict(__store=store) if store else {}
+    for rec in rows:
+        yield kind(
+            ((k, v) for k, v in zip(names, rec)),
+            **set_store
+        )
+
+
+class Result(object):
+    """rows resulting from a method call"""
+    # pylint: disable=too-few-public-methods
+
+    def __init__(self, rows, kind, name=None):
+        self.rows = rows
+        self.kind = kind
+        self.name = name
+
+    def __iter__(self):
+        store = gitdata.table_of(self.kind, name=self.name) if self.name else None
+        return get_result_iterator(self.rows, self.kind, store)
+
+    # def __len__(self):
+    #     return self.rows.cursor.rowcount
+
+    def __repr__(self):
+        return repr(list(self))
+
+    def __str__(self):
+        return str(RecordList(self))
+
+
+class SqlExpression:
+
+    def __init__(self, *terms):
+        self.terms = terms
+
+    def __str__(self):
+        return ' '.join(str(t) for t in self.terms)
+
+    def __and__(self, value):
+        return SqlExpression(self, 'and', str(value))
+
+    def __or__(self, value):
+        return SqlExpression(self, 'or', str(value))
+
+
+class SqlTerm:
+
+    def __init__(self, name):
+        self.name = name
+        print(name)
+
+    def __str__(self):
+        return '`{}`'.format(self.name)
+
+    def __eq__(self, value):
+        return SqlExpression(str(self), '=', repr(value))
+
+    def __ne__(self, value):
+        return SqlExpression(str(self), '!=', repr(value))
+
+    def __gt__(self, value):
+        return SqlExpression(str(self), '>', repr(value))
+
+    def __ge__(self, value):
+        return SqlExpression(str(self), '>=', repr(value))
+
+    def __lt__(self, value):
+        return SqlExpression(str(self), '<', repr(value))
+
+    def __le__(self, value):
+        return SqlExpression(str(self), '<=', repr(value))
+
+
+# class SqlTable:
+
+#     def __init__(self, db, table_name, kind=Record):
+#         self.db = db
+#         self.table_name = table_name
+#         self.kind = kind
+
+#     def __iter__(self):
+#         for row in gitdata.table_of(self.kind, name=self.table_name, db=self.db):
+#             yield row
+
+#     def __call__(self, *args):
+#         columns = ', '.join(a for a in args if not isinstance(a, SqlExpression))
+#         clauses = ' and '.join(
+#             str(a) for a in args if isinstance(a, SqlExpression)
+#         )
+#         if clauses:
+#             clauses = ' where ' + clauses
+#         cmd = 'select %s from %s%s' % (columns, self.table_name, clauses)
+#         for row in Result(self.db(cmd), self.kind):
+#             yield row
+
+#     def __len__(self):
+#         cmd = f'select count(*) from {self.table_name}'
+#         return list(self.db(cmd))[0][0]
+
+#     def select(
+#             self, *args,
+#             where=None, group_by=None, order_by=None, limit=None
+#         ):
+#         columns = ', '.join(args)
+#         cmd = [
+#             'select %s from %s' % (columns, self.table_name)
+#         ]
+#         if where:
+#             cmd.append('where ', where)
+#         if group_by:
+#             cmd.append('group by ', group_by)
+#         if order_by:
+#             cmd.append('order by ', order_by)
+#         if limit:
+#             cmd.append('limit ', limit)
+#         for row in Result(self.db(cmd), self.kind):
+#             yield row
+
+#     def __getattr__(self, name):
+#         return SqlTerm(name)
+
+#     def get(self, ids):
+#         cmd = f'select * from {self.table_name} where id in %s'
+#         as_list = True
+#         if isinstance(ids, (int, str)):
+#             ids = list(map(int, [ids]))
+#             as_list = False
+#         rows = Result(self.db(cmd, ids), self.kind)
+#         if as_list:
+#             return rows
+#         else:
+#             if rows:
+#                 return list(rows)[0]
+
+#     def __getitem__(self, key):
+#         return self.get(key)
+
+#     def __str__(self):
+#         return str(gitdata.table_of(self.kind, name=self.table_name, db=self.db))
+
+
+class SqlQuery:
+
+    def __init__(self, db, pathname):
+        self.db = db
+        self.pathname = pathname
+        self.kind = Record
+
+    def __call__(self, *args, **kwargs):
+        code = load(self.pathname)
+        return Result(self.db(code, *args or kwargs), self.kind)
+
+    def __iter__(self):
+        for row in self():
+            yield row
+
+    # def to(self, kind):
+    #     for item in self:
+    #         return kind(item)
+
+    # def execute(self, *args, **kwargs):
+    #     code = load(self.pathname)
+    #     cursor = self.db.cursor()
+    #     result = cursor.execute(code, *args or kwargs)
+    #     return result
+
+    # def __str__(self):
+    #     return str(self())
+
+
+class SQL:
+    """SQL Class
+
+    The SQL class provides a convenient way to work with SQL databases.  It
+    provides access to database tables and can load and execute SQL queries
+    stored in .sql files by referring to the name of the file.  Parameters
+    can be passed to SQL queries as required by treating the attribute as
+    a callable attribute.
+    """
+
+    def __init__(self, db, path='sql'):
+        self.db = db
+        if os.path.isfile(path):
+            dirname = os.path.dirname(path)
+        elif os.path.isdir(path):
+            dirname = path
+        else:
+            raise Exception(f'{path} is not a directory or filename')
+
+        if os.path.isdir(os.path.join(dirname, 'sql')):
+            path = os.path.join(dirname, 'sql')
+        else:
+            path = dirname
+
+        self.path = os.path.realpath(path)
+
+    def __getattr__(self, name):
+        pathname = os.path.join(self.path, name + '.sql')
+
+        if os.path.isfile(pathname):
+            return SqlQuery(self.db, pathname)
+
+        elif name in self.db.table_names:
+            return table_of(Record, name=name, db=self.db)
+
+
+def get_sql(path=None, db=None):
+    """Return a SQL object"""
+
+    path = path or os.path.abspath((inspect.stack()[1])[1])
+
+    db = db or gitdata.database.connect()
+    return SQL(db, path)

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata/utils.py

@@ -842,3 +842,12 @@ class OrderedSet(collections.MutableSet):
         if isinstance(other, OrderedSet):
             return len(self) == len(other) and list(self) == list(other)
         return set(self) == set(other)
+
+
+def load(pathname, encoding='utf-8'):
+    """Read a file and return the contents"""
+
+    logger = logging.getLogger(__name__)
+    logger.debug('load %r', pathname)
+    with open(pathname, encoding=encoding) as reader:
+        return reader.read()

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12/gitdata_lib.egg-info}/PKG-INFO

@@ -1,12 +1,10 @@
 Metadata-Version: 2.1
 Name: gitdata-lib
-Version: 0.0.11
+Version: 0.0.12
 Summary: Data extraction and analysis library
 Home-page: https://github.com/gitdata/gitdata-lib
 Author: DSI Labs
 Author-email: support@gitdata.com
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: Development Status :: 1 - Planning
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -22,11 +20,17 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Database :: Front-Ends
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: python-decouple>=3.4
+Requires-Dist: Unipath>=1.1
+Requires-Dist: PyMySQL==0.10.1
+Requires-Dist: psycopg2-binary==2.9.1
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Requires-Dist: docopt
+Requires-Dist: faker
 
 # GitData Lib
 Data Wrangling for Everyone.
 
 GitData is a fast, scalable, distributed data exploration system
 with a rich set of commands that provide ways to gather, manage and query data in an unusually straightforward way.
-
-

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata_lib.egg-info/SOURCES.txt

@@ -14,14 +14,17 @@ gitdata/json.py
 gitdata/queues.py
 gitdata/repositories.py
 gitdata/solutions.py
+gitdata/sql.py
 gitdata/utils.py
 gitdata/assets/README.md
 gitdata/cli/__init__.py
 gitdata/cli/gitdata_get.py
+gitdata/cli/gitdata_scan.py
 gitdata/connectors/__init__.py
 gitdata/connectors/common.py
 gitdata/connectors/fake.py
 gitdata/connectors/gitlab.py
+gitdata/connectors/http.py
 gitdata/connectors/local.py
 gitdata/database/__init__.py
 gitdata/database/common.py

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata_lib.egg-info/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 gitdata = gitdata.cli:main
-

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/gitdata_lib.egg-info/requires.txt

@@ -5,5 +5,4 @@ psycopg2-binary==2.9.1
 python-dotenv
 requests
 docopt
-webdavclient3>=3.14.5
 faker

{gitdata-lib-0.0.11 → gitdata_lib-0.0.12}/requirements.txt

@@ -5,5 +5,4 @@ psycopg2-binary==2.9.1
 python-dotenv
 requests
 docopt
-webdavclient3>=3.14.5
 faker

gitdata-lib-0.0.11/gitdata/__version__.py

@@ -1 +0,0 @@
-__version__ = '0.0.11'

gitdata-lib-0.0.11/gitdata/connectors/__init__.py

@@ -1,5 +0,0 @@
-"""
-    connectors
-"""
-
-from .common import explore, connect, fetch