data-finder 0.1.0__tar.gz

data_finder-0.1.0/PKG-INFO

@@ -0,0 +1,38 @@
+Metadata-Version: 2.4
+Name: data-finder
+Version: 0.1.0
+Summary: Model driven data finders
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: atpublic==4.1.0
+Requires-Dist: bidict==0.23.1
+Requires-Dist: duckdb==1.0.0
+Requires-Dist: greenlet==3.0.3
+Requires-Dist: ibis-framework==9.0.0
+Requires-Dist: iniconfig==2.1.0
+Requires-Dist: jinja2==3.1.6
+Requires-Dist: markdown-it-py==3.0.0
+Requires-Dist: markupsafe==3.0.2
+Requires-Dist: mdurl==0.1.2
+Requires-Dist: numpy==1.26.4
+Requires-Dist: packaging==25.0
+Requires-Dist: pandas==2.2.2
+Requires-Dist: parsy==2.1
+Requires-Dist: pluggy==1.6.0
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: pyarrow-hotfix==0.7
+Requires-Dist: pygments==2.19.1
+Requires-Dist: pytest==8.3.4
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: pytz==2024.1
+Requires-Dist: rich==13.9.4
+Requires-Dist: six==1.16.0
+Requires-Dist: sqlglot==23.12.2
+Requires-Dist: toml==0.10.2
+Requires-Dist: toolz==0.12.1
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: tzdata==2024.1
+
+# Experimental typed finders
+
+This generates helper classes to query data using a logical model and mapping

data_finder-0.1.0/README.md

@@ -0,0 +1,3 @@
+# Experimental typed finders
+
+This generates helper classes to query data using a logical model and mapping

data_finder-0.1.0/calc/src/calc/calc_protocol.py

@@ -0,0 +1,56 @@
+from abc import ABC
+
+from numpy import ndarray
+
+from datafinder import Attribute
+
+
+class DomainAwareCalcProtocol:
+    def name(self):
+        raise NotImplementedError
+
+    # TODO should this be an ordered set ? to avoid duplicates
+    def inputs_spec(self) -> list[Attribute]:
+        raise NotImplementedError
+
+    def output_spec(self) -> Attribute:
+        raise NotImplementedError
+
+    #TODO should have key and time?
+    def calculate(self, inputs:ndarray) -> []:
+        raise NotImplementedError
+
+
+# From https://charlesreid1.github.io/python-patterns-the-registry.html
+class RegistryBase(type):
+
+    REGISTRY = {}
+
+    def __new__(cls, name, bases, attrs):
+        # instantiate a new type corresponding to the type of class being defined
+        # this is currently RegisterBase but in child classes will be the child class
+        new_cls = type.__new__(cls, name, bases, attrs)
+        cls.REGISTRY[new_cls.__name__] = new_cls
+        return new_cls
+
+    @classmethod
+    def get_registry(cls):
+        return dict(cls.REGISTRY)
+
+
+class BaseRegisteredClass(metaclass=RegistryBase):
+    pass
+
+
+class CalcEngineRegistry(metaclass=RegistryBase):
+    calcs = {}
+
+    @staticmethod
+    def register(calc:DomainAwareCalcProtocol):
+        CalcEngineRegistry.calcs[calc.name()] = calc
+
+
+class DomainAwareCalc(DomainAwareCalcProtocol, ABC):
+
+    def __init__(self):
+        CalcEngineRegistry.register(self)

data_finder-0.1.0/calc/src/calc/simple_price.py

@@ -0,0 +1,24 @@
+from calc.calc_protocol import DomainAwareCalc
+from contractualposition_finder import ContractualPositionFinder
+from datafinder import Attribute
+from numpy import array
+
+
+def npv(quantity: array, price: array):
+    return quantity * price
+
+
+class PositionNPVCalc(DomainAwareCalc):
+
+    def name(self) -> str:
+        return 'npv'
+
+    def inputs_spec(self) -> list[Attribute]:
+        return [ContractualPositionFinder.quantity(),
+                ContractualPositionFinder.instrument().price()]
+
+    def calculate(self, inputs):
+        return npv(inputs[0], inputs[1])
+
+    def output_spec(self) -> Attribute:
+        return ContractualPositionFinder.npv()

data_finder-0.1.0/calc/tests/test_calc.py

@@ -0,0 +1,52 @@
+import datetime
+
+import duckdb
+import numpy
+
+from calc.calc_protocol import CalcEngineRegistry
+from numpy.testing import assert_array_almost_equal
+
+from datafinder import QueryRunnerBase
+from mappings import generate_mappings
+
+
+class TestCalc:
+
+    def setup(self):
+        #Register the Ibis engine
+        from datafinder_ibis.ibis_engine import IbisConnect
+        assert QueryRunnerBase.get_runner() == IbisConnect
+
+        generate_mappings()
+        con = duckdb.connect('test.db')
+        con.execute("DROP TABLE IF EXISTS contractualposition;")
+        con.execute(
+            "CREATE TABLE contractualposition(DATE DATE, INSTRUMENT VARCHAR, CPTY_ID INT, QUANTITY DOUBLE); COPY contractualposition FROM 'data/contractualpositions.csv'")
+        con.sql("SELECT * from contractualposition").show()
+
+        con.execute("DROP TABLE IF EXISTS price;")
+        con.execute(
+            "CREATE TABLE price(DATE_TIME DATETIME, SYM VARCHAR, PRICE DOUBLE); COPY price FROM 'data/prices.csv'")
+
+    def test_price(self):
+        self.setup()
+        # TODO - we have to import this first for it to register due to Python's dynamic nature
+        # need to do imports to force the load = https://stackoverflow.com/questions/73829483/register-classes-in-different-files-to-a-class-factory
+        from calc.simple_price import PositionNPVCalc
+        calc = PositionNPVCalc()
+        assert len(CalcEngineRegistry.calcs) == 1
+
+        inputs = calc.inputs_spec()
+
+        #Calc run
+        from contractualposition_finder import ContractualPositionFinder
+        input_data = ContractualPositionFinder.find_all(datetime.date.today(), datetime.date.today(), "LATEST",
+                                           inputs).to_numpy()
+
+        output = calc.calculate(input_data)
+
+        assert_array_almost_equal(output, numpy.array([200000.0, 54999.95]), decimal=2)
+
+
+
+

data_finder-0.1.0/data_finder.egg-info/PKG-INFO

@@ -0,0 +1,38 @@
+Metadata-Version: 2.4
+Name: data-finder
+Version: 0.1.0
+Summary: Model driven data finders
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: atpublic==4.1.0
+Requires-Dist: bidict==0.23.1
+Requires-Dist: duckdb==1.0.0
+Requires-Dist: greenlet==3.0.3
+Requires-Dist: ibis-framework==9.0.0
+Requires-Dist: iniconfig==2.1.0
+Requires-Dist: jinja2==3.1.6
+Requires-Dist: markdown-it-py==3.0.0
+Requires-Dist: markupsafe==3.0.2
+Requires-Dist: mdurl==0.1.2
+Requires-Dist: numpy==1.26.4
+Requires-Dist: packaging==25.0
+Requires-Dist: pandas==2.2.2
+Requires-Dist: parsy==2.1
+Requires-Dist: pluggy==1.6.0
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: pyarrow-hotfix==0.7
+Requires-Dist: pygments==2.19.1
+Requires-Dist: pytest==8.3.4
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: pytz==2024.1
+Requires-Dist: rich==13.9.4
+Requires-Dist: six==1.16.0
+Requires-Dist: sqlglot==23.12.2
+Requires-Dist: toml==0.10.2
+Requires-Dist: toolz==0.12.1
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: tzdata==2024.1
+
+# Experimental typed finders
+
+This generates helper classes to query data using a logical model and mapping

data_finder-0.1.0/data_finder.egg-info/SOURCES.txt

@@ -0,0 +1,73 @@
+README.md
+pyproject.toml
+./calc/src/calc/__init__.py
+./calc/src/calc/calc_protocol.py
+./calc/src/calc/simple_price.py
+./calc/tests/test_calc.py
+./datafinder/src/datafinder/__init__.py
+./datafinder/src/datafinder/attribute.py
+./datafinder/src/datafinder/finder.py
+./datafinder/src/datafinder/operation.py
+./datafinder/src/datafinder/output.py
+./datafinder/src/datafinder/runner.py
+./datafinder/src/datafinder/typed_attributes.py
+./datafinder/src/datafinder/typed_operations.py
+./datafinder_duckdb/__init__.py
+./datafinder_duckdb/duckdb_engine.py
+./datafinder_generator/src/datafinder_generator/generator.py
+./datafinder_ibis/src/datafinder_ibis/__init__.py
+./datafinder_ibis/src/datafinder_ibis/ibis_engine.py
+./datafinder_ibis/tests/test_ibis_engine.py
+./datafinder_ibis_duckdb/tests/test_datafinder_ibis_duckdb.py
+./example/__init__.py
+./example/account_finder.py
+./example/contractualposition_finder.py
+./example/instrument_finder.py
+./example/mappings.py
+./example/queries.py
+./example/trade_finder.py
+./example_duckdb/__init__.py
+./example_duckdb/duckdb_example.py
+./example_duckdb/duckdb_trade_finder.py
+./model/src/model/__init__.py
+./model/src/model/m3.py
+./model/src/model/mapping.py
+./model/src/model/relational.py
+calc/src/calc/__init__.py
+calc/src/calc/calc_protocol.py
+calc/src/calc/simple_price.py
+calc/tests/test_calc.py
+data_finder.egg-info/PKG-INFO
+data_finder.egg-info/SOURCES.txt
+data_finder.egg-info/dependency_links.txt
+data_finder.egg-info/requires.txt
+data_finder.egg-info/top_level.txt
+datafinder/src/datafinder/__init__.py
+datafinder/src/datafinder/attribute.py
+datafinder/src/datafinder/finder.py
+datafinder/src/datafinder/operation.py
+datafinder/src/datafinder/output.py
+datafinder/src/datafinder/runner.py
+datafinder/src/datafinder/typed_attributes.py
+datafinder/src/datafinder/typed_operations.py
+datafinder_duckdb/__init__.py
+datafinder_duckdb/duckdb_engine.py
+datafinder_generator/src/datafinder_generator/generator.py
+datafinder_ibis/src/datafinder_ibis/__init__.py
+datafinder_ibis/src/datafinder_ibis/ibis_engine.py
+datafinder_ibis/tests/test_ibis_engine.py
+datafinder_ibis_duckdb/tests/test_datafinder_ibis_duckdb.py
+example/__init__.py
+example/account_finder.py
+example/contractualposition_finder.py
+example/instrument_finder.py
+example/mappings.py
+example/queries.py
+example/trade_finder.py
+example_duckdb/__init__.py
+example_duckdb/duckdb_example.py
+example_duckdb/duckdb_trade_finder.py
+model/src/model/__init__.py
+model/src/model/m3.py
+model/src/model/mapping.py
+model/src/model/relational.py

data_finder-0.1.0/data_finder.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data_finder-0.1.0/data_finder.egg-info/requires.txt

@@ -0,0 +1,28 @@
+atpublic==4.1.0
+bidict==0.23.1
+duckdb==1.0.0
+greenlet==3.0.3
+ibis-framework==9.0.0
+iniconfig==2.1.0
+jinja2==3.1.6
+markdown-it-py==3.0.0
+markupsafe==3.0.2
+mdurl==0.1.2
+numpy==1.26.4
+packaging==25.0
+pandas==2.2.2
+parsy==2.1
+pluggy==1.6.0
+pyarrow==16.1.0
+pyarrow-hotfix==0.7
+pygments==2.19.1
+pytest==8.3.4
+python-dateutil==2.9.0.post0
+pytz==2024.1
+rich==13.9.4
+six==1.16.0
+sqlglot==23.12.2
+toml==0.10.2
+toolz==0.12.1
+typing-extensions==4.12.2
+tzdata==2024.1

data_finder-0.1.0/data_finder.egg-info/top_level.txt

@@ -0,0 +1 @@
+data-finder

data_finder-0.1.0/datafinder/src/datafinder/__init__.py

@@ -0,0 +1,6 @@
+from .attribute import *
+from .operation import *
+from .typed_attributes import *
+from .typed_operations import *
+from .output import *
+from .runner import *

data_finder-0.1.0/datafinder/src/datafinder/attribute.py

@@ -0,0 +1,26 @@
+from typing import Any
+
+
+class Attribute:
+    __name: str
+    __column_db_type: str
+    __owner: str
+    __parent: Any
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        self.__name = name
+        self.__column_db_type = column_db_type
+        self.__owner = owner
+        self.__parent = parent
+
+    def column_name(self) -> str:
+        return self.__name
+
+    def column_type(self) -> str:
+        return self.__column_db_type
+
+    def owner(self) -> str:
+        return self.__owner
+
+    def parent(self) -> Any:
+        return self.__parent

data_finder-0.1.0/datafinder/src/datafinder/finder.py

@@ -0,0 +1,7 @@
+from datafinder import JoinOperation, Attribute
+
+
+class RelatedFinder:
+    def __init__(self, source: Attribute, target: Attribute):
+        self.__join = JoinOperation(source, target)
+

data_finder-0.1.0/datafinder/src/datafinder/operation.py

@@ -0,0 +1,162 @@
+import datetime
+
+from datafinder.attribute import Attribute
+
+
+class TableAlias:
+    def __init__(self, table: str, alias: str):
+        self.table = table
+        self.alias = alias
+
+
+class ColumnAlias:
+    def __init__(self, column_name: str, table_alias: TableAlias):
+        self.column_name = column_name
+        self.table_alias = table_alias
+
+
+class Join:
+    def __init__(self, source: ColumnAlias, target: ColumnAlias):
+        self.source = source
+        self.target = target
+
+
+class QueryEngine:
+    _select: list[ColumnAlias]
+    _from: set[TableAlias]
+    _where: list[str]
+    _join: list[Join]
+    __table_alias_incr: int
+
+    def __init__(self):
+        self._where = []
+        self._select = []
+        self._from = set()
+        self._join = []
+        self.__table_alias_incr = 0
+        self.__table_aliases_by_table = {}
+
+    def select(self, cols: list[Attribute]):
+        for col in cols:
+            table = col.owner()
+            ta = self.__table_alias_for_table(table)
+            parent: JoinOperation = col.parent()
+            if parent is not None:
+                left = parent.left
+                sc = ColumnAlias(left.column_name(), self.__table_alias_for_table(left.owner()))
+                right = parent.right
+                tc = ColumnAlias(right.column_name(), self.__table_alias_for_table(right.owner()))
+                self._join.append(Join(sc, tc))
+            else:
+                self._from.add(ta)
+            ca = ColumnAlias(col.column_name(), ta)
+            self._select.append(ca)
+
+    def __table_alias_for_table(self, table: str) -> TableAlias:
+        ta = None
+        if table in self.__table_aliases_by_table:
+            ta = self.__table_aliases_by_table[table]
+        else:
+            ta = TableAlias(table, "t" + str(self.__table_alias_incr))
+            self.__table_alias_incr = self.__table_alias_incr + 1
+            self.__table_aliases_by_table[table] = ta
+        return ta
+
+    def append_where_binary_clause(self, op: str):
+        self._where.append(op)
+
+    def append_where_clause(self, attr: Attribute, op: str, value: str):
+        ta = self.__table_alias_for_table(attr.owner())
+        self._where.append(ta.alias + '.' + attr.column_name() + ' ' + op + ' ' + value)
+
+    def build_query_string(self) -> str:
+        joins = map(lambda j: ' LEFT OUTER JOIN ' + j.target.table_alias.table + ' AS ' + j.target.table_alias.alias +
+                              ' ON ' + j.source.table_alias.alias + '.' + j.source.column_name + ' = ' +
+                              j.target.table_alias.alias + '.' + j.target.column_name, self._join)
+        return 'SELECT ' + ','.join(map(lambda ca: ca.table_alias.alias + '.' + ca.column_name, self._select)) \
+            + ' FROM ' + ','.join(map(lambda ta: ta.table + ' AS ' + ta.alias, self._from)) \
+            + ''.join(joins) \
+            + self.__build_where()
+
+    def __build_where(self) -> str:
+        if len(self._where) > 0:
+            return ' WHERE ' + ''.join(self._where)
+        else:
+            return ''
+
+    def where_clauses(self):
+        return self._where
+
+    def start_and(self):
+        pass
+
+    def end_and(self):
+        pass
+
+
+# Interface
+class Operation:
+
+    def generate_query(self, query: QueryEngine):
+        pass
+
+
+class NoOperation(Operation):
+    def __init__(self):
+        pass
+
+
+class SelectOperation(Operation):
+    def __init__(self, display: list[Attribute], table: str, filter: Operation):
+        self.__display = display
+        self.__table = table
+        self.__filter = filter
+
+    def generate_query(self, qe: QueryEngine):
+        qe.select(self.__display)
+        self.__filter.generate_query(qe)
+
+
+class AndOperation(Operation):
+    __left: Operation
+    __right: Operation
+
+    def __init__(self, lhs: Operation, rhs: Operation):
+        self.__left = lhs
+        self.__right = rhs
+
+    def generate_query(self, query: QueryEngine):
+        query.start_and()
+        self.__left.generate_query(query)
+        query.append_where_binary_clause(" and ")
+        self.__right.generate_query(query)
+        query.end_and()
+
+
+class BusinessTemporalOperation(Operation):
+    # TODO - which date format should we use
+    __business_date_from_inclusive: datetime.date
+    __business_date_to_inclusive: datetime.date
+
+
+class BaseOperation(Operation):
+
+    def and_op(self, rhs: Operation):
+        return AndOperation(self, rhs)
+
+
+class JoinOperation(Operation):
+    left: Attribute
+    right: Attribute
+
+    def __init__(self, lhs: Attribute, rhs: Attribute):
+        self.left = lhs
+        self.right = rhs
+
+
+def select_sql_to_string(columns: list[Attribute], table: str, op: Operation) -> str:
+    qe = QueryEngine()
+    select = SelectOperation(columns, table, op)
+    select.generate_query(qe)
+    return qe.build_query_string()
+

data_finder-0.1.0/datafinder/src/datafinder/output.py

@@ -0,0 +1,15 @@
+import numpy as np
+import pandas as pd
+
+
+class ToNumpy:
+
+    def to_numpy(self) -> np.array:
+        pass
+  
+    
+class DataFrame(ToNumpy):
+
+    def to_pandas(self) -> pd.DataFrame:
+        pass
+

data_finder-0.1.0/datafinder/src/datafinder/runner.py

@@ -0,0 +1,30 @@
+from datafinder import Attribute, Operation, DataFrame
+
+
+class RegistryBase(type):
+    REGISTRY = {}
+
+    def __new__(cls, name, bases, attrs):
+        # instantiate a new type corresponding to the type of class being defined
+        # this is currently RegisterBase but in child classes will be the child class
+        new_cls = type.__new__(cls, name, bases, attrs)
+        cls.REGISTRY[new_cls.__name__] = new_cls
+        return new_cls
+
+    @classmethod
+    def get_registry(cls):
+        return dict(cls.REGISTRY)
+
+
+class QueryRunnerBase(metaclass=RegistryBase):
+
+    @staticmethod
+    def select(columns: list[Attribute], table: str, op: Operation) -> DataFrame:
+        pass
+
+    @staticmethod
+    def get_runner():
+        for k in RegistryBase.REGISTRY.keys():
+            if k != 'QueryRunnerBase':
+                return RegistryBase.REGISTRY[k]
+        raise Exception("No query runner registered")

data_finder-0.1.0/datafinder/src/datafinder/typed_attributes.py

@@ -0,0 +1,43 @@
+from .operation import *
+from .typed_operations import *
+
+
+class StringAttribute(Attribute):
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        super().__init__(name, column_db_type, owner, parent)
+
+    def eq(self, value: str) -> Operation:
+        return StringEqOperation(self, value)
+
+    def __eq__(self, value: str) -> Operation:
+        return StringEqOperation(self, value)
+
+
+class FloatAttribute(Attribute):
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        super().__init__(name, column_db_type, owner, parent)
+
+    def eq(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __eq__(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __gt__(self, value: float) -> Operation:
+        return PrimitiveGreaterThanOperation(self, value)
+
+class IntegerAttribute(Attribute):
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        super().__init__(name, column_db_type, owner, parent)
+
+    def eq(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __eq__(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __gt__(self, value: float) -> Operation:
+        return PrimitiveGreaterThanOperation(self, value)