data-finder 0.1.0__py3-none-any.whl

data-finder/calc/src/calc/calc_protocol.py

@@ -0,0 +1,56 @@
+from abc import ABC
+
+from numpy import ndarray
+
+from datafinder import Attribute
+
+
+class DomainAwareCalcProtocol:
+    def name(self):
+        raise NotImplementedError
+
+    # TODO should this be an ordered set ? to avoid duplicates
+    def inputs_spec(self) -> list[Attribute]:
+        raise NotImplementedError
+
+    def output_spec(self) -> Attribute:
+        raise NotImplementedError
+
+    #TODO should have key and time?
+    def calculate(self, inputs:ndarray) -> []:
+        raise NotImplementedError
+
+
+# From https://charlesreid1.github.io/python-patterns-the-registry.html
+class RegistryBase(type):
+
+    REGISTRY = {}
+
+    def __new__(cls, name, bases, attrs):
+        # instantiate a new type corresponding to the type of class being defined
+        # this is currently RegisterBase but in child classes will be the child class
+        new_cls = type.__new__(cls, name, bases, attrs)
+        cls.REGISTRY[new_cls.__name__] = new_cls
+        return new_cls
+
+    @classmethod
+    def get_registry(cls):
+        return dict(cls.REGISTRY)
+
+
+class BaseRegisteredClass(metaclass=RegistryBase):
+    pass
+
+
+class CalcEngineRegistry(metaclass=RegistryBase):
+    calcs = {}
+
+    @staticmethod
+    def register(calc:DomainAwareCalcProtocol):
+        CalcEngineRegistry.calcs[calc.name()] = calc
+
+
+class DomainAwareCalc(DomainAwareCalcProtocol, ABC):
+
+    def __init__(self):
+        CalcEngineRegistry.register(self)

data-finder/calc/src/calc/simple_price.py

@@ -0,0 +1,24 @@
+from calc.calc_protocol import DomainAwareCalc
+from contractualposition_finder import ContractualPositionFinder
+from datafinder import Attribute
+from numpy import array
+
+
+def npv(quantity: array, price: array):
+    return quantity * price
+
+
+class PositionNPVCalc(DomainAwareCalc):
+
+    def name(self) -> str:
+        return 'npv'
+
+    def inputs_spec(self) -> list[Attribute]:
+        return [ContractualPositionFinder.quantity(),
+                ContractualPositionFinder.instrument().price()]
+
+    def calculate(self, inputs):
+        return npv(inputs[0], inputs[1])
+
+    def output_spec(self) -> Attribute:
+        return ContractualPositionFinder.npv()

data-finder/calc/tests/test_calc.py

@@ -0,0 +1,52 @@
+import datetime
+
+import duckdb
+import numpy
+
+from calc.calc_protocol import CalcEngineRegistry
+from numpy.testing import assert_array_almost_equal
+
+from datafinder import QueryRunnerBase
+from mappings import generate_mappings
+
+
+class TestCalc:
+
+    def setup(self):
+        #Register the Ibis engine
+        from datafinder_ibis.ibis_engine import IbisConnect
+        assert QueryRunnerBase.get_runner() == IbisConnect
+
+        generate_mappings()
+        con = duckdb.connect('test.db')
+        con.execute("DROP TABLE IF EXISTS contractualposition;")
+        con.execute(
+            "CREATE TABLE contractualposition(DATE DATE, INSTRUMENT VARCHAR, CPTY_ID INT, QUANTITY DOUBLE); COPY contractualposition FROM 'data/contractualpositions.csv'")
+        con.sql("SELECT * from contractualposition").show()
+
+        con.execute("DROP TABLE IF EXISTS price;")
+        con.execute(
+            "CREATE TABLE price(DATE_TIME DATETIME, SYM VARCHAR, PRICE DOUBLE); COPY price FROM 'data/prices.csv'")
+
+    def test_price(self):
+        self.setup()
+        # TODO - we have to import this first for it to register due to Python's dynamic nature
+        # need to do imports to force the load = https://stackoverflow.com/questions/73829483/register-classes-in-different-files-to-a-class-factory
+        from calc.simple_price import PositionNPVCalc
+        calc = PositionNPVCalc()
+        assert len(CalcEngineRegistry.calcs) == 1
+
+        inputs = calc.inputs_spec()
+
+        #Calc run
+        from contractualposition_finder import ContractualPositionFinder
+        input_data = ContractualPositionFinder.find_all(datetime.date.today(), datetime.date.today(), "LATEST",
+                                           inputs).to_numpy()
+
+        output = calc.calculate(input_data)
+
+        assert_array_almost_equal(output, numpy.array([200000.0, 54999.95]), decimal=2)
+
+
+
+

data-finder/datafinder/src/datafinder/__init__.py

@@ -0,0 +1,6 @@
+from .attribute import *
+from .operation import *
+from .typed_attributes import *
+from .typed_operations import *
+from .output import *
+from .runner import *

data-finder/datafinder/src/datafinder/attribute.py

@@ -0,0 +1,26 @@
+from typing import Any
+
+
+class Attribute:
+    __name: str
+    __column_db_type: str
+    __owner: str
+    __parent: Any
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        self.__name = name
+        self.__column_db_type = column_db_type
+        self.__owner = owner
+        self.__parent = parent
+
+    def column_name(self) -> str:
+        return self.__name
+
+    def column_type(self) -> str:
+        return self.__column_db_type
+
+    def owner(self) -> str:
+        return self.__owner
+
+    def parent(self) -> Any:
+        return self.__parent

data-finder/datafinder/src/datafinder/finder.py

@@ -0,0 +1,7 @@
+from datafinder import JoinOperation, Attribute
+
+
+class RelatedFinder:
+    def __init__(self, source: Attribute, target: Attribute):
+        self.__join = JoinOperation(source, target)
+

data-finder/datafinder/src/datafinder/operation.py

@@ -0,0 +1,162 @@
+import datetime
+
+from datafinder.attribute import Attribute
+
+
+class TableAlias:
+    def __init__(self, table: str, alias: str):
+        self.table = table
+        self.alias = alias
+
+
+class ColumnAlias:
+    def __init__(self, column_name: str, table_alias: TableAlias):
+        self.column_name = column_name
+        self.table_alias = table_alias
+
+
+class Join:
+    def __init__(self, source: ColumnAlias, target: ColumnAlias):
+        self.source = source
+        self.target = target
+
+
+class QueryEngine:
+    _select: list[ColumnAlias]
+    _from: set[TableAlias]
+    _where: list[str]
+    _join: list[Join]
+    __table_alias_incr: int
+
+    def __init__(self):
+        self._where = []
+        self._select = []
+        self._from = set()
+        self._join = []
+        self.__table_alias_incr = 0
+        self.__table_aliases_by_table = {}
+
+    def select(self, cols: list[Attribute]):
+        for col in cols:
+            table = col.owner()
+            ta = self.__table_alias_for_table(table)
+            parent: JoinOperation = col.parent()
+            if parent is not None:
+                left = parent.left
+                sc = ColumnAlias(left.column_name(), self.__table_alias_for_table(left.owner()))
+                right = parent.right
+                tc = ColumnAlias(right.column_name(), self.__table_alias_for_table(right.owner()))
+                self._join.append(Join(sc, tc))
+            else:
+                self._from.add(ta)
+            ca = ColumnAlias(col.column_name(), ta)
+            self._select.append(ca)
+
+    def __table_alias_for_table(self, table: str) -> TableAlias:
+        ta = None
+        if table in self.__table_aliases_by_table:
+            ta = self.__table_aliases_by_table[table]
+        else:
+            ta = TableAlias(table, "t" + str(self.__table_alias_incr))
+            self.__table_alias_incr = self.__table_alias_incr + 1
+            self.__table_aliases_by_table[table] = ta
+        return ta
+
+    def append_where_binary_clause(self, op: str):
+        self._where.append(op)
+
+    def append_where_clause(self, attr: Attribute, op: str, value: str):
+        ta = self.__table_alias_for_table(attr.owner())
+        self._where.append(ta.alias + '.' + attr.column_name() + ' ' + op + ' ' + value)
+
+    def build_query_string(self) -> str:
+        joins = map(lambda j: ' LEFT OUTER JOIN ' + j.target.table_alias.table + ' AS ' + j.target.table_alias.alias +
+                              ' ON ' + j.source.table_alias.alias + '.' + j.source.column_name + ' = ' +
+                              j.target.table_alias.alias + '.' + j.target.column_name, self._join)
+        return 'SELECT ' + ','.join(map(lambda ca: ca.table_alias.alias + '.' + ca.column_name, self._select)) \
+            + ' FROM ' + ','.join(map(lambda ta: ta.table + ' AS ' + ta.alias, self._from)) \
+            + ''.join(joins) \
+            + self.__build_where()
+
+    def __build_where(self) -> str:
+        if len(self._where) > 0:
+            return ' WHERE ' + ''.join(self._where)
+        else:
+            return ''
+
+    def where_clauses(self):
+        return self._where
+
+    def start_and(self):
+        pass
+
+    def end_and(self):
+        pass
+
+
+# Interface
+class Operation:
+
+    def generate_query(self, query: QueryEngine):
+        pass
+
+
+class NoOperation(Operation):
+    def __init__(self):
+        pass
+
+
+class SelectOperation(Operation):
+    def __init__(self, display: list[Attribute], table: str, filter: Operation):
+        self.__display = display
+        self.__table = table
+        self.__filter = filter
+
+    def generate_query(self, qe: QueryEngine):
+        qe.select(self.__display)
+        self.__filter.generate_query(qe)
+
+
+class AndOperation(Operation):
+    __left: Operation
+    __right: Operation
+
+    def __init__(self, lhs: Operation, rhs: Operation):
+        self.__left = lhs
+        self.__right = rhs
+
+    def generate_query(self, query: QueryEngine):
+        query.start_and()
+        self.__left.generate_query(query)
+        query.append_where_binary_clause(" and ")
+        self.__right.generate_query(query)
+        query.end_and()
+
+
+class BusinessTemporalOperation(Operation):
+    # TODO - which date format should we use
+    __business_date_from_inclusive: datetime.date
+    __business_date_to_inclusive: datetime.date
+
+
+class BaseOperation(Operation):
+
+    def and_op(self, rhs: Operation):
+        return AndOperation(self, rhs)
+
+
+class JoinOperation(Operation):
+    left: Attribute
+    right: Attribute
+
+    def __init__(self, lhs: Attribute, rhs: Attribute):
+        self.left = lhs
+        self.right = rhs
+
+
+def select_sql_to_string(columns: list[Attribute], table: str, op: Operation) -> str:
+    qe = QueryEngine()
+    select = SelectOperation(columns, table, op)
+    select.generate_query(qe)
+    return qe.build_query_string()
+

data-finder/datafinder/src/datafinder/output.py

@@ -0,0 +1,15 @@
+import numpy as np
+import pandas as pd
+
+
+class ToNumpy:
+
+    def to_numpy(self) -> np.array:
+        pass
+  
+    
+class DataFrame(ToNumpy):
+
+    def to_pandas(self) -> pd.DataFrame:
+        pass
+

data-finder/datafinder/src/datafinder/runner.py

@@ -0,0 +1,30 @@
+from datafinder import Attribute, Operation, DataFrame
+
+
+class RegistryBase(type):
+    REGISTRY = {}
+
+    def __new__(cls, name, bases, attrs):
+        # instantiate a new type corresponding to the type of class being defined
+        # this is currently RegisterBase but in child classes will be the child class
+        new_cls = type.__new__(cls, name, bases, attrs)
+        cls.REGISTRY[new_cls.__name__] = new_cls
+        return new_cls
+
+    @classmethod
+    def get_registry(cls):
+        return dict(cls.REGISTRY)
+
+
+class QueryRunnerBase(metaclass=RegistryBase):
+
+    @staticmethod
+    def select(columns: list[Attribute], table: str, op: Operation) -> DataFrame:
+        pass
+
+    @staticmethod
+    def get_runner():
+        for k in RegistryBase.REGISTRY.keys():
+            if k != 'QueryRunnerBase':
+                return RegistryBase.REGISTRY[k]
+        raise Exception("No query runner registered")

data-finder/datafinder/src/datafinder/typed_attributes.py

@@ -0,0 +1,43 @@
+from .operation import *
+from .typed_operations import *
+
+
+class StringAttribute(Attribute):
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        super().__init__(name, column_db_type, owner, parent)
+
+    def eq(self, value: str) -> Operation:
+        return StringEqOperation(self, value)
+
+    def __eq__(self, value: str) -> Operation:
+        return StringEqOperation(self, value)
+
+
+class FloatAttribute(Attribute):
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        super().__init__(name, column_db_type, owner, parent)
+
+    def eq(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __eq__(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __gt__(self, value: float) -> Operation:
+        return PrimitiveGreaterThanOperation(self, value)
+
+class IntegerAttribute(Attribute):
+
+    def __init__(self, name: str, column_db_type: str, owner:str, parent=None):
+        super().__init__(name, column_db_type, owner, parent)
+
+    def eq(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __eq__(self, value: float) -> Operation:
+        return PrimitiveEqOperation(self, value)
+
+    def __gt__(self, value: float) -> Operation:
+        return PrimitiveGreaterThanOperation(self, value)

data-finder/datafinder/src/datafinder/typed_operations.py

@@ -0,0 +1,80 @@
+from .operation import BaseOperation, QueryEngine
+from .attribute import Attribute
+
+
+class EqOperation(BaseOperation):
+    __attribute: Attribute
+
+    def __init__(self, attrib: Attribute):
+        self.__attribute = attrib
+
+    def attribute(self) -> Attribute:
+        return self.__attribute
+
+    def column_type(self) -> str:
+        return self.__attribute.column_type()
+
+    def column_name(self) -> str:
+        return self.__attribute.column_name()
+
+    def prepare_value(self) -> str:
+        pass
+
+
+class PrimitiveEqOperation(EqOperation):
+    __value: []
+
+    def __init__(self, attrib: Attribute, value):
+        super().__init__(attrib)
+        self.__value = value
+
+    def generate_query(self, query: QueryEngine):
+        query.append_where_clause(self.attribute(), '=', self.prepare_value())
+
+    def prepare_value(self) -> str:
+        return str(self.__value)
+
+
+class StringEqOperation(EqOperation):
+    __value: str
+
+    def __init__(self, attrib: Attribute, value: str):
+        super().__init__(attrib)
+        self.__value = value
+
+    def generate_query(self, query: QueryEngine):
+        query.append_where_clause(self.attribute(), 'LIKE', self.prepare_value())
+
+    def prepare_value(self) -> str:
+        #TODO - String escaper
+        if self.column_type() == 'kx_symbol':
+            return "`" + self.__value
+        else:
+            return "'" + self.__value + "'"
+
+
+class GreaterThanOperation(BaseOperation):
+    __attribute: Attribute
+
+    def column_type(self) -> str:
+        return self.__attribute.column_type()
+
+    def __init__(self, attrib: Attribute):
+        self.__attribute = attrib
+
+    def generate_query(self, query: QueryEngine):
+        query.append_where_clause(self.__attribute, '>', self.prepare_value())
+
+    def prepare_value(self) -> str:
+        pass
+
+
+class PrimitiveGreaterThanOperation(GreaterThanOperation):
+    __value: []
+
+    def __init__(self, attrib: Attribute, value):
+        super().__init__(attrib)
+        self.__value = value
+
+    def prepare_value(self) -> str:
+        return str(self.__value)

data-finder/datafinder_duckdb/duckdb_engine.py

@@ -0,0 +1,30 @@
+from datafinder import Operation, DataFrame, Attribute, select_sql_to_string
+
+import duckdb
+import numpy as np
+import pandas as pd
+
+
+class DuckDbConnect:
+
+    @staticmethod
+    def select(columns: list[Attribute], table: str, op: Operation) -> list:
+        conn = duckdb.connect('test.db')
+        query = select_sql_to_string(columns, table, op)
+        print(query)
+        # TODO this is inefficient, could convert straight to desired output - such as numpy, instead of list
+        return conn.sql(query).fetchall()
+
+
+class DuckDbOutput(DataFrame):
+    __table: list
+
+    def __init__(self, t: list):
+        self.__table = t
+
+    def to_numpy(self) -> np.array:
+        return np.array(self.__table)
+
+    def to_pandas(self) -> pd.DataFrame:
+        #todo - this needs to be better, to ensure types and column names
+        return pd.DataFrame(self.__table)

data-finder/datafinder_generator/src/datafinder_generator/generator.py

@@ -0,0 +1,25 @@
+import os
+
+from model.m3 import PrimitiveType, Property
+from jinja2 import Environment, PackageLoader
+
+from model.mapping import Mapping
+
+
+def is_primitive(prop: Property) -> bool:
+    return isinstance(prop.type, PrimitiveType)
+
+
+def generate(mapping:Mapping, output_directory):
+    environment = Environment(loader=PackageLoader("datafinder_generator"), trim_blocks=True, lstrip_blocks=True)
+    template = environment.get_template("finder_template.txt")
+
+    for rcm in mapping.mappings:
+        filename = f"{rcm.clazz.name.lower()}_finder.py"
+        filepath = os.path.join(output_directory, filename)
+        content = template.render(rcm=rcm,is_primitive=is_primitive)
+        with open(filepath, mode="w", encoding="utf-8") as message:
+            message.write(content)
+            print(f"... wrote {filename}")
+
+

data-finder/datafinder_ibis/src/datafinder_ibis/ibis_engine.py

@@ -0,0 +1,31 @@
+from datafinder import Operation, DataFrame, Attribute, select_sql_to_string
+
+import ibis
+import numpy as np
+import pandas as pd
+
+from datafinder import QueryRunnerBase
+
+
+class IbisConnect(QueryRunnerBase):
+
+    @staticmethod
+    def select(columns: list[Attribute], table: str, op: Operation) -> DataFrame:
+        conn = ibis.connect('duckdb://test.db')
+        query = select_sql_to_string(columns, table, op)
+        print(query)
+        t = conn.table(table)
+        #todo - can also do this with the dataframe API
+        return IbisOutput(t.sql(query))
+
+
+class IbisOutput(DataFrame):
+
+    def __init__(self, t: ibis.Table):
+        self.__table = t
+
+    def to_numpy(self) -> np.array:
+        return self.__table.__array__()
+
+    def to_pandas(self) -> pd.DataFrame:
+        return self.__table.to_pandas()

data-finder/datafinder_ibis/tests/test_ibis_engine.py

@@ -0,0 +1,8 @@
+from datafinder import QueryRunnerBase
+
+class TestIbisEngine:
+
+    def test_initialization(self):
+        from datafinder_ibis.ibis_engine import IbisConnect
+        out = QueryRunnerBase.get_runner()
+        assert out == IbisConnect

data-finder/datafinder_ibis_duckdb/tests/test_datafinder_ibis_duckdb.py

@@ -0,0 +1,53 @@
+import duckdb
+import datetime
+
+import numpy as np
+
+from datafinder import QueryRunnerBase
+from example import queries
+from numpy.testing import assert_array_equal
+
+from mappings import generate_mappings
+
+
+class TestDataFinderIbisDuckDb:
+
+    def setup(self):
+        #Register the Ibis engine
+        from datafinder_ibis.ibis_engine import IbisConnect
+        assert QueryRunnerBase.get_runner() == IbisConnect
+        generate_mappings()
+        con = duckdb.connect('test.db')
+        con.sql("SELECT 42 AS x").show()
+        con.execute("DROP TABLE IF EXISTS trade;")
+        con.execute(
+            "CREATE TABLE trade(id INT, account_id INT, sym VARCHAR, price DOUBLE); COPY trade FROM 'data/trades.csv'")
+        con.sql("SELECT * from trade").show()
+        con.sql("SELECT * from trade where sym LIKE 'AAPL'").show()
+
+        con.execute("DROP TABLE IF EXISTS account;")
+        con.execute(
+            "CREATE TABLE account(id INT, name VARCHAR); COPY account FROM 'data/accounts.csv'")
+
+    def test_queries(self):
+        self.setup()
+        # Import after generation, so we get the latest version
+        from trade_finder import TradeFinder
+        queries.find_trades(TradeFinder)
+        from account_finder import AccountFinder
+        np_accts = AccountFinder \
+            .find_all(datetime.date.today(), datetime.date.today(), "LATEST",
+                      [AccountFinder.id(), AccountFinder.name()],
+                      AccountFinder.id().eq(211978)) \
+            .to_numpy()
+        print(np_accts)
+        assert_array_equal(np_accts, np.array([[211978, 'Trading Account 1']],dtype=object))
+
+
+        trades_with_account = TradeFinder.find_all(datetime.date.today(), datetime.date.today(), "LATEST",
+                                                   [TradeFinder.account().name(), TradeFinder.symbol(),
+                                                    TradeFinder.price()],
+                                                   TradeFinder.symbol().eq("AAPL"))
+        np_trades = trades_with_account.to_numpy()
+        print(np_trades)
+        assert_array_equal(np_trades, np.array([['Trading Account 1', 'AAPL', 84.11]], dtype=object))

data-finder/example/account_finder.py

@@ -0,0 +1,37 @@
+from datafinder.typed_attributes import *
+from datafinder import QueryRunnerBase, DataFrame
+
+
+class AccountFinder:
+    __table = 'account'
+
+    __id = IntegerAttribute('id', 'INT', 'account')
+    __name = StringAttribute('name', 'VARCHAR', 'account')
+
+    @staticmethod
+    def id() -> IntegerAttribute:
+        return AccountFinder.__id
+
+    @staticmethod
+    def name() -> StringAttribute:
+        return AccountFinder.__name
+
+    @staticmethod
+    def find_all(date_from: datetime.date, date_to: datetime.date, as_of: str,
+                 display_columns: list[Attribute],
+                 filter_op: Operation = NoOperation()) -> DataFrame:
+        return QueryRunnerBase.get_runner().select(display_columns, AccountFinder.__table, filter_op)
+
+
+class AccountRelatedFinder:
+    def __init__(self, source: Attribute, target: Attribute):
+        join = JoinOperation(source,target)
+        self.__id = IntegerAttribute('id', 'INT', 'account', join)
+        self.__name = StringAttribute('name', 'VARCHAR', 'account', join)
+
+    def id(self) -> IntegerAttribute:
+        return self.__id
+
+    def name(self) -> StringAttribute:
+        return self.__name
+

data-finder/example/contractualposition_finder.py

@@ -0,0 +1,56 @@
+from datafinder.typed_attributes import *
+from datafinder import QueryRunnerBase, DataFrame
+from instrument_finder import InstrumentRelatedFinder
+
+
+class ContractualPositionFinder:
+    __table = 'contractualposition'
+
+    __quantity = FloatAttribute('QUANTITY', 'DOUBLE', 'contractualposition')
+    __counterparty = IntegerAttribute('CPTY_ID', 'INT', 'contractualposition')
+    __npv = FloatAttribute('NPV', 'DOUBLE', 'contractualposition')
+    __instrument = InstrumentRelatedFinder(Attribute('INSTRUMENT', 'VARCHAR', 'contractualposition'),Attribute('SYM', 'VARCHAR', 'price'))
+
+    @staticmethod
+    def quantity() -> FloatAttribute:
+        return ContractualPositionFinder.__quantity
+
+    @staticmethod
+    def counterparty() -> IntegerAttribute:
+        return ContractualPositionFinder.__counterparty
+
+    @staticmethod
+    def instrument() -> InstrumentRelatedFinder:
+        return ContractualPositionFinder.__instrument
+
+    @staticmethod
+    def npv() -> FloatAttribute:
+        return ContractualPositionFinder.__npv
+
+    @staticmethod
+    def find_all(date_from: datetime.date, date_to: datetime.date, as_of: str,
+                 display_columns: list[Attribute],
+                 filter_op: Operation = NoOperation()) -> DataFrame:
+        return QueryRunnerBase.get_runner().select(display_columns, ContractualPositionFinder.__table, filter_op)
+
+
+class ContractualPositionRelatedFinder:
+    def __init__(self, source: Attribute, target: Attribute):
+        join = JoinOperation(source,target)
+        self.__quantity = FloatAttribute('QUANTITY', 'DOUBLE', 'contractualposition', join)
+        self.__counterparty = IntegerAttribute('CPTY_ID', 'INT', 'contractualposition', join)
+        self.__npv = FloatAttribute('NPV', 'DOUBLE', 'contractualposition', join)
+        self.__instrument = InstrumentRelatedFinder(Attribute('INSTRUMENT', 'VARCHAR', 'contractualposition'),Attribute('SYM', 'VARCHAR', 'price'))
+
+    def quantity(self) -> FloatAttribute:
+        return self.__quantity
+
+    def counterparty(self) -> IntegerAttribute:
+        return self.__counterparty
+
+    def instrument(self) -> InstrumentRelatedFinder:
+        return self.__instrument
+
+    def npv(self) -> FloatAttribute:
+        return self.__npv
+

data-finder/example/instrument_finder.py

@@ -0,0 +1,37 @@
+from datafinder.typed_attributes import *
+from datafinder import QueryRunnerBase, DataFrame
+
+
+class InstrumentFinder:
+    __table = 'price'
+
+    __symbol = StringAttribute('SYM', 'VARCHAR', 'price')
+    __price = FloatAttribute('PRICE', 'DOUBLE', 'price')
+
+    @staticmethod
+    def symbol() -> StringAttribute:
+        return InstrumentFinder.__symbol
+
+    @staticmethod
+    def price() -> FloatAttribute:
+        return InstrumentFinder.__price
+
+    @staticmethod
+    def find_all(date_from: datetime.date, date_to: datetime.date, as_of: str,
+                 display_columns: list[Attribute],
+                 filter_op: Operation = NoOperation()) -> DataFrame:
+        return QueryRunnerBase.get_runner().select(display_columns, InstrumentFinder.__table, filter_op)
+
+
+class InstrumentRelatedFinder:
+    def __init__(self, source: Attribute, target: Attribute):
+        join = JoinOperation(source,target)
+        self.__symbol = StringAttribute('SYM', 'VARCHAR', 'price', join)
+        self.__price = FloatAttribute('PRICE', 'DOUBLE', 'price', join)
+
+    def symbol(self) -> StringAttribute:
+        return self.__symbol
+
+    def price(self) -> FloatAttribute:
+        return self.__price
+

data-finder/example/mappings.py

@@ -0,0 +1,106 @@
+import os
+
+from datafinder_generator.generator import generate
+from model.m3 import Class, Property, String, Float, Package, Integer, Date
+from model.mapping import Mapping
+from model.relational import Column, Table, RelationalClassMapping, RelationalPropertyMapping, Join
+
+
+def create_account_class() -> Class:
+    p1 = Property('id', Integer)
+    p2 = Property('name', String)
+
+    account_c = Class('Account', [p1, p2], Package('finance'))
+    return account_c
+
+
+def create_instrument_class() -> Class:
+    p1 = Property('symbol', String)
+    #TODO this doesn't belong here, but using for simple example
+    p2 = Property('price', Float)
+
+    instrument_c = Class('Instrument', [p1, p2], Package('finance'))
+    return instrument_c
+
+
+def create_trade_class(account:Class) -> Class:
+    p1 = Property('symbol', String)
+    p2 = Property('price', Float)
+    p3 = Property('account', account)
+
+    trade_c = Class('Trade', [p1, p2, p3], Package('finance'))
+    return trade_c
+
+
+def create_contractual_position_class(instrument:Class) -> Class:
+    p1 = Property('date', Date)
+    p2 = Property('quantity', Float)
+    p3 = Property('counterparty', Integer)
+    p4 = Property('instrument', instrument)
+    p5 = Property('npv', Float)
+    pos_c = Class('ContractualPosition', [p1, p2, p3, p4, p5], Package('finance'))
+    return pos_c
+
+
+def create_mappings_normalized() -> Mapping:
+    account_c = create_account_class()
+
+    ac1 = Column('id', 'INT')
+    ac2 = Column('name', 'VARCHAR')
+    account_t = Table('account', [ac1, ac2])
+
+    instrument_c = create_instrument_class()
+    ic1 = Column('SYM', 'VARCHAR')
+    ic2 = Column('PRICE', 'DOUBLE')
+    instrument_t = Table('price', [ic1,ic2])
+
+    c_position_c = create_contractual_position_class(instrument_c)
+    p1 = Column('DATE', 'DATE')
+    p2 = Column('INSTRUMENT', 'VARCHAR')
+    p3 = Column('CPTY_ID', 'INT')
+    p4 = Column('QUANTITY', 'DOUBLE')
+    p5 = Column('NPV', 'DOUBLE')
+    pos_t = Table('contractualposition', [p1, p2, p3, p4, p5])
+
+    trade_c = create_trade_class(account_c)
+
+    c1 = Column('id', 'INT')
+    c2 = Column('account_id', 'INT')
+    c3 = Column('sym', 'VARCHAR')
+    c4 = Column('price', 'DOUBLE')
+
+    trade_t = Table('trade', [c1, c2, c3, c4])
+
+    pm1 = RelationalPropertyMapping(trade_c.property('symbol'), c3)
+    pm2 = RelationalPropertyMapping(trade_c.property('price'), c4)
+    pm3 = RelationalPropertyMapping(trade_c.property('account'),Join(c2,ac1))
+    rm_t = RelationalClassMapping(trade_c, [pm1, pm2, pm3])
+
+    a_pm1 = RelationalPropertyMapping(account_c.property('id'), ac1)
+    a_pm2 = RelationalPropertyMapping(account_c.property('name'), ac2)
+    rm_a = RelationalClassMapping(account_c, [a_pm1, a_pm2])
+
+    i_pm1 = RelationalPropertyMapping(instrument_c.property('symbol'), ic1)
+    i_pm2 = RelationalPropertyMapping(instrument_c.property('price'), ic2)
+    rm_i = RelationalClassMapping(instrument_c, [i_pm1, i_pm2])
+
+
+    cpm1 = RelationalPropertyMapping(c_position_c.property('quantity'), p4)
+    cpm2 = RelationalPropertyMapping(c_position_c.property('counterparty'), p3)
+    cpm3 = RelationalPropertyMapping(c_position_c.property('instrument'), Join(p2,ic1))
+    cpm4 = RelationalPropertyMapping(c_position_c.property('npv'), p5)
+    rm_cp = RelationalClassMapping(c_position_c, [cpm1, cpm2, cpm3, cpm4])
+
+    return Mapping('Test Mapping 1', [rm_t,rm_a,rm_i,rm_cp])
+
+
+def generate_mappings():
+    rcms = create_mappings_normalized()
+    import sys
+    mn = sys.modules[__name__]
+    directory = os.path.dirname(mn.__file__)
+    generate(rcms, directory)
+
+
+if __name__ == '__main__':
+    generate_mappings()

data-finder/example/queries.py

@@ -0,0 +1,29 @@
+import datetime
+
+
+def find_trades(trade_finder):
+    print(f'Finding trades')
+
+    trades = trade_finder.find_all(datetime.date.today(), datetime.date.today(), "LATEST",
+                                   [trade_finder.symbol(), trade_finder.price()],
+                                   trade_finder.symbol().eq("AAPL"))
+    np_trades = trades.to_numpy()
+    print(np_trades)
+    df = trades.to_pandas()
+    print(df)
+
+    trades = trade_finder.find_all(datetime.date.today(), datetime.date.today(), "LATEST",
+                                   [trade_finder.symbol(), trade_finder.price()],
+                                   trade_finder.price() > 200.0,)
+    np_trades = trades.to_numpy()
+    print(np_trades)
+    df = trades.to_pandas()
+    print(df)
+
+    trades = trade_finder.find_all(datetime.date.today(), datetime.date.today(), "LATEST",
+                                   [trade_finder.symbol(), trade_finder.price()],
+                                   (trade_finder.symbol() == "AAPL").and_op(trade_finder.price() == 84.11))
+    np_trades = trades.to_numpy()
+    print(np_trades)
+    df = trades.to_pandas()
+    print(df)

data-finder/example/trade_finder.py

@@ -0,0 +1,47 @@
+from datafinder.typed_attributes import *
+from datafinder import QueryRunnerBase, DataFrame
+from account_finder import AccountRelatedFinder
+
+
+class TradeFinder:
+    __table = 'trade'
+
+    __symbol = StringAttribute('sym', 'VARCHAR', 'trade')
+    __price = FloatAttribute('price', 'DOUBLE', 'trade')
+    __account = AccountRelatedFinder(Attribute('account_id', 'INT', 'trade'),Attribute('id', 'INT', 'account'))
+
+    @staticmethod
+    def symbol() -> StringAttribute:
+        return TradeFinder.__symbol
+
+    @staticmethod
+    def price() -> FloatAttribute:
+        return TradeFinder.__price
+
+    @staticmethod
+    def account() -> AccountRelatedFinder:
+        return TradeFinder.__account
+
+    @staticmethod
+    def find_all(date_from: datetime.date, date_to: datetime.date, as_of: str,
+                 display_columns: list[Attribute],
+                 filter_op: Operation = NoOperation()) -> DataFrame:
+        return QueryRunnerBase.get_runner().select(display_columns, TradeFinder.__table, filter_op)
+
+
+class TradeRelatedFinder:
+    def __init__(self, source: Attribute, target: Attribute):
+        join = JoinOperation(source,target)
+        self.__symbol = StringAttribute('sym', 'VARCHAR', 'trade', join)
+        self.__price = FloatAttribute('price', 'DOUBLE', 'trade', join)
+        self.__account = AccountRelatedFinder(Attribute('account_id', 'INT', 'trade'),Attribute('id', 'INT', 'account'))
+
+    def symbol(self) -> StringAttribute:
+        return self.__symbol
+
+    def price(self) -> FloatAttribute:
+        return self.__price
+
+    def account(self) -> AccountRelatedFinder:
+        return self.__account
+

data-finder/example_duckdb/duckdb_example.py

@@ -0,0 +1,20 @@
+import duckdb
+
+from duckdb_trade_finder import *
+from example import queries
+
+def duckdb_sample():
+    con = duckdb.connect('test.db')
+    con.sql("SELECT 42 AS x").show()
+    con.execute("DROP TABLE IF EXISTS trade;")
+    con.execute(
+        "CREATE TABLE trade(id INT, account_id INT, sym VARCHAR, price DOUBLE); COPY trade FROM '../data/trades.csv'")
+    con.sql("SELECT * from trade").show()
+    con.sql("SELECT * from trade where sym LIKE 'AAPL'").show()
+
+
+
+
+if __name__ == '__main__':
+    duckdb_sample()
+    queries.find_trades(TradeFinder)

data-finder/example_duckdb/duckdb_trade_finder.py

@@ -0,0 +1,24 @@
+from datafinder.typed_attributes import *
+from datafinder_duckdb.duckdb_engine import *
+
+
+class TradeFinder:
+    __table = 'trade'
+    __symbol = StringAttribute('sym', 'char')
+    __price = FloatAttribute('price', 'double precision')
+
+    @staticmethod
+    def symbol() -> StringAttribute:
+        return TradeFinder.__symbol
+
+    @staticmethod
+    def price() -> FloatAttribute:
+        return TradeFinder.__price
+
+    @staticmethod
+    def find_all(date_from: datetime.date, date_to: datetime.date, as_of: str,
+                 filter_op: Operation,
+                 display_columns: list[Attribute]) -> DataFrame:
+
+        out = DuckDbConnect.select(display_columns, TradeFinder.__table, filter_op)
+        return DuckDbOutput(out)

data-finder/model/src/model/m3.py

@@ -0,0 +1,51 @@
+class Package:
+    def __init__(self, name: str):
+        self.name = name
+
+
+class PackagableElement:
+    def __init__(self, package: Package):
+        self.package = package
+
+
+class Type:
+    def __init(self):
+        pass
+
+
+class PrimitiveType(Type):
+    def __init__(self, name: str):
+        self.name = name
+
+
+Integer = PrimitiveType("Integer")
+String = PrimitiveType("String")
+Float = PrimitiveType("Float")
+DateTime = PrimitiveType("DateTime")
+Date = PrimitiveType("Date")
+
+
+class Property:
+    def __init__(self, name: str, type: Type):
+        self.name = name
+        self.type = type
+
+
+class Class(PackagableElement, Type):
+    def __init__(self, name: str, properties: list[Property], package: Package):
+        super().__init__(package)
+        self.properties = {}
+        self.name = name
+        for prop in properties:
+            self.properties[prop.name] = prop
+
+    def property(self, name:str) -> Property:
+        return self.properties[name]
+
+
+class Association(PackagableElement):
+    def __init__(self, name: str, source: str, target: str, package: Package):
+        super().__init__(package)
+        self.name = name
+        self.source = source
+        self.target = target

data-finder/model/src/model/mapping.py

@@ -0,0 +1,19 @@
+from typing import Any
+
+from model.m3 import Class, Property
+
+
+class PropertyMapping:
+    def __init__(self, property: Property, target: Any):
+        self.property = property
+        self.target = target
+
+class ClassMapping:
+    def __init__(self, clazz: Class, property_mappings: list[PropertyMapping]):
+        self.clazz = clazz
+        self.property_mappings = property_mappings
+
+class Mapping:
+    def __init__(self, name: str, mappings: list[ClassMapping]):
+        self.name = name
+        self.mappings = mappings

data-finder/model/src/model/relational.py

@@ -0,0 +1,41 @@
+from model.m3 import Property, Class
+from model.mapping import ClassMapping, PropertyMapping
+
+
+class RelationalElement:
+    def __init(self):
+        pass
+
+
+class Column(RelationalElement):
+    def __init__(self, name: str, type: str):
+        self.name = name
+        self.type = type
+        self.table = None
+
+
+class Table:
+    def __init__(self, name: str, columns: list[Column]):
+        self.name = name
+        self.columns = columns
+        for col in columns:
+            col.table = self
+
+
+class Join(RelationalElement):
+    def __init__(self, lhs: Column, rhs: Column):
+        self.lhs = lhs
+        self.rhs = rhs
+
+
+class RelationalPropertyMapping(PropertyMapping):
+    def __init__(self, property: Property, target: RelationalElement):
+        super().__init__(property, target)
+
+
+class RelationalClassMapping(ClassMapping):
+    def __init__(self, clazz: Class, property_mappings: list[RelationalPropertyMapping]):
+        super().__init__(clazz, property_mappings)
+
+
+

data_finder-0.1.0.dist-info/METADATA

@@ -0,0 +1,38 @@
+Metadata-Version: 2.4
+Name: data-finder
+Version: 0.1.0
+Summary: Model driven data finders
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: atpublic==4.1.0
+Requires-Dist: bidict==0.23.1
+Requires-Dist: duckdb==1.0.0
+Requires-Dist: greenlet==3.0.3
+Requires-Dist: ibis-framework==9.0.0
+Requires-Dist: iniconfig==2.1.0
+Requires-Dist: jinja2==3.1.6
+Requires-Dist: markdown-it-py==3.0.0
+Requires-Dist: markupsafe==3.0.2
+Requires-Dist: mdurl==0.1.2
+Requires-Dist: numpy==1.26.4
+Requires-Dist: packaging==25.0
+Requires-Dist: pandas==2.2.2
+Requires-Dist: parsy==2.1
+Requires-Dist: pluggy==1.6.0
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: pyarrow-hotfix==0.7
+Requires-Dist: pygments==2.19.1
+Requires-Dist: pytest==8.3.4
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: pytz==2024.1
+Requires-Dist: rich==13.9.4
+Requires-Dist: six==1.16.0
+Requires-Dist: sqlglot==23.12.2
+Requires-Dist: toml==0.10.2
+Requires-Dist: toolz==0.12.1
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: tzdata==2024.1
+
+# Experimental typed finders
+
+This generates helper classes to query data using a logical model and mapping

data_finder-0.1.0.dist-info/RECORD

@@ -0,0 +1,37 @@
+data-finder/calc/src/calc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data-finder/calc/src/calc/calc_protocol.py,sha256=fEvRa27Jb2HyjrM9jKugUDhtwCUhxzyUWZ4bR4vhccE,1412
+data-finder/calc/src/calc/simple_price.py,sha256=K0xGKbo3F86xsMbLRXAgsTTrrm3JA8t0sBgDD-EHUwI,658
+data-finder/calc/tests/test_calc.py,sha256=lo_dPD-clIJhjlCrU-Wgm_2U0tPoURDZa4lSpIuYNt4,1866
+data-finder/datafinder/src/datafinder/__init__.py,sha256=z8UksHLDOjGDgPHlpufqpLWEI3-qmvHvhleq_aUTbfY,157
+data-finder/datafinder/src/datafinder/attribute.py,sha256=BCN_A3pOwQefOn1UVI7wUY7LiuJCfuW-LisnA8mMKDA,584
+data-finder/datafinder/src/datafinder/finder.py,sha256=jyo44zDnBX3FM8n-M1gX4NXhyuHxJe88kjeajXCX-7U,186
+data-finder/datafinder/src/datafinder/operation.py,sha256=txGzabAvX8lAG7wZ2WLvXcbQWzqyUO_0wUw-BKvEsoA,4812
+data-finder/datafinder/src/datafinder/output.py,sha256=CYbfNoVF8Hnq3G6rFcKfLHhdULXer-UON4CPcmHpTM4,196
+data-finder/datafinder/src/datafinder/runner.py,sha256=9IDbRZ_xIAlnjB02uqXRQDtF48b8d-fJDrnwj8OnyV0,911
+data-finder/datafinder/src/datafinder/typed_attributes.py,sha256=Bnx0PdQA77u6fmsgex9aNpbj3O4y5h90khZ0aD-JoN0,1381
+data-finder/datafinder/src/datafinder/typed_operations.py,sha256=9plTaVJewfrjnn39OJcrCrhfJh8wiGVCki9B7hmycEQ,2068
+data-finder/datafinder_duckdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data-finder/datafinder_duckdb/duckdb_engine.py,sha256=OAf1ym3AqY4_p7fohHfY1ENh5sIapFx377oyfiger-U,864
+data-finder/datafinder_generator/src/datafinder_generator/generator.py,sha256=yUyif6M0jYmFsSHrn18Yx-RTLeoiDS3q7LIkJQmbvWU,825
+data-finder/datafinder_ibis/src/datafinder_ibis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data-finder/datafinder_ibis/src/datafinder_ibis/ibis_engine.py,sha256=Zlh2yUtVNhpcRGb4HVu9x01oC-1_xFhwJknPDzsA5Fc,815
+data-finder/datafinder_ibis/tests/test_ibis_engine.py,sha256=Y0zAtk0wTpikzUR8FobBhEaVNFl2lD5kGEiVDGmeNAs,235
+data-finder/datafinder_ibis_duckdb/tests/test_datafinder_ibis_duckdb.py,sha256=eXPR_MZWfwhScsAUKRljaqFQbGZDOfUi3RUQkUi4xZQ,2172
+data-finder/example/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data-finder/example/account_finder.py,sha256=bYw3itpmjT5PhqM252pR7Kqze4hdcUxOTOcRXXbIx4Y,1157
+data-finder/example/contractualposition_finder.py,sha256=DzIoDNiJ5K7UBNzM6ajNvJ0GXR5lazcBYuAwVz07ti8,2235
+data-finder/example/instrument_finder.py,sha256=DDPXeOTmXRyW7M6eZIjF-VUDrRhfGs28jNIv1eJkEtk,1194
+data-finder/example/mappings.py,sha256=GtEcVwffcbxUXamKqasRVkEd4ZVtbovS49dfhUMUPkQ,3617
+data-finder/example/queries.py,sha256=THu-RLldl6tEaowWt3pZKCxrhGTf564eQ0W1sZWEhN4,1134
+data-finder/example/trade_finder.py,sha256=WLW62qbRAfxeJK_2OTvOAURz7BvCuadf9CvEQN8iaGk,1627
+data-finder/example_duckdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data-finder/example_duckdb/duckdb_example.py,sha256=tSV3uwH15XrGhefJldiIdKKX7Y84Bwu5Ny9auof5X-Y,548
+data-finder/example_duckdb/duckdb_trade_finder.py,sha256=on6ug-tkUdIZ8W1NgIo7-RvmCVwDAjm8BN702kTCOfI,736
+data-finder/model/src/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data-finder/model/src/model/m3.py,sha256=h1dyACsgcXDSc8bKXWXBDkZ8BvvPtLMdCVQ6CdnAWfg,1202
+data-finder/model/src/model/mapping.py,sha256=UPE8ZPZBFPQ_v79WCWsJ6fluHWxCZztAZKzxZCZZYaM,522
+data-finder/model/src/model/relational.py,sha256=mniAfpmVXojWtNjxDj6jGjUU2UIetl1Hc_R-qCxnKCo,983
+data_finder-0.1.0.dist-info/METADATA,sha256=REbMiO6hykkP48tzsZ3_D1I1B6K2qPCI90EcSxPIDcw,1138
+data_finder-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+data_finder-0.1.0.dist-info/top_level.txt,sha256=TkqwsWs5BMfXjQjd97MD6YQY8D5nmv5wZ7OlXupzHFQ,12
+data_finder-0.1.0.dist-info/RECORD,,

data_finder-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

data_finder-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+data-finder