finbourne-sdk-utils 0.0.24__py3-none-any.whl

finbourne_sdk_utils/lpt/map_instruments.py

@@ -0,0 +1,164 @@
+import os
+
+import pandas as pd
+
+from finbourne_sdk_utils.lpt import lpt
+from finbourne_sdk_utils.lpt import lse
+from finbourne_sdk_utils.lpt import stdargs
+from finbourne_sdk_utils.lpt.either import Either
+
+mapping_prefixes = {"Figi": "FIGI", "ClientInternal": "INT", "QuotePermId": "QPI"}
+
+mapping_table = {}
+
+
+def parse(extend=None, args=None):
+    return (
+        stdargs.Parser("Map Instruments", ["filename"])
+        .add("--folder", help="include all 'txn' files in the folder")
+        .add("input", nargs="*", metavar="file", help="file(s) containing instruments")
+        .add(
+            "--column",
+            metavar="input-column",
+            default="instrument_uid",
+            help="column name for instrument column",
+        )
+        .extend(extend)
+        .parse(args)
+    )
+
+
+def process_args(api, args):
+    if args.folder:
+        args.input.extend(
+            [
+                os.path.join(args.folder, f)
+                for f in os.listdir(args.folder)
+                if "-txn-" in f and f.endswith(".csv")
+            ]
+        )
+
+    df = (
+        pd.concat(
+            [lpt.read_csv(f)[[args.column]].drop_duplicates() for f in args.input],
+            ignore_index=True,
+            sort=True,
+        )
+        .drop_duplicates()
+        .reset_index(drop=True)
+    )
+
+    df.columns = ["FROM"]
+    df["TO"] = df["FROM"]
+
+    return map_instruments(api, df, "TO")
+
+
+def main():
+    lpt.standard_flow(parse, lse.connect, process_args)
+
+
+def map_instruments(api, df, column):
+    WORKING = "__:::__"  # temporary column name
+
+    # Apply any known mappings to avoid unecessary i/o
+
+    if len(mapping_table) > 0:
+        srs = df[column].map(mapping_table)
+        srs = srs[srs.notnull()]
+        if len(srs) > 0:
+            df.loc[srs.index, column] = srs
+
+    # updates the mappings table
+    def update_mappings(src, prefix):
+        mapping_table.update(
+            {prefix + k: v.lusid_instrument_id for k, v in src.items()}
+        )
+
+    def batch_query(instr_type, prefix, outstanding):
+        if len(outstanding) > 0:
+            batch = outstanding[:500]  # records to process now
+            remainder = outstanding[500:]  # remaining records
+
+            # called if the get_instruments() succeeds
+            def get_success(result):
+                get_found = result.content.values
+                get_failed = result.content.failed
+
+                # Update successfully found items
+                update_mappings(get_found, prefix)
+
+                if len(get_failed) > 0:
+                    if instr_type == "ClientInternal":
+                        # For un-mapped internal codes, we will try to add (upsert)
+
+                        # called if the upsert_instruments() succeeds
+                        def add_success(result):
+                            add_worked = result.content.values
+                            add_failed = result.content.failed
+
+                            if len(add_failed) > 0:
+                                return Either.Left("Failed to add internal instruments")
+
+                            # Update successfully added items
+                            update_mappings(add_worked, prefix)
+
+                            # Kick off the next batch
+                            return batch_query(instr_type, prefix, remainder)
+
+                        # Create the upsert request from the failed items
+                        request = {
+                            k: api.models.InstrumentDefinition(
+                                name=v.id, identifiers={"ClientInternal": v.id}
+                            )
+                            for k, v in get_failed.items()
+                        }
+
+                        return api.call.upsert_instruments(request).bind(add_success)
+                    else:
+                        # Instruments are not mapped. Nothing we cando.
+                        return Either.Left(
+                            "Failed to locate instruments of type {}".format(instr_type)
+                        )
+                else:
+                    # No failures, kick off the next batch
+                    return batch_query(instr_type, prefix, remainder)
+
+            return api.call.get_instruments(
+                instr_type, list(batch[WORKING].values)
+            ).bind(get_success)
+        else:
+            # No records remaining. Return the now-enriched dataframe
+            return Either.Right(df)
+
+    def map_type(key, instr_type):
+        prefix = key + ":"
+        subset = df[df[column].str.startswith(prefix)]
+
+        # See if there are any entries of this type
+        if len(subset) > 0:
+            width = len(prefix)
+            uniques = subset[[column]].drop_duplicates(column)
+            uniques[WORKING] = uniques[column].str.slice(width)
+
+            def map_success(v):
+                df.loc[subset.index, column] = subset[column].map(mapping_table)
+                return Either.Right(df)
+
+            return batch_query(instr_type, prefix, uniques).bind(map_success)
+        else:
+            # Nothing to be done, pass the full result back
+            return Either.Right(df)
+
+    return (
+        map_type("FIGI", "Figi")
+        .bind(lambda r: map_type("INT", "ClientInternal"))
+        .bind(lambda r: map_type("QPI", "QuotePermId"))
+    )
+
+
+def include_mappings(path):
+    if path:
+        df = lpt.read_csv(path)
+
+        mapping_table.update(df.set_index("FROM")["TO"].to_dict())

finbourne_sdk_utils/lpt/pager.py

@@ -0,0 +1,32 @@
+import pandas as pd
+import re
+import urllib.parse
+
+from finbourne_sdk_utils.lpt.either import Either
+
+rexp = re.compile(r".*page=([^=']{10,}).*")
+
+
+def page_all_results(fetch_page, page_handler):
+    results = []
+
+    def got_page(result):
+        results.append(page_handler(result))
+
+        links = [l for l in result.content.links if l.relation == "NextPage"]
+
+        if len(links) > 0:
+            match = rexp.match(links[0].href)
+            if match:
+                return urllib.parse.unquote(match.group(1))
+        return None
+
+    page = Either(None)
+    while True:
+        page = fetch_page(page.right).bind(got_page)
+        if page.is_left():
+            return page
+        if page.right == None:
+            break
+
+    return pd.concat(results, ignore_index=True, sort=False)

finbourne_sdk_utils/lpt/record.py

@@ -0,0 +1,13 @@
+# A simple key/value class with dot notation access
+class Rec:
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+    def __str__(self):
+        return str(self.__dict__)
+
+    def __iter__(self):
+        return self.__dict__.__iter__()
+
+    def to_dict(self):
+        return self.__dict__

finbourne_sdk_utils/lpt/refreshing_token.py

@@ -0,0 +1,43 @@
+import requests
+from datetime import datetime
+from datetime import timedelta
+from collections import UserString
+
+# Behaves like a string, but refreshes the
+# OKTA credentials should they expire
+
+
+class RefreshingToken(UserString):
+    def __init__(self, token_url, token_request_body, headers):
+
+        token_data = {"expires": datetime.now(), "credentials": ""}
+
+        def get_token():
+            if token_data["expires"] <= datetime.now():
+
+                okta_response = requests.post(
+                    token_url, data=token_request_body, headers=headers
+                )
+                if okta_response.status_code != 200:
+                    print("OKTA authentication failed")
+                    print(okta_response.text)
+                    exit()
+
+                d = dict(okta_response.json())
+                token_data["expires"] = datetime.now() + timedelta(
+                    seconds=d.get("expires_in", 3600) - 60
+                )
+                token_data["credentials"] = d["access_token"]
+
+            return token_data["credentials"]
+
+        self.token_fn = get_token
+
+    # Call the token function to get the credentials
+    # (refreshing them if necessary)
+    # and then return the attribute for the resulting token
+    def __getattribute__(self, name):
+        token = object.__getattribute__(self, "token_fn")()
+        if name == "data":
+            return token
+        return token.__getattribute__(name)

finbourne_sdk_utils/lpt/search_instruments.py

@@ -0,0 +1,48 @@
+import pandas as pd
+import dateutil
+from finbourne_sdk_utils.lpt import lpt
+from finbourne_sdk_utils.lpt import lse
+from finbourne_sdk_utils.lpt import stdargs
+
+TOOLNAME = "instr_search"
+TOOLTIP = "Search for Instruments"
+
+
+def parse(extend=None, args=None):
+    return (
+        stdargs.Parser("Search Instruments", ["filename", "limit"])
+        .add("--properties", nargs="*", help="properties to search")
+        .add("--date")
+        .extend(extend)
+        .parse(args)
+    )
+
+
+def process_args(api, args):
+    def success(result):
+        flat = [i for r in result.content for i in r.mastered_instruments]
+        if len(flat) > 0:
+            identifiers = sorted(set.union(*[set(i.identifiers.keys()) for i in flat]))
+            df = lpt.to_df(
+                flat,
+                ["name"] + ["identifiers.KEY:" + i + ".value" for i in identifiers],
+            )
+            df.columns = ["Name"] + identifiers
+            return df
+        else:
+            return "No Matches"
+
+    request = [
+        api.models.InstrumentSearchProperty(s[0], s[1])
+        for s in [p.split("=") for p in args.properties]
+    ]
+    return api.call.instruments_search(
+        instrument_search_property=request,
+        mastered_only=True,
+        mastered_effective_at=lpt.to_date(args.date),
+    ).bind(success)
+
+
+# Standalone tool
+def main(parse=parse, display_df=lpt.display_df):
+    lpt.standard_flow(parse, lse.connect, process_args, display_df)

finbourne_sdk_utils/lpt/stdargs.py

@@ -0,0 +1,154 @@
+import argparse
+
+# Ensure standardisation of commonly used arguments
+
+
+class Parser:
+
+    # Create a parser and add in the standard arguments
+    def __init__(self, description, sections=[]):
+        self.parser = argparse.ArgumentParser(
+            description=description, fromfile_prefix_chars="@"
+        )
+        self.post_processors = []
+        self.arguments = []
+
+        if "scope" in sections:
+            self.add("scope", help="Scope")
+
+        if "portfolio" in sections:
+            self.add("portfolio", help="Portfolio id")
+
+        if "date" in sections:
+            self.add("date", help="date YYYY-MM-DD")
+
+        if "input" in sections:
+            self.add("input", help="input filename")
+
+        if "properties" in sections:
+            self.add("--properties", nargs="+", help="List of propertykeys", default=[])
+
+        if "filename" in sections:
+            self.add(
+                "-f", "--filename", metavar="filename.csv", help="write to this file"
+            )
+
+        if "limit" in sections:
+            self.add(
+                "-l",
+                "--limit",
+                type=int,
+                default=0,
+                metavar="n",
+                help="limit the number of results",
+            )
+
+        if "date_range" in sections:
+            self.add("-s", "--start_date", dest="start_date", metavar="YYYY-MM-DD")
+            self.add("-e", "--end_date", dest="end_date", metavar="YYYY-MM-DD")
+
+        if "quiet" in sections:
+            self.add(
+                "-q",
+                "--quiet",
+                action="store_true",
+                help="Quiet mode. Doesn't show the progress bar",
+            )
+
+        if "group" in sections:
+            self.add(
+                "-g",
+                "--group",
+                action="store_true",
+                help="Indicates use of Portfolio Groups",
+            )
+
+        if "optional_portfolio" in sections:
+            self.add(
+                "-p",
+                "--portfolio",
+                dest="portfolio",
+                help="Optional Portfolio id"
+            )
+
+        if "flush_scope" in sections:
+            self.add(
+                "--flush_scope",
+                dest="flush_scope",
+                action="store_true",
+                help="Flush all transactions in scope"
+            )
+
+        self.add(
+            "--secrets-file",
+            dest="secrets",
+            default="secrets.json",
+            help="path to secrets file",
+        )
+
+        self.add(
+            "--environment",
+            dest="env",
+            default=["lusid"],
+            nargs="+",
+            help="choose a special LUSID environment. E.g. 'ipsum_lorem'",
+        )
+
+        self.add(
+            "--stats",
+            dest="stats",
+            metavar="stats-file.csv",
+            const="-",
+            nargs="?",
+            help="Write statistics to file. [Leave blank for stdout]",
+        )
+
+        self.add("-d", "--debug", help=r"print debug messages, expected input: 'debug'")
+
+        if "asat" in sections:
+            self.add(
+                "--asat",
+                dest="asat",
+                metavar="YYYY-MM-DDTHH:MM:SS.000",
+                help="as-at time",
+            )
+
+        if "test" in sections:
+            self.add("--test", action="store_true", help="Run in test mode")
+
+        # If a filename parameter is given then pass output via dfq
+        # Can be supressed by the NODFQ option
+        if "filename" in sections and "NODFQ" not in sections:
+            self.add(
+                "--dfq",
+                nargs=argparse.REMAINDER,
+                help="pass the output through to 'dfq' - see `finbourne_sdk_utils dfq --help` for options",
+            )
+
+    def add(self, *args, **kwargs):
+        # Add arguments to a list, so they can be removed
+        self.arguments.append((args[0], args, kwargs))
+        # self.parser.add_argument(*args,**kwargs)
+        return self
+
+    def remove(self, key):
+        self.arguments = [tpl for tpl in self.arguments if tpl[0] != key]
+
+    def post_process(self, fn):
+        self.post_processors.append(fn)
+        return self
+
+    def parse(self, args=None):
+        for arg in self.arguments:
+            self.parser.add_argument(*arg[1], **arg[2])
+
+        args = self.parser.parse_args(args)
+        for fn in self.post_processors:
+            fn(args)
+
+        return args
+
+    def extend(self, fn):
+        if fn:
+            fn(self)
+        return self

finbourne_sdk_utils/lpt/txn_config.py

@@ -0,0 +1,128 @@
+from finbourne_sdk_utils.lpt import lpt
+from finbourne_sdk_utils.lpt import lse
+from finbourne_sdk_utils.lpt import stdargs
+from finbourne_sdk_utils.lpt import txn_config_yaml as tcy
+
+TOOLNAME = "txn_cfg"
+TOOLTIP = "Get/Set the transaction configuration"
+
+
+def parse(extend=None, args=None):
+    return (
+        stdargs.Parser(
+            "Get/Set transaction configuration", ["filename", "limit", "NODFQ", "asat"]
+        )
+        .add(
+            "action",
+            choices=("get", "set", "try"),
+            help="get or set the config. 'Try' can be used to validate a custom encoding",
+        )
+        .add("--raw", action="store_true", help="use raw (non custom) encoding")
+        .add("--json", action="store_true", help="display the json to be sent")
+        .add("--group", action="store_true", help="set a single group")
+        .add(
+            "--force",
+            action="store_true",
+            help="set a single group, remove existing aliases for the group",
+        )
+        .extend(extend)
+        .parse(args)
+    )
+
+
+def validate_group(txn_types, group):
+    for txn in txn_types:
+        for alias in txn.aliases:
+            assert alias.transaction_group == group, "More than one group in the list"
+
+
+def rem_groups(txn_types_old, group, arg):
+    def still_valid(tt):
+        for cand in tt.aliases:
+            if cand.transaction_group != group:
+                return True
+        if arg is not True:
+            raise AssertionError(
+                "Existing group detected, use '--force' to remove them"
+            )
+        return False
+
+    def clear_out_group(tt):
+        check = len(tt.aliases)
+        tt.aliases = [cand for cand in tt.aliases if cand.transaction_group != group]
+
+        if len(tt.aliases) != check and arg != True:
+            raise AssertionError(
+                "Existing group detected, use '--force' to remove them"
+            )
+        return tt
+
+    return [clear_out_group(t) for t in txn_types_old if still_valid(t)]
+
+
+def merge_sets(txn_types_old, txn_types, arg):
+    group = txn_types[0].aliases[0].transaction_group
+
+    validate_group(txn_types, group)
+    txn_types_clean = rem_groups(txn_types_old, group, arg)
+
+    txn_types += txn_types_clean
+    return txn_types
+
+
+def process_args(api, args):
+    y = tcy.TxnConfigYaml(api.models)
+
+    if args.action == "get":
+
+        def get_success(result):
+            y.dump(
+                y.TransactionSetConfigurationDataNoLinks(
+                    result.content.transaction_configs, result.content.side_definitions
+                ),
+                args.filename,
+                args.raw,
+            )
+            return None
+
+        return api.call.list_configuration_transaction_types().bind(get_success)
+
+    if args.action == "try":
+        ffs = y.load(args.filename)
+        y.dump(ffs, "{}-try".format(args.filename))
+
+    if args.action == "set":
+
+        def set_success(result):
+            print(y.get_yaml(result.content))
+            return None
+
+        if args.group:
+            txn_types = y.load(args.filename)
+
+            result = api.call.list_configuration_transaction_types()
+
+            if result.right is not None:
+                txn_types_old = result.right.content
+            else:
+                raise ValueError("Api call did not return correct result")
+
+            txn_types = y.load_update_str(
+                y.get_yaml(merge_sets(txn_types_old, txn_types, args.force))
+            )
+        else:
+            txn_types = y.load_update(args.filename)
+
+        # y.dump(ffs,"{}-set".format(args.filename),True)
+        if args.json:
+            print(txn_types)
+            return None
+        else:
+            return api.call.set_configuration_transaction_types(
+                transaction_set_configuration_data_request=txn_types
+            ).bind(set_success)
+
+
+# Standalone tool
+def main(parse=parse):
+    lpt.standard_flow(parse, lse.connect, process_args)