csvpath 0.0.501__tar.gz → 0.0.502__tar.gz

{csvpath-0.0.501 → csvpath-0.0.502}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.501
+Version: 0.0.502
 Summary: A declarative language for validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -25,6 +25,7 @@ Classifier: Topic :: Utilities
 Provides-Extra: pandas
 Provides-Extra: smartopen
 Requires-Dist: bullet (>=2.2.0,<3.0.0)
+Requires-Dist: ckanapi (>=4.8,<5.0)
 Requires-Dist: inflect (>=7.4.0,<8.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)

csvpath-0.0.502/config/config.ini

@@ -0,0 +1,68 @@
+[csvpath_files]
+extensions = txt, csvpath, csvpaths
+
+[csv_files]
+extensions = txt, csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, stop, fail, print
+csvpaths = raise, collect
+
+[logging]
+csvpath = info
+csvpaths = info
+log_file = logs/csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+
+[config]
+path = config/config.ini
+
+[cache]
+path = cache
+
+[functions]
+imports = config/functions.imports
+
+[listeners]
+groups =
+#slack, marquez, ckan
+
+# add ckan to the list of groups above for alerts to slack webhooks
+ckan.paths = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+ckan.result = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+
+#add marquez to the list of groups above for OpenLineage events to a local Marquez
+marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
+marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
+marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
+marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
+
+# add slack to the list of groups above for alerts to slack webhooks
+slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+
+[marquez]
+base_url = http://localhost:5000
+endpoint = api/v1/lineage
+api_key = "none"
+timeout = 5
+verify = False
+
+[slack]
+# add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
+# on-valid-slack: webhook-minus-'https://' and/or
+# on-invalid-slack: webhook-minus-'https://'
+webhook_url =
+
+[results]
+archive = archive
+
+[inputs]
+files = inputs/named_files
+csvpaths = inputs/named_paths
+on_unmatched_file_fingerprints = halt
+

csvpath-0.0.502/csvpath/managers/integrations/ckan/ckan.py

@@ -0,0 +1,193 @@
+from ckanapi import RemoteCKAN
+from ckanapi.errors import NotFound
+from csvpath import CsvPaths
+from csvpath.util.config import Config
+from .dataset import Dataset
+from .datafile import Datafile
+import hashlib
+import os
+
+
+class CkanException(Exception):
+    pass
+
+
+class Ckan:
+    def __init__(self, *, config, manifest, csvpaths=None) -> None:
+        self.config = config
+        self.csvpaths = csvpaths
+        self._manifest = manifest
+        self._client = None
+
+    @property
+    def manifest(self):
+        return self._manifest
+
+    @property
+    def config(self):
+        return self._config
+
+    @config.setter
+    def config(self, config):
+        self._config = config
+
+    @property
+    def client(self):
+        if self._client is None:
+            server = self.config._get("ckan", "server")
+            if server is None:
+                raise CkanException(
+                    "CKAN server cannot be None. Check config for [ckan] server"
+                )
+            token = self.config._get("ckan", "api_token")
+            if token is None:
+                raise CkanException(
+                    "API token cannot be None. Check config for [ckan] api_token"
+                )
+            self._client = RemoteCKAN(server, apikey=token, user_agent="CsvPath")
+        return self._client
+
+    # ==========================
+    # ==========================
+
+    def create_or_update_dataset(self, dataset: Dataset) -> None:
+        if self._has_dataset(dataset.name):
+            self._update_dataset(dataset)
+        else:
+            self._create_dataset(dataset)
+
+    def upload_datafile(self, datafile: Datafile) -> dict[str, ...]:
+        with open(datafile.path, "rb") as file:
+            id = self._has_resource(datafile.dataset_id, datafile.name)
+            if id is not None:
+                resource = self.client.action.resource_update(
+                    id=id,
+                    package_id=datafile.dataset_id,
+                    url=datafile.url,
+                    name=datafile.name,
+                    mimetype=datafile.mime_type,
+                    upload=file,
+                )
+            else:
+                resource = self.client.action.resource_create(
+                    package_id=datafile.dataset_id,
+                    url=datafile.url,
+                    name=datafile.name,
+                    mimetype=datafile.mime_type,
+                    upload=file,
+                )
+            return resource
+
+    def create_group_if(self, *, name: str, title: str) -> None:
+        #
+        # the archive turns into a ckan group. orgs may run multiple CsvPaths.
+        # and each named-group can be a dataset with multiple files.
+        #
+        thename = self._fix_name_if(name)
+        if not self._has_group(thename):
+            if title is not None and title != thename:
+                self.client.action.group_create(name=thename, title=title)
+            else:
+                self.client.action.group_create(name=thename)
+        return thename
+
+    def _create_dataset(self, dataset: Dataset) -> None:
+        if not self._has_org(dataset.org):
+            raise CkanException(
+                "You must provide the name of an organization. Ask your CKAN admin to create one for you."
+            )
+        args = {
+            "name": self._fix_name_if(dataset.name),
+            "title": dataset.title,
+            "owner_org": dataset.org,
+            "private": not dataset.visible,
+            "url": dataset.url,
+        }
+        if dataset.tags is not None:
+            args["tags"] = dataset.tags
+        if dataset.fields is not None:
+            args["extras"] = dataset.fields
+        if dataset.groups is not None:
+            args["groups"] = dataset.groups()
+        #
+        # author = dataset.author
+        # author_email = dataset.author_email
+        #
+        # create dataset as a ckan package
+        #
+        self.client.action.package_create(**args)
+
+    def _update_dataset(self, dataset: Dataset) -> None:
+        args = {
+            "id": self._fix_name_if(dataset.name),
+            "title": dataset.title,
+            "owner_org": dataset.org,
+            "private": not dataset.visible,
+            "url": dataset.url,
+        }
+        if dataset.tags is not None:
+            args["tags"] = dataset.tags
+        if dataset.fields is not None:
+            args["extras"] = dataset.fields
+        if dataset.groups is not None:
+            args["groups"] = dataset.groups()
+        #
+        # version = dataset.version
+        # author = dataset.author
+        # author_email = dataset.author_email
+        #
+        self.client.action.package_patch(**args)
+
+    # ==========================
+    # ==========================
+
+    def _fix_name_if(self, name: str) -> str:
+        name = name.strip().lower()
+        thename = ""
+        for n in name:
+            if n.isalnum() or n in ["-", "_"]:
+                thename += n
+            else:
+                thename += "_"
+        return thename
+
+    def _has_org(self, name) -> bool:
+        try:
+            org = self.client.action.organization_show(id=name)
+        except NotFound:
+            #
+            # not found is expected so we do nothing
+            #
+            ...
+        return org is not None
+
+    def _has_dataset(self, name) -> bool:
+        dataset = None
+        try:
+            dataset = self.client.action.package_show(id=name)
+        except NotFound:
+            #
+            # not found is expected so we do nothing
+            #
+            ...
+        return dataset is not None
+
+    def _has_group(self, name: str) -> bool:
+        name = self._fix_name_if(name)
+        groups = self.client.action.group_list(groups=[name])
+        return groups and len(groups) == 1 and groups[0] == name
+
+    def _has_resource(self, package_name, resource_name: str) -> str:
+        dataset = None
+        try:
+            dataset = self.client.action.package_show(id=package_name)
+            if "resources" in dataset:
+                for r in dataset["resources"]:
+                    if r.get("name") == resource_name:
+                        return r["id"]
+        except NotFound:
+            #
+            # not found is expected so we do nothing
+            #
+            ...
+        return None

csvpath-0.0.502/csvpath/managers/integrations/ckan/ckan_listener.py

@@ -0,0 +1,390 @@
+import os
+import json
+from tempfile import NamedTemporaryFile
+from csvpath.managers.listener import Listener
+from csvpath.managers.metadata import Metadata
+from csvpath.managers.results.results_metadata import ResultsMetadata
+from csvpath.managers.results.result import Result
+from csvpath.matching.util.runtime_data_collector import RuntimeDataCollector
+from .ckan import Ckan, CkanException
+from .dataset import Dataset
+from .datafile import Datafile
+
+
+class CkanListener(Listener):
+
+    FILETYPE_MAP = {
+        "data": "data.csv",
+        "unmatched": "unmatched.csv",
+        "errors": "errors.json",
+        "vars": "vars.json",
+        "meta": "meta.json",
+        "manifest": "manifest.json",
+        "printouts": "printouts.txt",
+    }
+    FILETYPES = ["data", "unmatched", "meta", "vars", "errors", "printouts", "manifest"]
+
+    def __init__(self, config=None, csvpaths=None):
+        #
+        # we're unlikely to get either of these because
+        # we're loading dynamically. we'll get them after
+        # init.
+        #
+        super().__init__(config=config)
+        self._csvpaths = None
+
+    def manifest(self, metadata: Metadata) -> dict[str, ...]:
+        path = metadata.manifest_path
+        with open(path, "r", encoding="utf-8") as file:
+            return json.load(file)
+
+    @property
+    def csvpaths(self):
+        return self._csvpaths
+
+    @csvpaths.setter
+    def csvpaths(self, csvpaths) -> None:
+        self._csvpaths = csvpaths
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        #
+        # CKAN is mainly interested in the output file and/or metadata.
+        # we can assume that we don't need to react to file, paths,
+        # and run events.
+        #
+        # CKAN cares about individual csvpath instances within the
+        # named-paths group. however, it needs to be able to react to
+        # a csvpath based on the results of all the other csvpaths in
+        # the group. that means we need to work from the results event
+        # rather than the result events. using the results event is
+        # easy because it is at the right point in time, already
+        # collects the summary good/bad indicators, and we have all
+        # the data we need to run through the individual csvpaths to
+        # look for CKAN work.
+        #
+        if isinstance(mdata, ResultsMetadata):
+            return self._send_result_files(mdata)
+
+    def _send_result_files(self, mdata: Metadata) -> None:
+        mani = self.manifest(mdata)
+        if "time_completed" not in mani or mani["time_completed"] is None:
+            #
+            # the run has started but not completed
+            #
+            return
+        #
+        # if for some reason we don't get back our correct results -- the most recent
+        # results available -- we can use results_manager.get_named_results_for_run()
+        # to find the right set
+        #
+        results = self.csvpaths.results_manager.get_named_results(
+            mdata.named_paths_name
+        )
+        if results is None:
+            raise CkanException(f"Results cannot be none for {mdata.named_paths_name}")
+        if len(results) == 0:
+            raise CkanException(f"Results cannot be 0 for {mdata.named_paths_name}")
+        if not mdata.run_home.endswith(results[0].run_dir):
+            raise CkanException(
+                f"Results of {mdata.named_paths_name} from wrong run dir: {results[0].run_dir}"
+            )
+        #
+        # we're good. we iterate all and for each we check if the instance requires
+        # all other instances to be valid,complete,no-errors. if so, we iterate for
+        # that. if all is good we follow the metadata instructions.
+        #
+        for result in results:
+            self._handle_result(results, result, mdata)
+
+    def _handle_result(
+        self, results: list[Result], result: Result, mdata: Metadata
+    ) -> None:
+        #
+        # give the Dataset object a chance to find any new metadata
+        # we want to apply to the dataset in CKAN. we may need to
+        # pull the CKAN package to see what's already there, or not,
+        # or maybe leave that to the CKAN client to adjudicate, if
+        # we're ok with overwriting fields and don't lose existing
+        # metadata if we don't present it again during the update?
+        #
+        """
+        Metadata directives for CKAN:
+          - ckan-publish: always | on-valid | on-all-valid | never
+          - ckan-group: use-archive | use-named-results | some-literal
+          - ckan-dataset-name: use-instance | use-named-results | var-value:name | a literal
+          - ckan-dataset-title: a-metadata-field-name | var-value:name
+          - ckan-visibility: private
+          - ckan-tags: static-tag1-n | instance-identity | instance-home | var-value:name
+          - ckan-show-fields: a, b, c, d....
+          - ckan-send: all, printouts, data, metadata, unmatched, vars, errors, manifest
+          - ckan-printouts-title: Background
+          - ckan-data-title: Orders
+          - ckan-unmatched-title: Orders
+          - ckan-vars-title: Orders
+          - ckan-meta-title: Orders
+          - ckan-errors-title: Orders
+          - ckan-split-printouts: (no-)split
+        """
+        pub = self._publish(results=results, result=result)
+        if not pub:
+            return
+        #
+        # get manifest
+        #
+        mani = self.manifest(mdata)
+        dataset = Dataset(listener=self, manifest=mani, metadata=mdata)
+        #
+        # look in metadata for instructions as to if we use a group, and if so,
+        # how it is named.
+        #
+        self._set_group(dataset=dataset, result=result, mani=mani, mdata=mdata)
+        #
+        # look in metadata for instructions on how to map to the dataset; what name to use, etc.
+        #
+        self._set_dataset_name(dataset=dataset, result=result, mani=mani, mdata=mdata)
+        #
+        # find a title if any
+        #
+        self._set_dataset_title(dataset=dataset, result=result)
+        #
+        # find a visibility if any
+        #
+        self._set_visibility(dataset=dataset, result=result)
+        #
+        # tags to label the dataset with
+        #
+        self._set_tags(dataset=dataset, result=result, mdata=mdata)
+        #
+        # metadata fields to display in the dataset
+        #
+        self._set_metadata_fields(dataset=dataset, result=result)
+        #
+        # create the dataset before we load the files, or update it if existing
+        #
+        ckan = Ckan(config=self.config, manifest=mani, csvpaths=self.csvpaths)
+        ckan.create_or_update_dataset(dataset)
+        #
+        # send any files requested. one or more makes sense, but there is no requirement to send any.
+        #
+        self._send_files(result=result, mani=mani, mdata=mdata, dataset_id=dataset.name)
+
+    def _publish(self, *, results, result) -> bool:
+        pub = result.csvpath.metadata.get("ckan-publish")
+        ret = None
+        if pub is None:
+            ret = False
+        else:
+            pub = pub.strip().lower()
+        if pub == "never":
+            ret = False
+        elif pub == "always":
+            ret = True
+        elif pub == "on-valid":
+            ret = result.csvpath.is_valid
+        elif pub == "on-all-valid":
+            ret = True
+            for r in results:
+                if not r.is_valid:
+                    ret = False
+                    break
+        return ret
+
+    def _send_files(
+        self, *, result: Result, mani: dict, mdata: Metadata, dataset_id: str
+    ) -> None:
+        filesstr = result.csvpath.metadata.get("ckan-send")
+        files = []
+        if filesstr is not None:
+            if filesstr.strip() == "all":
+                files = CkanListener.FILETYPES[:]
+            else:
+                fs = filesstr.split(",")
+                for f in fs:
+                    f = f.strip().lower()
+                    if f not in CkanListener.FILETYPES:
+                        raise ValueError(
+                            f"Incorrect file identifier: {f}. Must be in {CkanListener.FILETYPES}."
+                        )
+                    files.append(f)
+        for file in files:
+            path = os.path.join(mani["run_home"], result.identity_or_index)
+            path = os.path.join(path, CkanListener.FILETYPE_MAP[file])
+            #
+            # move printouts out
+            #
+            if file == "printouts":
+                split = result.csvpath.metadata.get("ckan-split-printouts")
+                if split and split.strip().lower() == "split":
+                    if os.path.exists(path):
+                        printouts = ""
+                        with open(path, "r", encoding="utf-8") as file:
+                            printouts = file.read()
+                        printouts = printouts.split("---- PRINTOUT:")
+                        for p in printouts:
+                            self._send_a_printout(
+                                p=p,
+                                result=result,
+                                mani=mani,
+                                mdata=mdata,
+                                dataset_id=dataset_id,
+                            )
+            else:
+                datafile = Datafile(
+                    listener=self,
+                    result=result,
+                    manifest=mani,
+                    metadata=mdata,
+                    filetype=file,
+                    path=path,
+                )
+                datafile.dataset_id = dataset_id
+                title = result.csvpath.metadata.get(f"ckan-{file}-title")
+                if title is not None:
+                    datafile.name = title
+                ckan = Ckan(config=self.config, manifest=mani, csvpaths=self.csvpaths)
+                ckan.upload_datafile(datafile)
+
+    def _send_a_printout(
+        self, *, p: str, result: Result, mani: dict, mdata: Metadata, dataset_id: str
+    ) -> None:
+        i = p.find("\n")
+        name = p[0:i]
+        name = name.strip()
+        if name == "default":
+            name = result.csvpath.metadata.get("ckan-printouts-title")
+            if name is None:
+                name = "Default printer"
+        body = p[i + 1 :]
+        body = body.strip()
+        if body == "":
+            return
+        with NamedTemporaryFile(delete_on_close=False) as file:
+            file.write(body.encode("utf-8"))
+            file.close()
+            datafile = Datafile(
+                listener=self,
+                result=result,
+                manifest=mani,
+                metadata=mdata,
+                filetype="printouts",
+                path=file.name,
+            )
+            datafile.dataset_id = dataset_id
+            datafile.name = name
+            datafile.mime_type = "text/plain"
+            ckan = Ckan(config=self.config, manifest=mani, csvpaths=self.csvpaths)
+            ckan.upload_datafile(datafile)
+
+    def _set_visibility(self, dataset: Dataset, result: Result) -> None:
+        visibility = result.csvpath.metadata.get("ckan-visibility")
+        if visibility and visibility.strip().lower() == "public":  # not private
+            dataset.visible = False
+
+    def _set_group(
+        self, *, dataset: Dataset, result: Result, mani: dict, mdata: Metadata
+    ):
+        g = self._create_group_if(
+            dataset=dataset, result=result, mani=mani, mdata=mdata
+        )
+        dataset.group = g
+
+    def _create_group_if(
+        self, *, dataset: Dataset, result: Result, mani: dict, mdata: Metadata
+    ) -> str:
+        group, title = self._get_group_name(result, mdata)
+        if group is None:
+            return
+        ckan = Ckan(config=self.config, manifest=mani, csvpaths=self.csvpaths)
+        group = ckan.create_group_if(name=group, title=title)
+        return group
+
+    def _get_group_name(self, result: Result, mdata: Metadata) -> str:
+        group = result.csvpath.metadata.get("ckan-group")
+        title = group
+        if group is not None:
+            group = group.strip().lower()
+            if group == "use-archive":
+                group = mdata.archive_name
+                title = group
+            elif group == "use-named-results":
+                group = mdata.named_results_name
+                title = group
+            group = group.strip().lower()
+        return (group, title)
+
+    def _set_dataset_name(
+        self, *, dataset: Dataset, result: Result, mani: dict, mdata: Metadata
+    ) -> None:
+        slug = result.csvpath.metadata.get("ckan-dataset-name")
+        lslug = None
+        if slug is not None:
+            slug = slug.strip()
+            lslug = slug.lower()
+        if lslug == "use-instance":
+            dataset.name = result.csvpath.identity
+        elif lslug == "use-named-results":
+            dataset.name = mdata.named_results_name
+        elif lslug is not None and lslug.startswith("var-value:"):
+            v = lslug[10:]
+            val = result.csvpath.variables.get(v)
+            if val is not None:
+                dataset.name = val if val else v
+        elif slug is not None:
+            dataset.name = slug
+        else:
+            dataset.name = mdata.named_results_name
+
+    def _set_dataset_title(self, *, dataset: Dataset, result: Result) -> None:
+        title = result.csvpath.metadata.get("ckan-dataset-title")
+        if title is not None:
+            if title.startswith("var-value:"):
+                v = title[10:]
+                v = result.csvpath.variables.get(v)
+                if v is not None:
+                    title = v
+            dataset.title = title
+
+    def _set_tags(self, *, dataset: Dataset, result: Result, mdata: Metadata) -> None:
+        tagsstr = result.csvpath.metadata.get("ckan-tags")
+        tags = []
+        if tagsstr is not None:
+            ts = tagsstr.split(",")
+            for t in ts:
+                t = t.strip()
+                if t == "instance-identity":
+                    tags.append({"name": f"{result.csvpath.identity}"})
+                elif t == "instance-home":
+                    h = mdata.run_home
+                    if h is not None:
+                        tags.append({"name": f"{h}"})
+                elif t.startswith("var-value:"):
+                    v = t[10:]
+                    v = result.csvpath.variables.get(v)
+                    if v is not None:
+                        tags.append({"name": f"{v}"})
+                else:
+                    tags.append({"name": f"{t}"})
+        if len(tags) > 0:
+            dataset.tags = tags
+
+    def _set_metadata_fields(self, *, dataset: Dataset, result: Result) -> None:
+        fieldsstr = result.csvpath.metadata.get("ckan-show-fields")
+        fields = []
+        if fieldsstr is not None:
+            fs = fieldsstr.split(",")
+            runt = None
+            for f in fs:
+                f = f.strip()
+                v = result.csvpath.metadata.get(f)
+                if v is None and f.startswith("var-value:"):
+                    f = f[10:]
+                    v = result.csvpath.variables.get(f)
+                if v is None:
+                    if runt is None:
+                        runt = {}
+                        RuntimeDataCollector.collect(result.csvpath, runt, local=True)
+                    v = runt.get(f)
+                if v is not None:
+                    fields.append({"key": f"{f}", "value": f"{v}"})
+        if len(fields) > 0:
+            dataset.fields = fields