csvpath 0.0.612__tar.gz → 0.0.614__tar.gz

{csvpath-0.0.612 → csvpath-0.0.614}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.612
+Version: 0.0.614
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

csvpath-0.0.614/assets/config/jenkins-windows-sftp.ini

@@ -0,0 +1,114 @@
+[testing]
+azure.skip = yes
+gcs.skip = yes
+s3.skip = yes
+sftp.skip = no
+
+[extensions]
+csvpath_files = csvpath, csvpaths
+csv_files = csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, print
+csvpaths = raise, collect, print
+use_format = full
+pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
+
+[logging]
+csvpath = info
+csvpaths = debug
+log_file = logs\csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+handler = file
+
+[config]
+path = assets\config\jenkins-windows-sftp.ini
+allow_var_sub = True
+var_sub_source = env
+
+[cache]
+path = cache
+use_cache = no
+
+[functions]
+imports = 
+
+[results]
+archive = tmp\sftp\archive
+transfers = tmp\sftp\transfers
+
+[inputs]
+files = sftp://{SFTP_SERVER}:{SFTP_PORT}/windows/inputs/named_files
+csvpaths = tmp\sftp\inputs\named_paths
+on_unmatched_file_fingerprints = halt
+allow_http_files = True
+allow_local_files = True
+
+[listeners]
+groups = default
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
+otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
+sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+openlineage.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
+openlineage.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
+openlineage.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
+openlineage.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
+slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
+[otlp]
+
+[aws]
+
+[azure]
+
+[gcs]
+
+[sftp]
+server = SFTP_SERVER
+port = SFTP_PORT
+username = python
+password = hangzhou
+
+[sqlite]
+db = archive\csvpath.db
+
+[sftpplus]
+mailbox_user = MAILBOX_USER
+mailbox_password = MAILBOX_PASSWORD
+server = SFTPPLUS_SERVER
+port = SFTPPLUS_PORT
+admin_username = SFTPPLUS_ADMIN_USERNAME
+admin_password = SFTPPLUS_ADMIN_PASSWORD
+api_url = https://. . . :10020/json
+scripts_dir = 
+execute_timeout = 300
+
+[ckan]
+server = http://. . . :80
+api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
+
+[scripts]
+run_scripts = yes
+
+[openlineage]
+base_url = http://. . . :5000
+endpoint = api/v1/lineage
+api_key = "none"
+timeout = 5
+verify = False
+
+[slack]
+webhook_url = 
+

{csvpath-0.0.612 → csvpath-0.0.614}/csvpath/csvpaths.py

@@ -8,7 +8,7 @@ import threading
 
 from uuid import uuid4
 from abc import ABC, abstractmethod
-from typing import List, Any, NewType
+from typing import Any, NewType, Optional
 from datetime import datetime
 from .managers.errors.error import Error
 from .managers.errors.error_comms import ErrorCommunications
@@ -459,7 +459,7 @@ Cache: {cache}
         self._advance_all = lines
 
     @property
-    def errors(self) -> List[Error]:  # pylint: disable=C0116
+    def errors(self) -> list[Error]:  # pylint: disable=C0116
         """@private
         generally you should be looking at results_manager or error_manager for errors.
         """
@@ -758,7 +758,12 @@ Cache: {cache}
     # case data.csv is the default.
     #
     def collect_paths(
-        self, *, pathsname: str, filename: str, template: str = None
+        self,
+        *,
+        pathsname: str,
+        filename: str,
+        template: str = None,
+        extra_data: Optional[dict[str, str]] = None,
     ) -> Reference:
         """
         Sequentially does a CsvPath.collect() on filename for every named-path in the
@@ -783,11 +788,19 @@ Cache: {cache}
         if isinstance(files, list):
             for file in files:
                 ref = self._collect_paths(
-                    pathsname=pathsname, filename=filename, template=template, file=file
+                    pathsname=pathsname,
+                    filename=filename,
+                    template=template,
+                    file=file,
+                    extra_data=extra_data,
                 )
         else:
             ref = self._collect_paths(
-                pathsname=pathsname, filename=filename, template=template, file=files
+                pathsname=pathsname,
+                filename=filename,
+                template=template,
+                file=files,
+                extra_data=extra_data,
             )
         #
         # absolute reference to the results
@@ -795,7 +808,13 @@ Cache: {cache}
         return ref
 
     def _collect_paths(
-        self, *, pathsname: str, file: str, filename: str, template: str = None
+        self,
+        *,
+        pathsname: str,
+        file: str,
+        filename: str,
+        template: str = None,
+        extra_data: Optional[dict[str, str]] = None,
     ) -> Reference:
         #
         # if template is None we need to go find any template that was given when
@@ -828,7 +847,6 @@ Cache: {cache}
 
         results = []
         #
-        # exp
         # adding uuid for the run as a whole
         run_uuid = uuid4()
         #
@@ -841,6 +859,7 @@ Cache: {cache}
             file=file,
             run_uuid=run_uuid,
             method="collect_paths",
+            extra_data=extra_data,
         )
         #
         #
@@ -930,7 +949,12 @@ Cache: {cache}
         return ret
 
     def fast_forward_paths(
-        self, *, pathsname: str, filename: str, template: str = None
+        self,
+        *,
+        pathsname: str,
+        filename: str,
+        template: str = None,
+        extra_data: Optional[dict[str, str]] = None,
     ) -> Reference:
         """
         Sequentially does a CsvPath.fast_forward() on filename for every named path. No matches are collected.
@@ -942,16 +966,30 @@ Cache: {cache}
         if isinstance(files, list):
             for file in files:
                 ref = self._fast_forward_paths(
-                    pathsname=pathsname, filename=filename, template=template, file=file
+                    pathsname=pathsname,
+                    filename=filename,
+                    template=template,
+                    file=file,
+                    extra_data=extra_data,
                 )
         else:
             ref = self._fast_forward_paths(
-                pathsname=pathsname, filename=filename, template=template, file=files
+                pathsname=pathsname,
+                filename=filename,
+                template=template,
+                file=files,
+                extra_data=extra_data,
             )
         return ref
 
     def _fast_forward_paths(
-        self, *, pathsname: str, file: str, filename: str, template: str = None
+        self,
+        *,
+        pathsname: str,
+        file: str,
+        filename: str,
+        template: str = None,
+        extra_data: Optional[dict[str, str]] = None,
     ) -> Reference:
         """Sequentially does a CsvPath.fast_forward() on filename for every named path. No matches are collected."""
         paths = self._get_named_paths(pathsname)
@@ -991,6 +1029,7 @@ Cache: {cache}
             filename=filename,
             run_uuid=run_uuid,
             method="fast_forward_paths",
+            extra_data=extra_data,
         )
         #
         #
@@ -1066,7 +1105,8 @@ Cache: {cache}
         filename: str,
         collect: bool = False,
         template: str = None,
-    ) -> List[Any]:  # pylint: disable=R0914
+        extra_data: Optional[dict[str, str]] = None,
+    ) -> list[Any]:  # pylint: disable=R0914
         """
         Does a CsvPath.next() on filename for every line against every named path in sequence
         """
@@ -1107,6 +1147,7 @@ Cache: {cache}
             filename=filename,
             run_uuid=run_uuid,
             method="next_paths",
+            extra_data=extra_data,
         )
         #
         #
@@ -1200,6 +1241,7 @@ Cache: {cache}
         collect_when_not_matched=False,
         template: str = None,
         lines: list = None,
+        extra_data: Optional[dict[str, str]] = None,
     ) -> Reference:
         """
         Does a CsvPath.collect() on filename where each row is considered by
@@ -1228,6 +1270,7 @@ Cache: {cache}
                 file=file,
                 template=template,
                 method="collect_by_line",
+                extra_data=extra_data,
             ):
                 # re: W0612: we need 'line' in order to do the iteration. we have to iterate.
                 lines.append(line)
@@ -1254,6 +1297,7 @@ Cache: {cache}
         if_all_agree=False,
         collect_when_not_matched=False,
         template: str = None,
+        extra_data: Optional[dict[str, str]] = None,
     ) -> Reference:
         """
         Does a CsvPath.fast_forward() on filename where each row is considered by
@@ -1279,6 +1323,7 @@ Cache: {cache}
                 file=file,
                 template=template,
                 method="fast_forward_by_line",
+                extra_data=extra_data,
             ):
                 # re: W0612: we need 'line' in order to do the iteration. we have to iterate.
                 pass
@@ -1308,7 +1353,8 @@ Cache: {cache}
         collect_when_not_matched=False,
         template: str = None,
         method: str = None,
-    ) -> List[Any]:
+        extra_data: Optional[dict[str, str]] = None,
+    ) -> list[Any]:
         #
         # we're doing a programmatic use when we use next_by_line() so we don't allow
         # multiple file runs. we check that here; the other _by_line()s skip it.
@@ -1326,6 +1372,7 @@ Cache: {cache}
             collect_when_not_matched=collect_when_not_matched,
             template=template,
             file=file,
+            extra_data=extra_data,
         )
 
     def _next_by_line(  # pylint: disable=R0912,R0915,R0914
@@ -1339,7 +1386,8 @@ Cache: {cache}
         template: str = None,
         file: str,
         method: str = "next_by_line",
-    ) -> List[Any]:
+        extra_data: Optional[dict[str, str]] = None,
+    ) -> list[Any]:
         """Does a CsvPath.next() on filename where each row is considered
         by every named path before the next row starts.
 
@@ -1350,6 +1398,11 @@ Cache: {cache}
 
         collect_when_not_matched=True inverts the match so that lines
         which did not match are returned, rather than the default behavior.
+
+        extra_data is a flat dict of values that are meaningful to the caller.
+        the results registrar stores them as an extra.json file in the
+        run_dir within an _extra_data directory. the directory can be used
+        by any listener as a scratch space
         """
         # re: R0912 -- absolutely. plan to refactor.
         if isinstance(file, list):
@@ -1393,6 +1446,7 @@ Cache: {cache}
             pathsname=pathsname,
             crt=crt,
             method=method,
+            extra_data=extra_data,
         )
         #
         # setting file into the csvpath is less obviously useful at CsvPaths
@@ -1402,7 +1456,7 @@ Cache: {cache}
         reader = FileManager.get_reader(
             file, delimiter=self.delimiter, quotechar=self.quotechar
         )
-        stopped_count: List[int] = []
+        stopped_count: list[int] = []
         for line in reader.next():
             # for line in reader:  # pylint: disable=R1702
             # question to self: should this default be in a central place
@@ -1524,7 +1578,7 @@ Cache: {cache}
     def _load_csvpath_objects(
         self,
         *,
-        paths: List[str],
+        paths: list[str],
         named_file: str,
         collect_when_not_matched=False,
         filename,
@@ -1562,7 +1616,14 @@ Cache: {cache}
         return csvpath_objects
 
     def _prep_csvpath_results(
-        self, *, csvpath_objects, filename, pathsname, crt: str, method: str
+        self,
+        *,
+        csvpath_objects,
+        filename,
+        pathsname,
+        crt: str,
+        method: str,
+        extra_data: Optional[dict[str, str]],
     ):
         """@private"""
         #
@@ -1578,6 +1639,7 @@ Cache: {cache}
             filename=filename,
             run_uuid=run_uuid,
             method="next_paths",
+            extra_data=extra_data,
         )
         #
         #

{csvpath-0.0.612 → csvpath-0.0.614}/csvpath/managers/files/file_manager.py

@@ -1,6 +1,7 @@
 import re
 import json
 import traceback
+from uuid import UUID
 from typing import NewType
 from json import JSONDecodeError
 from csvpath.util.file_readers import DataFileReader
@@ -53,18 +54,6 @@ class FileManager:
     def activator(self) -> NamedFileActivator:
         return NamedFileActivator(self)
 
-    """
-    @property
-    def nos(self) -> Nos:
-        box = Box()
-        if self._nos is None:
-            self._nos = box.get("boto_s3_nos")
-            if self._nos is None:
-                self._nos = Nos(None)
-                box.add("boto_s3_nos", self._nos)
-        return self._nos
-    """
-
     @property
     def csvpaths(self):
         """@private"""
@@ -87,7 +76,6 @@ class FileManager:
     def files_root_manifest(self) -> dict:
         """@private"""
         p = self.files_root_manifest_path
-        # nos = self.nos
         nos = Nos(p)
         if nos.exists():
             with DataFileReader(p) as reader:
@@ -187,6 +175,13 @@ class FileManager:
                         return _["uuid"]
         raise ValueError(f"No matching UUID found for file {file} in {name}")
 
+    def get_reference_for_uuid(self, name:NamedFileName, uuid:str) -> str|None:
+        mani = self.get_manifest(name)
+        for _ in mani:
+            if uuid == _["uuid"]:
+                return _["reference"]
+        return None
+
     def get_manifest(self, name: NamedFileName) -> json:
         if name is None:
             raise ValueError("Paths name cannot be None")
@@ -344,28 +339,7 @@ class FileManager:
         # and then, once fingerprinted:
         #   -> d/b/myfile.csv/0b849c9c1ef....csv
         #
-        """
-        _t = t
-        parts = pathu.parts(path)
-        for i, part in enumerate(parts):
-            t = t.replace(f":{i}", part)
-            #
-            # TODO: replace dynamic tokens: :day_of_week, :day, :month, :month_of_year, :year, :quarter
-            #
-        t = t.replace(":filename", parts[-1])
-        """
-        #
-        # this is what we need to do instead of the above ^^^^
-        #
         t = temu.transform_file_template(template=t, file=path)
-        """
-        print(f"xt2: path: {path}")
-        print(f"xt2: parts: {parts}")
-        print(f"xt2: _t: {_t}")
-        print(f"xt2: _2: {_t2}")
-        print(f"xt2: t: {t}")
-        print("")
-        """
         #
         #
         #
@@ -624,6 +598,7 @@ class FileManager:
         path: str,
         template: str = None,
         config: Config = None,
+        registration_uuid:str = None
     ) -> str | None:
         #
         # note that we can pass in a Config with a template that is different
@@ -631,6 +606,15 @@ class FileManager:
         # using the arg, not the template from the config.
         #
         self.csvpaths.logger.info("Adding named file %s from %s", name, path)
+        if registration_uuid is not None:
+            #
+            # check that we have a real uuid and if not blow up before registration
+            # blowing up seems right here, rather than checking should_i_raise because
+            # we're looking at a serious development problem, rather than a runtime
+            # issue driven by the 3rd party or business rules.
+            #
+            UUID(registration_uuid)
+            self.csvpaths.logger.info("Registration of %s will be associated with UUID %s", path, registration_uuid)
         if not self.can_load(path):
             #
             # if False, can_load() will have already raised an error and/or, minimally,
@@ -738,7 +722,6 @@ class FileManager:
             #
             if mark is not None:
                 self._validate_xlsx_mark(path, mark)
-
             #
             # exp: moving docs and descriptor to above copy_in()
             #
@@ -751,13 +734,9 @@ class FileManager:
             self._copy_in(name=name, path=path, home=home, template=template)
             self.csvpaths.logger.debug("Done copying in")
             #
-            #
-            #
             # self.assure_docs_and_discriptor(name)
             # self.csvpaths.logger.debug("Assured docs and discriptor")
             #
-            # end exp!
-            #
             # create the reference to the bytes/version added to the named-file.
             # this is the most specific reference possible. we return it when we're
             # done here.
@@ -776,6 +755,15 @@ class FileManager:
             name_home = self.named_file_home(name)
             self.csvpaths.logger.debug("Name home is %s", name_home)
             mdata = FileMetadata(self.csvpaths.config)
+            #
+            # add the ability to pass in a UUID. if received, use it here to
+            # set the FileMetadata's UUID. this will allow the caller to give a
+            # trackable ticket. for e.g. we may want to spin-up a registration in
+            # a thread, create our own UUID, pass it in for the registration, and
+            # return the UUID to the user so they can retrieve the reference later
+            #
+            if registration_uuid is not None:
+                mdata.uuid_string = registration_uuid
             mdata.named_file_name = name
             mdata.named_file_ref = ret
             #
@@ -789,7 +777,6 @@ class FileManager:
             mdata.fingerprint = h
             mdata.file_path = rpath
             mdata.file_home = file_home
-            # nos = self.nos
             nos.path = file_home
             mdata.file_name = file_home[file_home.rfind(nos.sep) + 1 :]
             mdata.name_home = name_home

{csvpath-0.0.612 → csvpath-0.0.614}/csvpath/managers/files/files_listener.py

@@ -1,7 +1,4 @@
-import os
 import json
-from datetime import datetime
-from csvpath.util.exceptions import InputException, FileException
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.nos import Nos
@@ -87,4 +84,6 @@ class FilesListener(Listener):  # Registrar,
         jdata = self.get_manifest(manifest_path)
         jdata.append(mani)
         with DataFileWriter(path=manifest_path, mode="w") as writer:
-            json.dump(jdata, writer.sink, indent=2)
+            j = json.dumps(jdata, indent=2)
+            writer.sink.write(j)
+            writer.sink.flush()

{csvpath-0.0.612 → csvpath-0.0.614}/csvpath/managers/paths/paths_listener.py

@@ -1,10 +1,7 @@
-import os
 import json
-from csvpath.util.exceptions import InputException
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.nos import Nos
-from .paths_metadata import PathsMetadata
 from ..listener import Listener
 from ..metadata import Metadata
 
@@ -37,14 +34,13 @@ class PathsListener(Listener):
         mpath = self.manifest_path
         with DataFileReader(mpath, encoding="utf-8") as file:
             contents = file.read()
+            # print(f"pathslist: mpath: {mpath}: {contents}")
             j = json.loads(contents)
-            # j = json.load(file.source)
             return j
 
     @property
     def manifest_path(self) -> None:
         mf = Nos(self.csvpaths.config.inputs_csvpaths_path).join("manifest.json")
-        # mf = os.path.join(self.csvpaths.config.inputs_csvpaths_path, "manifest.json")
         if not Nos(mf).exists():
             with DataFileWriter(path=mf) as file:
                 file.append("[]")
@@ -74,4 +70,6 @@ class PathsListener(Listener):
         jdata = self.manifest
         jdata.append(mani)
         with DataFileWriter(path=self.manifest_path) as file:
-            json.dump(jdata, file.sink, indent=2)
+            j = json.dumps(jdata, indent=2)
+            file.sink.write(j)
+            file.sink.flush()

{csvpath-0.0.612 → csvpath-0.0.614}/csvpath/managers/paths/paths_manager.py

@@ -616,7 +616,8 @@ class PathsManager:
         return t
 
     def get_template_for_paths(self, name: NamedPathsName) -> str:
-        return self.describer.get_template(name)
+        t = self.describer.get_template(name)
+        return "" if t is None else t
 
     def store_template_for_paths(self, name: NamedPathsName, template: str) -> None:
         self.describer.store_template(name, template)

{csvpath-0.0.612 → csvpath-0.0.614}/csvpath/managers/results/result_registrar.py

@@ -209,7 +209,6 @@ class ResultRegistrar(Registrar, Listener):
     @property
     def manifest_path(self) -> str:
         h = Nos(self.result_path).join("manifest.json")
-        # h = os.path.join(self.result_path, "manifest.json")
         return h
 
     @property

{csvpath-0.0.612 → csvpath-0.0.614}/csvpath/managers/results/results_manager.py

@@ -3,7 +3,7 @@ import os
 import re
 from uuid import UUID
 import dateutil.parser
-from typing import Dict, List, Any
+from typing import Any, Optional
 from csvpath.util.line_spooler import LineSpooler
 from csvpath.util.exceptions import InputException, CsvPathsException
 from csvpath.util.references.reference_parser import ReferenceParser
@@ -112,6 +112,7 @@ class ResultsManager:  # pylint: disable=C0115
         file: str = None,
         run_uuid: UUID,
         method: str,
+        extra_data: Optional[dict[str, str]] = None,
     ) -> ResultsMetadata:
         """@private"""
         rr = ResultsRegistrar(
@@ -142,6 +143,12 @@ class ResultsManager:  # pylint: disable=C0115
         #
         #
         mdata = ResultsMetadata(self.csvpaths.config)
+        #
+        # extra data is a flat list of dict[str,str] determined by the user and saved by some
+        # listener. we don't do anything with it, other than save it in the default listener
+        # when we first create the run manifest.json.
+        #
+        mdata.extra_data = extra_data
         mdata.archive_name = self.csvpaths.config.archive_name
         mdata.run_home = run_dir
         mdata.run_uuid = run_uuid
@@ -410,13 +417,13 @@ class ResultsManager:  # pylint: disable=C0115
             rr.add_internal_listener(_)
         rr.register_start(mdata)
 
-    def set_named_results(self, results: Dict[str, List[Result]]) -> None:
+    def set_named_results(self, results: dict[str, list[Result]]) -> None:
         """@private"""
         self.named_results = {}
         for value in results.values():
             self.add_named_results(value)
 
-    def add_named_results(self, results: List[Result]) -> None:
+    def add_named_results(self, results: list[Result]) -> None:
         """@private"""
         for r in results:
             self.add_named_result(r)
@@ -522,7 +529,7 @@ class ResultsManager:  # pylint: disable=C0115
             mydirs[os.path.dirname(path)] = path
         return mydirs
 
-    def _get_named_results_for_reference(self, name: str) -> List[List[Any]]:
+    def _get_named_results_for_reference(self, name: str) -> list[list[Any]]:
         #
         # $mygroup.results.orders/acme/2025:first.myinstance
         #
@@ -592,7 +599,7 @@ class ResultsManager:  # pylint: disable=C0115
     #
     # this gets the results of a single run. it does not get all runs under the name.
     #
-    def get_named_results(self, name) -> List[List[Any]]:
+    def get_named_results(self, name) -> list[list[Any]]:
         #
         # as it turns out, references are (finally!) the easiest, so let's do that
         # first.
@@ -748,7 +755,7 @@ class ResultsManager:  # pylint: disable=C0115
         instances = Nos(path).listdir()
         rs = [None for inst in instances if inst != "manifest.json"]
         for inst in instances:
-            if inst.endswith(".json"):
+            if inst.endswith(".json") or inst == "_extra_data":
                 #
                 # exp ^^^^
                 #