csvpath 0.0.508__tar.gz → 0.0.510__tar.gz

{csvpath-0.0.508 → csvpath-0.0.510}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.508
+Version: 0.0.510
 Summary: An edge governance framework for managing and validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -32,6 +32,7 @@ Requires-Dist: lark (>=1.2.2,<2.0.0)
 Requires-Dist: marquez-python (>=0.50.0,<0.51.0)
 Requires-Dist: metaphone (>=0.6,<0.7)
 Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
+Requires-Dist: opentelemetry-distro[otlp] (>=0.50b0,<0.51)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: ply (>=3.11,<4.0)
 Requires-Dist: pylightxl (>=1.61,<2.0)

{csvpath-0.0.508 → csvpath-0.0.510}/config/config.ini

@@ -26,7 +26,10 @@ imports = config/functions.imports
 
 [listeners]
 groups =
-#slack, marquez, ckan, sftp, sftpplus
+#slack, marquez, ckan, sftp, sftpplus, otlp
+
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
 
 # add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/csvpaths.py

@@ -148,6 +148,16 @@ class CsvPaths(CsvPathsPublic, CsvPathsCoordinator, ErrorCollector):
         self._advance_all = 0
         self._current_run_time = None
         self._run_time_str = None
+        #
+        # metrics is for OTLP OpenTelemetry. it should only
+        # be used by the OTLP listener. it is here because
+        # the integration may need a long-lived presence. if
+        # needed, the first OTLP listener will set it up
+        # before spinning up a thread. any other OTLP
+        # listener threads that need to use a long-lived metric
+        # will work with this property.
+        #
+        self.metrics = None
 
     def run_time_str(self, pathsname=None) -> str:
         """adds the stringified current run time to the named-paths

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/managers/files/file_manager.py

@@ -36,7 +36,37 @@ class FileManager:
     # named-file homes are a dir like: inputs/named_files/March-2024/March-2024.csv
     #
     def named_file_home(self, name: str) -> str:
-        home = os.path.join(self.named_files_dir, name)
+        #
+        # not a named-file name
+        #
+        if name.find("/") > -1:
+            #
+            # this is definitely not what we should be returning. but it is what
+            # works in the new world of remote and fully-qualified local paths.
+            # for now, going with it. the previous implementation was wonky too,
+            # in a different and not visible way, but not good, so this is a step
+            # up in multiple ways.
+            #
+            return ""
+        #
+        # added
+        #
+        home = None
+        if name.startswith("/"):
+            home = name
+        else:
+            #
+            # done add
+            #
+            home = os.path.join(self.named_files_dir, name)
+        #
+        # added
+        #
+        if Nos(home).isfile():
+            home = home[0 : home.rfind(Nos(home).sep)]
+        #
+        # done add
+        #
         return home
 
     def assure_named_file_home(self, name: str) -> str:

csvpath-0.0.510/csvpath/managers/integrations/otlp/metrics.py

@@ -0,0 +1,52 @@
+from opentelemetry import metrics
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+from csvpath.managers.listener import Listener
+
+
+class Metrics:
+    def __init__(self, listener: Listener, exporting=True):
+        self.listener = listener
+        self.reader = None
+        self.provider = None
+        self.meter = None
+        if exporting:
+            self.reader = PeriodicExportingMetricReader(OTLPMetricExporter())
+            self.provider = MeterProvider(metric_readers=[self.reader])
+            metrics.set_meter_provider(self.provider)
+        else:
+            self.reader = InMemoryMetricReader()
+            self.provider = MeterProvider(metric_readers=[self.reader])
+            metrics.set_meter_provider(self.provider)
+
+        self.meter = metrics.get_meter("csvpath")
+        # Create metrics
+        self.runs_started = self.meter.create_counter(
+            "runs_started", description="Number of named-files started"
+        )
+        self.runs_completed = self.meter.create_counter(
+            "runs_completed", description="Number of named-files completed"
+        )
+        self.errors = self.meter.create_counter(
+            "errors", description="Number of errors found at the end of a csvpath's run"
+        )
+        self.files = self.meter.create_counter(
+            "files_generated",
+            description="Number of result files generated by an instance",
+        )
+        self.valid = self.meter.create_counter(
+            "valid_files", description="Number of named-files considered valid"
+        )
+        self.files_expected = self.meter.create_counter(
+            "files_expected",
+            description="Were all the files expected actually produced?",
+        )
+        self.size = self.meter.create_histogram(
+            "named_file_size_distribution",
+            description="Distribution of processed file sizes",
+        )
+        self.bytes = self.meter.create_counter(
+            "bytes", description="Number of bytes in the named-files"
+        )

csvpath-0.0.510/csvpath/managers/integrations/otlp/otlp_listener.py

@@ -0,0 +1,38 @@
+from csvpath.managers.listener import Listener
+from csvpath.managers.metadata import Metadata
+from csvpath.managers.results.results_registrar import ResultsRegistrar
+from .metrics import Metrics
+
+
+class OtlpListener(Listener):
+    def __init__(self, config=None):
+        super().__init__(config=config)
+        self.csvpaths = None
+        self.result = None
+
+    def core_meta(self, mdata: Metadata) -> dict:
+        cmeta = {}
+        if mdata.named_file_name:
+            cmeta["file"] = mdata.named_file_name
+        if mdata.named_results_name:
+            cmeta["paths"] = mdata.named_results_name
+        if mdata.archive_name:
+            cmeta["archive"] = mdata.archive_name
+        if mdata.archive_path:
+            cmeta["archive_path"] = mdata.archive_path
+        if mdata.named_files_root:
+            cmeta["named_files_root"] = mdata.named_files_root
+        if mdata.named_paths_root:
+            cmeta["named_paths_root"] = mdata.named_paths_root
+        if mdata.uuid_string:
+            cmeta["uuid"] = mdata.uuid_string
+        if mdata.run_home:
+            cmeta["run_home"] = mdata.run_home
+        if mdata.hostname:
+            cmeta["hostname"] = mdata.hostname
+        if mdata.username:
+            cmeta["username"] = mdata.username
+        if mdata.ip_address:
+            cmeta["ip_address"] = mdata.ip_address
+
+        return cmeta

csvpath-0.0.510/csvpath/managers/integrations/otlp/otlp_result_listener.py

@@ -0,0 +1,46 @@
+from csvpath.managers.listener import Listener
+from csvpath.managers.metadata import Metadata
+from csvpath.managers.results.results_registrar import ResultsRegistrar
+from .otlp_listener import OtlpListener
+from .metrics import Metrics
+
+
+class OpenTelemetryResultListener(OtlpListener):
+    def __init__(self, config=None):
+        super().__init__(config=config)
+        self.csvpaths = None
+        self.result = None
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        if self.csvpaths and self.csvpaths.metrics is None:
+            self.csvpaths.metrics = Metrics(self)
+        self.csvpaths.metrics.errors.add(mdata.error_count, self.core_meta(mdata))
+        if mdata.error_count:
+            self.csvpaths.metrics.errors.add(mdata.error_count, self.core_meta(mdata))
+        if mdata.valid:
+            v = 0
+            if mdata.valid is True:
+                v = 1
+            self.csvpaths.metrics.valid.add(v, self.core_meta(mdata))
+        if mdata.file_count:
+            self.csvpaths.metrics.files.add(mdata.file_count, self.core_meta(mdata))
+
+        yn = mdata.files_expected
+        if yn and not isinstance(yn, bool):
+            raise ValueError("you must convert")
+        if yn is True:
+            yn = 1
+        else:
+            yn = 0
+        self.csvpaths.metrics.files_expected.add(yn, self.core_meta(mdata))
+
+    def core_meta(self, mdata):
+        cmeta = super().core_meta(mdata)
+        if mdata.instance_identity:
+            cmeta["instance"] = mdata.instance_identity
+        #
+        # named_paths_uuid_string should be on results as well
+        #
+        if mdata.named_paths_uuid_string:
+            cmeta["named_paths_uuid_string"] = mdata.named_paths_uuid_string
+        return cmeta

csvpath-0.0.510/csvpath/managers/integrations/otlp/otlp_results_listener.py

@@ -0,0 +1,32 @@
+from csvpath.managers.listener import Listener
+from csvpath.managers.metadata import Metadata
+from csvpath.managers.results.results_registrar import ResultsRegistrar
+from .otlp_listener import OtlpListener
+from .metrics import Metrics
+
+
+class OpenTelemetryResultsListener(OtlpListener):
+    def __init__(self, config=None):
+        super().__init__(config=config)
+        self.csvpaths = None
+
+    def metadata_update(self, mdata: Metadata) -> None:
+
+        if self.csvpaths and self.csvpaths.metrics is None:
+            self.csvpaths.metrics = Metrics(self)
+
+        if mdata.status == ResultsRegistrar.COMPLETE:
+            print("completing!")
+            self.csvpaths.metrics.runs_completed.add(1, self.core_meta(mdata))
+        else:
+            print("starting")
+            self.csvpaths.metrics.runs_started.add(1, self.core_meta(mdata))
+        size = mdata.named_file_size
+        if size:
+            try:
+                size = int(size)
+            except ValueError:
+                size = None
+        if size is not None:
+            self.csvpaths.metrics.bytes.add(size, self.core_meta(mdata))
+            self.csvpaths.metrics.size.record(size, self.core_meta(mdata))

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/managers/integrations/sftp/sftp_sender.py

@@ -7,6 +7,7 @@ from csvpath.managers.results.result import Result
 from csvpath.managers.listener import Listener
 from csvpath.util.nos import Nos
 from csvpath.util.var_utility import VarUtility
+from csvpath.util.file_readers import DataFileReader
 
 
 #
@@ -105,10 +106,24 @@ class SftpSender(Listener, threading.Thread):
                     raise ValueError("File name {file} is not in {files}")
                 if to is None:
                     raise ValueError("File name {file} has no destination")
-                path = f"/Users/davidkershaw/dev/csvpath/{self.metadata.run_home}{sep}{self.result.identity_or_index}{sep}{file}"
+                path = f"{self.metadata.run_home}{sep}{self.result.identity_or_index}{sep}{file}"
                 remote_path = f"{self._target_path}/{to}"
                 self.csvpaths.logger.info("Putting %s to %s", path, remote_path)
-                sftp.put(path, remote_path)
+                #
+                # need to use data reader and stream. if we're
+                # sftping from s3 or something this is going to
+                # be slow
+                #
+                # sftp.put(path, remote_path)
+                try:
+                    sftp.stat(self._target_path)
+                except FileNotFoundError:
+                    sftp.mkdir(self._target_path)
+                with DataFileReader(path) as reader:
+                    flo = reader.source
+                    f = sftp.open(remote_path, "wb")
+                    f.write(flo.read())
+                    f.close()
             #
             # send the original file if we need to. this will always be the normative
             # original, w/o regard to source-mode or by_line chaining or to rewind.

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/managers/metadata.py

@@ -3,6 +3,8 @@ import os
 from abc import ABC
 from datetime import datetime, timezone
 from dateutil import parser
+import getpass
+import socket
 
 
 class Metadata(ABC):
@@ -18,6 +20,19 @@ class Metadata(ABC):
         self._base_path = None
         self._named_files_root: str = None
         self._named_paths_root: str = None
+        self.username = None
+        try:
+            self.username = getpass.getuser()
+        except Exception:
+            ...
+        self.hostname = None
+        self.ip_address = None
+        try:
+            self.hostname = socket.gethostname()
+            self.ip_address = socket.gethostbyname(self.hostname)
+        except Exception:
+            ...
+
         #
         # find base dir so we can add file:// refs, if needed
         #

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/managers/results/result_serializer.py

@@ -24,6 +24,9 @@ class ResultSerializer:
     def save_result(self, result) -> None:
         self.result = result
         runtime_data = {}
+        result.csvpath.csvpaths.logger.debug(
+            "Saving result of %s.%s", result.paths_name, result.identity_or_index
+        )
         RuntimeDataCollector.collect(result.csvpath, runtime_data, local=True)
         runtime_data["run_index"] = result.run_index
         es = []

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/managers/results/results_metadata.py

@@ -1,5 +1,4 @@
 from csvpath.managers.metadata import Metadata
-
 from datetime import datetime
 
 

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/managers/results/results_registrar.py

@@ -106,6 +106,9 @@ class ResultsRegistrar(Registrar, Listener):
         m["named_file_last_change"] = mdata.named_file_last_change
         m["named_file_fingerprint"] = mdata.named_file_fingerprint
         m["named_file_fingerprint_on_file"] = mdata.named_file_fingerprint_on_file
+        m["hostname"] = mdata.hostname
+        m["username"] = mdata.username
+        m["ip_address"] = mdata.ip_address
         mp = mdata.manifest_path
         m["manifest_path"] = mp
         with DataFileWriter(path=mp) as file:
@@ -119,16 +122,14 @@ class ResultsRegistrar(Registrar, Listener):
     def _size(self, path) -> str:
         try:
             fi = FileInfo.info(path)
-            return fi["bytes"] if fi and "bytes" in fi else -1
+            return fi["bytes"]
         except FileNotFoundError:
             return 0
 
     def _last_change(self, path) -> str:
         try:
             fi = FileInfo.info(path)
-            if fi and "last_mod" in fi:
-                return fi["last_mod"]
-            return None
+            return fi["last_mod"]
         except FileNotFoundError:
             return -1
 

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/util/box.py

@@ -1,6 +1,6 @@
 #
 # just a box to put things in. initial use is
-# sharing a boto3 client under "boto_client".
+# sharing a boto3 client under "boto_s3_client".
 #
 class Box:
     STUFF = {}

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/util/config.py

@@ -175,10 +175,11 @@ path =
 [listeners]
 # add listener group names to send events to the channel they represent
 groups =
-#slack, marquez, ckan, sftp, sftpplus
+#slack, marquez, ckan, sftp, sftpplus, otlp
 
-# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
-sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
+# add otlp to the list of groups above to push observability metrics to an OpenTelemetry endpoint
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
 
 # add sftp to the list of groups above to push content and metadata to an SFTP account
 sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener

csvpath-0.0.510/csvpath/util/file_info.py

@@ -0,0 +1,49 @@
+import os
+
+
+class FileInfo:
+    @classmethod
+    def info(cls, path) -> dict[str, str | int | float]:
+        if path is None:
+            raise ValueError("Path cannot be None")
+        try:
+            if path.find("://") > -1:
+                return cls._remote(path)
+            return cls._local(path)
+        except Exception:
+            # this shouldn't happen, but it also shouldn't be the
+            # main source of friction. If we have a wrong path or
+            # an asset we don't know how to get info on, we'll know it
+            # above and can handle it there in a more meaningful
+            # context.
+            return cls._empty()
+
+    @classmethod
+    def _remote(cls, path):
+        return cls._empty()
+
+    @classmethod
+    def _empty(cls) -> dict:
+        return {
+            "mode": "",
+            "device": "",
+            "bytes": -1,
+            "created": None,
+            "last_read": None,
+            "last_mod": None,
+            "flags": None,
+        }
+
+    @classmethod
+    def _local(cls, path):
+        s = os.stat(path)
+        meta = {
+            "mode": s.st_mode,
+            "device": s.st_dev,
+            "bytes": s.st_size,
+            "created": s.st_ctime,
+            "last_read": s.st_atime,
+            "last_mod": s.st_mtime,
+            "flags": s.st_flags,
+        }
+        return meta

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/util/nos.py

@@ -139,6 +139,12 @@ class S3Do:
     def isfile(self) -> bool:
         bucket, key = S3Utils.path_to_parts(self.path)
         client = boto3.client("s3")
+        #
+        # boto3 uses a deprecated feature. pytest doesn't like it. this is a quick fix.
+        #
+        import warnings
+
+        warnings.filterwarnings(action="ignore", message=r"datetime.datetime.utcnow")
         try:
             client.head_object(Bucket=bucket, Key=key)
         except ClientError as e:

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/util/s3/s3_data_reader.py

@@ -16,13 +16,16 @@ from csvpath.util.box import Box
 class S3DataReader(CsvDataReader):
     def load_if(self) -> None:
         if self.source is None:
-            client = Box.STUFF.get("boto_client")
+            client = Box.STUFF.get("boto_s3_client")
             if client is None:
+                client = S3Utils.make_client()
+                """
                 session = boto3.Session(
                     aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                     aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                 )
                 client = session.client("s3")
+                """
             try:
                 self.source = open(self._path, "r", transport_params={"client": client})
             except DeprecationWarning:

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/util/s3/s3_data_writer.py

@@ -5,23 +5,26 @@ import boto3
 from smart_open import open
 from ..file_writers import DataFileWriter
 from csvpath.util.box import Box
+from csvpath.util.s3.s3_utils import S3Utils
 
 
 class S3DataWriter(DataFileWriter):
     def load_if(self) -> None:
         if self.sink is None:
-            client = Box.STUFF.get("boto_client")
+            client = Box.STUFF.get("boto_s3_client")
             if client is None:
+                client = S3Utils.make_client()
+                """
                 session = boto3.Session(
                     aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                     aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                 )
                 import warnings
-
                 warnings.filterwarnings(
                     action="ignore", message=r"datetime.datetime.utcnow"
                 )
                 client = session.client("s3")
+                """
             self.sink = open(
                 self._path,
                 self._mode,
@@ -33,13 +36,16 @@ class S3DataWriter(DataFileWriter):
         as a context manager for this method. for multiple write
         calls to the same file handle use append().
         """
-        client = Box.STUFF.get("boto_client")
+        client = Box.STUFF.get("boto_s3_client")
         if client is None:
+            client = S3Utils.make_client()
+            """
             session = boto3.Session(
                 aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                 aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
             )
             client = session.client("s3")
+            """
         with open(self._path, "wb", transport_params={"client": client}) as file:
             file.write(data.encode("utf-8"))
 

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/util/s3/s3_utils.py

@@ -1,9 +1,23 @@
+import os
 import boto3
 import uuid
 from botocore.exceptions import ClientError
+from csvpath.util.box import Box
 
 
 class S3Utils:
+    @classmethod
+    def make_client(cls):
+        import warnings
+
+        warnings.filterwarnings(action="ignore", message=r"datetime.datetime.utcnow")
+        session = boto3.Session(
+            aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+            aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+        )
+        client = session.client("s3")
+        return client
+
     @classmethod
     def path_to_parts(self, path) -> tuple[str, str]:
         if path.startswith("s3://"):
@@ -53,14 +67,5 @@ class S3Utils:
 
     @classmethod
     def rename(self, bucket: str, key: str, new_key: str) -> None:
-        """
-        client = boto3.client("s3")
-        r = client.copy_object(
-            Bucket=bucket,
-            CopySource={'Bucket': bucket, 'Key': key},
-            Key=new_key,
-            ChecksumAlgorithm='SHA256'
-        )
-        """
         S3Utils.copy(bucket, key, bucket, new_key)
         S3Utils.remove(bucket, key)

{csvpath-0.0.508 → csvpath-0.0.510}/csvpath/util/s3/s3_xlsx_data_reader.py

@@ -6,6 +6,7 @@ from smart_open import open
 from ..file_readers import XlsxDataReader
 from .s3_fingerprinter import S3Fingerprinter
 from csvpath.util.box import Box
+from csvpath.util.s3.s3_utils import S3Utils
 
 
 class S3XlsxDataReader(XlsxDataReader):
@@ -19,13 +20,9 @@ class S3XlsxDataReader(XlsxDataReader):
 
     def load_if(self) -> None:
         if self.source is None:
-            client = Box.STUFF.get("boto_client")
+            client = Box.STUFF.get("boto_s3_client")
             if client is None:
-                session = boto3.Session(
-                    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
-                    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
-                )
-                client = session.client("s3")
+                client = S3Utils.make_client()
             try:
                 self.source = open(
                     self._path, "rb", transport_params={"client": client}

{csvpath-0.0.508 → csvpath-0.0.510}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.508"
+version = "0.0.510"
 description = "An edge governance framework for managing and validating CSV, Excel, and other tabular data files"
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"
@@ -49,6 +49,7 @@ ckanapi = "^4.8"
 smart-open = "^7.1.0"
 paramiko = "^3.5.0"
 boto3 = "^1.35.91"
+opentelemetry-distro = {extras = ["otlp"], version = "^0.50b0"}
 
 [tool.poetry.extras]
 pandas = ["pandas"]
@@ -62,6 +63,7 @@ pydeps = "^1.12.20"
 pylint = "^3.2.7"
 coverage = "^7.6.1"
 pytest = "^8.2.2"
+pytest-profiling = "^1.8.1"
 
 [build-system]
 requires = ["poetry-core"]

csvpath-0.0.508/csvpath/util/file_info.py

@@ -1,29 +0,0 @@
-import os
-
-
-class FileInfo:
-    @classmethod
-    def info(cls, path) -> dict[str, str | int | float]:
-        if path is None:
-            raise ValueError("Path cannot be None")
-        if path.find("://") > -1:
-            return cls._remote(path)
-        return cls._local(path)
-
-    @classmethod
-    def _remote(cls, path):
-        return {}
-
-    @classmethod
-    def _local(cls, path):
-        s = os.stat(path)
-        meta = {
-            "mode": s.st_mode,
-            "device": s.st_dev,
-            "bytes": s.st_size,
-            "created": s.st_ctime,
-            "last_read": s.st_atime,
-            "last_mod": s.st_mtime,
-            "flags": s.st_flags,
-        }
-        return meta