csvpath 0.0.502__tar.gz → 0.0.504__tar.gz

{csvpath-0.0.502 → csvpath-0.0.504}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.502
+Version: 0.0.504
 Summary: A declarative language for validating CSV, Excel, and other tabular data files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -24,6 +24,7 @@ Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
 Provides-Extra: pandas
 Provides-Extra: smartopen
+Requires-Dist: boto3 (>=1.35.91,<2.0.0)
 Requires-Dist: bullet (>=2.2.0,<3.0.0)
 Requires-Dist: ckanapi (>=4.8,<5.0)
 Requires-Dist: inflect (>=7.4.0,<8.0.0)
@@ -38,6 +39,7 @@ Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: pytz (>=2024.2,<2025.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: smart-open (>=7.1.0,<8.0.0) ; extra == "smartopen"
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git
@@ -61,13 +63,16 @@ CsvPath's validation is inspired by:
 - XPath for XML files
 - The ISO standard <a href='https://schematron.com/'>Schematron validation</a>
 
-CsvPath is intended to fit with other DataOps and data quality tools. Files are streamed. The interface is simple. Metadata is plentiful. New functions are easy to create. CsvPath can stream events to an OpenLineage server, such as the open source Marquez server.
+CsvPath is intended to fit tightly with other DataOps and data quality tools. Files are streamed. The interface is simple. Metadata is plentiful. New functions and listeners are easy to create.
 
-<a href='https://openlineage.io' >
+CsvPath can stream lineage events to an OpenLineage server, such as the open source Marquez server. Read about <a href="https://www.csvpath.org/getting-started/getting-started-with-csvpath-+-openlineage" target="_blank">CsvPath and OpenLineage here</a>.
+<br/><a href='https://openlineage.io' >
 <img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/openlineage-logo-sm.png" alt="OpenLineage"/></a>
 <a href='https://peppy-sprite-186812.netlify.app/' >
 <img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/marquez-logo-sm.png" alt="Marquez Server"/></a>
 
+Need to publish validated datasets to a CKAN data portal? <a href="https://www.csvpath.org/getting-started/getting-started-with-csvpath-+-ckan" target="_blank">Read about how CsvPath is integrated with CKAN</a>.
+<a href="https://ckan.org/" target="_blank"><img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/ckan-logo-sm.png" alt="CKAN Data Portal"/></a>
 
 Read more about CsvPath and see CSV, Excel, and Data Frames validation examples at <a href='https://www.csvpath.org'>https://www.csvpath.org</a>.
 

{csvpath-0.0.502 → csvpath-0.0.504}/README.md

@@ -16,13 +16,16 @@ CsvPath's validation is inspired by:
 - XPath for XML files
 - The ISO standard <a href='https://schematron.com/'>Schematron validation</a>
 
-CsvPath is intended to fit with other DataOps and data quality tools. Files are streamed. The interface is simple. Metadata is plentiful. New functions are easy to create. CsvPath can stream events to an OpenLineage server, such as the open source Marquez server.
+CsvPath is intended to fit tightly with other DataOps and data quality tools. Files are streamed. The interface is simple. Metadata is plentiful. New functions and listeners are easy to create.
 
-<a href='https://openlineage.io' >
+CsvPath can stream lineage events to an OpenLineage server, such as the open source Marquez server. Read about <a href="https://www.csvpath.org/getting-started/getting-started-with-csvpath-+-openlineage" target="_blank">CsvPath and OpenLineage here</a>.
+<br/><a href='https://openlineage.io' >
 <img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/openlineage-logo-sm.png" alt="OpenLineage"/></a>
 <a href='https://peppy-sprite-186812.netlify.app/' >
 <img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/marquez-logo-sm.png" alt="Marquez Server"/></a>
 
+Need to publish validated datasets to a CKAN data portal? <a href="https://www.csvpath.org/getting-started/getting-started-with-csvpath-+-ckan" target="_blank">Read about how CsvPath is integrated with CKAN</a>.
+<a href="https://ckan.org/" target="_blank"><img target='_blank' src="https://github.com/csvpath/csvpath/blob/main/docs/images/ckan-logo-sm.png" alt="CKAN Data Portal"/></a>
 
 Read more about CsvPath and see CSV, Excel, and Data Frames validation examples at <a href='https://www.csvpath.org'>https://www.csvpath.org</a>.
 

{csvpath-0.0.502 → csvpath-0.0.504}/config/config.ini

@@ -29,8 +29,6 @@ groups =
 #slack, marquez, ckan
 
 # add ckan to the list of groups above for alerts to slack webhooks
-ckan.paths = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
-ckan.result = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
 
 #add marquez to the list of groups above for OpenLineage events to a local Marquez
@@ -45,6 +43,10 @@ slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
 
+[ckan]
+server = http://localhost:80
+api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
+
 [marquez]
 base_url = http://localhost:5000
 endpoint = api/v1/lineage

{csvpath-0.0.502 → csvpath-0.0.504}/csvpath/csvpath.py

@@ -5,7 +5,7 @@ import time
 import os
 import hashlib
 from datetime import datetime, timezone
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Self
 from collections.abc import Iterator
 from abc import ABC, abstractmethod
 from .util.config import Config
@@ -72,7 +72,7 @@ class CsvPathPublic(ABC):
         """Advances the iteration by ff rows. -1 means to the end of the file."""
 
     @abstractmethod
-    def fast_forward(self, csvpath: str = None) -> None:  # pragma: no cover
+    def fast_forward(self, csvpath: str = None) -> Self:  # pragma: no cover
         """Scans to the end of the CSV file. All scanned rows will be
         considered for match and variables and side effects will happen,
         but no rows will be returned or stored. -1 means to the end of
@@ -965,7 +965,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
             self.lines = None
         return lines
 
-    def fast_forward(self, csvpath=None) -> None:
+    def fast_forward(self, csvpath=None) -> Self:
         """Runs the path for all rows of the file. Variables are collected
         and side effects like print happen. No lines are collected.
         """
@@ -973,6 +973,7 @@ class CsvPath(CsvPathPublic, ErrorCollector, Printer):  # pylint: disable=R0902,
             self.parse(csvpath)
         for _ in self.next():
             pass
+        return self
 
     def next(self, csvpath=None):
         """Iterates over the lines in the CSV file returning those that match

{csvpath-0.0.502 → csvpath-0.0.504}/csvpath/managers/files/file_cacher.py

@@ -13,6 +13,8 @@ class FileCacher:
         self.pathed_lines_and_headers = {}
 
     def get_new_line_monitor(self, filename: str) -> LineMonitor:
+        if filename is None:
+            raise ValueError("Filename cannot be None")
         if filename not in self.pathed_lines_and_headers:
             self._find_lines_and_headers(filename)
         lm = self.pathed_lines_and_headers[filename][0]
@@ -25,6 +27,8 @@ class FileCacher:
         return self.pathed_lines_and_headers[filename][1][:]
 
     def _find_lines_and_headers(self, filename: str) -> None:
+        if filename is None:
+            raise ValueError("Filename cannot be None")
         lm, headers = self._cached_lines_and_headers(filename)
         if lm is None or headers is None:
             lc = LineCounter(self.csvpaths)
@@ -33,6 +37,8 @@ class FileCacher:
         self.pathed_lines_and_headers[filename] = (lm, headers)
 
     def _cached_lines_and_headers(self, filename: str) -> Tuple[LineMonitor, List[str]]:
+        if filename is None:
+            raise ValueError("Filename cannot be None")
         lm = LineMonitor()
         json = self.cache.cached_text(filename, "json")
         if json is not None and not json.strip() == "":

{csvpath-0.0.502 → csvpath-0.0.504}/csvpath/managers/files/file_manager.py

@@ -1,21 +1,20 @@
 import os
 import json
 import csv
-import hashlib
-import shutil
 from json import JSONDecodeError
-from typing import Dict, List, Tuple
 from csvpath.util.error import ErrorHandler
 from csvpath.util.file_readers import DataFileReader
+from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.reference_parser import ReferenceParser
 from csvpath.util.exceptions import InputException, FileException
+from csvpath.util.nos import Nos
 from .file_registrar import FileRegistrar
 from .file_cacher import FileCacher
 from .file_metadata import FileMetadata
 
 
 class FileManager:
-    def __init__(self, *, named_files: Dict[str, str] = None, csvpaths=None):
+    def __init__(self, *, named_files: dict[str, str] = None, csvpaths=None):
         if named_files is None:
             named_files = {}
         self._csvpaths = csvpaths
@@ -43,24 +42,25 @@ class FileManager:
     def assure_named_file_home(self, name: str) -> str:
         home = self.named_file_home(name)
         if not os.path.exists(home):
-            os.makedirs(home)
+            Nos(home).makedirs()
         return home
 
     #
     # file homes are paths to files like:
     #   inputs/named_files/March-2024/March-2024.csv/March-2024.csv
-    # which become paths to hash-named file versions like:
+    # which become paths to fingerprint-named file versions like:
     #   inputs/named_files/March-2024/March-2024.csv/12467d811d1589ede586e3a42c41046641bedc1c73941f4c21e2fd2966f188b4.csv
     # once the files have been fingerprinted
     #
     def assure_file_home(self, name: str, path: str) -> str:
         if path.find("#") > -1:
             path = path[0 : path.find("#")]
-        fname = path if path.rfind(os.sep) == -1 else path[path.rfind(os.sep) + 1 :]
+        sep = Nos(path).sep
+        fname = path if path.rfind(sep) == -1 else path[path.rfind(sep) + 1 :]
         home = self.named_file_home(name)
         home = os.path.join(home, fname)
-        if not os.path.exists(home):
-            os.makedirs(home)
+        if not Nos(home).exists():
+            Nos(home).makedirs()
         return home
 
     @property
@@ -70,28 +70,35 @@ class FileManager:
     @property
     def named_file_names(self) -> list:
         b = self.named_files_dir
-        ns = [n for n in os.listdir(b) if not os.path.isfile(os.path.join(b, n))]
+        ns = [n for n in Nos(b).listdir() if not Nos(os.path.join(b, n)).isfile()]
         return ns
 
     def name_exists(self, name: str) -> bool:
         p = self.named_file_home(name)
-        return os.path.exists(p)
+        b = Nos(p).dir_exists()
+        return b
 
     def remove_named_file(self, name: str) -> None:
         p = os.path.join(self.named_files_dir, name)
-        shutil.rmtree(p)
+        Nos(p).remove()
 
     def remove_all_named_files(self) -> None:
         names = self.named_file_names
         for name in names:
             self.remove_named_file(name)
 
-    def set_named_files(self, nf: Dict[str, str]) -> None:
+    def set_named_files(self, nf: dict[str, str]) -> None:
         for k, v in nf.items():
             self.add_named_file(name=k, path=v)
 
     def set_named_files_from_json(self, filename: str) -> None:
+        """named-files from json files are always local"""
         try:
+            #
+            # TODO: named-files json files are always local. they should
+            # be able to be on s3 so that we are completely independent of
+            # the local disk w/re file manager
+            #
             with open(filename, "r", encoding="utf-8") as f:
                 j = json.load(f)
                 self.set_named_files(j)
@@ -99,7 +106,7 @@ class FileManager:
             ErrorHandler(csvpaths=self._csvpaths).handle_error(ex)
 
     def add_named_files_from_dir(self, dirname: str):
-        dlist = os.listdir(dirname)
+        dlist = Nos(dirname).listdir()
         base = dirname
         for p in dlist:
             _ = p.lower()
@@ -115,7 +122,6 @@ class FileManager:
 
     #
     # -------------------------------------
-    # move functions to this class and file_data_filesystem_storekeeper
     #
     def add_named_file(self, *, name: str, path: str) -> None:
         #
@@ -156,30 +162,41 @@ class FileManager:
         mdata.fingerprint = h
         mdata.file_path = rpath
         mdata.file_home = file_home
-        mdata.file_name = file_home[file_home.rfind(os.sep) + 1 :]
+        mdata.file_name = file_home[file_home.rfind(Nos(file_home).sep) + 1 :]
         mdata.name_home = name_home
         mdata.mark = mark
         self.registrar.register_complete(mdata)
 
     def _copy_in(self, path, home) -> None:
-        fname = path if path.rfind(os.sep) == -1 else path[path.rfind(os.sep) + 1 :]
+        sep = Nos(path).sep
+        fname = path if path.rfind(sep) == -1 else path[path.rfind(sep) + 1 :]
         # creates
         #   a/file.csv -> named_files/name/file.csv/file.csv
         # the dir name matching the resulting file name is correct
         # once the file is landed and fingerprinted, the file
         # name is changed.
         temp = os.path.join(home, fname)
-        if path.startswith("s3:"):
-            self._copy_down(path, temp)
+        #
+        # this is another place that is too s3 vs. local. we'll have
+        # other source/sinks to support.
+        #
+        if path.startswith("s3:") and not home.startswith("s3"):
+            self._copy_down(path, temp, mode="wb")
+        elif path.startswith("s3:") and home.startswith("s3"):
+            Nos(path).copy(temp)
+        elif not path.startswith("s3:") and not home.startswith("s3"):
+            self._copy_down(path, temp, mode="wb")
+        elif not path.startswith("s3:") and home.startswith("s3"):
+            self._copy_down(path, temp, mode="wb")
         else:
-            shutil.copy(path, temp)
+            ...  # not possible. just being explicit for the moment.
         return temp
 
-    def _copy_down(self, path, temp) -> None:
-        reader = DataFileReader(path)
-        with open(temp, "w", encoding="utf-8") as file:
-            for line in reader.next_raw():
-                file.write(line)
+    def _copy_down(self, path, temp, mode="wb") -> None:
+        with DataFileReader(path) as reader:
+            with DataFileWriter(path=temp, mode=mode) as writer:
+                for line in reader.next_raw():
+                    writer.append(line)
 
     #
     # can take a reference. the ref would only be expected to point
@@ -201,18 +218,21 @@ class FileManager:
         else:
             if not self.name_exists(name):
                 return None
-            ret = self.registrar.registered_file(self.named_file_home(name))
+            n = self.named_file_home(name)
+            ret = self.registrar.registered_file(n)
         return ret
 
     def get_fingerprint_for_name(self, name) -> str:
         if name.startswith("$"):
             # atm, we don't give fingerprints for references doing rewind/replay
             return ""
+        #
+        # note: this is not creating fingerprints, just getting existing ones.
+        #
         return self.registrar.get_fingerprint(self.named_file_home(name))
 
     #
     # -------------------------------------
-    # move to file_data_filesystem_storekeeper?
     #
     def get_named_file_reader(self, name: str) -> DataFileReader:
         path = self.get_named_file(name)
@@ -228,7 +248,8 @@ class FileManager:
         )
 
     def _fingerprint(self, path) -> str:
-        fname = path if path.rfind(os.sep) == -1 else path[path.rfind(os.sep) + 1 :]
+        sep = Nos(path).sep
+        fname = path if path.rfind(sep) == -1 else path[path.rfind(sep) + 1 :]
         t = None
         i = fname.find(".")
         if i > -1:
@@ -240,25 +261,29 @@ class FileManager:
         # creating the initial file name, where the file starts
         #
         fpath = os.path.join(path, fname)
-        with open(fpath, "rb") as f:
-            h = hashlib.file_digest(f, hashlib.sha256)
-            h = h.hexdigest()
-        #
-        # creating the new path using the hash as filename
-        #
-        hpath = os.path.join(path, h)
-        if t is not None:
-            hpath = f"{hpath}.{t}"
-        #
-        # if we're re-adding the file we don't need to make
-        # another copy of it. re-adds are fine.
-        #
-        b = os.path.exists(hpath)
-        if b:
-            os.remove(fpath)
-            return hpath, h
+        h = None
         #
-        # if a first add, rename the file to the hash + ext
+        # this version should work local and minimize traffic when in S3
         #
-        os.rename(fpath, hpath)
+        with DataFileReader(fpath) as f:
+            h = f.fingerprint()
+            #
+            # creating the new path using the fingerprint as filename
+            #
+            hpath = os.path.join(path, h)
+            if t is not None:
+                hpath = f"{hpath}.{t}"
+            #
+            # if we're re-adding the file we don't need to make
+            # another copy of it. re-adds are fine.
+            #
+            # need an s3 way to do this
+            b = Nos(hpath).exists()
+            if b:
+                Nos(fpath).remove()
+                return hpath, h
+            #
+            # if a first add, rename the file to the fingerprint + ext
+            #
+            Nos(fpath).rename(hpath)
         return hpath, h

{csvpath-0.0.502 → csvpath-0.0.504}/csvpath/managers/files/file_registrar.py

@@ -1,10 +1,10 @@
 import os
 import json
-import hashlib
-import shutil
 from datetime import datetime
 from csvpath.util.exceptions import InputException, FileException
 from csvpath.util.file_readers import DataFileReader
+from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.nos import Nos
 from csvpath.managers.registrar import Registrar
 from csvpath.managers.listener import Listener
 from csvpath.managers.metadata import Metadata
@@ -30,17 +30,17 @@ class FileRegistrar(Registrar, Listener):
         return man[len(man) - 1]["fingerprint"]
 
     def manifest_path(self, home) -> str:
-        if not os.path.exists(home):
+        if not Nos(home).dir_exists():
             raise InputException(f"Named file home does not exist: {home}")
         mf = os.path.join(home, "manifest.json")
-        if not os.path.exists(mf):
-            with open(mf, "w", encoding="utf-8") as file:
-                file.write("[]")
+        if not Nos(mf).exists():
+            with DataFileWriter(path=mf, mode="w") as writer:
+                writer.append("[]")
         return mf
 
     def get_manifest(self, mpath) -> list:
-        with open(mpath, "r", encoding="utf-8") as file:
-            return json.load(file)
+        with DataFileReader(mpath) as reader:
+            return json.load(reader.source)
 
     def metadata_update(self, mdata: Metadata) -> None:
         path = mdata.origin_path
@@ -60,8 +60,8 @@ class FileRegistrar(Registrar, Listener):
             mani["mark"] = mark
         jdata = self.get_manifest(manifest_path)
         jdata.append(mani)
-        with open(manifest_path, "w", encoding="utf-8") as file:
-            json.dump(jdata, file, indent=2)
+        with DataFileWriter(path=manifest_path, mode="w") as writer:
+            json.dump(jdata, writer.sink, indent=2)
 
     def register_complete(self, mdata: Metadata) -> None:
         path = mdata.origin_path
@@ -75,7 +75,8 @@ class FileRegistrar(Registrar, Listener):
             raise InputException(
                 f"File mgr and registrar marks should match: {mdata.mark}, {mark}"
             )
-        if not path.startswith("s3:") and not os.path.exists(path):
+        if not path.startswith("s3:") and not Nos(path).exists():
+            # if not path.startswith("s3:") and not os.path.exists(path):
             #
             # try for a data reader in case we're smart-opening
             #
@@ -134,8 +135,8 @@ class FileRegistrar(Registrar, Listener):
 
     def registered_file(self, home: str) -> str:
         mpath = self.manifest_path(home)
-        with open(mpath, "r", encoding="utf-8") as file:
-            mdata = json.load(file)
+        with DataFileReader(mpath) as reader:
+            mdata = json.load(reader.source)
             if mdata is None or len(mdata) == 0:
                 raise InputException(f"Manifest for {home} at {mpath} is empty")
             m = mdata[len(mdata) - 1]

{csvpath-0.0.502 → csvpath-0.0.504}/csvpath/managers/integrations/ckan/ckan.py

@@ -4,7 +4,6 @@ from csvpath import CsvPaths
 from csvpath.util.config import Config
 from .dataset import Dataset
 from .datafile import Datafile
-import hashlib
 import os
 
 

{csvpath-0.0.502 → csvpath-0.0.504}/csvpath/managers/metadata.py

@@ -14,6 +14,7 @@ class Metadata(ABC):
         self._uuid = uuid4()
         self.manifest_path: str = None
         self.archive_name: str = None
+        self.archive_path: str = None
         self._base_path = None
         self._named_files_root: str = None
         self._named_paths_root: str = None
@@ -54,6 +55,8 @@ class Metadata(ABC):
             self.manifest_path = m.get("manifest_path")
         if m.get("archive_name") is not None:
             self.archive_name = m.get("archive_name")
+        if m.get("archive_path") is not None:
+            self.archive_path = m.get("archive_path")
 
     @property
     def uuid(self) -> UUID:
@@ -68,7 +71,6 @@ class Metadata(ABC):
     @property
     def uuid_string(self) -> str:
         return str(self._uuid)
-        # return self._uuid.hex
 
     @uuid_string.setter
     def uuid_string(self, u: str) -> None:
@@ -128,7 +130,6 @@ class Metadata(ABC):
 
     @time_completed_string.setter
     def time_completed_string(self, s: str) -> None:
-        # self._time_completed = datetime.date.fromisoformat(s)
         self._time_completed = parser.parse(s)
 
     @property

{csvpath-0.0.502 → csvpath-0.0.504}/csvpath/managers/paths/paths_manager.py

@@ -1,7 +1,6 @@
 # pylint: disable=C0114
 import os
 import json
-import shutil
 from typing import NewType
 from json import JSONDecodeError
 from csvpath import CsvPath
@@ -9,6 +8,9 @@ from csvpath.util.exceptions import InputException
 from csvpath.util.error import ErrorHandler
 from csvpath.util.metadata_parser import MetadataParser
 from csvpath.util.reference_parser import ReferenceParser
+from csvpath.util.file_readers import DataFileReader
+from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.nos import Nos
 from .paths_registrar import PathsRegistrar
 from .paths_metadata import PathsMetadata
 
@@ -37,8 +39,8 @@ class PathsManager:
 
     def named_paths_home(self, name: NamedPathsName) -> str:
         home = os.path.join(self.named_paths_dir, name)
-        if not os.path.exists(home):
-            os.makedirs(home)
+        if not Nos(home).exists():
+            Nos(home).makedirs()
         return home
 
     @property
@@ -61,8 +63,8 @@ class PathsManager:
         if directory is None:
             ie = InputException("Named paths collection name needed")
             ErrorHandler(csvpaths=self.csvpaths).handle_error(ie)
-        if os.path.isdir(directory):
-            dlist = os.listdir(directory)
+        if not Nos(directory).isfile():
+            dlist = Nos(directory).listdir()
             base = directory
             for p in dlist:
                 if p[0] == ".":
@@ -142,8 +144,9 @@ class PathsManager:
         mdata = PathsMetadata(self.csvpaths.config)
         mdata.archive_name = self.csvpaths.config.archive_name
         mdata.named_paths_name = name
-        mdata.named_paths_home = f"{mdata.named_paths_root}{os.sep}{name}"
-        mdata.group_file_path = f"{mdata.named_paths_home}{os.sep}group.csvpaths"
+        sep = Nos(mdata.named_paths_root).sep
+        mdata.named_paths_home = f"{mdata.named_paths_root}{sep}{name}"
+        mdata.group_file_path = f"{mdata.named_paths_home}{sep}group.csvpaths"
         mdata.named_paths = paths
         mdata.named_paths_identities = ids
         mdata.named_paths_count = len(ids)
@@ -193,15 +196,15 @@ class PathsManager:
     def store_json_paths_file(self, name: str, jsonpath: str) -> None:
         home = self.named_paths_home(name)
         j = ""
-        with open(jsonpath, "r", encoding="utf-8") as file:
+        with DataFileReader(jsonpath) as file:
             j = file.read()
-        with open(os.path.join(home, "definition.json"), "w", encoding="utf-8") as file:
-            file.write(j)
+        with DataFileWriter(path=os.path.join(home, "definition.json")) as writer:
+            writer.write(j)
 
     @property
     def named_paths_names(self) -> list[str]:
         path = self.named_paths_dir
-        names = [n for n in os.listdir(path) if not n.startswith(".")]
+        names = [n for n in Nos(path).listdir() if not n.startswith(".")]
         return names
 
     def remove_named_paths(self, name: NamedPathsName, strict: bool = False) -> None:
@@ -210,7 +213,7 @@ class PathsManager:
         if not self.has_named_paths(name):
             return
         home = self.named_paths_home(name)
-        shutil.rmtree(home)
+        Nos(home).remove()
 
     def remove_all_named_paths(self) -> None:
         names = self.named_paths_names
@@ -219,7 +222,7 @@ class PathsManager:
 
     def has_named_paths(self, name: NamedPathsName) -> bool:
         path = os.path.join(self.named_paths_dir, name)
-        return os.path.exists(path)
+        return Nos(path).dir_exists()
 
     def number_of_named_paths(self, name: NamedPathsName) -> int:
         return len(self._get_named_paths(name))
@@ -237,9 +240,9 @@ class PathsManager:
         s = ""
         path = self.named_paths_home(name)
         grp = os.path.join(path, "group.csvpaths")
-        if os.path.exists(grp):
-            with open(grp, "r", encoding="utf-8") as file:
-                s = file.read()
+        if Nos(grp).exists():
+            with DataFileReader(grp) as reader:
+                s = reader.read()
         cs = s.split("---- CSVPATH ----")
         cs = [s for s in cs if s.strip() != ""]
         #
@@ -258,8 +261,11 @@ class PathsManager:
 
     def _copy_in(self, name, csvpathstr) -> None:
         temp = self._group_file_path(name)
-        with open(temp, "w", encoding="utf-8") as file:
-            file.write(csvpathstr)
+        #
+        # TODO: use a DataFileWriter that supports S3 and local to write.
+        #
+        with DataFileWriter(path=temp, mode="w") as writer:
+            writer.append(csvpathstr)
         return temp
 
     def _group_file_path(self, name: NamedPathsName) -> str:
@@ -267,8 +273,11 @@ class PathsManager:
         return temp
 
     def _get_csvpaths_from_file(self, file_path: str) -> list[str]:
-        with open(file_path, "r", encoding="utf-8") as f:
-            cp = f.read()
+        #
+        # TODO: use DataFileReader to support S3 and local
+        #
+        with DataFileReader(file_path) as reader:
+            cp = reader.read()
             _ = [
                 apath.strip()
                 for apath in cp.split(PathsManager.MARKER)