fairdatanow 0.2.0__tar.gz

fairdatanow-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Frank Ligterink
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fairdatanow-0.2.0/MANIFEST.in

@@ -0,0 +1,5 @@
+include settings.ini
+include LICENSE
+include CONTRIBUTING.md
+include README.md
+recursive-exclude * __pycache__

fairdatanow-0.2.0/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.2
+Name: fairdatanow
+Version: 0.2.0
+Summary: Experimental Python package to support working with data from (y)our Nextcloud server
+Home-page: https://github.com/fligt/fairdatanow
+Author: Frank Ligterink
+Author-email: frank.ligterink@gmail.com
+License: MIT License
+Keywords: nbdev jupyter notebook python
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: nc-py-api
+Requires-Dist: polars[all]
+Requires-Dist: itables
+Requires-Dist: anywidget
+Provides-Extra: dev
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Welcome to fairdatanow
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Effective interdisciplinary cooperation in data science projects
+requires smooth access to both data and code. And with the recent
+development of two amazing open source frameworks (self hosted Nextcloud
+for data storage and Jupyter notebooks for computing), we are now in a
+position to actually make this happen. Recently our lab (the
+Rijkserfgoedlaboratorium in Amsterdam) has installed a Nextcloud server
+that gives our lab the possibility to share data files organized in
+project folders with research partners. These can be accessed manually
+via a web interface, but also programmatically with the webdav protocol!
+With `fairdatanow` we want to learn how to easily find en download data
+files from a Nextcloud storage directly in Jupyter notebooks in order to
+explore them and create visualizations.
+
+Under the hood the `fairdatanow` makes use of the Nextcloud Python
+Framework [nc-py-api](https://pypi.org/project/nc-py-api) for
+communication with the cloud server, and the powerful Python packages
+[polars](https://pypi.org/project/polars/),
+[itables](https://pypi.org/project/itables/) and
+[anywidget](https://pypi.org/project/anywidget/) and for creating
+interactive tables.
+
+Although the package is still **under construction** it is already
+available for installation from the python package index.

fairdatanow-0.2.0/README.md

@@ -0,0 +1,28 @@
+# Welcome to fairdatanow
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Effective interdisciplinary cooperation in data science projects
+requires smooth access to both data and code. And with the recent
+development of two amazing open source frameworks (self hosted Nextcloud
+for data storage and Jupyter notebooks for computing), we are now in a
+position to actually make this happen. Recently our lab (the
+Rijkserfgoedlaboratorium in Amsterdam) has installed a Nextcloud server
+that gives our lab the possibility to share data files organized in
+project folders with research partners. These can be accessed manually
+via a web interface, but also programmatically with the webdav protocol!
+With `fairdatanow` we want to learn how to easily find en download data
+files from a Nextcloud storage directly in Jupyter notebooks in order to
+explore them and create visualizations.
+
+Under the hood the `fairdatanow` makes use of the Nextcloud Python
+Framework [nc-py-api](https://pypi.org/project/nc-py-api) for
+communication with the cloud server, and the powerful Python packages
+[polars](https://pypi.org/project/polars/),
+[itables](https://pypi.org/project/itables/) and
+[anywidget](https://pypi.org/project/anywidget/) and for creating
+interactive tables.
+
+Although the package is still **under construction** it is already
+available for installation from the python package index.

fairdatanow-0.2.0/fairdatanow/__init__.py

@@ -0,0 +1,3 @@
+__version__ = "0.2.0"
+
+from .webdav import *

fairdatanow-0.2.0/fairdatanow/_modidx.py

@@ -0,0 +1,15 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'master',
+                'doc_baseurl': '/fairdatanow',
+                'doc_host': 'https://fligt.github.io',
+                'git_url': 'https://github.com/fligt/fairdatanow',
+                'lib_path': 'fairdatanow'},
+  'syms': { 'fairdatanow.webdav': { 'fairdatanow.webdav.RemoteData': ( 'exploring-your-remote-data.html#remotedata',
+                                                                       'fairdatanow/webdav.py'),
+                                    'fairdatanow.webdav.RemoteData.__init__': ( 'exploring-your-remote-data.html#remotedata.__init__',
+                                                                                'fairdatanow/webdav.py'),
+                                    'fairdatanow.webdav.RemoteData.download_selected': ( 'exploring-your-remote-data.html#remotedata.download_selected',
+                                                                                         'fairdatanow/webdav.py'),
+                                    'fairdatanow.webdav._node_to_dataframe': ( 'exploring-your-remote-data.html#_node_to_dataframe',
+                                                                               'fairdatanow/webdav.py')}}}

fairdatanow-0.2.0/fairdatanow/webdav.py

@@ -0,0 +1,139 @@
+"""Instantly find and access all your Nextcloud data files"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../notebooks/10_exploring-your-remote-data.ipynb.
+
+# %% auto 0
+__all__ = ['RemoteData']
+
+# %% ../notebooks/10_exploring-your-remote-data.ipynb 19
+import nc_py_api 
+from nc_py_api import Nextcloud 
+
+import polars as pl 
+import itables
+from itables.widget import ITable
+import os 
+from pathlib import Path 
+import time 
+import re 
+from IPython.display import HTML, display 
+
+# %% ../notebooks/10_exploring-your-remote-data.ipynb 20
+def _node_to_dataframe(fsnode): 
+    '''Convert `fsnode` object to polars a single row polars dataframe.'''
+
+    df = pl.DataFrame({'path': fsnode.user_path, 'size': fsnode.info.size, 'mimetype': fsnode.info.mimetype, 'modified': fsnode.info.last_modified, 
+                   'isdir': fsnode.is_dir, 'ext': os.path.splitext(fsnode.user_path)[1]})
+
+    return df 
+
+
+class RemoteData(object): 
+    
+    # See: https://help.nextcloud.com/t/using-nc-py-api-i-cant-download-any-file-due-to-ssl-certificte-verify-failed/194019 
+    nc_py_api.options.NPA_NC_CERT = False 
+    
+    # keep full dataframe 
+    itables.options.maxBytes = 0
+    itables.init_notebook_mode()
+
+    def __init__(self, configuration, searchBuilder={}): 
+        '''Recursively scan the contents of a remote webdav server as specified by `configuration`. 
+        '''
+
+        # parse configuration 
+        m = re.match('(^https://[^/]+/)(.*)', configuration['url'])
+        nextcloud_url, self.cache_dir = m.groups()
+        nc_auth_user = configuration['user']
+        nc_auth_pass = configuration['password'] 
+
+        print(f'Please wait while scanning all file paths in remote folder...')
+        
+        # Instantiate Nextcloud client 
+        self.nc = Nextcloud(nextcloud_url=nextcloud_url, nc_auth_user=nc_auth_user, nc_auth_pass=nc_auth_pass)
+
+        # query webdav server to obtain file listing 
+        fs_nodes_list = self.nc.files.listdir(self.cache_dir, depth=-1, exclude_self=False) 
+        
+        n_paths = len(fs_nodes_list)
+
+        # initialize polars dataframe with first row to fix schema 
+        self.df = _node_to_dataframe(fs_nodes_list[0])
+
+        for fsnode in fs_nodes_list[1:]: 
+            self.df.extend(_node_to_dataframe(fsnode))
+
+        # create interactive table 
+        self.itable = ITable(
+                    self.df,
+                    layout={"top1": "searchBuilder"},
+                    select=True,
+                    searchBuilder=searchBuilder, 
+                    scrollY="500px", scrollCollapse=True, paging=False, 
+                ) 
+
+        print(f"Ready building file table for '{self.cache_dir}', Total number of files and directories: {n_paths}   ")
+
+    
+    def download_selected(self, cache_dir=None): 
+        '''Download selected files (blue rows) from `table` to default local cache directory. 
+        
+        A custom `cache_dir` can be specified. '''
+        
+        # create cache path 
+        if cache_dir is None: 
+            cache_path = Path.home().joinpath('.cache', 'fairdatanow')
+        else: 
+            cache_path = Path.home().joinpath('.cache', cache_dir)
+    
+        os.makedirs(cache_path, exist_ok=True)
+    
+        # obtain remote paths and remote timestamps 
+        remote_path_list = [self.itable.df['path'][n] for n in self.itable.selected_rows]
+        remote_modified_list = [self.itable.df['modified'][n] for n in self.itable.selected_rows]
+        remote_isdir_list = [self.itable.df['isdir'][n] for n in self.itable.selected_rows]
+        
+        n_files = len(remote_path_list)
+       
+        for i, [remote_path, remote_modified, remote_isdir] in enumerate(zip(remote_path_list, remote_modified_list, remote_isdir_list)): 
+    
+            # only download actual files 
+            if not remote_isdir:   
+                remote_directory = os.path.dirname(remote_path)
+                local_directory = cache_path.joinpath(remote_directory) # I guess this will not yet work for Windows
+                
+                # create directory structure inside cache 
+                os.makedirs(local_directory, exist_ok=True) 
+            
+                # get remote epoch time  
+                remote_modified_epoch_time = remote_modified.timestamp()
+            
+                # construct corresponding local path 
+                local_path = cache_path.joinpath(remote_path) 
+            
+                # check if local file exists and if modification times are similar 
+                is_local = local_path.exists()  
+            
+                is_similar = False 
+                local_modified_epoch_time = None 
+                if is_local: 
+                    local_modified_epoch_time = os.stat(local_path).st_mtime
+                    if local_modified_epoch_time == remote_modified_epoch_time: 
+                        is_similar = True 
+                        
+                # download from nextcloud 
+                if not is_similar: 
+                    print(f'[{i}/{n_files - 1}] Timestamps do no match: {remote_modified_epoch_time} vs {local_modified_epoch_time}', end='\r')
+                    print(f'[{i}/{n_files - 1}] Downloading to: {local_path}                                                       ' , end='\r')
+                      
+                    # write to cache 
+                    with open(local_path, 'bw') as fh: 
+                        self.nc.files.download2stream(remote_path, fh) 
+                        
+                    # adjust last modified timestamp 
+                    now = int(time.time())
+                    os.utime(local_path, (now, remote_modified_epoch_time)) 
+                    
+        print(f"Ready with downloading {n_files} selected remote files to local cache: {cache_path}                                                                      ")
+
+

fairdatanow-0.2.0/fairdatanow.egg-info/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.2
+Name: fairdatanow
+Version: 0.2.0
+Summary: Experimental Python package to support working with data from (y)our Nextcloud server
+Home-page: https://github.com/fligt/fairdatanow
+Author: Frank Ligterink
+Author-email: frank.ligterink@gmail.com
+License: MIT License
+Keywords: nbdev jupyter notebook python
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: nc-py-api
+Requires-Dist: polars[all]
+Requires-Dist: itables
+Requires-Dist: anywidget
+Provides-Extra: dev
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Welcome to fairdatanow
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Effective interdisciplinary cooperation in data science projects
+requires smooth access to both data and code. And with the recent
+development of two amazing open source frameworks (self hosted Nextcloud
+for data storage and Jupyter notebooks for computing), we are now in a
+position to actually make this happen. Recently our lab (the
+Rijkserfgoedlaboratorium in Amsterdam) has installed a Nextcloud server
+that gives our lab the possibility to share data files organized in
+project folders with research partners. These can be accessed manually
+via a web interface, but also programmatically with the webdav protocol!
+With `fairdatanow` we want to learn how to easily find en download data
+files from a Nextcloud storage directly in Jupyter notebooks in order to
+explore them and create visualizations.
+
+Under the hood the `fairdatanow` makes use of the Nextcloud Python
+Framework [nc-py-api](https://pypi.org/project/nc-py-api) for
+communication with the cloud server, and the powerful Python packages
+[polars](https://pypi.org/project/polars/),
+[itables](https://pypi.org/project/itables/) and
+[anywidget](https://pypi.org/project/anywidget/) and for creating
+interactive tables.
+
+Although the package is still **under construction** it is already
+available for installation from the python package index.

fairdatanow-0.2.0/fairdatanow.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+settings.ini
+setup.py
+fairdatanow/__init__.py
+fairdatanow/_modidx.py
+fairdatanow/webdav.py
+fairdatanow.egg-info/PKG-INFO
+fairdatanow.egg-info/SOURCES.txt
+fairdatanow.egg-info/dependency_links.txt
+fairdatanow.egg-info/entry_points.txt
+fairdatanow.egg-info/not-zip-safe
+fairdatanow.egg-info/requires.txt
+fairdatanow.egg-info/top_level.txt

fairdatanow-0.2.0/fairdatanow.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

fairdatanow-0.2.0/fairdatanow.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[nbdev]
+fairdatanow = fairdatanow._modidx:d

fairdatanow-0.2.0/fairdatanow.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

fairdatanow-0.2.0/fairdatanow.egg-info/requires.txt

@@ -0,0 +1,6 @@
+nc-py-api
+polars[all]
+itables
+anywidget
+
+[dev]

fairdatanow-0.2.0/fairdatanow.egg-info/top_level.txt

@@ -0,0 +1 @@
+fairdatanow

fairdatanow-0.2.0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=64.0"]
+build-backend = "setuptools.build_meta"

fairdatanow-0.2.0/settings.ini

@@ -0,0 +1,46 @@
+[DEFAULT]
+# All sections below are required unless otherwise specified.
+# See https://github.com/AnswerDotAI/nbdev/blob/main/settings.ini for examples.
+
+### Python library ###
+repo = fairdatanow
+lib_name = %(repo)s
+version = 0.2.0
+min_python = 3.9
+license = mit
+black_formatting = False
+
+### nbdev ###
+doc_path = _docs
+lib_path = fairdatanow
+nbs_path = notebooks
+recursive = True
+tst_flags = notest
+put_version_in_init = True
+update_pyproject = True
+
+### Docs ###
+branch = master
+custom_sidebar = False
+doc_host = https://%(user)s.github.io
+doc_baseurl = /%(repo)s
+git_url = https://github.com/%(user)s/%(repo)s
+title = %(lib_name)s
+
+### PyPI ###
+audience = Developers
+author = Frank Ligterink
+author_email = frank.ligterink@gmail.com
+copyright = 2025 onwards, %(author)s
+description = Experimental Python package to support working with data from (y)our Nextcloud server 
+keywords = nbdev jupyter notebook python
+language = English
+status = 3
+user = fligt
+
+### Optional ###
+requirements = nc-py-api polars[all] itables anywidget
+# dev_requirements = 
+# console_scripts =
+# conda_user = 
+# package_data =

fairdatanow-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fairdatanow-0.2.0/setup.py

@@ -0,0 +1,64 @@
+from pkg_resources import parse_version
+from configparser import ConfigParser
+import setuptools, shlex
+assert parse_version(setuptools.__version__)>=parse_version('36.2')
+
+# note: all settings are in settings.ini; edit there, not here
+config = ConfigParser(delimiters=['='])
+config.read('settings.ini', encoding='utf-8')
+cfg = config['DEFAULT']
+
+cfg_keys = 'version description keywords author author_email'.split()
+expected = cfg_keys + "lib_name user branch license status min_python audience language".split()
+for o in expected: assert o in cfg, "missing expected setting: {}".format(o)
+setup_cfg = {o:cfg[o] for o in cfg_keys}
+
+licenses = {
+    'apache2': ('Apache Software License 2.0','OSI Approved :: Apache Software License'),
+    'mit': ('MIT License', 'OSI Approved :: MIT License'),
+    'gpl2': ('GNU General Public License v2', 'OSI Approved :: GNU General Public License v2 (GPLv2)'),
+    'gpl3': ('GNU General Public License v3', 'OSI Approved :: GNU General Public License v3 (GPLv3)'),
+    'bsd3': ('BSD License', 'OSI Approved :: BSD License'),
+}
+statuses = [ '1 - Planning', '2 - Pre-Alpha', '3 - Alpha',
+    '4 - Beta', '5 - Production/Stable', '6 - Mature', '7 - Inactive' ]
+py_versions = '3.6 3.7 3.8 3.9 3.10 3.11 3.12'.split()
+
+requirements = shlex.split(cfg.get('requirements', ''))
+if cfg.get('pip_requirements'): requirements += shlex.split(cfg.get('pip_requirements', ''))
+min_python = cfg['min_python']
+lic = licenses.get(cfg['license'].lower(), (cfg['license'], None))
+dev_requirements = (cfg.get('dev_requirements') or '').split()
+
+package_data = dict()
+pkg_data = cfg.get('package_data', None)
+if pkg_data:
+    package_data[cfg['lib_name']] =  pkg_data.split() # split as multiple files might be listed
+# Add package data to setup_cfg for setuptools.setup(..., **setup_cfg)
+setup_cfg['package_data'] = package_data
+
+setuptools.setup(
+    name = cfg['lib_name'],
+    license = lic[0],
+    classifiers = [
+        'Development Status :: ' + statuses[int(cfg['status'])],
+        'Intended Audience :: ' + cfg['audience'].title(),
+        'Natural Language :: ' + cfg['language'].title(),
+    ] + ['Programming Language :: Python :: '+o for o in py_versions[py_versions.index(min_python):]] + (['License :: ' + lic[1] ] if lic[1] else []),
+    url = cfg['git_url'],
+    packages = setuptools.find_packages(),
+    include_package_data = True,
+    install_requires = requirements,
+    extras_require={ 'dev': dev_requirements },
+    dependency_links = cfg.get('dep_links','').split(),
+    python_requires  = '>=' + cfg['min_python'],
+    long_description = open('README.md', encoding='utf-8').read(),
+    long_description_content_type = 'text/markdown',
+    zip_safe = False,
+    entry_points = {
+        'console_scripts': cfg.get('console_scripts','').split(),
+        'nbdev': [f'{cfg.get("lib_path")}={cfg.get("lib_path")}._modidx:d']
+    },
+    **setup_cfg)
+
+