fairdatanow 0.2.0__py3-none-any.whl

fairdatanow/__init__.py

@@ -0,0 +1,3 @@
+__version__ = "0.2.0"
+
+from .webdav import *

fairdatanow/_modidx.py

@@ -0,0 +1,15 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'master',
+                'doc_baseurl': '/fairdatanow',
+                'doc_host': 'https://fligt.github.io',
+                'git_url': 'https://github.com/fligt/fairdatanow',
+                'lib_path': 'fairdatanow'},
+  'syms': { 'fairdatanow.webdav': { 'fairdatanow.webdav.RemoteData': ( 'exploring-your-remote-data.html#remotedata',
+                                                                       'fairdatanow/webdav.py'),
+                                    'fairdatanow.webdav.RemoteData.__init__': ( 'exploring-your-remote-data.html#remotedata.__init__',
+                                                                                'fairdatanow/webdav.py'),
+                                    'fairdatanow.webdav.RemoteData.download_selected': ( 'exploring-your-remote-data.html#remotedata.download_selected',
+                                                                                         'fairdatanow/webdav.py'),
+                                    'fairdatanow.webdav._node_to_dataframe': ( 'exploring-your-remote-data.html#_node_to_dataframe',
+                                                                               'fairdatanow/webdav.py')}}}

fairdatanow/webdav.py

@@ -0,0 +1,139 @@
+"""Instantly find and access all your Nextcloud data files"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../notebooks/10_exploring-your-remote-data.ipynb.
+
+# %% auto 0
+__all__ = ['RemoteData']
+
+# %% ../notebooks/10_exploring-your-remote-data.ipynb 19
+import nc_py_api 
+from nc_py_api import Nextcloud 
+
+import polars as pl 
+import itables
+from itables.widget import ITable
+import os 
+from pathlib import Path 
+import time 
+import re 
+from IPython.display import HTML, display 
+
+# %% ../notebooks/10_exploring-your-remote-data.ipynb 20
+def _node_to_dataframe(fsnode): 
+    '''Convert `fsnode` object to polars a single row polars dataframe.'''
+
+    df = pl.DataFrame({'path': fsnode.user_path, 'size': fsnode.info.size, 'mimetype': fsnode.info.mimetype, 'modified': fsnode.info.last_modified, 
+                   'isdir': fsnode.is_dir, 'ext': os.path.splitext(fsnode.user_path)[1]})
+
+    return df 
+
+
+class RemoteData(object): 
+    
+    # See: https://help.nextcloud.com/t/using-nc-py-api-i-cant-download-any-file-due-to-ssl-certificte-verify-failed/194019 
+    nc_py_api.options.NPA_NC_CERT = False 
+    
+    # keep full dataframe 
+    itables.options.maxBytes = 0
+    itables.init_notebook_mode()
+
+    def __init__(self, configuration, searchBuilder={}): 
+        '''Recursively scan the contents of a remote webdav server as specified by `configuration`. 
+        '''
+
+        # parse configuration 
+        m = re.match('(^https://[^/]+/)(.*)', configuration['url'])
+        nextcloud_url, self.cache_dir = m.groups()
+        nc_auth_user = configuration['user']
+        nc_auth_pass = configuration['password'] 
+
+        print(f'Please wait while scanning all file paths in remote folder...')
+        
+        # Instantiate Nextcloud client 
+        self.nc = Nextcloud(nextcloud_url=nextcloud_url, nc_auth_user=nc_auth_user, nc_auth_pass=nc_auth_pass)
+
+        # query webdav server to obtain file listing 
+        fs_nodes_list = self.nc.files.listdir(self.cache_dir, depth=-1, exclude_self=False) 
+        
+        n_paths = len(fs_nodes_list)
+
+        # initialize polars dataframe with first row to fix schema 
+        self.df = _node_to_dataframe(fs_nodes_list[0])
+
+        for fsnode in fs_nodes_list[1:]: 
+            self.df.extend(_node_to_dataframe(fsnode))
+
+        # create interactive table 
+        self.itable = ITable(
+                    self.df,
+                    layout={"top1": "searchBuilder"},
+                    select=True,
+                    searchBuilder=searchBuilder, 
+                    scrollY="500px", scrollCollapse=True, paging=False, 
+                ) 
+
+        print(f"Ready building file table for '{self.cache_dir}', Total number of files and directories: {n_paths}   ")
+
+    
+    def download_selected(self, cache_dir=None): 
+        '''Download selected files (blue rows) from `table` to default local cache directory. 
+        
+        A custom `cache_dir` can be specified. '''
+        
+        # create cache path 
+        if cache_dir is None: 
+            cache_path = Path.home().joinpath('.cache', 'fairdatanow')
+        else: 
+            cache_path = Path.home().joinpath('.cache', cache_dir)
+    
+        os.makedirs(cache_path, exist_ok=True)
+    
+        # obtain remote paths and remote timestamps 
+        remote_path_list = [self.itable.df['path'][n] for n in self.itable.selected_rows]
+        remote_modified_list = [self.itable.df['modified'][n] for n in self.itable.selected_rows]
+        remote_isdir_list = [self.itable.df['isdir'][n] for n in self.itable.selected_rows]
+        
+        n_files = len(remote_path_list)
+       
+        for i, [remote_path, remote_modified, remote_isdir] in enumerate(zip(remote_path_list, remote_modified_list, remote_isdir_list)): 
+    
+            # only download actual files 
+            if not remote_isdir:   
+                remote_directory = os.path.dirname(remote_path)
+                local_directory = cache_path.joinpath(remote_directory) # I guess this will not yet work for Windows
+                
+                # create directory structure inside cache 
+                os.makedirs(local_directory, exist_ok=True) 
+            
+                # get remote epoch time  
+                remote_modified_epoch_time = remote_modified.timestamp()
+            
+                # construct corresponding local path 
+                local_path = cache_path.joinpath(remote_path) 
+            
+                # check if local file exists and if modification times are similar 
+                is_local = local_path.exists()  
+            
+                is_similar = False 
+                local_modified_epoch_time = None 
+                if is_local: 
+                    local_modified_epoch_time = os.stat(local_path).st_mtime
+                    if local_modified_epoch_time == remote_modified_epoch_time: 
+                        is_similar = True 
+                        
+                # download from nextcloud 
+                if not is_similar: 
+                    print(f'[{i}/{n_files - 1}] Timestamps do no match: {remote_modified_epoch_time} vs {local_modified_epoch_time}', end='\r')
+                    print(f'[{i}/{n_files - 1}] Downloading to: {local_path}                                                       ' , end='\r')
+                      
+                    # write to cache 
+                    with open(local_path, 'bw') as fh: 
+                        self.nc.files.download2stream(remote_path, fh) 
+                        
+                    # adjust last modified timestamp 
+                    now = int(time.time())
+                    os.utime(local_path, (now, remote_modified_epoch_time)) 
+                    
+        print(f"Ready with downloading {n_files} selected remote files to local cache: {cache_path}                                                                      ")
+
+

fairdatanow-0.2.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Frank Ligterink
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fairdatanow-0.2.0.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.2
+Name: fairdatanow
+Version: 0.2.0
+Summary: Experimental Python package to support working with data from (y)our Nextcloud server
+Home-page: https://github.com/fligt/fairdatanow
+Author: Frank Ligterink
+Author-email: frank.ligterink@gmail.com
+License: MIT License
+Keywords: nbdev jupyter notebook python
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: nc-py-api
+Requires-Dist: polars[all]
+Requires-Dist: itables
+Requires-Dist: anywidget
+Provides-Extra: dev
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Welcome to fairdatanow
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Effective interdisciplinary cooperation in data science projects
+requires smooth access to both data and code. And with the recent
+development of two amazing open source frameworks (self hosted Nextcloud
+for data storage and Jupyter notebooks for computing), we are now in a
+position to actually make this happen. Recently our lab (the
+Rijkserfgoedlaboratorium in Amsterdam) has installed a Nextcloud server
+that gives our lab the possibility to share data files organized in
+project folders with research partners. These can be accessed manually
+via a web interface, but also programmatically with the webdav protocol!
+With `fairdatanow` we want to learn how to easily find en download data
+files from a Nextcloud storage directly in Jupyter notebooks in order to
+explore them and create visualizations.
+
+Under the hood the `fairdatanow` makes use of the Nextcloud Python
+Framework [nc-py-api](https://pypi.org/project/nc-py-api) for
+communication with the cloud server, and the powerful Python packages
+[polars](https://pypi.org/project/polars/),
+[itables](https://pypi.org/project/itables/) and
+[anywidget](https://pypi.org/project/anywidget/) and for creating
+interactive tables.
+
+Although the package is still **under construction** it is already
+available for installation from the python package index.

fairdatanow-0.2.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+fairdatanow/__init__.py,sha256=ck9MDNa13_UNZFFApaxdA-9ySmsdjfwm_QCtnJQWgUk,45
+fairdatanow/_modidx.py,sha256=0qQSdqRZmE87iEA-niRDh_LUAk5BzlU3EH6Xg00B_yo,1242
+fairdatanow/webdav.py,sha256=JcLYfZoc3WpNSXPUCUEX2UmrKRzHXNxUAdW1GR9H-K0,5852
+fairdatanow-0.2.0.dist-info/LICENSE,sha256=j-59fryWPhEIb36jnuX7LMesZyQnDOftm4PbTqf0Vj8,1072
+fairdatanow-0.2.0.dist-info/METADATA,sha256=R3tCdxn4lCFUoCsvoYNim7fWXU9a5oy4ybQhaWOPr5Q,2564
+fairdatanow-0.2.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+fairdatanow-0.2.0.dist-info/entry_points.txt,sha256=Ssb-NQaPCHs9cNvXjv0u9YaHXQiz1YrM_tiQo7p_v-E,44
+fairdatanow-0.2.0.dist-info/top_level.txt,sha256=1UDSldUj8-hHz6ejbIcoGulNtowBRLKrMqyA_cECAsg,12
+fairdatanow-0.2.0.dist-info/RECORD,,

fairdatanow-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fairdatanow-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[nbdev]
+fairdatanow = fairdatanow._modidx:d

fairdatanow-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fairdatanow