ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.2__py3-none-any.whl

ckanapi_harvesters/builder/example/builder_example_generate_data.py

@@ -0,0 +1,73 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Code to generate sample data for the dataset example
+"""
+from typing import Tuple
+import os
+import re
+
+import pandas as pd
+import numpy as np
+
+from ckanapi_harvesters.builder.example import example_package_dir
+self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+
+
+GPS_DIGITS = 6
+def degrees_DMS(lat:str, lon:str) -> Tuple[float, float]:
+    """
+    Returns angles for GPS coordinates in the form
+    :param lat: example: 48° 51' 12.24845" N
+    :param lon: example: 2° 20' 55.62563" E
+    :return:
+    """
+    deg, minutes, seconds, direction = re.split('[°\'"]', lat)
+    lat_val = (float(deg) + float(minutes) / 60 + float(seconds) / (60 * 60)) * (-1 if direction in ['W', 'S'] else 1)
+    deg, minutes, seconds, direction = re.split('[°\'"]', lon)
+    lon_val = (float(deg) + float(minutes) / 60 + float(seconds) / (60 * 60)) * (-1 if direction in ['W', 'S'] else 1)
+    return (lat_val, lon_val)
+
+
+def run():
+    N = 100
+    df_users = pd.DataFrame([{"user_id": 1, "age": 30},
+                             {"user_id": 2, "age": 80},
+                             ])
+    point_0 = degrees_DMS('48° 51\' 12.24845" N', '2° 20\' 55.62563" E')
+    traces = [{"user_id": 1, "origin": point_0, "destination": degrees_DMS('48°53\'37.6"N', '2°23\'24.9"E')},
+              {"user_id": 1, "origin": point_0, "destination": degrees_DMS('48°50\'01.7"N', '2°19\'57.1"E')},
+              {"user_id": 2, "origin": point_0, "destination": degrees_DMS('48°52\'45.2"N', '2°18\'32.8"E')},
+    ]
+    users_file = os.path.join(example_package_dir, "users_local.csv")
+    df_users.to_csv(users_file, index=False)
+    traces_dir = os.path.join(example_package_dir, "traces")
+    traces_dir_multi = os.path.join(example_package_dir, "traces_multi")
+    for trace_id, trace in enumerate(traces):
+        table_index = np.array([0, N-1])
+        table_lat = np.array([trace["origin"][0], trace["destination"][0]])
+        table_lon = np.array([trace["origin"][1], trace["destination"][1]])
+        df_trace = pd.DataFrame()
+        index = np.arange(N)
+        df_trace["index_in_trace"] = index
+        df_trace.insert(loc=0, column="trace_id", value=trace_id)
+        df_trace["user_id"] = trace["user_id"]
+        df_trace["timestamp"] = pd.Timestamp(year=2025, month=1, day=1, hour=12, minute=0, second=0, microsecond=0) + index.astype('timedelta64[s]')
+        df_trace["timestamp_local"] = df_trace["timestamp"] + pd.Timedelta(hours=1)  # local = UTC+1 for winter light saving time in Paris
+        df_trace["timestamp"] = df_trace["timestamp"].apply(pd.Timestamp.isoformat)  # ISO-8601 format
+        df_trace["timestamp_local"] = df_trace["timestamp_local"].apply(pd.Timestamp.isoformat)  # ISO-8601 format
+        df_trace["latitude"] = np.interp(index, xp=table_lat, fp=table_lat)
+        df_trace["longitude"] = np.interp(index, xp=table_lat, fp=table_lon)
+        df_trace["latitude"] = df_trace["latitude"].values.round(GPS_DIGITS)
+        df_trace["longitude"] = df_trace["longitude"].values.round(GPS_DIGITS)
+        df_trace.set_index(keys=["trace_id", "index_in_trace"], drop=False, inplace=True, verify_integrity=True)
+        trace_file = os.path.join(traces_dir, f"trace_{trace_id:03d}.csv")
+        df_trace.to_csv(trace_file, index=False)
+        trace_file_multi = os.path.join(traces_dir_multi, f"trace_{trace_id:03d}.csv")
+        df_trace.to_csv(trace_file_multi, index=False)
+    print("Example data regenerated")
+
+
+if __name__ == '__main__':
+    run()
+

ckanapi_harvesters/builder/example/builder_example_patch_upload.py

@@ -0,0 +1,51 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Example code to upload the builder example to a CKAN server
+"""
+from typing import Tuple
+import os
+import re
+import getpass
+
+import pandas as pd
+import numpy as np
+
+from ckanapi_harvesters.builder.builder_package import BuilderPackage
+from ckanapi_harvesters.ckan_api import CkanApi
+
+from ckanapi_harvesters.builder.example import example_package_xls
+self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+
+
+def run(ckan:CkanApi = None):
+    BuilderPackage.unlock_external_code_execution()
+
+    mdl = BuilderPackage.from_excel(example_package_xls)
+    ckan = mdl.init_ckan(ckan)
+    ckan.input_missing_info(input_args_if_necessary=True, input_owner_org=True)
+    ckan.set_limits(10000)  # reduce if server hangs up
+    ckan.set_submit_timeout(5)
+    ckan.set_verbosity(True)
+    # ckan.set_default_map_mode(datastore_info=True)  # uncomment to query DataStore information
+
+    # Test re-encoding the Excel file from the loaded model
+    example_package_xls_reencoded = os.path.abspath("builder_package_example-reencoded.xlsx")
+    mdl.to_excel(example_package_xls_reencoded)
+
+    # Patch package: apply metadata and upload small resources
+    reupload = True  # True: reuploads all documents and resets large datasets to the first document
+    mdl.patch_request_full(ckan, reupload=reupload)
+
+    # Upload large datasets
+    threads = 3  # > 1: multi-threading mode - reduce if HTTP 502 errors
+    mdl.upload_large_datasets(ckan, threads=threads)
+
+    print("Update done.")
+
+
+if __name__ == '__main__':
+    ckan = CkanApi(None)
+    ckan.initialize_from_cli_args()
+    run(ckan)
+

ckanapi_harvesters/builder/example/builder_example_policy.py

@@ -0,0 +1,114 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Tests to perform after the example package was uploaded
+"""
+from typing import Tuple
+import os
+import re
+import json
+import io
+
+import pandas as pd
+import numpy as np
+
+from ckanapi_harvesters.auxiliary import CkanMap
+from ckanapi_harvesters.builder.builder_package import BuilderPackage
+from ckanapi_harvesters.ckan_api import CkanApi
+from ckanapi_harvesters.policies.data_format_policy import CkanPackageDataFormatPolicy
+from ckanapi_harvesters.policies.data_format_policy import (SingleValueListPolicy, ValueListPolicy, StringValueSpecification,
+                                                            ListChoiceMode, CustomFieldsPolicy, CustomFieldSpecification,
+                                                            GroupedValueListPolicy, ErrorLevel, DataPolicyError,
+                                                            StringMatchMode, TagListPolicy, TagGroupsListPolicy)
+
+from ckanapi_harvesters.builder.specific.configuration_builder import ConfigurationBuilder
+from ckanapi_harvesters.builder.example import example_package_xls
+self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+
+
+enable_write = False  # be careful, setting this to True could erase your CKAN data format policy !!!
+
+
+def run(ckan:CkanApi = None):
+    BuilderPackage.unlock_external_code_execution()  # comment to test if the safety feature is enabled
+
+    mdl = BuilderPackage.from_excel(example_package_xls)
+    ckan = mdl.init_ckan(ckan)
+    ckan.initialize_from_cli_args()
+    ckan.input_missing_info(input_args_if_necessary=True, input_owner_org=True)
+    ckan.set_verbosity(False)
+    ckan.verbose_policy = True
+
+    policy = CkanPackageDataFormatPolicy()
+    policy.resource_format = SingleValueListPolicy(ValueListPolicy([StringValueSpecification("CSV")]), extra_values=ListChoiceMode.NoExtra)
+    policy.package_custom_fields = CustomFieldsPolicy([
+                                                        CustomFieldSpecification(key="New field", mandatory=True),
+                                                        CustomFieldSpecification(key="Algorithm", values=["Random"], match_mode=StringMatchMode.Match, mandatory=True)], error_level=ErrorLevel.Error)
+    policy.package_tags = TagGroupsListPolicy([TagListPolicy([StringValueSpecification("Test")], group_name="Vocabulary_Test")])
+    policy.package_mandatory_attributes = {"description", "author"}
+    ckan.policy = policy
+
+    policy_dict = policy.to_dict(sets_as_lists=False)
+    policy_from_dict = CkanPackageDataFormatPolicy.from_dict(policy_dict)
+    policy_dict_bis = policy_from_dict.to_dict(sets_as_lists=False)
+    assert(policy_dict == policy_dict_bis)
+
+    # serialisation
+    policy_dict = policy.to_dict(sets_as_lists=True)
+    policy_json_file = os.path.abspath("policy_py.json")
+    with open(policy_json_file, "w") as f:
+        json.dump(policy_dict, f, indent=4)
+    with open(policy_json_file, "r") as f:
+        policy_json_dict = json.load(f)
+    # policy_json = CkanPackageDataFormatPolicy.from_dict(policy_json_dict)
+    policy_json = CkanPackageDataFormatPolicy.from_json(policy_json_file)
+    assert(policy_dict == policy_json_dict)
+
+    # test on local definition (offline)
+    buffer = {}
+    success = mdl.local_policy_check(policy, buffer=buffer)
+    # test if an error is raised (this mode does not display all messages)
+    try:
+        success = mdl.local_policy_check(policy)
+    except DataPolicyError as e:
+        print("Exception: " + str(e))
+        assert(not success)
+    else:
+        print(f"No exception / success={success}")
+
+    # test on remote definition (CKAN API)
+    print("Test on remote")
+    buffer = {}
+    ckan.map_resources(mdl.package_name)
+    success = ckan.policy_check(buffer=buffer)
+
+    ckan.set_verbosity(True)
+
+    # update vocabularies: deprecated
+    # if enable_write:
+    #     ckan.vocabularies_clear()
+    #     ckan.initiate_vocabularies_from_policy(policy, remove_others=True)
+
+    # upload default policy
+    config_ckan = ConfigurationBuilder(ckan, ckan.owner_org)
+    if enable_write:
+        config_ckan.patch_policy(ckan, policy, reduced_size=False)
+
+    # download default policy
+    # default_policy = ckan.load_default_policy(force=True)
+    config_ckan.load_default_policy(ckan)
+
+    # check all packages against policy
+    ckan.owner_org = None
+    print(" ")
+    buffer = {}
+    config_ckan.policy_check(ckan, policy=policy, buffer=buffer, verbose=True)
+
+    print("Tests done.")
+
+
+if __name__ == '__main__':
+    ckan = CkanApi(None)
+    ckan.initialize_from_cli_args()
+    run(ckan)
+

ckanapi_harvesters/builder/example/builder_example_test_sql.py

@@ -0,0 +1,53 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Tests to perform after the example package was uploaded
+"""
+from typing import Tuple
+import os
+import re
+import getpass
+import json
+
+import pandas as pd
+import numpy as np
+
+from ckanapi_harvesters.auxiliary import CkanMap
+from ckanapi_harvesters.builder.builder_package import BuilderPackage
+from ckanapi_harvesters.ckan_api import CkanApi
+from ckanapi_harvesters.auxiliary.ckan_action import CkanNotFoundError, CkanSqlCapabilityError
+
+from ckanapi_harvesters.builder.example import example_package_xls
+self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+
+
+def run(ckan:CkanApi = None):
+    BuilderPackage.unlock_external_code_execution()  # comment to test if the safety feature is enabled
+
+    mdl = BuilderPackage.from_excel(example_package_xls)
+    ckan = mdl.init_ckan(ckan)
+    ckan.initialize_from_cli_args()
+    ckan.input_missing_info(input_args_if_necessary=True, input_owner_org=True)
+    ckan.set_verbosity(True)
+
+    # ckan.apikey.clear()  # uncomment for specific test
+
+    capability = ckan.test_sql_capabilities()
+    try:
+        ckan.api_help_show("datastore_search_sql")
+    except CkanNotFoundError:
+        print("No datastore_search_sql help")
+    users_table_id = mdl.get_or_query_resource_id(ckan, "users.csv")
+    try:
+        ckan.datastore_search_sql(f'SELECT * FROM "{users_table_id}"')
+    except CkanSqlCapabilityError:
+        print("datastore_search_sql is not accessible")
+
+    print("Tests done.")
+
+
+if __name__ == '__main__':
+    ckan = CkanApi(None)
+    ckan.initialize_from_cli_args()
+    run(ckan)
+

ckanapi_harvesters/builder/example/builder_example_tests.py

@@ -0,0 +1,87 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Tests to perform after the example package was uploaded
+"""
+from typing import Tuple
+import os
+import re
+import getpass
+import json
+
+import pandas as pd
+import numpy as np
+
+from ckanapi_harvesters.auxiliary import CkanMap
+from ckanapi_harvesters.builder.builder_package import BuilderPackage
+from ckanapi_harvesters.ckan_api import CkanApi
+
+from ckanapi_harvesters.builder.example import example_package_xls
+self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+
+
+def run(ckan:CkanApi = None):
+    BuilderPackage.unlock_external_code_execution()  # comment to test if the safety feature is enabled
+
+    mdl = BuilderPackage.from_excel(example_package_xls)
+    ckan = mdl.init_ckan(ckan)
+    ckan.initialize_from_cli_args()
+    ckan.input_missing_info(input_args_if_necessary=True, input_owner_org=True)
+    ckan.set_verbosity(True)
+
+    # Test re-encoding the Excel file from the loaded model
+    example_package_xls_reencoded = os.path.abspath("builder_package_example-reencoded.xlsx")
+    mdl.to_excel(example_package_xls_reencoded)
+
+    # map package
+    ckan.map_resources(mdl.package_name, datastore_info=True, license_list=True)
+    map_init = ckan.map.copy()
+    ckan.purge(purge_map=True)
+
+    # Test using the model to update CKAN map (update_ckan_map)
+    mdl.info_request_full(ckan)
+    map_queried = ckan.map.copy()
+    ckan.purge(purge_map=True)
+    mdl.update_ckan_map(ckan)
+    map_from_mdl = ckan.map.copy()
+
+    # Test saving the map to a dictionary
+    dict_map_queried = map_queried.to_dict()
+    map_queried_from_dict = CkanMap.from_dict(dict_map_queried)
+
+    dict_map_from_mdl = map_from_mdl.to_dict()
+    map_from_mdl_from_dict = CkanMap.from_dict(dict_map_from_mdl)
+
+    # test the function that recreates the Excel file from the online information
+    ckan.purge(purge_map=True)
+    mdl_api = BuilderPackage.from_ckan(ckan, mdl.package_name)
+    example_package_xls_from_api = os.path.abspath("builder_package_example-from-api.xlsx")
+    mdl_api.to_excel(example_package_xls_from_api)
+
+    # export mdl to dict and re-import
+    base_dir = os.path.abspath(".")
+    mdl_dict = mdl.to_dict(base_dir=base_dir)
+    mdl_from_dict = BuilderPackage.from_dict(mdl_dict, base_dir=base_dir)
+    mdl_dict_bis = mdl_from_dict.to_dict(base_dir=base_dir)
+    assert(mdl_dict == mdl_dict_bis)
+
+    # test json serialization
+    example_package_json_reencoded = os.path.abspath("builder_package_example-reencoded.json")
+    with open(example_package_json_reencoded, "w") as json_file:
+        json.dump(mdl_dict, json_file, indent=4)
+        json_file.close()
+
+    # test copy constructors
+    ckan_copy = ckan.copy()
+    mdl_copy = mdl.copy()
+    mdl_copy_dict = mdl_copy.to_dict(base_dir=base_dir)
+    assert(mdl_dict == mdl_copy_dict)
+
+    print("Tests done.")
+
+
+if __name__ == '__main__':
+    ckan = CkanApi(None)
+    ckan.initialize_from_cli_args()
+    run(ckan)
+

ckanapi_harvesters/builder/example/builder_example_tests_offline.py

@@ -0,0 +1,57 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Tests to perform after the example package was uploaded
+"""
+from typing import Tuple
+import os
+import re
+import getpass
+import json
+
+import pandas as pd
+import numpy as np
+
+from ckanapi_harvesters.auxiliary import CkanMap
+from ckanapi_harvesters.builder.builder_package import BuilderPackage
+from ckanapi_harvesters.ckan_api import CkanApi
+
+from ckanapi_harvesters.builder.example import example_package_xls
+self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+
+
+if __name__ == '__main__':
+    BuilderPackage.unlock_external_code_execution()  # comment to test if the safety feature is enabled
+
+    mdl = BuilderPackage.from_excel(example_package_xls)
+
+    ckan = CkanApi()
+    ckan = mdl.init_ckan(ckan)
+    # mdl.update_ckan_map(ckan)
+    # map_from_mdl = ckan.map.copy()
+
+    # Test re-encoding the Excel file from the loaded model
+    example_package_xls_reencoded = os.path.abspath("builder_package_example-reencoded.xlsx")
+    mdl.to_excel(example_package_xls_reencoded)
+
+    # export mdl to dict and re-import
+    base_dir = os.path.abspath(".")
+    mdl_dict = mdl.to_dict(base_dir=base_dir)
+    mdl_from_dict = BuilderPackage.from_dict(mdl_dict, base_dir=base_dir)
+    mdl_dict_bis = mdl_from_dict.to_dict(base_dir=base_dir)
+    assert(mdl_dict == mdl_dict_bis)
+
+    # test json serialization
+    example_package_json_reencoded = os.path.abspath("builder_package_example-reencoded.json")
+    with open(example_package_json_reencoded, "w") as json_file:
+        json.dump(mdl_dict, json_file, indent=4)
+
+    # test copy constructors
+    ckan_copy = ckan.copy()
+    mdl_copy = mdl.copy()
+    mdl_copy_dict = mdl_copy.to_dict(base_dir=base_dir)
+    assert(mdl_dict == mdl_copy_dict)
+
+    print("Tests done.")
+
+

ckanapi_harvesters/builder/example/package/ckan-dpg.svg

@@ -0,0 +1,74 @@
+<svg width="236" height="50" viewBox="0 0 236 50" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M228.17 41H199.236L196 34.7557L196.071 34.6044L201.995 22H230.902L234.129 28.2621L228.17 41ZM199.641 40.3324H227.742L233.386 28.2799L230.497 22.6721H202.422L196.743 34.7423L199.641 40.3324Z" fill="#2C269A"/>
+<path d="M228.17 41H199.236L196 34.7557L196.071 34.6044L201.995 22H230.902L234.129 28.2621L228.17 41ZM199.641 40.3324H227.742L233.386 28.2799L230.497 22.6721H202.422L196.743 34.7423L199.641 40.3324Z" fill="#82DBE1"/>
+<path d="M201.996 22H230.498L233.387 28.2755L227.743 40.3324H199.642L196.744 34.7423L201.996 22Z" fill="#82DBE1"/>
+<path d="M203.482 28.0137H205.312C207.154 28.0137 208.659 29.5091 208.659 31.3606C208.659 33.2121 207.154 34.7075 205.312 34.7075H203.482V28.0137ZM205.303 33.9954C206.789 33.9954 207.946 32.8694 207.946 31.3606C207.946 29.8518 206.789 28.7258 205.303 28.7258H204.186V33.9909H205.303V33.9954Z" fill="#2C269A"/>
+<path d="M214.289 29.8741C214.289 30.9111 213.461 31.7255 212.424 31.7255H210.738V34.7075H210.025V28.0137H212.424C213.461 28.0137 214.289 28.8504 214.289 29.8741ZM213.577 29.8874C213.577 29.1753 213.03 28.7035 212.362 28.7035H210.742V31.049H212.362C213.03 31.049 213.577 30.6218 213.577 29.8874Z" fill="#2C269A"/>
+<path d="M221.934 31.1595V31.3598C221.934 33.2558 220.394 34.7957 218.498 34.7957C216.589 34.7957 215.062 33.2558 215.062 31.3598C215.062 29.4638 216.594 27.9238 218.498 27.9238C219.78 27.9238 220.897 28.627 221.476 29.664L220.862 30.0112C220.403 29.1834 219.513 28.6404 218.498 28.6404C216.99 28.6404 215.775 29.8465 215.775 31.3642C215.775 32.8819 216.99 34.088 218.498 34.088C219.847 34.088 220.964 33.1267 221.178 31.8315H218.49V31.1639H221.934V31.1595Z" fill="#2C269A"/>
+<path d="M234.319 26.0703H225.453V34.9361H234.319V26.0703Z" fill="#2C269A"/>
+<path d="M235.922 28.847L228.244 24.4141L223.811 32.0918L231.489 36.5247L235.922 28.847Z" fill="#2C269A"/>
+<path d="M231.49 24.4128L223.812 28.8457L228.245 36.5235L235.923 32.0906L231.49 24.4128Z" fill="#2C269A"/>
+<path d="M227.643 30.5719L229.325 32.2542L232.601 28.9785" stroke="white" stroke-miterlimit="10"/>
+<path d="M87.9128 10.8496L65.283 45.6255L27.2637 34.0335L87.9128 10.8496Z" fill="#D4E5F1"/>
+<path d="M34.4015 49.5221L27.2637 34.2207L65.378 45.5987L34.4015 49.5221Z" fill="#CADBE9"/>
+<path d="M0 8.3457L27.2643 33.9402L19.8699 38.9485L0 8.3457Z" fill="#ABC1D7"/>
+<path d="M34.2187 49.5221L27.0459 34.2207L19.752 39.1676L34.2187 49.5221Z" fill="#95B0C4"/>
+<path d="M22.8125 30.1048L38.9485 0L36.4491 30.6257L27.0973 34.2195L22.8125 30.1048Z" fill="#EAE735"/>
+<path d="M55.9198 23.1603L38.978 0L36.4453 30.6028L55.9198 23.1603Z" fill="#ED5248"/>
+<path d="M87.9137 10.8496L53.416 19.5607L56.0418 23.0907L87.9137 10.8496Z" fill="#DEEAF5"/>
+<path d="M30.3246 15.9509L0 8.3457L22.6739 29.7676L30.3246 15.9509Z" fill="#E0E9F5"/>
+<path opacity="0.1" d="M26.4659 15.0215L22.8125 29.7665L30.3241 15.9988L26.4659 15.0215Z" fill="black"/>
+<path opacity="0.1" d="M54.8327 19.1953L55.9199 23.0902L53.416 19.5638L54.8327 19.1953Z" fill="black"/>
+<path opacity="0.1" d="M38.9408 0L40.3402 29.0935L36.4453 30.6028L38.9408 0Z" fill="black"/>
+<path opacity="0.1" d="M34.498 31.1592L38.9494 0L36.4445 30.4177L34.498 31.1592Z" fill="black"/>
+<path d="M26.8979 34.219L0 8.58703L0.224005 8.3457L26.9668 33.8225L87.7928 10.8624L87.9134 11.1726L26.8979 34.219Z" fill="white"/>
+<path d="M98.2051 46.461C99.5465 46.2935 100.718 45.9583 101.686 45.4219C103.095 44.6678 104.335 43.6622 105.438 42.4555L102.416 39.5059L98.2051 46.461Z" fill="url(#paint0_linear_1251_237)"/>
+<path d="M102.098 30.6036L105.442 27.0822C104.345 25.8909 103.051 24.9449 101.595 24.209C101.145 23.9813 100.66 23.8061 100.156 23.6484L101.9 30.3759C101.954 30.4635 102.026 30.5335 102.098 30.6036Z" fill="url(#paint1_linear_1251_237)"/>
+<path d="M122.382 32.8567L131.314 23.6484H125.061L115.734 33.6687V39.7844L118.878 36.5019L122.382 32.8567Z" fill="url(#paint2_linear_1251_237)"/>
+<path d="M115.733 31.605V15.0215H110.447V45.9025H115.733V42.9678V39.5044V33.4647V31.605Z" fill="url(#paint3_linear_1251_237)"/>
+<path d="M118.793 36.3835L125.45 45.9039H131.312L122.24 32.8281L118.793 36.3835Z" fill="url(#paint4_linear_1251_237)"/>
+<path d="M143.858 23.0918C141.999 23.0918 140.401 23.2635 139.08 23.6068C137.761 23.9501 136.474 24.3964 135.207 24.9628L136.631 29.0826C137.657 28.6534 138.681 28.3101 139.689 28.0699C140.697 27.8124 141.843 27.6922 143.128 27.6922C144.935 27.6922 146.325 28.1041 147.28 28.9282C148.236 29.7521 148.722 30.9365 148.722 32.4985V33.0478V36.3436C148.757 36.3608 148.774 36.3608 148.809 36.378V37.7855C148.809 38.0945 148.774 38.3863 148.722 38.661V45.9048H153.847V32.5843C153.847 29.5804 153.031 27.2458 151.397 25.5809C149.765 23.9158 147.245 23.0918 143.858 23.0918Z" fill="#414242"/>
+<path d="M148.203 39.8615C147.912 40.4427 147.484 40.9385 146.952 41.3487C146.421 41.759 145.787 42.0838 145.068 42.306C144.348 42.5283 143.56 42.648 142.704 42.648C141.487 42.648 140.458 42.3573 139.619 41.7932C138.779 41.229 138.368 40.4085 138.368 39.3316V39.246C138.368 38.0836 138.814 37.1948 139.722 36.5793C140.63 35.9639 141.881 35.6391 143.508 35.6391C144.502 35.6391 145.445 35.7246 146.336 35.8955C147.192 36.0493 147.929 36.2544 148.563 36.4938V33.2117C147.689 32.9382 146.764 32.6988 145.822 32.5278C144.879 32.3569 143.714 32.2715 142.36 32.2715C141.058 32.2715 139.841 32.4253 138.728 32.7159C137.614 33.0066 136.655 33.4509 135.832 34.0493C135.027 34.6476 134.393 35.3998 133.947 36.2887C133.502 37.1948 133.262 38.2546 133.262 39.4683V39.5538C133.262 40.682 133.484 41.6735 133.914 42.5453C134.359 43.4001 134.924 44.1352 135.661 44.6992C136.38 45.2805 137.22 45.7078 138.18 46.0155C139.139 46.3061 140.167 46.46 141.23 46.46C142.926 46.46 144.365 46.1523 145.565 45.5198C146.747 44.9044 147.723 44.1352 148.495 43.2291V45.9814H148.528V38.7674C148.495 39.1776 148.374 39.5367 148.203 39.8615Z" fill="url(#paint5_linear_1251_237)"/>
+<path d="M177.918 25.4141C176.521 23.8602 174.557 23.0918 171.991 23.0918C171.112 23.0918 170.319 23.1943 169.63 23.4162C168.941 23.6383 168.304 23.9285 167.718 24.2871C167.132 24.6457 166.632 25.0726 166.185 25.5507C165.736 26.0289 165.34 26.524 164.979 27.0704V33.3201C164.979 31.5784 165.444 30.2123 166.375 29.2391C167.304 28.2657 168.511 27.7876 170.009 27.7876C171.526 27.7876 172.714 28.2657 173.541 29.2049C174.369 30.144 174.799 31.4931 174.799 33.2347V45.9048H180.002V31.6809C180.002 29.0512 179.313 26.968 177.918 25.4141Z" fill="url(#paint6_linear_1251_237)"/>
+<path d="M164.976 23.6484H159.969V45.905H164.976V35.7374V33.374V27.134V23.6484Z" fill="url(#paint7_linear_1251_237)"/>
+<path d="M91.6826 29.7369C92.2576 29.081 92.9545 28.5632 93.7559 28.1834C94.5573 27.8038 95.4459 27.6139 96.3866 27.6139C97.7108 27.6139 98.8258 27.8728 99.7318 28.4079C100.568 28.8911 101.352 29.4952 102.101 30.2374L100.411 23.6095C99.2614 23.2643 97.9372 23.0918 96.4389 23.0918C94.7315 23.0918 93.1635 23.4024 91.7 24.0411C90.2366 24.6624 88.9821 25.5254 87.9368 26.6128C86.874 27.7001 86.0551 28.9602 85.4629 30.3755C84.8705 31.808 84.5742 33.3269 84.5742 34.9494C84.5742 36.5718 84.8705 38.0907 85.4629 39.506L90.4107 31.877C90.7244 31.0486 91.1425 30.3582 91.6826 29.7369Z" fill="#414242"/>
+<path d="M102.346 39.4096C101.519 40.2355 100.641 40.9065 99.7276 41.4399C98.7972 41.9732 97.729 42.2312 96.5057 42.2312C95.5064 42.2312 94.5933 42.0421 93.7663 41.6635C92.9392 41.285 92.2329 40.7689 91.6299 40.115C91.0269 39.4612 90.5617 38.687 90.2343 37.8095C89.9069 36.932 89.7347 35.9685 89.7347 34.9534C89.7347 33.9555 89.8897 33.0264 90.2343 32.1661C90.2515 32.0973 90.286 32.0457 90.3032 31.9941L85.4102 39.5988C85.996 41.0097 86.8229 42.2485 87.8567 43.3153C88.9076 44.3819 90.1482 45.225 91.561 45.8272C92.9909 46.4466 94.5415 46.7391 96.2128 46.7391C96.8847 46.7391 97.5049 46.7046 98.1079 46.6187L102.381 39.4784L102.346 39.4096Z" fill="url(#paint8_linear_1251_237)"/>
+<defs>
+<linearGradient id="paint0_linear_1251_237" x1="98.2042" y1="42.9851" x2="105.433" y2="42.9851" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint1_linear_1251_237" x1="100.138" y1="27.1344" x2="105.438" y2="27.1344" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint2_linear_1251_237" x1="112.012" y1="40.1492" x2="126.924" y2="25.3243" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint3_linear_1251_237" x1="113.092" y1="5.02253" x2="113.092" y2="39.0676" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint4_linear_1251_237" x1="116.49" y1="27.8655" x2="128.674" y2="46.8393" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint5_linear_1251_237" x1="154.218" y1="39.3965" x2="139.738" y2="39.3965" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint6_linear_1251_237" x1="159.59" y1="34.4929" x2="177.454" y2="34.4929" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint7_linear_1251_237" x1="162.475" y1="16.2046" x2="162.475" y2="47.9144" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+<linearGradient id="paint8_linear_1251_237" x1="93.2522" y1="30.0485" x2="94.5713" y2="49.2974" gradientUnits="userSpaceOnUse">
+<stop stop-color="#999999"/>
+<stop offset="1"/>
+</linearGradient>
+</defs>
+</svg>

ckanapi_harvesters/builder/example/package/users_local.csv

@@ -0,0 +1,3 @@
+user_id,age
+1,30
+2,80

ckanapi_harvesters/builder/mapper_datastore.py

@@ -0,0 +1,93 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Code to upload metadata to the CKAN server to create/update an existing package
+The metadata is defined by the user in an Excel worksheet
+This file implements functions to convert formats between database and local files.
+"""
+from typing import Dict, List, Callable, Any, Tuple, Union, Set
+import copy
+
+import pandas as pd
+
+from ckanapi_harvesters.builder.builder_errors import MissingCodeFileError
+from ckanapi_harvesters.auxiliary.external_code_import import PythonUserCode
+from ckanapi_harvesters.auxiliary.list_records import ListRecords
+
+
+def simple_upload_fun(df_local: pd.DataFrame) -> None:
+    for field in df_local.columns:
+        if df_local[field].dtype == pd.Timestamp:
+            df_local[field] = df_local[field].apply(pd.Timestamp.isoformat)  # ISO-8601 format
+
+
+class DataSchemeConversion:
+    def __init__(self, *, df_upload_fun:Callable[[pd.DataFrame], Any] = None,
+                 df_download_fun:Callable[[pd.DataFrame], Any] = None):
+        """
+        Class to convert between local data formats and database formats
+
+        :param df_upload_fun:
+        :param df_download_fun:
+        """
+        self.df_upload_fun:Union[Callable[[Any, Any], Union[ListRecords, pd.DataFrame]], None] = df_upload_fun
+        self.df_download_fun:Union[Callable[[pd.DataFrame, Any], pd.DataFrame], None] = df_download_fun
+
+    def copy(self):
+        return copy.deepcopy(self)
+
+    def df_upload_alter(self, df_local: Union[pd.DataFrame, Any], file_name:str=None, mapper_kwargs:dict=None, **kwargs) -> pd.DataFrame:
+        """
+        Apply used-defined df_upload_fun if present
+
+        :param df_local: the dataframe to upload
+        :return: the dataframe ready for upload, converted in the format of the database
+        """
+        if mapper_kwargs is None: mapper_kwargs = {}
+        mapper_kwargs["file_name"] = file_name
+        if file_name is not None and isinstance(df_local, pd.DataFrame):
+            df_local.attrs["source"] = file_name
+        df_database = df_local
+        if self.df_upload_fun is not None:
+            # df_database = df_local.copy()  # unnecessary copy
+            df_upload_fun = self.df_upload_fun
+            df_database = df_upload_fun(df_database, **mapper_kwargs, **kwargs)
+        if not isinstance(df_database, pd.DataFrame):
+            if isinstance(df_database, ListRecords):
+                pass  # also accept ListRecords (List[dict])
+            elif self.df_upload_fun is None:
+                raise TypeError("No upload function was defined to convert the data format to a DataFrame")
+            else:
+                raise TypeError("df_upload_fun must return a DataFrame")
+        return df_database
+
+    def df_download_alter(self, df_database:pd.DataFrame, file_query:dict=None, mapper_kwargs:dict=None, **kwargs) -> pd.DataFrame:
+        """
+        Apply used-defined df_download_fun if present.
+        df_download_fun should be the reverse function of df_upload_fun
+
+        :param df_database: the downloaded dataframe from the database
+        :return: the dataframe ready to save, converted in the local format
+        """
+        if mapper_kwargs is None: mapper_kwargs = {}
+        mapper_kwargs["file_query"] = file_query
+        if file_query is not None:
+            df_database.attrs["query"] = file_query
+        df_local = df_database
+        if self.df_download_fun is not None:
+            # df_local = df_database.copy()  # unnecessary copy
+            df_download_fun = self.df_download_fun
+            df_local = df_download_fun(df_local, **mapper_kwargs, **kwargs)
+        return df_local
+
+    def _connect_aux_functions(self, module: PythonUserCode, aux_upload_fun_name:str, aux_download_fun_name:str) -> None:
+        if (aux_upload_fun_name or aux_download_fun_name) and module is None:
+            raise MissingCodeFileError()
+        if aux_upload_fun_name:
+            self.df_upload_fun = module.function_pointer(aux_upload_fun_name)
+        if aux_download_fun_name:
+            self.df_download_fun = module.function_pointer(aux_download_fun_name)
+
+    def get_necessary_fields(self) -> Set[str]:
+        return set()
+