labkey 2.6.0__tar.gz → 3.0.0__tar.gz

{labkey-2.6.0 → labkey-3.0.0}/CHANGE.txt

@@ -2,6 +2,23 @@
 LabKey Python Client API News
 +++++++++++
 
+What's New in the LabKey 3.0.0 package
+==============================
+
+*Release date: 12/14/2023*
+- Query API - WAF encode "sql" parameter for execute_sql
+    - WAF encoding of parameters is initially supported with LabKey Server v23.09
+    - WAF encoding can be opted out of on execute_sql calls by specifying waf_encode_sql=False
+- Query API - add optional parameters to insert_rows, update_rows, and delete_rows
+- Query API - add move_rows()
+    - earliest compatible LabKey Server version: 24.1.0
+
+What's New in the LabKey 2.6.1 package
+==============================
+
+*Release date: 10/09/2023*
+- Query API - Change max_rows default value to -1 in select_rows
+
 What's New in the LabKey 2.6.0 package
 ==============================
 

{labkey-2.6.0/labkey.egg-info → labkey-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: labkey
-Version: 2.6.0
+Version: 3.0.0
 Summary: Python client API for LabKey Server
 Home-page: https://github.com/LabKey/labkey-api-python
 Author: LabKey
@@ -8,9 +8,7 @@ Author-email: alanv@labkey.com
 Maintainer: Alan Vezina
 Maintainer-email: alanv@labkey.com
 License: Apache License 2.0
-Description: Python client API for LabKey Server. Supports query and experiment APIs.
 Keywords: labkey api client
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Science/Research
@@ -21,4 +19,12 @@ Classifier: Operating System :: Microsoft
 Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering
+License-File: LICENSE.txt
+Requires-Dist: requests
 Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: requests; extra == "test"
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+
+Python client API for LabKey Server. Supports query and experiment APIs.

{labkey-2.6.0 → labkey-3.0.0}/README.md

@@ -16,6 +16,7 @@ Query API - [sample code](samples/query_examples.py)
 - **insert_rows()** - Insert rows into a table.
 - **select_rows()** - Query and get results sets.
 - **update_rows()** - Update rows in a table.
+- **move_rows()()** - Move rows in a table.
 - **truncate_table()** - Delete all rows from a table.
 
 Domain API - [sample code](samples/domain_example.py)

{labkey-2.6.0 → labkey-3.0.0}/labkey/__init__.py

@@ -14,6 +14,6 @@
 # limitations under the License.
 #
 __title__ = "labkey"
-__version__ = "2.6.0"
+__version__ = "3.0.0"
 __author__ = "LabKey"
 __license__ = "Apache License 2.0"

{labkey-2.6.0 → labkey-3.0.0}/labkey/query.py

@@ -44,6 +44,7 @@ import functools
 from typing import List
 
 from .server_context import ServerContext
+from .utils import waf_encode
 
 _default_timeout = 60 * 5  # 5 minutes
 
@@ -164,12 +165,25 @@ class QueryFilter:
         return "<QueryFilter [{} {} {}]>".format(self.column_name, self.filter_type, self.value)
 
 
+class AuditBehavior:
+    """
+    Enum of different auditing levels
+    """
+
+    DETAILED = "DETAILED"
+    NONE = "NONE"
+    SUMMARY = "SUMMARY"
+
+
 def delete_rows(
     server_context: ServerContext,
     schema_name: str,
     query_name: str,
     rows: any,
     container_path: str = None,
+    transacted: bool = True,
+    audit_behavior: AuditBehavior = None,
+    audit_user_comment: str = None,
     timeout: int = _default_timeout,
 ):
     """
@@ -179,12 +193,25 @@ def delete_rows(
     :param query_name: table name to delete from
     :param rows: Set of rows to delete
     :param container_path: labkey container path if not already set in context
+    :param transacted: whether all of the updates should be done in a single transaction
+    :param audit_behavior: used to override the audit behavior for the update. See class query.AuditBehavior
+    :param audit_user_comment: used to provide a comment that will be attached to certain detailed audit log records
     :param timeout: timeout of request in seconds (defaults to 30s)
     :return:
     """
     url = server_context.build_url("query", "deleteRows.api", container_path=container_path)
+
     payload = {"schemaName": schema_name, "queryName": query_name, "rows": rows}
 
+    if transacted is False:
+        payload["transacted"] = transacted
+
+    if audit_behavior is not None:
+        payload["auditBehavior"] = audit_behavior
+
+    if audit_user_comment is not None:
+        payload["auditUserComment"] = audit_user_comment
+
     return server_context.make_request(
         url,
         json=payload,
@@ -231,6 +258,7 @@ def execute_sql(
     parameters: dict = None,
     required_version: float = None,
     timeout: int = _default_timeout,
+    waf_encode_sql: bool = True
 ):
     """
     Execute sql query against a LabKey server.
@@ -248,11 +276,12 @@ def execute_sql(
     :param parameters: parameter values to pass through to a parameterized query
     :param required_version: Api version of response
     :param timeout: timeout of request in seconds (defaults to 30s)
+    :param waf_encode_sql: WAF encode sql in request (defaults to True)
     :return:
     """
     url = server_context.build_url("query", "executeSql.api", container_path=container_path)
 
-    payload = {"schemaName": schema_name, "sql": sql}
+    payload = {"schemaName": schema_name, "sql": waf_encode(sql) if waf_encode_sql else sql}
 
     if container_filter is not None:
         payload["containerFilter"] = container_filter
@@ -285,6 +314,10 @@ def insert_rows(
     query_name: str,
     rows: List[any],
     container_path: str = None,
+    skip_reselect_rows: bool = False,
+    transacted: bool = True,
+    audit_behavior: AuditBehavior = None,
+    audit_user_comment: str = None,
     timeout: int = _default_timeout,
 ):
     """
@@ -294,6 +327,10 @@ def insert_rows(
     :param query_name: table name to insert into
     :param rows: set of rows to insert
     :param container_path: labkey container path if not already set in context
+    :param skip_reselect_rows: whether the full detailed response for the insert can be skipped
+    :param transacted: whether all of the updates should be done in a single transaction
+    :param audit_behavior: used to override the audit behavior for the update. See class query.AuditBehavior
+    :param audit_user_comment: used to provide a comment that will be attached to certain detailed audit log records
     :param timeout: timeout of request in seconds (defaults to 30s)
     :return:
     """
@@ -301,6 +338,18 @@ def insert_rows(
 
     payload = {"schemaName": schema_name, "queryName": query_name, "rows": rows}
 
+    if skip_reselect_rows is True:
+        payload["skipReselectRows"] = skip_reselect_rows
+
+    if transacted is False:
+        payload["transacted"] = transacted
+
+    if audit_behavior is not None:
+        payload["auditBehavior"] = audit_behavior
+
+    if audit_user_comment is not None:
+        payload["auditUserComment"] = audit_user_comment
+
     return server_context.make_request(
         url,
         json=payload,
@@ -316,7 +365,7 @@ def select_rows(
     filter_array: List[QueryFilter] = None,
     container_path: str = None,
     columns=None,
-    max_rows: int = None,
+    max_rows: int = -1,
     sort: str = None,
     offset: int = None,
     container_filter: str = None,
@@ -339,7 +388,7 @@ def select_rows(
     :param filter_array: set of filter objects to apply
     :param container_path: folder path if not already part of server_context
     :param columns: set of columns to retrieve
-    :param max_rows: max number of rows to retrieve
+    :param max_rows: max number of rows to retrieve, defaults to -1 (unlimited)
     :param sort: comma separated list of column names to sort by, prefix a column with '-' to sort descending
     :param offset: number of rows to offset results by
     :param container_filter: enumeration of the various container filters available. See:
@@ -419,6 +468,9 @@ def update_rows(
     query_name: str,
     rows: List[any],
     container_path: str = None,
+    transacted: bool = True,
+    audit_behavior: AuditBehavior = None,
+    audit_user_comment: str = None,
     timeout: int = _default_timeout,
 ):
     """
@@ -429,6 +481,9 @@ def update_rows(
     :param query_name: table name to update
     :param rows: Set of rows to update
     :param container_path: labkey container path if not already set in context
+    :param transacted: whether all of the updates should be done in a single transaction
+    :param audit_behavior: used to override the audit behavior for the update. See class query.AuditBehavior
+    :param audit_user_comment: used to provide a comment that will be attached to certain detailed audit log records
     :param timeout: timeout of request in seconds (defaults to 30s)
     :return:
     """
@@ -436,6 +491,61 @@ def update_rows(
 
     payload = {"schemaName": schema_name, "queryName": query_name, "rows": rows}
 
+    if transacted is False:
+        payload["transacted"] = transacted
+
+    if audit_behavior is not None:
+        payload["auditBehavior"] = audit_behavior
+
+    if audit_user_comment is not None:
+        payload["auditUserComment"] = audit_user_comment
+
+    return server_context.make_request(
+        url,
+        json=payload,
+        timeout=timeout,
+    )
+
+
+def move_rows(
+    server_context: ServerContext,
+    target_container_path: str,
+    schema_name: str,
+    query_name: str,
+    rows: any,
+    container_path: str = None,
+    transacted: bool = True,
+    audit_behavior: AuditBehavior = None,
+    audit_user_comment: str = None,
+    timeout: int = _default_timeout,
+):
+    """
+    Move a set of rows from the schema.query
+    :param server_context: A LabKey server context. See utils.create_server_context.
+    :param target_container_path: target labkey container path for the move
+    :param schema_name: schema of table
+    :param query_name: table name to move from
+    :param rows: Set of rows to move
+    :param container_path: source labkey container path if not already set in context
+    :param transacted: whether all of the updates should be done in a single transaction
+    :param audit_behavior: used to override the audit behavior for the update. See class query.AuditBehavior
+    :param audit_user_comment: used to provide a comment that will be attached to certain detailed audit log records
+    :param timeout: timeout of request in seconds (defaults to 30s)
+    :return:
+    """
+    url = server_context.build_url("query", "moveRows.api", container_path=container_path)
+
+    payload = {"targetContainerPath": target_container_path, "schemaName": schema_name, "queryName": query_name, "rows": rows}
+
+    if transacted is False:
+        payload["transacted"] = transacted
+
+    if audit_behavior is not None:
+        payload["auditBehavior"] = audit_behavior
+
+    if audit_user_comment is not None:
+        payload["auditUserComment"] = audit_user_comment
+
     return server_context.make_request(
         url,
         json=payload,
@@ -458,10 +568,21 @@ class QueryWrapper:
         query_name: str,
         rows: any,
         container_path: str = None,
+        transacted: bool = True,
+        audit_behavior: AuditBehavior = None,
+        audit_user_comment: str = None,
         timeout: int = _default_timeout,
     ):
         return delete_rows(
-            self.server_context, schema_name, query_name, rows, container_path, timeout
+            self.server_context,
+            schema_name,
+            query_name,
+            rows,
+            container_path,
+            transacted,
+            audit_behavior,
+            audit_user_comment,
+            timeout
         )
 
     @functools.wraps(truncate_table)
@@ -484,6 +605,7 @@ class QueryWrapper:
         parameters: dict = None,
         required_version: float = None,
         timeout: int = _default_timeout,
+        waf_encode_sql: bool = True
     ):
         return execute_sql(
             self.server_context,
@@ -498,6 +620,7 @@ class QueryWrapper:
             parameters,
             required_version,
             timeout,
+            waf_encode_sql
         )
 
     @functools.wraps(insert_rows)
@@ -507,10 +630,23 @@ class QueryWrapper:
         query_name: str,
         rows: List[any],
         container_path: str = None,
+        skip_reselect_rows: bool = False,
+        transacted: bool = True,
+        audit_behavior: AuditBehavior = None,
+        audit_user_comment: str = None,
         timeout: int = _default_timeout,
     ):
         return insert_rows(
-            self.server_context, schema_name, query_name, rows, container_path, timeout
+            self.server_context,
+            schema_name,
+            query_name,
+            rows,
+            container_path,
+            skip_reselect_rows,
+            transacted,
+            audit_behavior,
+            audit_user_comment,
+            timeout
         )
 
     @functools.wraps(select_rows)
@@ -522,7 +658,7 @@ class QueryWrapper:
         filter_array: List[QueryFilter] = None,
         container_path: str = None,
         columns=None,
-        max_rows: int = None,
+        max_rows: int = -1,
         sort: str = None,
         offset: int = None,
         container_filter: str = None,
@@ -566,8 +702,45 @@ class QueryWrapper:
         query_name: str,
         rows: List[any],
         container_path: str = None,
+        transacted: bool = True,
+        audit_behavior: AuditBehavior = None,
+        audit_user_comment: str = None,
         timeout: int = _default_timeout,
     ):
         return update_rows(
-            self.server_context, schema_name, query_name, rows, container_path, timeout
+            self.server_context,
+            schema_name,
+            query_name,
+            rows,
+            container_path,
+            transacted,
+            audit_behavior,
+            audit_user_comment,
+            timeout
+        )
+
+    @functools.wraps(move_rows)
+    def move_rows(
+        self,
+        target_container_path: str,
+        schema_name: str,
+        query_name: str,
+        rows: any,
+        container_path: str = None,
+        transacted: bool = True,
+        audit_behavior: AuditBehavior = None,
+        audit_user_comment: str = None,
+        timeout: int = _default_timeout,
+    ):
+        return move_rows(
+            self.server_context,
+            target_container_path,
+            schema_name,
+            query_name,
+            rows,
+            container_path,
+            transacted,
+            audit_behavior,
+            audit_user_comment,
+            timeout
         )

{labkey-2.6.0 → labkey-3.0.0}/labkey/utils.py

@@ -16,6 +16,8 @@
 import json
 from functools import wraps
 from datetime import date, datetime
+from base64 import b64encode
+from urllib import parse
 
 
 # Issue #14: json.dumps on datetime throws TypeError
@@ -71,3 +73,21 @@ def transform_helper(user_transform_func, file_path_run_properties):
             row = [str(el).strip() for el in row]
             row = "\t".join(row)
             file_out.write(row + "\n")
+
+
+def btoa(value: str) -> str:
+    if not value:
+        return value
+    binary = value.encode("utf-8")
+    return b64encode(binary).decode()
+
+
+def encode_uri_component(value: str) -> str:
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
+    return parse.quote(value, encoding="utf-8", safe="-_.!~*'()")
+
+
+def waf_encode(value: str) -> str:
+    if value:
+        return "/*{{base64/x-www-form-urlencoded/wafText}}*/" + btoa(encode_uri_component(value))
+    return value

{labkey-2.6.0 → labkey-3.0.0/labkey.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: labkey
-Version: 2.6.0
+Version: 3.0.0
 Summary: Python client API for LabKey Server
 Home-page: https://github.com/LabKey/labkey-api-python
 Author: LabKey
@@ -8,9 +8,7 @@ Author-email: alanv@labkey.com
 Maintainer: Alan Vezina
 Maintainer-email: alanv@labkey.com
 License: Apache License 2.0
-Description: Python client API for LabKey Server. Supports query and experiment APIs.
 Keywords: labkey api client
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Science/Research
@@ -21,4 +19,12 @@ Classifier: Operating System :: Microsoft
 Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering
+License-File: LICENSE.txt
+Requires-Dist: requests
 Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: requests; extra == "test"
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+
+Python client API for LabKey Server. Supports query and experiment APIs.

{labkey-2.6.0 → labkey-3.0.0}/labkey.egg-info/SOURCES.txt

@@ -2,10 +2,6 @@ CHANGE.txt
 LICENSE.txt
 MANIFEST.in
 README.md
-download_file_example.txt
-lundbeck_file_download_notes.txt
-perf_testing_notes.txt
-playground.py.txt
 pyproject.toml
 pytest.ini
 setup.cfg

labkey-2.6.0/download_file_example.txt

@@ -1,69 +0,0 @@
-from labkey.api_wrapper import APIWrapper
-
-
-def get_base_url(api: APIWrapper) -> str:
-    ctx = api.server_context
-
-    # The URL returned from LabKey Server's select rows API isn't a full URL, so we need to add The scheme
-    # (e.g. https://), the domain, and the context path. We don't need to add the container path, because that is
-    # already on the URL returned from the server.
-    base_url = ctx._scheme + ctx._domain
-
-    if ctx._context_path is not None:
-        base_url += "/" + ctx._context_path
-
-    return base_url
-
-
-def get_file(api: APIWrapper, file_url: str):
-    """
-    Downloads a file given a file_url from a select_rows response. File is stored in memory and returned (as bytes).
-    Response from this function could be passed to something like pandas. This is useful when you know the file is small
-    enough to fit into memory, but will cause problems if you have a large file (see download_file below).
-    """
-    ctx = api.server_context
-    full_url = get_base_url(api) + file_url
-    resp = ctx._session.get(full_url)
-    return resp.content
-
-
-def download_file(api: APIWrapper, file_name: str, file_url: str, destination_path: str):
-    """
-    Downloads a file from LabKey Server to disk. This doesn't put the whole file in memory, which is good for larger
-    files. After the file is saved to disk you can open it with another tool such as pandas.
-    """
-    ctx = api.server_context
-    full_url = get_base_url(api) + file_url
-
-    # This with block is needed so we clean up the connection created by requests when we're done
-    with ctx._session.get(full_url, stream=True) as req:
-        # This with block opens and closes the file handle for us
-        with open(destination_path + "/" + file_name, "wb") as f:
-            # There is no standard correct size for chunk size here, you can play around with it and see if it has an
-            # impact on perf.
-            for chunk in req.iter_content(chunk_size=16*1024):
-                f.write(chunk)
-
-
-def main():
-    # Create your API wrapper, the variables here will depend on your server configuration
-    domain = "localhost:8080"
-    container = "api_sandbox"
-    api = APIWrapper(domain, container, use_ssl=False, verify_ssl=False)
-    # The name of the column that is a file, this will depend on the table you're querying    
-    file_column = "file"
-    
-    # Select your data, using required_version=17.1 is important here, older versions of the API return the file URLs,
-    # but not in away that are associated with the actual file column.
-    resp = api.query.select_rows("lists", "list of files", required_version=17.1)
-    
-    # Here we're just grabbing the first row of data, but you could easily iterate over all of the URLs. You could even
-    # use a thread pool to load multiple files in parallel, which would improve perf because it's I/O bound.
-    data = resp["rows"][0]["data"]
-    file_name = data[file_column]["value"]
-    file_url = data[file_column]["url"]
-    download_file(api, file_name, file_url, "./downloads")
-
-
-if __name__ == "__main__":
-    main()

labkey-2.6.0/lundbeck_file_download_notes.txt

@@ -1,10 +0,0 @@
-- make assay with file field in assay results
-- make python script to download said file
-- the crux here is converting the "url" attribute of the column in the select rows response to something
-that our labkey api can use to download the file
-    - Storing the file in memory should be fine
-    - Storing in /tmp may be better because this will be done in Docker, and if it's on disk it can
-    be streamed which should theory be better perf-wise
-
-
-

labkey-2.6.0/perf_testing_notes.txt

@@ -1,20 +0,0 @@
-Run     User Time       System Time     Total CPU Time     Wall Time   Total Rows      Rows Retrieved
-1       .42             .14             .56                1:36.6      118,000         118,000
-2       .38             .12             .50                5:42.9      25,000,000      26,000
-
-Hypothesis:
-The server is taking a long time to respond to the request, which is causing the long response time. Both scripts spend
-very little time running, which means they're probably spending more time waiting for data from the server. I suspect
-that the R script may be faster because they're possibly running on the same machine as the server, so the network
-request stays local to the server, whereas the Python script is being run externally from the server, so the network is
-necessarily slower.
-
-Steps to reproduce:
-1. Create some tables to emulate the query provided by the customer
-2. Populate these tables with a similar size of data (25M total rows, that can be filtered to 26k rows)
-3. Run the script locally with profiler.
-
-
-API time: 38.3407 seconds
-Pandas time: 1.5355 seconds
-Total time: 39.8762 seconds

labkey-2.6.0/playground.py.txt

@@ -1,46 +0,0 @@
-from labkey.query import QueryFilter
-from labkey.api_wrapper import APIWrapper
-
-
-def get_webdav_url(server_context, container_path=None):
-    parts = [server_context._scheme + server_context._domain]
-
-    if server_context._context_path is not None:
-        parts.append(server_context._context_path)
-
-    parts.append("_webdav")
-
-    if container_path is not None:
-        parts.append(container_path)
-    elif server_context._container_path is not None:
-        parts.append(server_context._container_path)
-
-    parts.append("@files")
-    parts.append("")
-
-    return "/".join(parts)
-
-
-def main():
-    # Create your API wrapper, the variables here will depend on your server configuration
-    domain = "localhost:8080"
-    container = "NIAD Python"
-    api = APIWrapper(domain, container, use_ssl=False, verify_ssl=False)
-    url = get_webdav_url(api.server_context)
-    file_name = "api_wrapper.py"
-    file_path = "./labkey/api_wrapper.py"
-    
-    # Open the file you want to upload, and upload it via the webdav API
-    with open(file_path, 'r') as file:
-        resp = api.server_context.make_request(url, payload={"createIntermediates": 'true'}, file_payload={"file": file}, non_json_response=True)
-    
-    # Find the RowId of the file we just uploaded, by using select_rows and filtering by the file name.
-    resp = api.query.select_rows('exp', 'files', filter_array=[QueryFilter('name', file_name)])
-    row_id = resp["rows"][0]["RowId"]
-    # Update the file metadata, the "RowId" field here is required, the rest of the fields will depend on the custom fields you have defined 
-    resp = api.query.update_rows('exp', 'files', [{ "RowId": row_id, "Site": "My Test Site", "Visit": "A Visit", "Form": "Some Form" }])
-    print(resp)
-
-
-if __name__ == "__main__":
-    main()