datachain 0.31.2__tar.gz → 0.31.3__tar.gz

{datachain-0.31.2 → datachain-0.31.3}/.gitignore

@@ -149,3 +149,5 @@ cython_debug/
 # for local dev, e.g. LLM generated files, .env.test to override
 # test variables, local scripts to try, etc
 local/
+docs/openapi.json
+docs/studio/api/index.md

{datachain-0.31.2 → datachain-0.31.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.2
+Version: 0.31.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

datachain-0.31.3/docs/api_hooks.py

@@ -0,0 +1,35 @@
+# ruff: noqa: INP001
+def on_pre_build(**kwargs):
+    import os
+
+    import requests
+
+    # Download OpenAPI spec
+    response = requests.get(
+        "https://studio.datachain.ai/api/openapi.json",
+        timeout=30,
+    )
+
+    # Write to file
+    print("Writing OpenAPI spec to docs/openapi.json")
+    with open("docs/openapi.json", "w") as f:
+        f.write(response.text)
+
+    # Generate API docs using widdershins
+    print("Generating API docs using widdershins")
+    cmd = [
+        "npx widdershins",
+        "docs/openapi.json",
+        "-o docs/studio/api/index.md",
+        "--language_tabs 'python:Python'",
+        "--language_tabs 'shell:curl'",
+        "--expandBody true",
+        "--summary",
+        "--shallowSchemas",
+        "--omitBody",
+        "--resolve",
+        "--httpsnippet",
+        "-u docs/templates",
+        "--omitHeader",
+    ]
+    os.system(" ".join(cmd))  # noqa: S605

datachain-0.31.3/docs/templates/main.dot

@@ -0,0 +1,81 @@
+<h1 id="{{=data.title_prefix||''}}">{{=data.api.info && data.api.info.title}} {{=data.version||''}}</h1>
+
+> Scroll down for {{? data.header.language_tabs.length}}code samples, {{?}}example requests and responses.{{? data.header.language_tabs.length}} .{{?}}
+
+{{? data.api.info && data.api.info.description}}{{=data.api.info.description||''}}{{?}}
+
+
+{{? data.api.info && data.api.info.termsOfService}}<a href="{{=data.api.info.termsOfService||''}}">Terms of service</a>{{?}}
+{{? data.api.info && data.api.info.contact}}{{? data.api.info.contact.email}}Email: <a href="mailto:{{=data.api.info.contact.email||''}}">{{=data.api.info.contact.name || 'Support'}}</a> {{?}}{{? data.api.info.contact.url}}Web: <a href="{{=data.api.info.contact.url||''}}">{{= data.api.info.contact.name || 'Support'}}</a> {{?}}{{?}}
+{{? data.api.info && data.api.info.license}}{{? data.api.info.license.url}}License: <a href="{{=data.api.info.license.url||''}}">{{=data.api.info.license.name||''}}</a>{{??}} License: {{=data.api.info.license.name||''}}{{?}}{{?}}
+{{= data.tags.endSection }}
+
+{{? data.api.components && data.api.components.securitySchemes && data.def && data.def.security}}
+{{#def.security}}
+{{?}}
+
+{{ for (var r in data.resources) { }}
+{{ data.resource = data.resources[r]; }}
+
+{{= data.tags.section }}
+<h1 id="{{=data.title_prefix||''+'-'+data.utils.slugify(r)}}">{{= r||''}}</h1>
+
+{{? data.resource.description }}{{= data.resource.description||''}}{{?}}
+
+{{? data.resource.externalDocs}}
+<a href="{{=data.resource.externalDocs.url||''}}">{{=data.resource.externalDocs.description||'External documentation'}}</a>
+{{?}}
+
+{{ for (var m in data.resource.methods) { }}
+{{ data.operationUniqueName = m; }}
+{{ data.method = data.resource.methods[m]; }}
+{{ data.operationUniqueSlug = data.method.slug; }}
+{{ data.operation = data.method.operation; }}
+{{= data.templates.operation(data) }}
+{{ } /* of methods */ }}
+
+{{= data.tags.endSection }}
+{{ } /* of resources */ }}
+
+{{? data.api.components && data.api.components.schemas }}
+{{= data.tags.section }}
+
+# Schemas
+
+{{ for (var s in data.components.schemas) { }}
+{{ var origSchema = data.components.schemas[s]; }}
+{{ var schema = data.api.components.schemas[s]; }}
+
+{{= data.tags.section }}
+<h2 id="tocS_{{=s||''}}">{{=s||''}}</h2>
+{{ /* backwards compatibility */ }}
+<a id="schema{{=s.toLowerCase()||''}}"></a>
+<a id="schema_{{=s||''}}"></a>
+<a id="tocS{{=s.toLowerCase()||''}}"></a>
+<a id="tocs{{=s.toLowerCase()||''}}"></a>
+
+{{? data.options.yaml }}
+```yaml
+{{=data.utils.yaml.stringify(data.utils.getSample(schema,data.options,{quiet:true},data.api))}}
+{{??}}
+```json
+{{=data.utils.safejson(data.utils.getSample(schema,data.options,{quiet:true},data.api),null,2)}}
+{{?}}
+```
+
+
+
+{{= data.tags.endSection }}
+{{= data.tags.endSection }}
+
+{{ } /* of schemas */ }}
+
+{{?}}
+
+{{? data.options.footer}}
+{{#def.footer}}
+{{?}}
+
+{{? data.options.discovery}}
+{{#def.discovery}}
+{{?}}

datachain-0.31.3/docs/templates/operation.dot

@@ -0,0 +1,47 @@
+{{= data.tags.section }}
+
+## {{= data.operationUniqueName||''}}
+
+{{? data.operation.operationId}}
+<a id="opId{{=data.operation.operationId||''}}"></a>
+{{?}}
+
+{{ data.methodUpper = data.method.verb.toUpperCase(); }}
+{{ data.url = data.utils.slashes(data.baseUrl + data.method.path); }}
+{{ data.parameters = data.operation.parameters; }}
+{{ data.enums = []; }}
+{{ data.utils.fakeProdCons(data); }}
+{{ data.utils.fakeBodyParameter(data); }}
+{{ data.utils.mergePathParameters(data); }}
+{{ data.utils.getParameters(data); }}
+
+{{? data.options.codeSamples || data.operation["x-code-samples"] }}
+> Code samples
+
+{{? data.utils.getCodeSamples(data) }}{{= data.utils.getCodeSamples(data) }}{{?}}
+{{?}}
+
+`{{= data.methodUpper||''}} {{=data.method.path||''}}`
+
+{{? data.operation.summary && !data.options.tocSummary}}*{{= data.operation.summary||'' }}*{{?}}
+
+{{? data.operation.description}}{{= data.operation.description||'' }}{{?}}
+
+{{? data.operation.requestBody}}
+> Body parameter
+
+{{? data.bodyParameter.exampleValues.description }}
+> {{= data.bodyParameter.exampleValues.description||'' }}
+{{?}}
+
+{{? data.utils.getBodyParameterExamples(data) }}{{= data.utils.getBodyParameterExamples(data) }}{{?}}
+{{?}}
+
+{{#def.responses}}
+
+{{ data.security = data.operation.security ? data.operation.security : data.api.security; }}
+{{? data.security && data.security.length }}
+{{??}}
+{{#def.authentication_none}}
+{{?}}
+{{= data.tags.endSection }}

datachain-0.31.3/docs/templates/responses.def

@@ -0,0 +1,61 @@
+{{ data.responses = data.utils.getResponses(data); }}
+{{ data.responseSchemas = false; }}
+{{~ data.responses :response }}
+{{ if (response.content) data.responseSchemas = true; }}
+{{~}}
+
+{{? data.responseSchemas }}
+> Example responses
+
+{{? data.utils.getResponseExamples(data) }}{{= data.utils.getResponseExamples(data) }}{{?}}
+{{?}}
+
+{{= data.tags.section }}
+<h3 id="{{=data.operationUniqueSlug||''}}-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+{{~ data.responses :r}}|{{=r.status}}|{{=r.meaning}}|{{=r.description || 'none'}}|{{=r.schema||'none'}}|
+{{~}}
+
+{{ data.responseSchemas = false; }}
+{{~ data.responses :response }}
+{{ if (response.content && !response.$ref && !data.utils.isPrimitive(response.type)) data.responseSchemas = true; }}
+{{~}}
+{{? data.responseSchemas }}
+<h3 id="{{=data.operationUniqueSlug||''}}-responseschema">Response Schema</h3>
+{{~ data.responses :response}}
+{{? response.content && !response.$ref && !data.utils.isPrimitive(response.type)}}
+{{? Object.keys(response.content).length }}
+{{ var responseKey = Object.keys(response.content)[0]; }}
+{{ var responseSchema = response.content[responseKey].schema; }}
+{{ var enums = []; }}
+{{ var blocks = data.utils.schemaToArray(responseSchema,0,{trim:true,join:true},data); }}
+{{ for (var block of blocks) {
+     for (var p of block.rows) {
+       if (p.schema && p.schema.enum) {
+         for (var e of p.schema.enum) {
+           enums.push({name:p.name,value:e});
+         }
+       }
+     }
+   }
+}}
+
+Status Code **{{=response.status||''}}**
+
+{{ data.response = response; }}
+
+{{?}}
+{{?}}
+{{~}}
+{{?}}
+{{ data.responseHeaders = data.utils.getResponseHeaders(data); }}
+{{? data.responseHeaders.length }}
+### Response Headers
+|Status|Header|Type|Format|Description|
+|---|---|---|---|---|
+{{~ data.responseHeaders :h}}|{{=h.status}}|{{=h.header}}|{{=h.type}}|{{=h.format||''}}|{{=h.description||'none'}}|
+{{~}}
+{{?}}
+{{= data.tags.endSection }}

{datachain-0.31.2 → datachain-0.31.3}/mkdocs.yml

@@ -117,6 +117,8 @@ nav:
           - Environment Variables: guide/env.md
           - Namespaces: guide/namespaces.md
           - Local DB Migrations: guide/db_migrations.md
+      - 🔗 Studio:
+          - API: studio/api/index.md
       - 🤝 Contributing: contributing.md
 
   - DataChain Website ↗: https://datachain.ai" target="_blank"
@@ -161,6 +163,8 @@ extra:
     - icon: fontawesome/brands/linkedin
       link: https://www.linkedin.com/company/dvc-ai
 
+hooks:
+  - docs/api_hooks.py
 plugins:
   - search
   - section-index

{datachain-0.31.2 → datachain-0.31.3}/src/datachain/catalog/catalog.py

@@ -144,19 +144,26 @@ def shutdown_process(
             return proc.wait()
 
 
-def _process_stream(stream: "IO[bytes]", callback: Callable[[str], None]) -> None:
+def process_output(stream: IO[bytes], callback: Callable[[str], None]) -> None:
     buffer = b""
-    while byt := stream.read(1):  # Read one byte at a time
-        buffer += byt
 
-        if byt in (b"\n", b"\r"):  # Check for newline or carriage return
-            line = buffer.decode("utf-8")
-            callback(line)
-            buffer = b""  # Clear buffer for next line
+    try:
+        while byt := stream.read(1):  # Read one byte at a time
+            buffer += byt
 
-    if buffer:  # Handle any remaining data in the buffer
-        line = buffer.decode("utf-8")
-        callback(line)
+            if byt in (b"\n", b"\r"):  # Check for newline or carriage return
+                line = buffer.decode("utf-8", errors="replace")
+                callback(line)
+                buffer = b""  # Clear buffer for the next line
+
+        if buffer:  # Handle any remaining data in the buffer
+            line = buffer.decode("utf-8", errors="replace")
+            callback(line)
+    finally:
+        try:
+            stream.close()  # Ensure output is closed
+        except Exception:  # noqa: BLE001, S110
+            pass
 
 
 class DatasetRowsFetcher(NodesThreadPool):
@@ -1760,13 +1767,13 @@ class Catalog:
             recursive=recursive,
         )
 
+    @staticmethod
     def query(
-        self,
         query_script: str,
         env: Optional[Mapping[str, str]] = None,
         python_executable: str = sys.executable,
-        capture_output: bool = False,
-        output_hook: Callable[[str], None] = noop,
+        stdout_callback: Optional[Callable[[str], None]] = None,
+        stderr_callback: Optional[Callable[[str], None]] = None,
         params: Optional[dict[str, str]] = None,
         job_id: Optional[str] = None,
         interrupt_timeout: Optional[int] = None,
@@ -1781,13 +1788,18 @@ class Catalog:
             },
         )
         popen_kwargs: dict[str, Any] = {}
-        if capture_output:
-            popen_kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT}
+
+        if stdout_callback is not None:
+            popen_kwargs = {"stdout": subprocess.PIPE}
+        if stderr_callback is not None:
+            popen_kwargs["stderr"] = subprocess.PIPE
 
         def raise_termination_signal(sig: int, _: Any) -> NoReturn:
             raise TerminationSignal(sig)
 
-        thread: Optional[Thread] = None
+        stdout_thread: Optional[Thread] = None
+        stderr_thread: Optional[Thread] = None
+
         with subprocess.Popen(cmd, env=env, **popen_kwargs) as proc:  # noqa: S603
             logger.info("Starting process %s", proc.pid)
 
@@ -1801,10 +1813,20 @@ class Catalog:
             orig_sigterm_handler = signal.getsignal(signal.SIGTERM)
             signal.signal(signal.SIGTERM, raise_termination_signal)
             try:
-                if capture_output:
-                    args = (proc.stdout, output_hook)
-                    thread = Thread(target=_process_stream, args=args, daemon=True)
-                    thread.start()
+                if stdout_callback is not None:
+                    stdout_thread = Thread(
+                        target=process_output,
+                        args=(proc.stdout, stdout_callback),
+                        daemon=True,
+                    )
+                    stdout_thread.start()
+                if stderr_callback is not None:
+                    stderr_thread = Thread(
+                        target=process_output,
+                        args=(proc.stderr, stderr_callback),
+                        daemon=True,
+                    )
+                    stderr_thread.start()
 
                 proc.wait()
             except TerminationSignal as exc:
@@ -1822,8 +1844,22 @@ class Catalog:
             finally:
                 signal.signal(signal.SIGTERM, orig_sigterm_handler)
                 signal.signal(signal.SIGINT, orig_sigint_handler)
-                if thread:
-                    thread.join()  # wait for the reader thread
+                # wait for the reader thread
+                thread_join_timeout_seconds = 30
+                if stdout_thread is not None:
+                    stdout_thread.join(timeout=thread_join_timeout_seconds)
+                    if stdout_thread.is_alive():
+                        logger.warning(
+                            "stdout thread is still alive after %s seconds",
+                            thread_join_timeout_seconds,
+                        )
+                if stderr_thread is not None:
+                    stderr_thread.join(timeout=thread_join_timeout_seconds)
+                    if stderr_thread.is_alive():
+                        logger.warning(
+                            "stderr thread is still alive after %s seconds",
+                            thread_join_timeout_seconds,
+                        )
 
         logger.info("Process %s exited with return code %s", proc.pid, proc.returncode)
         if proc.returncode in (

{datachain-0.31.2 → datachain-0.31.3}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.2
+Version: 0.31.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.2 → datachain-0.31.3}/src/datachain.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ pyproject.toml
 .github/workflows/tests-studio.yml
 .github/workflows/tests.yml
 .github/workflows/update-template.yaml
+docs/api_hooks.py
 docs/contributing.md
 docs/examples.md
 docs/index.md
@@ -70,6 +71,10 @@ docs/references/functions/path.md
 docs/references/functions/random.md
 docs/references/functions/string.md
 docs/references/functions/window.md
+docs/studio/api/.gitkeep
+docs/templates/main.dot
+docs/templates/operation.dot
+docs/templates/responses.def
 examples/computer_vision/iptc_exif_xmp_lib.py
 examples/computer_vision/llava2_image_desc_lib.py
 examples/computer_vision/openimage-detect.py

{datachain-0.31.2 → datachain-0.31.3}/tests/unit/test_query.py

@@ -42,12 +42,31 @@ def test_args(catalog, mock_popen):
     mock_popen.assert_called_once_with(["mypython", "-c", "pass"], env=expected_env)
 
 
+def test_capture_stdout(catalog, mock_popen):
+    mock_popen.stdout = io.BytesIO(b"Hello, World!\rLorem Ipsum\nDolor Sit Amet\nconse")
+    stdout = []
+
+    catalog.query("pass", stdout_callback=stdout.append)
+    assert stdout == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
+
+
+def test_capture_stderr(catalog, mock_popen):
+    mock_popen.stderr = io.BytesIO(b"Hello, World!\rLorem Ipsum\nDolor Sit Amet\nconse")
+    stderr = []
+
+    catalog.query("pass", stderr_callback=stderr.append)
+    assert stderr == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
+
+
 def test_capture_output(catalog, mock_popen):
     mock_popen.stdout = io.BytesIO(b"Hello, World!\rLorem Ipsum\nDolor Sit Amet\nconse")
-    lines = []
+    mock_popen.stderr = io.BytesIO(b"foo\nbar")
+    stdout = []
+    stderr = []
 
-    catalog.query("pass", capture_output=True, output_hook=lines.append)
-    assert lines == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
+    catalog.query("pass", stdout_callback=stdout.append, stderr_callback=stderr.append)
+    assert stdout == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
+    assert stderr == ["foo\n", "bar"]
 
 
 def test_canceled_by_user(catalog, mock_popen):