datachain 0.31.2__tar.gz → 0.31.4__tar.gz

{datachain-0.31.2 → datachain-0.31.4}/.gitignore

@@ -149,3 +149,5 @@ cython_debug/
 # for local dev, e.g. LLM generated files, .env.test to override
 # test variables, local scripts to try, etc
 local/
+docs/openapi.json
+docs/studio/api/index.md

{datachain-0.31.2 → datachain-0.31.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.2
+Version: 0.31.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

datachain-0.31.4/docs/api_hooks.py

@@ -0,0 +1,35 @@
+# ruff: noqa: INP001
+def on_pre_build(**kwargs):
+    import os
+
+    import requests
+
+    # Download OpenAPI spec
+    response = requests.get(
+        "https://studio.datachain.ai/api/openapi.json",
+        timeout=30,
+    )
+
+    # Write to file
+    print("Writing OpenAPI spec to docs/openapi.json")
+    with open("docs/openapi.json", "w") as f:
+        f.write(response.text)
+
+    # Generate API docs using widdershins
+    print("Generating API docs using widdershins")
+    cmd = [
+        "npx widdershins",
+        "docs/openapi.json",
+        "-o docs/studio/api/index.md",
+        "--language_tabs 'python:Python'",
+        "--language_tabs 'shell:curl'",
+        "--expandBody true",
+        "--summary",
+        "--shallowSchemas",
+        "--omitBody",
+        "--resolve",
+        "--httpsnippet",
+        "-u docs/templates",
+        "--omitHeader",
+    ]
+    os.system(" ".join(cmd))  # noqa: S605

datachain-0.31.4/docs/templates/main.dot

@@ -0,0 +1,81 @@
+<h1 id="{{=data.title_prefix||''}}">{{=data.api.info && data.api.info.title}} {{=data.version||''}}</h1>
+
+> Scroll down for {{? data.header.language_tabs.length}}code samples, {{?}}example requests and responses.{{? data.header.language_tabs.length}} .{{?}}
+
+{{? data.api.info && data.api.info.description}}{{=data.api.info.description||''}}{{?}}
+
+
+{{? data.api.info && data.api.info.termsOfService}}<a href="{{=data.api.info.termsOfService||''}}">Terms of service</a>{{?}}
+{{? data.api.info && data.api.info.contact}}{{? data.api.info.contact.email}}Email: <a href="mailto:{{=data.api.info.contact.email||''}}">{{=data.api.info.contact.name || 'Support'}}</a> {{?}}{{? data.api.info.contact.url}}Web: <a href="{{=data.api.info.contact.url||''}}">{{= data.api.info.contact.name || 'Support'}}</a> {{?}}{{?}}
+{{? data.api.info && data.api.info.license}}{{? data.api.info.license.url}}License: <a href="{{=data.api.info.license.url||''}}">{{=data.api.info.license.name||''}}</a>{{??}} License: {{=data.api.info.license.name||''}}{{?}}{{?}}
+{{= data.tags.endSection }}
+
+{{? data.api.components && data.api.components.securitySchemes && data.def && data.def.security}}
+{{#def.security}}
+{{?}}
+
+{{ for (var r in data.resources) { }}
+{{ data.resource = data.resources[r]; }}
+
+{{= data.tags.section }}
+<h1 id="{{=data.title_prefix||''+'-'+data.utils.slugify(r)}}">{{= r||''}}</h1>
+
+{{? data.resource.description }}{{= data.resource.description||''}}{{?}}
+
+{{? data.resource.externalDocs}}
+<a href="{{=data.resource.externalDocs.url||''}}">{{=data.resource.externalDocs.description||'External documentation'}}</a>
+{{?}}
+
+{{ for (var m in data.resource.methods) { }}
+{{ data.operationUniqueName = m; }}
+{{ data.method = data.resource.methods[m]; }}
+{{ data.operationUniqueSlug = data.method.slug; }}
+{{ data.operation = data.method.operation; }}
+{{= data.templates.operation(data) }}
+{{ } /* of methods */ }}
+
+{{= data.tags.endSection }}
+{{ } /* of resources */ }}
+
+{{? data.api.components && data.api.components.schemas }}
+{{= data.tags.section }}
+
+# Schemas
+
+{{ for (var s in data.components.schemas) { }}
+{{ var origSchema = data.components.schemas[s]; }}
+{{ var schema = data.api.components.schemas[s]; }}
+
+{{= data.tags.section }}
+<h2 id="tocS_{{=s||''}}">{{=s||''}}</h2>
+{{ /* backwards compatibility */ }}
+<a id="schema{{=s.toLowerCase()||''}}"></a>
+<a id="schema_{{=s||''}}"></a>
+<a id="tocS{{=s.toLowerCase()||''}}"></a>
+<a id="tocs{{=s.toLowerCase()||''}}"></a>
+
+{{? data.options.yaml }}
+```yaml
+{{=data.utils.yaml.stringify(data.utils.getSample(schema,data.options,{quiet:true},data.api))}}
+{{??}}
+```json
+{{=data.utils.safejson(data.utils.getSample(schema,data.options,{quiet:true},data.api),null,2)}}
+{{?}}
+```
+
+
+
+{{= data.tags.endSection }}
+{{= data.tags.endSection }}
+
+{{ } /* of schemas */ }}
+
+{{?}}
+
+{{? data.options.footer}}
+{{#def.footer}}
+{{?}}
+
+{{? data.options.discovery}}
+{{#def.discovery}}
+{{?}}

datachain-0.31.4/docs/templates/operation.dot

@@ -0,0 +1,47 @@
+{{= data.tags.section }}
+
+## {{= data.operationUniqueName||''}}
+
+{{? data.operation.operationId}}
+<a id="opId{{=data.operation.operationId||''}}"></a>
+{{?}}
+
+{{ data.methodUpper = data.method.verb.toUpperCase(); }}
+{{ data.url = data.utils.slashes(data.baseUrl + data.method.path); }}
+{{ data.parameters = data.operation.parameters; }}
+{{ data.enums = []; }}
+{{ data.utils.fakeProdCons(data); }}
+{{ data.utils.fakeBodyParameter(data); }}
+{{ data.utils.mergePathParameters(data); }}
+{{ data.utils.getParameters(data); }}
+
+{{? data.options.codeSamples || data.operation["x-code-samples"] }}
+> Code samples
+
+{{? data.utils.getCodeSamples(data) }}{{= data.utils.getCodeSamples(data) }}{{?}}
+{{?}}
+
+`{{= data.methodUpper||''}} {{=data.method.path||''}}`
+
+{{? data.operation.summary && !data.options.tocSummary}}*{{= data.operation.summary||'' }}*{{?}}
+
+{{? data.operation.description}}{{= data.operation.description||'' }}{{?}}
+
+{{? data.operation.requestBody}}
+> Body parameter
+
+{{? data.bodyParameter.exampleValues.description }}
+> {{= data.bodyParameter.exampleValues.description||'' }}
+{{?}}
+
+{{? data.utils.getBodyParameterExamples(data) }}{{= data.utils.getBodyParameterExamples(data) }}{{?}}
+{{?}}
+
+{{#def.responses}}
+
+{{ data.security = data.operation.security ? data.operation.security : data.api.security; }}
+{{? data.security && data.security.length }}
+{{??}}
+{{#def.authentication_none}}
+{{?}}
+{{= data.tags.endSection }}

datachain-0.31.4/docs/templates/responses.def

@@ -0,0 +1,61 @@
+{{ data.responses = data.utils.getResponses(data); }}
+{{ data.responseSchemas = false; }}
+{{~ data.responses :response }}
+{{ if (response.content) data.responseSchemas = true; }}
+{{~}}
+
+{{? data.responseSchemas }}
+> Example responses
+
+{{? data.utils.getResponseExamples(data) }}{{= data.utils.getResponseExamples(data) }}{{?}}
+{{?}}
+
+{{= data.tags.section }}
+<h3 id="{{=data.operationUniqueSlug||''}}-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+{{~ data.responses :r}}|{{=r.status}}|{{=r.meaning}}|{{=r.description || 'none'}}|{{=r.schema||'none'}}|
+{{~}}
+
+{{ data.responseSchemas = false; }}
+{{~ data.responses :response }}
+{{ if (response.content && !response.$ref && !data.utils.isPrimitive(response.type)) data.responseSchemas = true; }}
+{{~}}
+{{? data.responseSchemas }}
+<h3 id="{{=data.operationUniqueSlug||''}}-responseschema">Response Schema</h3>
+{{~ data.responses :response}}
+{{? response.content && !response.$ref && !data.utils.isPrimitive(response.type)}}
+{{? Object.keys(response.content).length }}
+{{ var responseKey = Object.keys(response.content)[0]; }}
+{{ var responseSchema = response.content[responseKey].schema; }}
+{{ var enums = []; }}
+{{ var blocks = data.utils.schemaToArray(responseSchema,0,{trim:true,join:true},data); }}
+{{ for (var block of blocks) {
+     for (var p of block.rows) {
+       if (p.schema && p.schema.enum) {
+         for (var e of p.schema.enum) {
+           enums.push({name:p.name,value:e});
+         }
+       }
+     }
+   }
+}}
+
+Status Code **{{=response.status||''}}**
+
+{{ data.response = response; }}
+
+{{?}}
+{{?}}
+{{~}}
+{{?}}
+{{ data.responseHeaders = data.utils.getResponseHeaders(data); }}
+{{? data.responseHeaders.length }}
+### Response Headers
+|Status|Header|Type|Format|Description|
+|---|---|---|---|---|
+{{~ data.responseHeaders :h}}|{{=h.status}}|{{=h.header}}|{{=h.type}}|{{=h.format||''}}|{{=h.description||'none'}}|
+{{~}}
+{{?}}
+{{= data.tags.endSection }}

{datachain-0.31.2 → datachain-0.31.4}/mkdocs.yml

@@ -117,6 +117,8 @@ nav:
           - Environment Variables: guide/env.md
           - Namespaces: guide/namespaces.md
           - Local DB Migrations: guide/db_migrations.md
+      - 🔗 Studio:
+          - API: studio/api/index.md
       - 🤝 Contributing: contributing.md
 
   - DataChain Website ↗: https://datachain.ai" target="_blank"
@@ -161,6 +163,8 @@ extra:
     - icon: fontawesome/brands/linkedin
       link: https://www.linkedin.com/company/dvc-ai
 
+hooks:
+  - docs/api_hooks.py
 plugins:
   - search
   - section-index

{datachain-0.31.2 → datachain-0.31.4}/src/datachain/lib/file.py

@@ -35,6 +35,7 @@ if TYPE_CHECKING:
     from datachain.catalog import Catalog
     from datachain.client.fsspec import Client
     from datachain.dataset import RowDict
+    from datachain.query.session import Session
 
 sha256 = partial(hashlib.sha256, usedforsecurity=False)
 
@@ -252,6 +253,15 @@ class File(DataModel):
         "last_modified",
     ]
 
+    # Allowed kwargs we forward to TextIOWrapper
+    _TEXT_WRAPPER_ALLOWED: ClassVar[tuple[str, ...]] = (
+        "encoding",
+        "errors",
+        "newline",
+        "line_buffering",
+        "write_through",
+    )
+
     @staticmethod
     def _validate_dict(
         v: Optional[Union[str, dict, list[dict]]],
@@ -328,7 +338,6 @@ class File(DataModel):
             from datachain.catalog.loader import get_catalog
 
             catalog = get_catalog()
-
         from datachain.client.fsspec import Client
 
         client_cls = Client.get_implementation(path)
@@ -341,6 +350,27 @@ class File(DataModel):
         file._set_stream(catalog)
         return file
 
+    @classmethod
+    def at(cls, uri: str, session: Optional["Session"] = None) -> "Self":
+        """Construct a File from a full URI in one call.
+
+        Example:
+            file = File.at("s3://bucket/path/to/output.png")
+            with file.open("wb") as f: ...
+        """
+        from datachain.client.fsspec import Client
+        from datachain.query.session import Session
+
+        if session is None:
+            session = Session.get()
+        catalog = session.catalog
+
+        client_cls = Client.get_implementation(uri)
+        source, rel_path = client_cls.split_url(uri)
+        file = cls(source=client_cls.get_uri(source), path=rel_path)
+        file._set_stream(catalog)
+        return file
+
     @classmethod
     def _from_row(cls, row: "RowDict") -> "Self":
         return cls(**{key: row[key] for key in cls._datachain_column_types})
@@ -354,28 +384,70 @@ class File(DataModel):
         return str(PurePosixPath(self.path).parent)
 
     @contextmanager
-    def open(self, mode: Literal["rb", "r"] = "rb") -> Iterator[Any]:
-        """Open the file and return a file object."""
-        if self.location:
-            with VFileRegistry.open(self, self.location) as f:  # type: ignore[arg-type]
-                yield f
+    def open(self, mode: str = "rb", **open_kwargs) -> Iterator[Any]:
+        """Open the file and return a file-like object.
 
-        else:
+        Supports both read ("rb", "r") and write modes (e.g. "wb", "w", "ab").
+        When opened in a write mode, metadata is refreshed after closing.
+        """
+        writing = any(ch in mode for ch in "wax+")
+        if self.location and writing:
+            raise VFileError(
+                "Writing to virtual file is not supported",
+                self.source,
+                self.path,
+            )
+
+        if self._catalog is None:
+            raise RuntimeError("Cannot open file: catalog is not set")
+
+        client: Client = self._catalog.get_client(self.source)
+
+        if not writing:
+            if self.location:
+                with VFileRegistry.open(self, self.location) as f:  # type: ignore[arg-type]
+                    yield self._wrap_text(f, mode, open_kwargs)
+                return
             if self._caching_enabled:
                 self.ensure_cached()
-            client: Client = self._catalog.get_client(self.source)
             with client.open_object(
                 self, use_cache=self._caching_enabled, cb=self._download_cb
             ) as f:
-                yield io.TextIOWrapper(f) if mode == "r" else f
+                yield self._wrap_text(f, mode, open_kwargs)
+            return
+
+        # write path
+        full_path = client.get_full_path(self.get_path_normalized())
+        with client.fs.open(full_path, mode, **open_kwargs) as f:
+            yield self._wrap_text(f, mode, open_kwargs)
+
+        # refresh metadata
+        info = client.fs.info(full_path)
+        refreshed = client.info_to_file(info, self.get_path_normalized())
+        for k, v in refreshed.model_dump().items():
+            setattr(self, k, v)
+
+    def _wrap_text(self, f: Any, mode: str, open_kwargs: dict[str, Any]) -> Any:
+        """Return stream possibly wrapped for text."""
+        if "b" in mode or isinstance(f, io.TextIOBase):
+            return f
+        filtered = {
+            k: open_kwargs[k] for k in self._TEXT_WRAPPER_ALLOWED if k in open_kwargs
+        }
+        return io.TextIOWrapper(f, **filtered)
 
     def read_bytes(self, length: int = -1):
         """Returns file contents as bytes."""
         with self.open() as stream:
             return stream.read(length)
 
-    def read_text(self):
-        """Returns file contents as text."""
+    def read_text(self, **open_kwargs):
+        """Return file contents decoded as text.
+
+        **open_kwargs : Any
+            Extra keyword arguments forwarded to ``open(mode="r", ...)``
+            (e.g. ``encoding="utf-8"``, ``errors="ignore"``)
+        """
         if self.location:
             raise VFileError(
                 "Reading text from virtual file is not supported",
@@ -383,7 +455,7 @@ class File(DataModel):
                 self.path,
             )
 
-        with self.open(mode="r") as stream:
+        with self.open(mode="r", **open_kwargs) as stream:
             return stream.read()
 
     def read(self, length: int = -1):
@@ -701,14 +773,19 @@ class TextFile(File):
     """`DataModel` for reading text files."""
 
     @contextmanager
-    def open(self, mode: Literal["rb", "r"] = "r"):
-        """Open the file and return a file object (default to text mode)."""
-        with super().open(mode=mode) as stream:
+    def open(self, mode: str = "r", **open_kwargs) -> Iterator[Any]:
+        """Open the file and return a file-like object.
+        Default to text mode"""
+        with super().open(mode=mode, **open_kwargs) as stream:
             yield stream
 
-    def read_text(self):
-        """Returns file contents as text."""
-        with self.open() as stream:
+    def read_text(self, **open_kwargs):
+        """Return file contents as text.
+
+        **open_kwargs : Any
+            Extra keyword arguments forwarded to ``open()`` (e.g. encoding).
+        """
+        with self.open(**open_kwargs) as stream:
             return stream.read()
 
     def save(self, destination: str, client_config: Optional[dict] = None):

{datachain-0.31.2 → datachain-0.31.4}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.2
+Version: 0.31.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.2 → datachain-0.31.4}/src/datachain.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ pyproject.toml
 .github/workflows/tests-studio.yml
 .github/workflows/tests.yml
 .github/workflows/update-template.yaml
+docs/api_hooks.py
 docs/contributing.md
 docs/examples.md
 docs/index.md
@@ -70,6 +71,10 @@ docs/references/functions/path.md
 docs/references/functions/random.md
 docs/references/functions/string.md
 docs/references/functions/window.md
+docs/studio/api/.gitkeep
+docs/templates/main.dot
+docs/templates/operation.dot
+docs/templates/responses.def
 examples/computer_vision/iptc_exif_xmp_lib.py
 examples/computer_vision/llava2_image_desc_lib.py
 examples/computer_vision/openimage-detect.py

{datachain-0.31.2 → datachain-0.31.4}/tests/func/test_file.py

@@ -1,9 +1,12 @@
+import io
+
 import pytest
 import pytz
 
 import datachain as dc
 from datachain.data_storage.sqlite import SQLiteWarehouse
 from datachain.lib.file import File, FileError
+from datachain.query import C
 from datachain.utils import TIME_ZERO
 
 
@@ -91,3 +94,68 @@ def test_upload(cloud_test_catalog):
     assert f.read() == img_bytes
 
     client.fs.rm(dest, recursive=True)
+
+
+def test_open_write_binary(cloud_test_catalog):
+    ctc = cloud_test_catalog
+    catalog = ctc.catalog
+    src_uri = ctc.src_uri
+    data = b"hello via open()"
+    file_path = f"{src_uri}/test-open-write-bytes.bin"
+
+    file = File.at(file_path, ctc.session)
+    with file.open("wb") as f:
+        f.write(data)
+
+    assert file.size == len(data)
+    assert file.read() == data
+
+    # Query storage for exactly that relative path.
+    # Metadata already refreshed by open() write path.
+    rel_path = file.path
+    chain = dc.read_storage(src_uri, session=ctc.session).filter(
+        C("file.path") == rel_path
+    )
+    results = list(chain.to_values("file"))
+    assert len(results) == 1
+    match = results[0]
+    for field_name in File.model_fields:
+        if field_name == "last_modified":
+            # Allow up to 1s difference across backends
+            # (some backends don't keep microsecond precision, we keep it simple here)
+            assert match.last_modified.timestamp() == pytest.approx(
+                file.last_modified.timestamp(), abs=1
+            )
+        else:
+            assert getattr(match, field_name) == getattr(file, field_name), (
+                f"Mismatch in field '{field_name}'"
+            )
+
+    catalog.get_client(src_uri).fs.rm(file_path)
+
+
+def test_open_write_text(cloud_test_catalog):
+    ctc = cloud_test_catalog
+    catalog = ctc.catalog
+    src_uri = ctc.src_uri
+    file_path = f"{src_uri}/test-open-write-text.txt"
+    # Unicode content to exercise non-default (utf-16) encoding round trip
+    content = "Привет Мир\nSecond line"
+
+    file = File.at(file_path, ctc.session)
+    with file.open("w", encoding="utf-16-le") as f:
+        written_chars = f.write(content)
+
+    assert written_chars == len(content)
+    assert file.read_text(encoding="utf-16-le") == content
+
+    # Compute expected byte size using identical TextIOWrapper logic
+    buf = io.BytesIO()
+    tw = io.TextIOWrapper(buf, encoding="utf-16-le")
+    tw.write(content)
+    tw.flush()
+    expected_size = len(buf.getvalue())
+    tw.close()
+    assert file.size == expected_size
+
+    catalog.get_client(src_uri).fs.rm(file_path)