datachain 0.2.14__tar.gz → 0.2.16__tar.gz

{datachain-0.2.14 → datachain-0.2.16}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.5.0'
+    rev: 'v0.5.5'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
@@ -35,7 +35,7 @@ repos:
       - id: codespell
         additional_dependencies: ["tomli"]
   - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-    rev: v2.13.0
+    rev: v2.14.0
     hooks:
       - id: pretty-format-toml
         args: [--autofix, --no-sort]

{datachain-0.2.14/src/datachain.egg-info → datachain-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.14
+Version: 0.2.16
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.14 → datachain-0.2.16}/docs/index.md

@@ -278,10 +278,9 @@ images_with_dogs.select("annotations", "file.name").show()
 
 [Limited by 20 rows]
 ```
+For in-depth review of working with JSON metadata, please follow this tutorial:
 
-To see more examples of working with metadata formats, please follow this tutorial:
-
-[https://github.com/iterative/datachain/blob/main/examples/getting_started/datasets.ipynb](https://github.com/iterative/datachain/blob/main/examples/getting_started/datasets.ipynb)
+[https://github.com/iterative/datachain/blob/main/examples/get_started/json-metadata-tutorial.ipynb](https://github.com/iterative/datachain/blob/main/examples/get_started/json-metadata-tutorial.ipynb)
 
 ### Passing data to training
 

{datachain-0.2.14 → datachain-0.2.16}/docs/references/udf.md

@@ -13,6 +13,8 @@ steps need to happen before or after the processing function runs.
 
 ::: datachain.lib.udf.Aggregator
 
+::: datachain.lib.udf.BatchMapper
+
 ::: datachain.lib.udf.Generator
 
 ::: datachain.lib.udf.Mapper

{datachain-0.2.14 → datachain-0.2.16}/examples/computer_vision/iptc_exif_xmp_lib.py

@@ -1,3 +1,4 @@
+# pip install defusedxml
 import json
 
 from PIL import (
@@ -63,7 +64,7 @@ if __name__ == "__main__":
         DataChain.from_storage(source, type="image")
         .settings(parallel=-1)
         .filter(C("file.name").glob("*.jpg"))
-        .limit(10000)
+        .limit(5000)
         .map(
             image_description,
             params=["file"],

{datachain-0.2.14 → datachain-0.2.16}/examples/computer_vision/openimage-detect.py

@@ -48,7 +48,7 @@ def openimage_detect(args):
         yield fstream, bbox
 
 
-source = "gs://datachain-demo/openimages-v6-test-jsonpairs"
+source = "gs://datachain-demo/openimages-v6-test-jsonpairs/"
 
 (
     DataChain.from_storage(source)

{datachain-0.2.14 → datachain-0.2.16}/examples/get_started/json-csv-reader.py

@@ -35,8 +35,8 @@ def main():
     print("Dynamic JSONl schema from 2 objects")
     print("========================================================================")
     uri = "gs://datachain-demo/jsonl/object.jsonl"
-    jsonl_ds = DataChain.from_json(uri, meta_type="jsonl", show_schema=True)
-    print(jsonl_ds.to_pandas())
+    jsonl_ds = DataChain.from_json(uri, meta_type="jsonl", print_schema=True)
+    jsonl_ds.show()
 
     print()
     print("========================================================================")
@@ -49,8 +49,7 @@ def main():
     json_pairs_ds = DataChain.from_json(
         uri, schema_from=schema_uri, jmespath="@", model_name="OpenImage"
     )
-    print(json_pairs_ds.to_pandas())
-    # print(list(json_pairs_ds.collect())[0])
+    json_pairs_ds.show()
 
     uri = "gs://datachain-demo/coco2017/annotations_captions/"
 
@@ -61,7 +60,7 @@ def main():
     chain = (
         DataChain.from_storage(uri)
         .filter(C("file.name").glob("*.json"))
-        .show_json_schema(jmespath="@", model_name="Coco")
+        .print_json_schema(jmespath="@", model_name="Coco")
     )
     chain.save()
 
@@ -72,13 +71,13 @@ def main():
     static_json_ds = DataChain.from_json(
         uri, jmespath="licenses", spec=LicenseFeature, nrows=3
     )
-    print(static_json_ds.to_pandas())
+    static_json_ds.show()
 
     print()
     print("========================================================================")
     print("dynamic JSON schema test parsing 5K objects")
     print("========================================================================")
-    dynamic_json_ds = DataChain.from_json(uri, jmespath="images", show_schema=True)
+    dynamic_json_ds = DataChain.from_json(uri, jmespath="images", print_schema=True)
     print(dynamic_json_ds.to_pandas())
 
     uri = "gs://datachain-demo/chatbot-csv/"
@@ -88,16 +87,7 @@ def main():
     print("========================================================================")
     static_csv_ds = DataChain.from_csv(uri, output=ChatDialog, object_name="chat")
     static_csv_ds.print_schema()
-    print(static_csv_ds.to_pandas())
-
-    uri = "gs://datachain-demo/laion-aesthetics-csv"
-    print()
-    print("========================================================================")
-    print("dynamic CSV with header schema test parsing 3/3M objects")
-    print("========================================================================")
-    dynamic_csv_ds = DataChain.from_csv(uri, object_name="laion", nrows=3)
-    dynamic_csv_ds.print_schema()
-    print(dynamic_csv_ds.to_pandas())
+    static_csv_ds.show()
 
 
 if __name__ == "__main__":