palimpzest 0.5.4__py3-none-any.whl → 0.6.1__py3-none-any.whl

palimpzest/utils/udfs.py

@@ -3,17 +3,12 @@ This file collects a sample of useful UDFs to convert schemata.
 """
 
 import io
-import json
 from datetime import datetime
 
-import modal
 import pandas as pd
 import requests
-from papermage import Document
 
 from palimpzest.constants import MAX_ROWS
-from palimpzest.datamanager.datamanager import DataDirectory
-from palimpzest.tools.pdfparser import get_text_from_pdf
 
 
 def url_to_file(candidate: dict):
@@ -30,29 +25,6 @@ def url_to_file(candidate: dict):
     return {"filename": filename, "timestamp": timestamp, "contents": contents}
 
 
-def file_to_pdf(candidate: dict):
-    pdfprocessor = DataDirectory().current_config.get("pdfprocessor")
-    if pdfprocessor == "modal":
-        print("handling PDF processing remotely")
-        remote_func = modal.Function.lookup("palimpzest.tools", "processPapermagePdf")
-    else:
-        remote_func = None
-
-    pdf_bytes = candidate["contents"]
-    # generate text_content from PDF
-    if remote_func is not None:
-        doc_json_str = remote_func.remote([pdf_bytes])
-        docdict = json.loads(doc_json_str[0])
-        doc = Document.from_json(docdict)
-        text_content = ""
-        for p in doc.pages:
-            text_content += p.text
-    else:
-        text_content = get_text_from_pdf(candidate["filename"], candidate["contents"])
-
-    return {"text_contents": text_content[:10000]}  # TODO Very hacky
-
-
 def file_to_xls(candidate: dict):
     """Function used to convert a DataRecord instance of File to a XLSFile DataRecord."""
     xls = pd.ExcelFile(io.BytesIO(candidate["contents"]), engine="openpyxl")

{palimpzest-0.5.4.dist-info → palimpzest-0.6.1.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.2
 Name: palimpzest
-Version: 0.5.4
+Version: 0.6.1
 Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
 Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
-Project-URL: homepage, https://github.com/mitdbg/palimpzest/
+Project-URL: homepage, https://palimpzest.org
 Project-URL: repository, https://github.com/mitdbg/palimpzest/
+Project-URL: documentation, https://palimpzest.org
 Keywords: relational,optimization,llm,AI programming,extraction,tools,document,search,integration
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -28,6 +29,10 @@ Requires-Dist: opencv-python-headless<4.9.0,>=4.8.0
 Requires-Dist: jupyter
 Requires-Dist: layoutparser>=0.3.4
 Requires-Dist: lxml-html-clean>=0.1.1
+Requires-Dist: mkdocs>=1.6.1
+Requires-Dist: mkdocs-material>=9.6.3
+Requires-Dist: mkdocs-material[imaging]
+Requires-Dist: mkdocstrings-python>=1.15.0
 Requires-Dist: modal>=0.62.198
 Requires-Dist: ncls==0.0.68
 Requires-Dist: necessary>=0.3.2
@@ -38,7 +43,6 @@ Requires-Dist: pandas>=2.1.1
 Requires-Dist: papermage>=0.16.0
 Requires-Dist: pdf2image
 Requires-Dist: pytest>=8.2.2
-Requires-Dist: pypdf==4.3.1
 Requires-Dist: python-Levenshtein
 Requires-Dist: pdfplumber==0.7.4
 Requires-Dist: pillow>=10.2.0
@@ -46,6 +50,7 @@ Requires-Dist: prettytable>=3.9.0
 Requires-Dist: PyLD>=2.0.4
 Requires-Dist: pyarrow<15.0.0,>=13.0.0; python_version < "3.12"
 Requires-Dist: pyarrow<19.0.0,>=15.0.0; python_version >= "3.12"
+Requires-Dist: pypdf>=5.1.0
 Requires-Dist: pytest-mock>=3.14.0
 Requires-Dist: python-Levenshtein>=0.25.1
 Requires-Dist: pyyaml>=6.0.1
@@ -66,12 +71,16 @@ Requires-Dist: sphinx>=8.1.3
 ![pz-banner](https://palimpzest-workloads.s3.us-east-1.amazonaws.com/palimpzest-cropped.png)
 
 # Palimpzest (PZ)
-[![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b?logo=arxiv)](https://arxiv.org/pdf/2405.14696)
-[![Blog Post](https://img.shields.io/badge/Blog-PZ-green)](https://dsg.csail.mit.edu/projects/palimpzest/)
+[![Discord](https://img.shields.io/discord/1245561987480420445?logo=discord)](https://discord.gg/dN85JJ6jaH)
+[![Docs](https://img.shields.io/badge/Read_the_Docs-purple?logo=readthedocs)](https://palimpzest.org/)
 [![Colab Demo](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1zqOxnh_G6eZ8_xax6PvDr-EjMt7hp4R5?usp=sharing)
-[![Video](https://img.shields.io/badge/YouTube-Talk-red?logo=youtube)](https://youtu.be/T8VQfyBiki0?si=eiph57DSEkDNbEIu)
 [![PyPI](https://img.shields.io/pypi/v/palimpzest)](https://pypi.org/project/palimpzest/)
-[![PyPI - Monthly Downloads](https://img.shields.io/pypi/dm/palimpzest)](https://pypi.org/project/palimpzest/)
+[![PyPI - Monthly Downloads](https://img.shields.io/pypi/dm/palimpzest?color=teal)](https://pypi.org/project/palimpzest/)
+<!-- [![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b?logo=arxiv)](https://arxiv.org/pdf/2405.14696) -->
+<!-- [![Video](https://img.shields.io/badge/YouTube-Talk-red?logo=youtube)](https://youtu.be/T8VQfyBiki0?si=eiph57DSEkDNbEIu) -->
+
+## Learn How to Use PZ
+Our [full documentation](https://palimpzest.org) is the definitive resource for learning how to use PZ. It contains all of the installation and quickstart materials on this page, as well as user guides, full API documentation, and much more.
 
 ## Getting started
 You can find a stable version of the PZ package on PyPI [here](https://pypi.org/project/palimpzest/). To install the package, run:
@@ -86,6 +95,17 @@ $ cd palimpzest
 $ pip install .
 ```
 
+## Join the PZ Community
+We are actively hacking on PZ and would love to have you join our community [![Discord](https://img.shields.io/discord/1245561987480420445?logo=discord)](https://discord.gg/dN85JJ6jaH)
+
+[Our Discord server](https://discord.gg/dN85JJ6jaH) is the best place to:
+- Get help with your PZ program(s)
+- Give feedback to the maintainers
+- Discuss the future direction(s) of the project
+- Discuss anything related to data processing with LLMs!
+
+We are eager to learn more about your workloads and use cases, and will take them into consideration in planning our future roadmap.
+
 ## Quick Start
 The easiest way to get started with Palimpzest is to run the `quickstart.ipynb` jupyter notebook. We demonstrate the full workflow of working with PZ, including registering a dataset, composing and executing a pipeline, and accessing the results.
 To run the notebook, you can use the following command:
@@ -97,48 +117,28 @@ And then access the notebook from the jupyter interface in your browser at `loca
 ### Even Quicker Start
 For eager readers, the code in the notebook can be found in the following condensed snippet. However, we do suggest reading the notebook as it contains more insight into each element of the program.
 ```python
-import pandas as pd
-import palimpzest.datamanager.datamanager as pzdm
-from palimpzest.sets import Dataset
-from palimpzest.core.lib.fields import Field
-from palimpzest.core.lib.schemas import Schema, TextFile
-from palimpzest.policy import MinCost, MaxQuality
-from palimpzest.query.processor.config import QueryProcessorConfig
-
-# Dataset registration
-dataset_path = "testdata/enron-tiny"
-dataset_name = "enron-tiny"
-pzdm.DataDirectory().register_local_directory(dataset_path, dataset_name)
-
-# Dataset loading
-dataset = Dataset(dataset_name, schema=TextFile)
-
-# Schema definition for the fields we wish to compute
-class Email(Schema):
-    """Represents an email, which in practice is usually from a text file"""
-    sender = Field(desc="The email address of the sender")
-    subject = Field(desc="The subject of the email")
-    date = Field(desc="The date the email was sent")
-
-# Lazy construction of computation to filter for emails about holidays sent in July
-dataset = dataset.convert(Email, desc="An email from the Enron dataset")
-dataset = dataset.filter("The email was sent in July")
-dataset = dataset.filter("The email is about holidays")
-
-# Executing the compuation
-policy = MinCost()
-config = QueryProcessorConfig(
-    policy=policy,
-    verbose=True,
-    processing_strategy="no_sentinel",
-    execution_strategy="sequential",
-    optimizer_strategy="pareto",
-)
-results, execution_stats = dataset.run(config)
-
-# Writing output to disk
-output_df = pd.DataFrame([r.to_dict() for r in results])[["date","sender","subject"]]
-output_df.to_csv("july_holiday_emails.csv")
+import palimpzest as pz
+
+# define the fields we wish to compute
+email_cols = [
+    {"name": "sender", "type": str, "desc": "The email address of the sender"},
+    {"name": "subject", "type": str, "desc": "The subject of the email"},
+    {"name": "date", "type": str, "desc": "The date the email was sent"},
+]
+
+# lazily construct the computation to get emails about holidays sent in July
+dataset = pz.Dataset("testdata/enron-tiny/")
+dataset = dataset.sem_add_columns(email_cols)
+dataset = dataset.sem_filter("The email was sent in July")
+dataset = dataset.sem_filter("The email is about holidays")
+
+# execute the computation w/the MinCost policy
+config = pz.QueryProcessorConfig(policy=pz.MinCost(), verbose=True)
+output = dataset.run(config)
+
+# display output (if using Jupyter, otherwise use print(output_df))
+output_df = output.to_df(cols=["date", "sender", "subject"])
+display(output_df)
 ```
 
 ## Palimpzest CLI

palimpzest-0.6.1.dist-info/RECORD

@@ -0,0 +1,87 @@
+palimpzest/__init__.py,sha256=ZcxM-zzT3sX7cDTcx1tN7-Udet0lqoiDgnvlOs7nMhY,766
+palimpzest/constants.py,sha256=SumDHOKDot25Sld0tCzWF6rs3oeLp42DFrDFNJQ8uoM,14035
+palimpzest/policy.py,sha256=2cMio_AUfZv6lksr_klfP747G4w1nsZJtfmt6zjeaMk,12656
+palimpzest/sets.py,sha256=LPbYSXf0LhsFXHCR7xq9nyBER0DBzcWM95Q6vhd3RvA,14180
+palimpzest/core/__init__.py,sha256=XJQxijqc68kWa44-me5ZIij6PKjOspxaeZKKrVhEjVo,1472
+palimpzest/core/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/core/data/dataclasses.py,sha256=biRtaxwy1ALeo50jykKx2_J3bXE9qbu0cwM2RtoZlzo,19486
+palimpzest/core/data/datareaders.py,sha256=9ZNLp2j7g8_f0YDTqavLpMhXtxh5O4efprSZNkwpDkA,17323
+palimpzest/core/elements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/core/elements/filters.py,sha256=zBBYRUnPfEYb85IfC04TJkS45GxgL5KfXASIHWwlwow,1554
+palimpzest/core/elements/groupbysig.py,sha256=1qHuR2-fcW-E4rxPSieYGSXZYwvFaPwf1ld9VPWvWjw,2233
+palimpzest/core/elements/records.py,sha256=wEQnk2-TygII-2h0j9ag91AUE2heDAD5ohEBCSjhNZ8,13801
+palimpzest/core/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/core/lib/fields.py,sha256=QW99ya_VffHMnaef0LP97isYp8w59YI9FInGMy_M8M8,4072
+palimpzest/core/lib/schemas.py,sha256=g3yH6RxgN0sbsW7YkklHAnrHjlbK_2Am-Mt6inw7f6U,16977
+palimpzest/prompts/__init__.py,sha256=klF8bYcNZWTQyuX8ZaqUXmD8Syq-MOHSRKemiwyM4N4,680
+palimpzest/prompts/code_synthesis_prompts.py,sha256=8mlMTPAI5WsoG0LVohoBFL-dnOro-mP3VJgEAiwgxnU,1472
+palimpzest/prompts/convert_prompts.py,sha256=mUt2TkSerAYuYyDg7LC4AQ195Zz-zoZjA0AN_yMH9MQ,3595
+palimpzest/prompts/critique_and_refine_convert_prompts.py,sha256=WoXExBxQ7twswd9VCCST26c-2ehZtpD2iQoBi7sqDnQ,7814
+palimpzest/prompts/filter_prompts.py,sha256=iQjn-39h3L0E5wng_UPgAXRHrP1ok329TXpOgZ6Wn1w,2372
+palimpzest/prompts/moa_aggregator_convert_prompts.py,sha256=BQRrtGdr53PTqvXzmFh8kfQ_w9KoKw-zTtmdo-8RFjo,2887
+palimpzest/prompts/moa_proposer_convert_prompts.py,sha256=d_hOh0-0m6HWBDAxUu7W3WyQtSTlUvqio3nzpnX2bxM,3642
+palimpzest/prompts/prompt_factory.py,sha256=VzZNH9kblFXYn4YKVKudJ21Y5Q-3tL6ZgFmNhBNTGjQ,31921
+palimpzest/prompts/util_phrases.py,sha256=NWrcHfjJyiOY16Jyt7R50moVnlJDyvSBZ9kBqyX2WQo,751
+palimpzest/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/execution/execution_strategy.py,sha256=HdiZ0nhMON0FyCzNoPhUgZnJgRstUAIZ99a33k_04BI,2443
+palimpzest/query/execution/parallel_execution_strategy.py,sha256=8_0B_MYoLERVhQqllLHOkQ6OTIJqc6VRHPftRLlx7_s,10974
+palimpzest/query/execution/single_threaded_execution_strategy.py,sha256=7SWr-cOJARBkLeDE_0UF45XQNwetWZi76p_iIUXO0xU,13469
+palimpzest/query/generators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/generators/generators.py,sha256=ktd-NDwaDf7W-t3d8qa3G2DX3YvuaPXVr9ZqGPr7AkE,18784
+palimpzest/query/operators/__init__.py,sha256=a_Jk_1LzaoNQHY4b5qdHZmF6kG4g1xY8lO-ZxZHkrUQ,3285
+palimpzest/query/operators/aggregate.py,sha256=nVLgJkB8oWj4Urclr8TV2w0fL9LZwU6x0Cf4dxMy19A,10212
+palimpzest/query/operators/code_synthesis_convert.py,sha256=wWHAH83fpC99uleI3tVHPBGyrqAZax5jU7xJ5MgNntI,20948
+palimpzest/query/operators/convert.py,sha256=aVZCo5R-nDs2n8J9h8x7JCk7WIYgx5zwldUlQZOyD3I,19905
+palimpzest/query/operators/critique_and_refine_convert.py,sha256=nrNUX9rtZRQ40XdUXbqxS5_R3ThnpXN9d95Vy4XWGCI,5229
+palimpzest/query/operators/filter.py,sha256=TXZAjNW9xiMV17Z4E83m5OTXT_0BOdLcfmFEbBOUqO4,10372
+palimpzest/query/operators/limit.py,sha256=xnGC6zmHdPm_2YCtsVRBL2iwXcUB1lP_vsEkuHV4nmY,2103
+palimpzest/query/operators/logical.py,sha256=Mx7Q12Cf2sr6Xr-PbtLlNLAVjC6lwV7WLk77pC3Gvs4,14708
+palimpzest/query/operators/mixture_of_agents_convert.py,sha256=_RVmZ4gRzgM6bRgsSzDwU8JrGSuA1zotxKO1wRhUDhc,6628
+palimpzest/query/operators/physical.py,sha256=2AOqj0D79-g3IVVnsyBH9g_F9NNHb1j-iOhogIEx8eY,8346
+palimpzest/query/operators/project.py,sha256=djlKXCkz2b-h1phsD8tWqewcTLKBfWMgsyZ52oFN2MY,2084
+palimpzest/query/operators/rag_convert.py,sha256=vUcmgNyrJO8KJYRbs15BVUNHDi8xF-n_UO5Pou9V5zc,8445
+palimpzest/query/operators/retrieve.py,sha256=VrwtCT4Bgv4ipPmFoAgSLK93dilquKMi_hE6GKy1UNs,3972
+palimpzest/query/operators/scan.py,sha256=z6wUVxuhr5VqPIeUxb3hxhkaljKpDc_exzGMe4NMgxY,5728
+palimpzest/query/operators/token_reduction_convert.py,sha256=yy9GYMPt-LQxPdwIgVyhCb9hi_8FRorGU8XqK_3jq9g,8513
+palimpzest/query/optimizer/__init__.py,sha256=pl1co0dCwDZkAQ-0oiwT81GjvB0Oc59WiwmmYF8k73s,3109
+palimpzest/query/optimizer/cost_model.py,sha256=zSK2Nsya96pR5Zh67cr_O5q0qtPa08--Tchn0cYvE58,44837
+palimpzest/query/optimizer/optimizer.py,sha256=lBfNYgbyyE_0bdZCGnz9oicyG3gFUdkRnzcyJ31_36o,20644
+palimpzest/query/optimizer/optimizer_strategy.py,sha256=-1xx_cviSJw6PH8XiQQK9qe4YPnAmxZEAhNVKdxRgH4,12894
+palimpzest/query/optimizer/plan.py,sha256=xlWB3sY5qDac3o6IHoWcuGK5Azv-4C2_zKKx4PzxEh4,5768
+palimpzest/query/optimizer/primitives.py,sha256=ikaX8YcDM3IrxKt98OX-mYujRYQtdMlDgsFKyjchMMA,4061
+palimpzest/query/optimizer/rules.py,sha256=jpwSI_xCzkdML4PNQScDNGzXExqont2AhQFfL4Eumdg,44059
+palimpzest/query/optimizer/tasks.py,sha256=ORyPpAbbVAUjkxh3WyDYw2I8Z6RfQLUsLGOh5987zTI,28058
+palimpzest/query/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/processor/config.py,sha256=kOhBxAZ3OeDDlQ2qMII1i2EorFpSSQbEFVFrNk-3F-o,3226
+palimpzest/query/processor/mab_sentinel_processor.py,sha256=xShPVW8ejhNR_wQ8ofPF2hL7iGj8tlFhsc2wiiIPb9Y,45600
+palimpzest/query/processor/nosentinel_processor.py,sha256=alPq1tnZvqxCSO5LYRCjlF4CB4v7NbzH_BHB-DSuehI,26478
+palimpzest/query/processor/query_processor.py,sha256=aR0OBmaZZt4_KSBjHy_KCT6pIBI8WTfT8TTcIkgPBt4,11109
+palimpzest/query/processor/query_processor_factory.py,sha256=663_V-AJK0VsBZNwgnqYu84g0rmtKf-U-xJWnps3XWs,8239
+palimpzest/query/processor/random_sampling_sentinel_processor.py,sha256=VFC0HFY3OLxAjpdmBt41K0rw8C_1ylECFjbRYsui3rU,30440
+palimpzest/query/processor/streaming_processor.py,sha256=4-XvgAjUTnO3Dgdxm9VSw4udREjNWTW526Rggy5Do7s,6501
+palimpzest/schemabuilder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/schemabuilder/schema_builder.py,sha256=kGEv-Adba-FNziRrlG0zwx317IuD7rmzNl2GecvnbDw,8528
+palimpzest/tools/README.md,sha256=56_6LPG80uc0CLVhTBP6I1wgIffNv9cyTr0TmVZqmrM,483
+palimpzest/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/tools/allenpdf.py,sha256=fXMOmSDdSSLXDKAPYYJ8k4egtWEBf_Me9Lq9tM3iyoA,1690
+palimpzest/tools/pdfparser.py,sha256=0DOVUZLxYfqjxM8WNEfYcyiXb1qW9BWVIHEB_B_YhWA,9570
+palimpzest/tools/skema_tools.py,sha256=HXUFpjMhbVxZwKKkATeK-FwtlTCawaCbeP-uHntI1Kg,669
+palimpzest/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/utils/datareader_helpers.py,sha256=-tkIf9iOF7mr-gyrrUQqnHWwRig4OGnowU0Wpx5HKG0,2121
+palimpzest/utils/demo_helpers.py,sha256=BcNgtTz4O9iGriefy6f26BtJd_G5SQPzD3oQg_qLUdU,2522
+palimpzest/utils/env_helpers.py,sha256=n81KzoJ459pRxo7QmJA7duazwWsfoMGTHc71D2LatFk,334
+palimpzest/utils/field_helpers.py,sha256=Op18ThAnDlALiAkquUQbelHodZZYg378Ct1I8eIkKio,2291
+palimpzest/utils/generation_helpers.py,sha256=jveE9iQQtUQ94nuU6c1zuWoQMkwizr037S8si4n35jo,3230
+palimpzest/utils/hash_helpers.py,sha256=3A8dA7SbXTwnnvZvPVNqqMLlVRhCKyKF_bjNNAu3Exk,334
+palimpzest/utils/index_helpers.py,sha256=7webOjV2vYF7UJ_YsNdoX5OyR1zJ6lSLWO1mQSGWz0Q,123
+palimpzest/utils/model_helpers.py,sha256=dZdMkZ6zOBqG3uBCkmzXG1yQAoaGL3wF6lNSgnvigEQ,2399
+palimpzest/utils/progress.py,sha256=GYmPUBdG7xmqbqj1UiSNP-pWZKmRMLX797MBgrOPugM,7214
+palimpzest/utils/sandbox.py,sha256=Ge96gmzqeOGlNkMCG9A95_PB8wRQbvTFua136of8FcA,6465
+palimpzest/utils/token_reduction_helpers.py,sha256=Ob95PcqCsbGLiBdQ-4YQsWGWRppb2hvQyt0gi1fzL-Y,3855
+palimpzest/utils/udfs.py,sha256=LjHic54B1az-rKgNLur0wOpaz2ko_UodjLEJrazkxvY,1854
+palimpzest-0.6.1.dist-info/LICENSE,sha256=5GUlHy9lr-Py9kvV38FF1m3yy3NqM18fefuE9wkWumo,1079
+palimpzest-0.6.1.dist-info/METADATA,sha256=VxPI4-vfq3Fm3l3PjxTpdHGbDclIQNHo1Ag1enfAyMU,7837
+palimpzest-0.6.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+palimpzest-0.6.1.dist-info/top_level.txt,sha256=raV06dJUgohefUn3ZyJS2uqp_Y76EOLA9Y2e_fxt8Ew,11
+palimpzest-0.6.1.dist-info/RECORD,,

{palimpzest-0.5.4.dist-info → palimpzest-0.6.1.dist-info}/top_level.txt

@@ -1,2 +1 @@
-cli
 palimpzest

cli/README.md

@@ -1,156 +0,0 @@
-## Palimpzest CLI
-Installing Palimpzest also installs its CLI tool `pz` which provides users with basic utilities for creating and managing their own Palimpzest system. Running `pz --help` diplays an overview of the CLI's commands:
-```bash
-$ pz --help
-Usage: pz [OPTIONS] COMMAND [ARGS]...
-
-  The CLI tool for Palimpzest.
-
-Options:
-  --help  Show this message and exit.
-
-Commands:
-  help (h)                        Print the help message for PZ.
-  init (i)                        Initialize data directory for PZ.
-  ls-data (ls,lsdata)             Print a table listing the datasets
-                                  registered with PZ.
-  register-data (r,reg,register)  Register a data file or data directory with
-                                  PZ.
-  rm-data (rm,rmdata)             Remove a dataset that was registered with
-                                  PZ.
-```
-
-Users can initialize their own system by running `pz init`. This will create Palimpzest's working directory in `~/.palimpzest`:
-```bash
-$ pz init
-Palimpzest system initialized in: /Users/matthewrusso/.palimpzest
-```
-
-If we list the set of datasets registered with Palimpzest, we'll see there currently are none:
-```bash
-$ pz ls
-+------+------+------+
-| Name | Type | Path |
-+------+------+------+
-+------+------+------+
-
-Total datasets: 0
-```
-
-### Registering Datasets
-To add (or "register") a dataset with Palimpzest, we can use the `pz register-data` command (also aliased as `pz reg`) to specify that a file or directory at a given `--path` should be registered as a dataset with the specified `--name`:
-```bash
-$ pz reg --path README.md --name rdme
-Registered rdme
-```
-
-If we list Palimpzest's datasets again we will see that `README.md` has been registered under the dataset named `rdme`:
-```bash
-$ pz ls
-+------+------+------------------------------------------+
-| Name | Type |                   Path                   |
-+------+------+------------------------------------------+
-| rdme | file | /Users/matthewrusso/palimpzest/README.md |
-+------+------+------------------------------------------+
-
-Total datasets: 1
-```
-
-To remove a dataset from Palimpzest, simply use the `pz rm-data` command (also aliased as `pz rm`) and specify the `--name` of the dataset you would like to remove:
-```bash
-$ pz rm --name rdme
-Deleted rdme
-```
-
-Finally, listing our datasets once more will show that the dataset has been deleted:
-```bash
-$ pz ls
-+------+------+------+
-| Name | Type | Path |
-+------+------+------+
-+------+------+------+
-
-Total datasets: 0
-```
-
-### Cache Management
-Palimpzest will cache intermediate results by default. It can be useful to remove them from the cache when trying to evaluate the performance improvement(s) of code changes. We provide a utility command `pz clear-cache` (also aliased as `pz clr`) to clear the cache:
-```bash
-$ pz clr
-Cache cleared
-```
-
-### Config Management
-You may wish to work with multiple configurations of Palimpzest in order to, e.g., evaluate the difference in performance between various LLM services for your data extraction task. To see the config Palimpzest is currently using, you can run the `pz print-config` command (also aliased as `pz config`):
-```bash
-$ pz config
---- default ---
-filecachedir: /some/local/filepath
-llmservice: openai
-name: default
-parallel: false
-```
-By default, Palimpzest uses the configuration named `default`. As shown above, if you run a script using Palimpzest out-of-the-box, it will use OpenAI endpoints for all of its API calls.
-
-Now, let's say you wanted to try using [together.ai's](https://www.together.ai/) for your API calls, you could do this by creating a new config with the `pz create-config` command (also aliased as `pz cc`):
-```bash
-$ pz cc --name together-conf --llmservice together --parallel True --set
-Created and set config: together-conf
-```
-The `--name` parameter is required and specifies the unique name for your config. The `--llmservice` and `--parallel` options specify the service to use and whether or not to process files in parallel. Finally, if the `--set` flag is present, Palimpzest will update its current config to point to the newly created config.
-
-We can confirm that Palimpzest checked out our new config by running `pz config`:
-```bash
-$ pz config
---- together-conf ---
-filecachedir: /some/local/filepath
-llmservice: together
-name: together-conf
-parallel: true
-```
-
-You can switch which config you are using at any time by using the `pz set-config` command (also aliased as `pz set`):
-```bash
-$ pz set --name default
-Set config: default
-
-$ pz config
---- default ---
-filecachedir: /some/local/filepath
-llmservice: openai
-name: default
-parallel: false
-
-$ pz set --name together-conf
-Set config: together-conf
-
-$ pz config
---- together-conf ---
-filecachedir: /some/local/filepath
-llmservice: together
-name: together-conf
-parallel: true
-```
-
-You can update an existing config using the `pz update` command (also aliased as `pz uc`):
-```bash
-$ pz update --name default --settings parallel=true,pdfprocessor=pdfplumber
-Updated config: default
-
-$ pz config
---- default ---
-filecachedir: /some/local/filepath
-llmservice: anthropic
-name: default
-parallel: true
-pdfprocessor: pdfplumber
-```
-
-The `--name` parameter specifies which config to update. `--settings` specifies all the parameter name and value pairs in the format `param_name=param_value`, separated by commas.
-
-Finally, you can delete a config with the `pz rm-config` command (also aliased as `pz rmc`):
-```bash
-$ pz rmc --name together-conf
-Deleted config: together-conf
-```
-Note that you cannot delete the `default` config, and if you delete the config that you currently have set, Palimpzest will set the current config to be `default`.