proceed 0.0.1__tar.gz

proceed-0.0.1/.gitignore

@@ -0,0 +1,140 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/build/
+docs/source/generated
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Visual Studio Code
+.vscode/
+
+# Proceed
+proceed_out/
+hello_world.yaml
+my/work/
+fizzbuzz.yaml
+summary.csv

proceed-0.0.1/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

proceed-0.0.1/PKG-INFO

@@ -0,0 +1,83 @@
+Metadata-Version: 2.3
+Name: proceed
+Version: 0.0.1
+Summary: Execute pipelines based on files and containers.
+Project-URL: Homepage, https://github.com/benjamin-heasly/proceed
+Project-URL: Bug Tracker, https://github.com/benjamin-heasly/proceed/issues
+Author-email: Ben Heasly <benjamin.heasly@gmail.com>
+License-File: LICENSE
+Classifier: License :: OSI Approved :: The Unlicense (Unlicense)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Requires-Dist: docker
+Requires-Dist: pandas
+Requires-Dist: pyyaml
+Description-Content-Type: text/markdown
+
+# Proceed
+Declarative file processing with YAML and containers.
+
+**Proceed** is a Python library and CLI tool for declarative batch processing.
+It reads a **pipeline** specification declared in [YAML](https://yaml.org/).
+A pipeline contains a list of **steps** that are based on [Docker](https://www.docker.com/) images and containers.
+
+Each pipeline execution accepts values for declared **args**, allowing controlled, explicit configuration of steps at runtime.
+Each execution produces an **execution record** that accounts for accepted arg values, step logs, and checksums of input and output files.
+
+Hopefully, Proceed will allow you to express everything you need to know about your processing pipeline in a *"nothing up my sleeves"* way.  The pipeline specification should be complete enough to share with others who have Proceed and Docker installed.
+The execution record should allow for auditing of expected outcomes and reproducibility.
+
+## docs
+Here are the [main docs](https://benjamin-heasly.github.io/proceed/index.html) for Proceed.
+
+# Installation
+Proceed requires [Python](https://www.python.org/) and [Docker](https://www.docker.com/) to be installed.
+With those, it should be able to run a wide variety pipelines and steps via containers.
+
+## pip
+Proceed itself is not yet available on [PyPI](https://pypi.org/).
+When it is, this will be the recommended way to install Proceed:
+
+```
+$ pip install proceed # TODO
+```
+
+## git and pip
+You can also install Proceed from source.
+
+```
+$ pip install git+https://https://github.com/benjamin-heasly/proceed.git
+
+# editable mode
+$ git checkout https://github.com/benjamin-heasly/proceed.git
+$ pip install -e ./proceed
+```
+
+## check installation
+You can check if Proceed installed correctly using the `proceed` command.
+
+```
+$ proceed --version
+Proceed x.y.z
+
+$ proceed --help
+usage etc...
+```
+
+## development and testing
+
+You can set up a development environment with [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) and [dev-environment.yml](./dev-environment.yml).
+
+```
+conda env create -f dev-environment.yml
+# or
+conda env update -f dev-environment.yml
+```
+
+With that, you should be able to run through the Proceed unit and integration tests.
+
+```
+conda activate proceed-dev
+hatch run test:cov
+```

proceed-0.0.1/README.md

@@ -0,0 +1,66 @@
+# Proceed
+Declarative file processing with YAML and containers.
+
+**Proceed** is a Python library and CLI tool for declarative batch processing.
+It reads a **pipeline** specification declared in [YAML](https://yaml.org/).
+A pipeline contains a list of **steps** that are based on [Docker](https://www.docker.com/) images and containers.
+
+Each pipeline execution accepts values for declared **args**, allowing controlled, explicit configuration of steps at runtime.
+Each execution produces an **execution record** that accounts for accepted arg values, step logs, and checksums of input and output files.
+
+Hopefully, Proceed will allow you to express everything you need to know about your processing pipeline in a *"nothing up my sleeves"* way.  The pipeline specification should be complete enough to share with others who have Proceed and Docker installed.
+The execution record should allow for auditing of expected outcomes and reproducibility.
+
+## docs
+Here are the [main docs](https://benjamin-heasly.github.io/proceed/index.html) for Proceed.
+
+# Installation
+Proceed requires [Python](https://www.python.org/) and [Docker](https://www.docker.com/) to be installed.
+With those, it should be able to run a wide variety pipelines and steps via containers.
+
+## pip
+Proceed itself is not yet available on [PyPI](https://pypi.org/).
+When it is, this will be the recommended way to install Proceed:
+
+```
+$ pip install proceed # TODO
+```
+
+## git and pip
+You can also install Proceed from source.
+
+```
+$ pip install git+https://https://github.com/benjamin-heasly/proceed.git
+
+# editable mode
+$ git checkout https://github.com/benjamin-heasly/proceed.git
+$ pip install -e ./proceed
+```
+
+## check installation
+You can check if Proceed installed correctly using the `proceed` command.
+
+```
+$ proceed --version
+Proceed x.y.z
+
+$ proceed --help
+usage etc...
+```
+
+## development and testing
+
+You can set up a development environment with [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) and [dev-environment.yml](./dev-environment.yml).
+
+```
+conda env create -f dev-environment.yml
+# or
+conda env update -f dev-environment.yml
+```
+
+With that, you should be able to run through the Proceed unit and integration tests.
+
+```
+conda activate proceed-dev
+hatch run test:cov
+```

proceed-0.0.1/dev-environment.yml

@@ -0,0 +1,10 @@
+name: proceed-dev
+dependencies:
+  - python=3.11
+  - pip=24.0
+  - pip:
+    - docker==7.1.0
+    - pyyaml==6.0.2
+    - pandas==2.2.2
+    - pytest==8.3.2
+    - hatch==1.12.0

proceed-0.0.1/pyproject.toml

@@ -0,0 +1,69 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "proceed"
+description = "Execute pipelines based on files and containers."
+keywords = []
+authors = [
+  { name="Ben Heasly", email="benjamin.heasly@gmail.com" }
+]
+readme = "README.md"
+requires-python = ">=3.11"
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: The Unlicense (Unlicense)",
+  "Operating System :: OS Independent",
+]
+dependencies = ["docker", "PyYAML", "pandas"]
+dynamic = ["version"]
+
+[project.urls]
+"Homepage" = "https://github.com/benjamin-heasly/proceed"
+"Bug Tracker" = "https://github.com/benjamin-heasly/proceed/issues"
+
+[project.scripts]
+proceed = "proceed.cli:main"
+
+[tool.pytest.ini_options]
+addopts = [
+    "--import-mode=importlib",
+]
+
+[tool.hatch]
+
+[tool.hatch.version]
+path = "src/proceed/__about__.py"
+
+[tool.hatch.envs.test]
+dependencies = [
+  "pytest",
+  "pytest-cov",
+]
+
+[tool.hatch.envs.test.scripts]
+cov = 'pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=proceed --cov=tests -vv {args}'
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "/.github",
+  "/docs",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/proceed"]
+
+[tool.hatch.envs.docs]
+dependencies = [
+  "sphinx",
+]
+
+[tool.hatch.envs.docs.scripts]
+# A long command!
+# This gets sphinx under control of hatch and pyproject.toml and eliminates the clunky sphinx Makefile.
+html = '''sphinx-build "docs/source" "docs/build" -b html -c docs/ -D project="$(hatch project metadata name)" -D release="$(hatch version)" -D author="Ben Heasly" {args}'''
+clean = [
+  "rm -rf docs/build",
+  "rm -rf docs/source/generated"
+]

proceed-0.0.1/src/fizzbuzz/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.7
+
+WORKDIR /fizzbuzz
+COPY ./fizzbuzz.py /fizzbuzz/fizzbuzz.py
+
+ENTRYPOINT ["python3", "fizzbuzz.py"]
+CMD ["--help"]

proceed-0.0.1/src/fizzbuzz/fizzbuzz.py

@@ -0,0 +1,61 @@
+import argparse
+import sys
+from typing import Optional, Sequence
+
+
+def classify(number):
+    suffix = ""
+    if number % 3 == 0:
+        suffix = suffix + "fizz"
+
+    if number % 5 == 0:
+        suffix = suffix + "buzz"
+
+    return suffix
+
+
+def append(line):
+    number = int(line)
+    suffix = classify(number)
+    if (suffix):
+        return f"{line} {suffix}"
+    else:
+        return line
+
+
+def classify_lines(in_file, out_file):
+    with open(out_file, 'w') as out_f:
+        with open(in_file) as in_f:
+            for in_line in in_f:
+                out_line = append(in_line.strip()) + "\n"
+                out_f.write(out_line)
+
+
+def filter_lines(in_file, out_file, substring):
+    with open(out_file, 'w') as out_f:
+        with open(in_file) as in_f:
+            for in_line in in_f:
+                if substring in in_line:
+                    out_f.write(in_line)
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description="Classify and filter lines of text files, according to fizzbuzz.")
+    parser.add_argument("in_file", type=str, help="input file to read")
+    parser.add_argument("out_file", type=str, help="output file to write")
+    parser.add_argument("operation", type=str, help="operation to perform", choices=["classify", "filter"])
+    parser.add_argument("--substring", type=str, help="filter substring for lines to keep", default="fizz")
+    args = parser.parse_args(argv)
+
+    if args.operation == "classify":
+        classify_lines(args.in_file, args.out_file)
+    elif args.operation == "filter":
+        filter_lines(args.in_file, args.out_file, args.substring)
+
+    print("OK.")
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

proceed-0.0.1/src/proceed/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

proceed-0.0.1/src/proceed/aggregator.py

@@ -0,0 +1,123 @@
+import logging
+from typing import Any
+from pathlib import Path
+from pandas import DataFrame
+import yaml
+from proceed.model import ExecutionRecord, Pipeline, Step, Timing, StepResult
+from proceed.file_matching import flatten_matches, file_summary, hash_contents
+
+def summarize_results(results_path: Path, columns: list[str] = None, sort_rows_by: list[str] = None) -> DataFrame:
+    summary_rows = []
+    group_paths = [path for path in results_path.iterdir() if path.is_dir()]
+    for group_path in group_paths:
+        id_paths = [path for path in group_path.iterdir() if path.is_dir()]
+        for id_path in id_paths:
+            for yaml_file in id_path.glob("execution_record.yaml"):
+                execution_record = safe_read_execution_record(yaml_file)
+                if execution_record:
+                    execution_summary = summarize_execution(id_path.stem, group_path.stem, execution_record)
+                    summary_rows = summary_rows + execution_summary
+
+    summary = DataFrame(summary_rows)
+
+    if columns:
+        summary_columns = list(summary.columns)
+        usable_columns = [column for column in columns if column in summary_columns]
+        summary = summary.filter(items=columns)
+
+    if sort_rows_by:
+        summary_columns = list(summary.columns)
+        usable_columns = [column for column in sort_rows_by if column in summary_columns]
+        summary = summary.sort_values(usable_columns)
+
+    return summary
+
+
+def safe_read_execution_record(yaml_file: Path) -> ExecutionRecord:
+    try:
+        with open(yaml_file) as f:
+            return ExecutionRecord.from_yaml(f.read())
+    except:
+        logging.error(f"Skipping file that seems not to be a Proceed execution record: {yaml_file}")
+        return None
+
+
+def summarize_execution(results_id: str, group: str, execution_record: ExecutionRecord) -> list[dict[str, str]]:
+    pipeline_summary = summarize_pipeline(results_id, group, execution_record.amended, execution_record.timing)
+
+    steps_and_results = zip(execution_record.amended.steps, execution_record.step_results)
+    step_summaries = [summarize_step_and_result(step, result) for step, result in steps_and_results]
+
+    combined_summary = [{**pipeline_summary, **file_summary} for step_summary in step_summaries for file_summary in step_summary]
+    return combined_summary
+
+
+def summarize_pipeline(results_id: str, group: str, pipeline: Pipeline, timing: Timing) -> dict[str, str]:
+    top_level_summary = {
+        "proceed_version": pipeline.version,
+        "results_id": results_id,
+        "results_group": group,
+        "pipeline_description": pipeline.description,
+        "pipeline_start": timing.start,
+        "pipeline_finish": timing.finish,
+        "pipeline_duration": timing.duration,
+    }
+
+    arg_summary = {f"arg_{key}": value for key, value in pipeline.args.items()}
+
+    combined_summary = {**top_level_summary, **arg_summary}
+    return combined_summary
+
+
+def summarize_step_and_result(step: Step, result: StepResult) -> list[dict[str, Any]]:
+    step_summary = {f"step_{key}": str(value) for key, value in step.to_dict().items()}
+
+    flattened_step_attributes = {"timing", "log_file", "files_done", "files_in", "files_out", "files_summary"}
+    result_summary = {f"step_{key}": str(value) for key, value in result.to_dict().items() if key not in flattened_step_attributes}
+
+    result_summary["step_start"] = result.timing.start
+    result_summary["step_finish"] = result.timing.finish
+    result_summary["step_duration"] = result.timing.duration
+
+    if result.log_file:
+        log_path = Path(result.log_file)
+        log_digest = hash_contents(log_path)
+        log_file = file_summary(volume=log_path.parent.as_posix(), path=log_path.name, digest=log_digest, file_role="log")
+    else:
+        log_file = file_summary(volume="", path="", digest="", file_role="log")
+
+    done_files = flatten_matches(result.files_done, file_role="done")
+    in_files = flatten_matches(result.files_in, file_role="in")
+    out_files = flatten_matches(result.files_out, file_role="out")
+    summary_files = flatten_matches(result.files_summary, file_role="summary")
+
+    all_files = [log_file] + done_files + in_files + out_files + summary_files
+
+    custom_summary = {}
+    for summary_file in summary_files:
+        custom_columns = collect_custom_columns(summary_file["file_volume"], summary_file["file_path"])
+        custom_summary.update(custom_columns)
+
+    combined_summary = [{**step_summary, **result_summary, **file_summary, **custom_summary} for file_summary in all_files]
+    return combined_summary
+
+
+def collect_custom_columns(file_volume: str, file_path: str) -> dict[str, str]:
+    path = Path(file_volume, file_path)
+    if not path.is_file() or not path.exists():
+        return {}
+
+    with open(path) as f:
+        content = f.read()
+
+    try:
+        parsed = yaml.safe_load(content)
+        if parsed and isinstance(parsed, dict):
+            return parsed
+
+        logging.info(f"Treating non-dictionary YAML as plain text: {path.as_posix()}")
+
+    except yaml.parser.ParserError:
+        logging.info(f"Treating non-YAML file as plain text: {path.as_posix()}")
+
+    return {path.stem: content.strip()}