koza 0.0.0__tar.gz

koza-0.0.0/.github/dependabot.yaml

@@ -0,0 +1,10 @@
+# Set update schedule for GitHub Actions
+
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

koza-0.0.0/.github/workflows/documentation.yaml

@@ -0,0 +1,51 @@
+name: Build and Deploy Docs to GitHub Pages
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  UV_VERSION: "0.7.x"
+
+jobs:
+  build-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@main
+        with:
+          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+
+      - name: Set up Python 3
+        uses: actions/setup-python@main
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          version: ${{ env.UV_VERSION }}
+
+      - name: Install library
+        run: |
+          make install
+
+      - name: Verify lockfile is up-to-date
+        run: |
+          uv lock --check
+
+      - name: Build Documentation
+        run: make docs
+
+      - name: Deploy to gh-pages
+        uses: JamesIves/github-pages-deploy-action@v4
+        if: github.ref == 'refs/heads/main'
+        with:
+          folder: site
+          target-folder: docs
+          clean: true
+          

koza-0.0.0/.github/workflows/publish.yaml

@@ -0,0 +1,40 @@
+name: publish on pypi
+
+on:
+  workflow_dispatch:
+  pull_request:
+  release:
+    types: [published]
+
+env:
+  UV_VERSION: "0.7.x"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          version: ${{ env.UV_VERSION }}
+
+      - name: Install
+        run: make install
+
+      - name: Verify lockfile is up-to-date
+        run: |
+          uv lock --check
+
+      - name: Build
+        run: make build
+
+      - name: Publish to PyPi
+        if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
+        env:
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          uv publish --token $UV_PUBLISH_TOKEN

koza-0.0.0/.github/workflows/test.yaml

@@ -0,0 +1,42 @@
+name: Test Koza
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches: [main]
+
+env:
+  UV_VERSION: "0.7.x"
+
+jobs:
+  test-python3-ubuntu-latest:
+    name: test py${{ matrix.python-version }} on linux
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    env:
+      OS: ubuntu
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          version: ${{ env.UV_VERSION }}
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install library
+        run: make install
+
+      - name: Verify lockfile is up-to-date
+        run: |
+          uv lock --check
+
+      - name: Run tests
+        run: make test

koza-0.0.0/.gitignore

@@ -0,0 +1,141 @@
+# Default output / Generated / Unpacked data files
+output/
+tests/resources/source-files/string.tsv*
+.ruff_cache
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# IDE
+.idea
+protein-links-detailed_edges.tsv
+protein-links-detailed_nodes.tsv
+uv.lock
+uv.lock

koza-0.0.0/CITATION.cff

@@ -0,0 +1,21 @@
+cff-version: '1.1.0'
+message: 'Please cite the following works when using this software.'
+abstract: 'Data transformation framework for LinkML data models'
+authors:
+  - family-names: 'Schaper'
+    given-names: 'Kevin'
+  - family-names: 'Ships'
+    given-names: 'Glass'
+  - family-names: 'Shefchek'
+    given-names: 'Kent'
+  - family-names: 'Moxon'
+    given-names: 'Sierra'
+  - family-names: 'Mungall'
+    given-names: 'Chris'
+date-released: 2022-06-15
+identifiers:
+  - type: 'url'
+    value: 'https://github.com/monarch-initiative/koza'
+title: 'monarch-initiative/koza'
+url: 'https://github.com/monarch-initiative/koza'
+version: '0.1.14'

koza-0.0.0/CONTRIBUTING.md

@@ -0,0 +1,44 @@
+##### Building locally
+
+First create a virtual environment with your favorite tool, and activate eg
+```bash
+python3.8 -m venv venv
+source venv/bin/activate
+```
+
+Install and test with make
+```bash
+make
+```
+
+Or with flit
+```
+pip install flit
+flit install --deps develop --symlink
+```
+
+##### Linting and Formatting
+TODO - write some docs on linting on formating
+
+Lint with flake8, black, and isort
+```bash
+make lint
+```
+
+Format with autoflake, black, and isort (updates files in place)
+```bash
+make format
+```
+
+##### Build and Publish to PyPI
+Building and publishing requires git >= 2.30
+
+Build a wheel and an sdist (tarball) from the package:
+```bash
+make build
+```
+
+Publish to PyPI
+```bash
+make publish
+```

koza-0.0.0/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Monarch Initiative
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

koza-0.0.0/Makefile

@@ -0,0 +1,66 @@
+# Note that uv is required, see https://docs.astral.sh/uv/getting-started/installation/
+
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+MAKEFLAGS += --no-builtin-variables
+
+ifneq (,$(wildcard ./.env))
+    include .env
+    export
+endif
+
+.DEFAULT_GOAL := all
+SHELL := bash
+RUN := uv run
+
+# This nifty grep/sort/awk pipeline collects all comments headed by the double "#" symbols next to each target and recycles them as comments
+.PHONY: help
+help: ## Print this help message
+	@grep -hE '^[[:alnum:]_/.-]+:.*## ' $(MAKEFILE_LIST) | \
+		awk 'BEGIN {FS = ":.*## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
+
+
+.PHONY: all
+all: install test clean ## Install, test, and clean
+
+.PHONY: install
+install:  ## Install development environment
+	uv venv --allow-existing
+	uv pip install -e .[dev]
+	uv lock
+
+.PHONY: build
+build:  ## Build the package
+	uv build
+
+.PHONY: test
+test:  ## Run the test suite
+	$(RUN) pytest tests
+
+.PHONY: docs
+docs:  ## Build the documentation
+	$(RUN) typer src/koza/main.py utils docs --name koza --output docs/Usage/CLI.md
+	$(RUN) mkdocs build
+
+.PHONY: clean
+clean:  ## Clean up build artifacts, etc.
+	rm -rf `find . -name __pycache__`
+	rm -f `find . -type f -name '*.py[co]' `
+	rm -rf .pytest_cache
+	rm -rf output test-output
+	rm -rf dist
+
+.PHONY: coverage
+coverage:  ## Run the test suite with coverage reporting
+	-$(RUN) coverage run -m pytest tests
+	$(RUN) coverage report -m
+
+.PHONY: lint
+lint:  ## Lint the codebase
+	$(RUN) ruff check --diff --exit-zero src/ tests/ examples/
+	$(RUN) ruff format --check --diff src/ tests/ examples/
+
+.PHONY: format
+format:  ## Format the codebase
+	$(RUN) ruff check --fix --exit-zero src/ tests/ examples/
+	$(RUN) ruff format src/ tests/ examples/

koza-0.0.0/PKG-INFO

@@ -0,0 +1,202 @@
+Metadata-Version: 2.4
+Name: koza
+Version: 0.0.0
+Summary: Data transformation framework for LinkML data models
+Author-email: The Monarch Initiative <info@monarchinitiative.org>, Kevin Schaper <kevinschaper@gmail.com>, Glass Elsarboukh <g.elsarboukh@gmail.com>, Kent Shefchek <kent@tislab.org>, Daniel Korn <daniel_korn@med.unc.edu>
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Requires-Python: <4,>=3.10
+Requires-Dist: biolink-model>=4.3.6
+Requires-Dist: coverage>=7.13.0
+Requires-Dist: duckdb
+Requires-Dist: linkml>=1.9.0
+Requires-Dist: loguru
+Requires-Dist: mergedeep==1.3.4
+Requires-Dist: ordered-set>=4.1.0
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: requests>=2.32.5
+Requires-Dist: sssom>=0.4
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: typer>=0.20.0
+Provides-Extra: dev
+Requires-Dist: mkdocs-material>=9.7.0; extra == 'dev'
+Requires-Dist: mkdocs>=1.6.1; extra == 'dev'
+Requires-Dist: mkdocstrings[python]>=1.0.0; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Koza - Knowledge Graph Transformation and Operations Toolkit
+
+[![Pyversions](https://img.shields.io/pypi/pyversions/koza.svg)](https://pypi.python.org/pypi/koza)
+[![PyPi](https://img.shields.io/pypi/v/koza.svg)](https://pypi.python.org/pypi/koza)
+![Github Action](https://github.com/monarch-initiative/koza/actions/workflows/test.yaml/badge.svg)
+
+![pupa](docs/img/pupa.png)  
+
+[**Documentation**](https://koza.monarchinitiative.org/)
+
+_Disclaimer_: Koza is in beta - we are looking for testers!
+
+## Overview
+
+Koza is a Python library and CLI tool for transforming biomedical data and performing graph operations on Knowledge Graph Exchange (KGX) files. It provides two main capabilities:
+
+### 📊 **Graph Operations** (New!)
+Powerful DuckDB-based operations for KGX knowledge graphs:
+- **Join** multiple KGX files with schema harmonization
+- **Split** files by field values with format conversion  
+- **Prune** dangling edges and handle singleton nodes
+- **Append** new data to existing databases with schema evolution
+- **Multi-format support** for TSV, JSONL, and Parquet files
+
+### 🔄 **Data Transformation** (Core)
+Transform biomedical data sources into KGX format:
+- Transform csv, json, yaml, jsonl, and xml to target formats
+- Output in [KGX format](https://github.com/biolink/kgx/blob/master/specification/kgx-format.md#kgx-format-as-tsv)
+- Write data transforms in semi-declarative Python
+- Configure source files, columns/properties, and metadata in YAML
+- Create mapping files and translation tables between vocabularies
+
+## Installation
+Koza is available on PyPi and can be installed via pip/pipx:
+```
+[pip|pipx] install koza
+```
+
+## Usage
+
+### Quick Start with Graph Operations
+
+Koza's graph operations work seamlessly across multiple KGX formats (TSV, JSONL, Parquet):
+
+```bash
+# Join multiple KGX files into a unified database
+koza join --nodes genes.tsv pathways.jsonl --edges interactions.parquet --output merged_graph.duckdb
+
+# Prune dangling edges and handle singleton nodes
+koza prune --database merged_graph.duckdb --keep-singletons
+
+# Append new data to existing database with schema evolution
+koza append --database merged_graph.duckdb --nodes new_genes.tsv --edges new_interactions.jsonl
+
+# Split database by source with format conversion
+koza split --database merged_graph.duckdb --split-on provided_by --output-format parquet
+```
+
+**NOTE: As of version 0.2.0, there is a new method for getting your ingest's `KozaApp` instance. Please see the [updated documentation](https://koza.monarchinitiative.org/Usage/configuring_ingests/#transform-code) for details.**
+
+See the [Koza documentation](https://koza.monarchinitiative.org/) for complete usage information
+
+### Examples
+
+#### Validate
+
+Give Koza a local or remote csv file, and get some basic information (headers, number of rows)
+
+```bash
+koza validate \
+  --file https://raw.githubusercontent.com/monarch-initiative/koza/main/examples/data/string.tsv \
+  --delimiter ' '
+```
+
+Sending a json or jsonl formatted file will confirm if the file is valid json or jsonl
+
+```bash
+koza validate \
+  --file ./examples/data/ZFIN_PHENOTYPE_0.jsonl.gz \
+  --format jsonl
+```
+
+```bash
+koza validate \
+  --file ./examples/data/ddpheno.json.gz \
+  --format json
+```
+
+#### Transform
+
+Run the example ingest, "string/protein-links-detailed"
+```bash
+koza transform \
+  --source examples/string/protein-links-detailed.yaml \
+  --global-table examples/translation_table.yaml
+
+koza transform \
+  --source examples/string-declarative/protein-links-detailed.yaml \
+  --global-table examples/translation_table.yaml
+```
+
+**Note**: 
+  Koza expects a directory structure as described in the above example  
+  with the source config file and transform code in the same directory: 
+  ```
+  .
+  ├── ...
+  │   ├── your_source
+  │   │   ├── your_ingest.yaml
+  │   │   └── your_ingest.py
+  │   └── some_translation_table.yaml
+  └── ...
+  ```
+
+#### Graph Operations
+
+Create and manipulate knowledge graphs from existing KGX files:
+
+```bash
+# Join heterogeneous KGX files with automatic schema harmonization
+koza join \
+  --nodes genes.tsv proteins.jsonl pathways.parquet \
+  --edges gene_protein.tsv protein_pathway.jsonl \
+  --output unified_graph.duckdb \
+  --schema-report
+
+# Clean up graph integrity issues
+koza prune \
+  --database unified_graph.duckdb \
+  --keep-singletons \
+  --dry-run  # Preview changes before applying
+
+# Incrementally add new data with schema evolution
+koza append \
+  --database unified_graph.duckdb \
+  --nodes new_genes.tsv updated_pathways.jsonl \
+  --deduplicate \
+  --show-progress
+
+# Export subsets with format conversion
+koza split \
+  --database unified_graph.duckdb \
+  --split-on provided_by \
+  --output-format parquet \
+  --output-dir ./split_graphs
+```
+
+## Key Features
+
+### 🔧 **Multi-Format Support**
+- Native support for TSV, JSONL, and Parquet KGX files
+- Automatic format detection and conversion
+- Mixed-format operations in single commands
+
+### 🛡️ **Schema Flexibility**
+- Automatic schema harmonization across heterogeneous files
+- Schema evolution with backward compatibility  
+- Comprehensive schema reporting and validation
+
+### ⚡ **High Performance**
+- DuckDB-powered operations for fast bulk processing
+- Memory-efficient handling of large knowledge graphs
+- Parallel processing and streaming where possible
+
+### 🔍 **Rich CLI Experience**
+- Progress indicators for long-running operations
+- Detailed statistics and operation summaries
+- Dry-run modes for safe operation preview
+
+### 🧹 **Data Integrity**
+- Dangling edge detection and preservation
+- Duplicate detection and removal strategies
+- Non-destructive operations with data archiving