imbed_data_prep 0.1.1__tar.gz

imbed_data_prep-0.1.1/.gitattributes

@@ -0,0 +1 @@
+*.ipynb linguist-documentation

imbed_data_prep-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,255 @@
+name: Continuous Integration (uv)
+on: [push, pull_request]
+
+# Workflow-level env vars from [tool.wads.ci.env] in pyproject.toml.
+# Populated by wads-migrate / wads init via template substitution.
+# Includes PROJECT_NAME, literal defaults from env.defaults, and secret-
+# backed vars from required_envvars / test_envvars / extra_envvars,
+# rendered as `KEY: secrets.KEY || ''` so missing secrets don't fail parsing.
+env:
+  PROJECT_NAME: imbed_data_prep
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || '' }}
+
+jobs:
+  # First job: Read configuration from pyproject.toml
+  setup:
+    name: Read Configuration
+    runs-on: ubuntu-latest
+    outputs:
+      project-name: ${{ steps.config.outputs.project-name }}
+      python-versions: ${{ steps.config.outputs.python-versions }}
+      pytest-args: ${{ steps.config.outputs.pytest-args }}
+      coverage-enabled: ${{ steps.config.outputs.coverage-enabled }}
+      exclude-paths: ${{ steps.config.outputs.exclude-paths }}
+      test-on-windows: ${{ steps.config.outputs.test-on-windows }}
+      tests-enabled: ${{ steps.config.outputs.tests-enabled }}
+      build-sdist: ${{ steps.config.outputs.build-sdist }}
+      build-wheel: ${{ steps.config.outputs.build-wheel }}
+      metrics-enabled: ${{ steps.config.outputs.metrics-enabled }}
+      metrics-config-path: ${{ steps.config.outputs.metrics-config-path }}
+      metrics-storage-branch: ${{ steps.config.outputs.metrics-storage-branch }}
+      metrics-python-version: ${{ steps.config.outputs.metrics-python-version }}
+      metrics-force-run: ${{ steps.config.outputs.metrics-force-run }}
+      ruff-enabled: ${{ steps.config.outputs.ruff-enabled }}
+      black-enabled: ${{ steps.config.outputs.black-enabled }}
+      mypy-enabled: ${{ steps.config.outputs.mypy-enabled }}
+      docs-enabled: ${{ steps.config.outputs.docs-enabled }}
+      publish-enabled: ${{ steps.config.outputs.publish-enabled }}
+      skip-ci-marker: ${{ steps.config.outputs.skip-ci-marker }}
+      publish-marker: ${{ steps.config.outputs.publish-marker }}
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Read CI Config
+        id: config
+        uses: i2mint/wads/actions/read-ci-config@master
+        with:
+          pyproject-path: .
+
+  # Second job: Validation using the config
+  validation:
+    name: Validation
+    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ${{ fromJson(needs.setup.outputs.python-versions) }}
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: i2mint/wads/actions/setup-python-uv@master
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install System Dependencies
+        uses: i2mint/wads/actions/install-system-deps@master
+        with:
+          pyproject-path: .
+
+      - name: Install Dependencies
+        uses: i2mint/wads/actions/install-deps-uv@master
+
+      - name: Format Source Code
+        if: needs.setup.outputs.ruff-enabled != 'false'
+        run: uvx ruff format .
+
+      - name: Format Source Code (black)
+        if: needs.setup.outputs.black-enabled == 'true'
+        run: uvx black .
+
+      - name: Lint Validation
+        if: needs.setup.outputs.ruff-enabled != 'false'
+        run: uvx ruff check --output-format=github ${{ needs.setup.outputs.project-name }}
+
+      - name: Type Check (mypy)
+        if: needs.setup.outputs.mypy-enabled == 'true'
+        run: uvx mypy ${{ needs.setup.outputs.project-name }}
+
+      - name: Run Tests
+        if: needs.setup.outputs.tests-enabled != 'false'
+        uses: i2mint/wads/actions/run-tests-uv@master
+        with:
+          root-dir: ${{ needs.setup.outputs.project-name }}
+          pytest-args: ${{ needs.setup.outputs.pytest-args }}
+          exclude-paths: ${{ needs.setup.outputs.exclude-paths }}
+          coverage: ${{ needs.setup.outputs.coverage-enabled }}
+
+      - name: Track Code Metrics
+        if: needs.setup.outputs.metrics-enabled == 'true'
+        uses: i2mint/umpyre/actions/track-metrics@master
+        continue-on-error: true
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          config-path: ${{ needs.setup.outputs.metrics-config-path }}
+          storage-branch: ${{ needs.setup.outputs.metrics-storage-branch }}
+          python-version: ${{ needs.setup.outputs.metrics-python-version }}
+          force-run: ${{ needs.setup.outputs.metrics-force-run }}
+
+  # Optional Windows testing (if enabled in config)
+  windows-validation:
+    name: Windows Tests
+    if: "!contains(github.event.head_commit.message, '[skip ci]') && needs.setup.outputs.test-on-windows == 'true' && needs.setup.outputs.tests-enabled != 'false'"
+    needs: setup
+    runs-on: windows-latest
+    continue-on-error: true
+    env:
+      # PEP 540 UTF-8 mode: avoid cp1252 UnicodeDecode/EncodeError when test
+      # code reads source files or scripts print non-ASCII characters.
+      PYTHONUTF8: "1"
+      PYTHONIOENCODING: "utf-8"
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        uses: i2mint/wads/actions/setup-python-uv@master
+        with:
+          python-version: ${{ fromJson(needs.setup.outputs.python-versions)[0] }}
+
+      - name: Install System Dependencies
+        uses: i2mint/wads/actions/install-system-deps@master
+        with:
+          pyproject-path: .
+
+      - name: Install Dependencies
+        uses: i2mint/wads/actions/install-deps-uv@master
+
+      - name: Run Tests
+        uses: i2mint/wads/actions/run-tests-uv@master
+        with:
+          root-dir: ${{ needs.setup.outputs.project-name }}
+          pytest-args: ${{ needs.setup.outputs.pytest-args }}
+          exclude-paths: ${{ needs.setup.outputs.exclude-paths }}
+
+  # Publishing job
+  #
+  # Gated by [tool.wads.ci.publish] in pyproject.toml (read via the setup job):
+  #   - skip-ci-marker  : when publishing is enabled, a commit message containing
+  #                       this substring skips the publish job (default "[skip ci]").
+  #   - publish-enabled : whether publishing runs at all (default true).
+  #   - publish-marker  : when publishing is disabled, a commit message containing
+  #                       this substring forces the publish job (default "[publish]").
+  # The publish-enabled check uses `== 'true'` (fail-closed): if an older wads
+  # without these outputs is installed by read-ci-config, publishing is skipped
+  # rather than run unintentionally.
+  publish:
+    name: Publish
+    permissions:
+      contents: write
+    if: "!contains(github.event.head_commit.message, needs.setup.outputs.skip-ci-marker) && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && (needs.setup.outputs.publish-enabled == 'true' || contains(github.event.head_commit.message, needs.setup.outputs.publish-marker))"
+    needs: [setup, validation]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Set up Python
+        uses: i2mint/wads/actions/setup-python-uv@master
+        with:
+          python-version: ${{ fromJson(needs.setup.outputs.python-versions)[0] }}
+          create-venv: "false"
+
+      - name: Format Source Code
+        if: needs.setup.outputs.ruff-enabled != 'false'
+        run: uvx ruff format .
+
+      - name: Format Source Code (black)
+        if: needs.setup.outputs.black-enabled == 'true'
+        run: uvx black .
+
+      - name: Update Version Number
+        id: version
+        uses: i2mint/isee/actions/bump-version-number@master
+
+      - name: Build Distribution
+        uses: i2mint/wads/actions/build-dist-uv@master
+        with:
+          sdist: ${{ needs.setup.outputs.build-sdist }}
+          wheel: ${{ needs.setup.outputs.build-wheel }}
+
+      - name: Publish to PyPI
+        uses: i2mint/wads/actions/pypi-publish-uv@master
+        with:
+          pypi-token: ${{ secrets.PYPI_PASSWORD }}
+
+      - name: Force SSH for git remote
+        run: git remote set-url origin git@github.com:${{ github.repository }}.git
+
+      - name: Commit Changes
+        uses: i2mint/wads/actions/git-commit@master
+        with:
+          commit-message: "**CI** Formatted code + Updated version to ${{ env.VERSION }} [skip ci]"
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          push: true
+
+      - name: Tag Repository
+        uses: i2mint/wads/actions/git-tag@master
+        with:
+          tag: ${{ env.VERSION }}
+          message: "Release version ${{ env.VERSION }}"
+          push: true
+
+  # Optional GitHub Pages (skipped when [tool.wads.ci.docs].enabled = false)
+  # Depends on validation (not publish) so docs still publish when the publish
+  # job is disabled via [tool.wads.ci.publish].enabled = false.
+  github-pages:
+    name: Publish GitHub Pages
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+    if: "!contains(github.event.head_commit.message, '[skip ci]') && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && needs.setup.outputs.docs-enabled != 'false'"
+    needs: [setup, validation]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: i2mint/epythet/actions/publish-github-pages@master
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          ignore: "tests/,scrap/,examples/"

imbed_data_prep-0.1.1/.gitignore

@@ -0,0 +1,115 @@
+wads_configs.json
+data/wads_configs.json
+wads/data/wads_configs.json
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+.DS_Store
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+_build
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/*
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# PyCharm
+.idea

imbed_data_prep-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 thorwhalen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

imbed_data_prep-0.1.1/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.4
+Name: imbed_data_prep
+Version: 0.1.1
+Summary: Modules to acquire and prepare data for the imbed package.
+Project-URL: Homepage, https://github.com/thorwhalen/imbed_data_prep
+Project-URL: Repository, https://github.com/thorwhalen/imbed_data_prep
+Project-URL: Documentation, https://thorwhalen.github.io/imbed_data_prep
+Author: thorwhalen
+License: mit
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4
+Requires-Dist: datasets
+Requires-Dist: dill
+Requires-Dist: dol
+Requires-Dist: duckdb
+Requires-Dist: graze
+Requires-Dist: haggle
+Requires-Dist: i2
+Requires-Dist: imbed
+Requires-Dist: lkj
+Requires-Dist: nltk
+Requires-Dist: numpy
+Requires-Dist: oa
+Requires-Dist: pandas
+Requires-Dist: requests
+Requires-Dist: scikit-learn
+Requires-Dist: tabled>=0.1.28
+Requires-Dist: vadersentiment
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
+Requires-Dist: sphinx>=6.0; extra == 'docs'
+Description-Content-Type: text/markdown
+
+# imbed_data_prep
+
+Modules to acquire and prepare data of different datasets

imbed_data_prep-0.1.1/README.md

@@ -0,0 +1,3 @@
+# imbed_data_prep
+
+Modules to acquire and prepare data of different datasets

imbed_data_prep-0.1.1/imbed_data_prep/__init__.py

@@ -0,0 +1,12 @@
+"""Modules to acquire and prepare data for the imbed package."""
+
+
+# # Use __getitem__ to protect access of the modules of list_of_modules
+
+# def __getitem__(name):
+#     # if name in list_of_modules:
+#     try:
+#         return globals()[name]
+#     except KeyError:
+#         pass  # will raise ImportError below
+#     raise ImportError(f"No module named {name}")

imbed_data_prep-0.1.1/imbed_data_prep/arxiv/README.md

@@ -0,0 +1,31 @@
+# ArXiv
+
+Data preparation for ArXiv papers.
+
+## Status
+
+This module has been migrated to the standalone
+[`xv`](https://pypi.org/project/xv/) package on PyPI.
+The module here is a thin wrapper that re-exports from `xv`.
+
+## Data source
+
+[ArXiv](https://arxiv.org/) is an open-access repository of scientific
+papers in physics, mathematics, computer science, and related fields, hosted
+by Cornell University.
+
+## Usage
+
+```bash
+pip install xv
+```
+
+```python
+from imbed_data_prep.arxiv import ...  # delegates to xv
+```
+
+## Files in this directory
+
+| File | Description |
+|---|---|
+| `__init__.py` | Wrapper module importing from the `xv` package |

imbed_data_prep-0.1.1/imbed_data_prep/arxiv/__init__.py

@@ -0,0 +1,10 @@
+"""Access to ArXiv data.
+
+Project moved to xv: https://pypi.org/project/xv/
+
+"""
+
+from contextlib import suppress
+
+with suppress(ImportError):
+    from xv.data_access import *  # pip install xv

imbed_data_prep-0.1.1/imbed_data_prep/embeddings_of_aggregations/README.md

@@ -0,0 +1,46 @@
+# Embeddings of Aggregations
+
+Experiments with aggregated embeddings over citation graphs.
+
+## Data source
+
+This module works with citation graph data and academic paper metadata.
+It takes a set of nodes (papers) with embeddings and citation links, then
+explores how aggregating the titles of cited papers and embedding those
+aggregated strings compares to the original embeddings.
+
+The data is expected to come from an external citation graph (e.g. Semantic
+Scholar, OpenAlex, or a custom corpus) loaded as DataFrames with paper IDs,
+titles, and citation edges.
+
+## What it does
+
+1. **Sample nodes** from a citation graph.
+2. **Permute citations** -- for each citing paper, generate multiple random
+   orderings of its cited papers.
+3. **Aggregate titles** -- concatenate cited-paper titles in each permutation
+   order into a single string.
+4. **Embed aggregated strings** -- compute embeddings of these concatenated
+   title strings.
+5. **Compare** -- measure how the aggregated embeddings relate to the
+   original paper embeddings, exploring whether citation context captures
+   similar semantic information.
+
+## Output
+
+A DataFrame with columns:
+
+| Column | Description |
+|---|---|
+| `citing_id` | ID of the citing paper |
+| `n_cited` | Number of papers cited |
+| `permutation_index` | Index of this particular citation ordering |
+| `aggregated_title` | Concatenated cited-paper titles |
+| `embedding` | Embedding vector of the aggregated title string |
+
+## Files in this directory
+
+| File | Description |
+|---|---|
+| `__init__.py` | Module code |
+| `embeddings_and_order.ipynb` | Notebook exploring aggregation experiments |