bagz 0.3.0__tar.gz

bagz-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,60 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: test (${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install system build dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev ninja-build
+
+      - name: Set up uv and Python ${{ matrix.python-version }}
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build wheels
+        run: |
+          uv build --wheel --out-dir dist .
+          uv build --wheel --out-dir dist plugins/gcs
+
+      - name: Verify plugin loading and run tests against built wheels
+        run: |
+          uv run --with dist/bagz-*.whl --with dist/bagz_gcs-*.whl --with pytest --with absl-py --with numpy bash -c '
+            python -c "import bagz; import bagz_gcs; print(\"Loaded bagz and bagz_gcs from wheel successfully\")"
+            python -m pytest src
+          '

bagz-0.3.0/.github/workflows/release.yml

@@ -0,0 +1,105 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Release
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      target:
+        description: 'Publishing target'
+        required: true
+        default: 'testpypi'
+        type: choice
+        options:
+          - testpypi
+          - pypi
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Build bagz core wheels
+        uses: pypa/cibuildwheel@v2.22.0
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+
+      - name: Build bagz-gcs plugin wheels
+        uses: pypa/cibuildwheel@v2.22.0
+        with:
+          package-dir: plugins/gcs
+          output-dir: wheelhouse
+
+      - name: Upload wheel artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels
+          path: wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distributions
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build sdists
+        run: |
+          uv build --sdist --out-dir dist .
+          uv build --sdist --out-dir dist plugins/gcs
+
+      - name: Upload sdist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: Publish to ${{ inputs.target || 'pypi' }}
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.target || 'pypi' }}
+      url: ${{ (inputs.target == 'testpypi') && 'https://test.pypi.org/p/bagz' || 'https://pypi.org/p/bagz' }}
+
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to PyPI or TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: ${{ (inputs.target == 'testpypi') && 'https://test.pypi.org/legacy/' || '' }}
+          packages-dir: dist/

bagz-0.3.0/.gitignore

@@ -0,0 +1,23 @@
+# System files
+.DS_Store
+
+# Python
+__pycache__/
+*.pyc
+*.pyd
+*.so
+.Python
+env/
+venv/
+.venv/
+uv.lock
+
+# BUILD
+build/
+wheelhouse/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo

bagz-0.3.0/CMakeLists.txt

@@ -0,0 +1,316 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.28)
+project(bagz LANGUAGES CXX)
+
+include(FetchContent)
+
+option(BUILD_SHARED_LIBS ON)
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
+set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS TRUE)
+set(CMAKE_SKIP_INSTALL_RULES ON)
+
+option(ENABLE_CORE "Enable Core bagz extension" ON)
+option(ENABLE_GCS "Enable GCS filesystem plugin" ON)
+
+# Temporarily disable warnings for third-party libraries
+set(SAVED_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -Wno-dev")
+
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module OPTIONAL_COMPONENTS Development.SABIModule)
+
+include(cmake/third_party_absl.cmake)
+include(cmake/third_party_nanobind.cmake)
+include(cmake/third_party_zstd.cmake)
+
+if(ENABLE_GCS)
+  include(cmake/third_party_gcs.cmake)
+endif()
+
+# Restore original CXX_FLAGS
+set(CMAKE_CXX_FLAGS "${SAVED_CMAKE_CXX_FLAGS}")
+
+set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS FALSE)
+
+set(CMAKE_SKIP_INSTALL_RULES OFF)
+
+include(cmake/bagz_cc.cmake)
+
+set(bagz_internal_sources
+  "src/internal/bagz_shard_reader.cc"
+  "src/internal/limits_name.cc"
+  "src/internal/parallel_do.cc"
+  "src/internal/records_limits.cc"
+  "src/internal/shard_indexing.cc"
+  "src/internal/zstd_compressor.cc"
+  "src/internal/zstd_decompressor.cc"
+)
+set(bagz_internal_headers
+  "src/internal/bagz_shard_reader.h"
+  "src/internal/limits_name.h"
+  "src/internal/parallel_do.h"
+  "src/internal/records_limits.h"
+  "src/internal/recycling_pool.h"
+  "src/internal/serialize.h"
+  "src/internal/shard_indexing.h"
+  "src/internal/zstd_compressor.h"
+  "src/internal/zstd_decompressor.h"
+)
+
+bagz_cc_library(
+  bagz_internal
+  SOURCES ${bagz_internal_sources}
+  HEADERS ${bagz_internal_headers}
+  DEPS
+    absl::strings
+    absl::status
+    absl::statusor
+    absl::flat_hash_map
+    absl::log
+    libzstd_static
+)
+
+set(bagz_file_system_sources
+  "src/file/file_system/file_system.cc"
+  "src/file/file_system/pread_file.cc"
+  "src/file/file_system/shard_spec.cc"
+  "src/file/file_system/string_pread_file.cc"
+)
+set(bagz_file_system_headers
+  "src/file/file_system/file_system.h"
+  "src/file/file_system/mock_file_system.h"
+  "src/file/file_system/pread_file.h"
+  "src/file/file_system/shard_spec.h"
+  "src/file/file_system/string_pread_file.h"
+  "src/file/file_system/write_file.h"
+)
+
+bagz_cc_library(
+  bagz_file_system
+  SOURCES ${bagz_file_system_sources}
+  HEADERS ${bagz_file_system_headers}
+  DEPS
+    absl::flat_hash_map
+    absl::log
+    absl::status
+    absl::statusor
+    absl::strings
+    bagz_internal
+)
+
+set(bagz_file_system_posix_sources
+  "src/file/file_systems/posix/posix_file_system.cc"
+)
+set(bagz_file_system_posix_headers
+  "src/file/file_systems/posix/posix_file_system.h"
+)
+
+bagz_cc_library(
+  bagz_file_system_posix
+  SOURCES ${bagz_file_system_posix_sources}
+  HEADERS ${bagz_file_system_posix_headers}
+  DEPS
+    absl::status
+    absl::statusor
+    absl::strings
+    bagz_file_system
+    bagz_internal
+)
+
+if(ENABLE_GCS)
+  set(bagz_file_system_gcs_sources
+    "src/file/file_systems/gcs/gcs_file_system.cc"
+  )
+  set(bagz_file_system_gcs_headers
+    "src/file/file_systems/gcs/gcs_file_system.h"
+  )
+
+  bagz_cc_library(
+    bagz_file_system_gcs
+    SOURCES ${bagz_file_system_gcs_sources}
+    HEADERS ${bagz_file_system_gcs_headers}
+    DEPS
+      absl::log
+      absl::status
+      absl::statusor
+      absl::strings
+      bagz_file_system
+      bagz_internal
+      google-cloud-cpp::storage
+  )
+endif()
+
+set(bagz_file_sources
+  "src/file/file.cc"
+  "src/file/registry/file_system_registry.cc"
+  "src/file/registry/register_file_systems.cc"
+)
+set(bagz_file_headers
+  "src/file/file.h"
+  "src/file/registry/file_system_registry.h"
+  "src/file/registry/register_file_systems.h"
+)
+
+bagz_cc_library(
+  bagz_file
+  SOURCES ${bagz_file_sources}
+  HEADERS ${bagz_file_headers}
+  DEPS
+    absl::log
+    absl::status
+    absl::statusor
+    absl::strings
+    bagz_internal
+    bagz_file_system
+    bagz_file_system_posix
+)
+
+set(bagz_core_sources
+  "src/bagz_index.cc"
+  "src/bagz_multi_index.cc"
+  "src/bagz_reader.cc"
+  "src/bagz_writer.cc"
+)
+set(bagz_core_headers
+  "src/bagz_index.h"
+  "src/bagz_iterator.h"
+  "src/bagz_multi_index.h"
+  "src/bagz_options.h"
+  "src/bagz_reader.h"
+  "src/bagz_writer.h"
+)
+
+bagz_cc_library(
+  bagz_core
+  SOURCES ${bagz_core_sources}
+  HEADERS ${bagz_core_headers}
+  DEPS
+    absl::flat_hash_map
+    absl::log
+    absl::status
+    absl::statusor
+    absl::strings
+    bagz_file
+    bagz_internal
+    libzstd_static
+)
+
+set(bagz_python_sources
+  "src/python/bagz.cc"
+  "src/python/bagz_index.cc"
+  "src/python/bagz_multi_index.cc"
+  "src/python/bagz_options.cc"
+  "src/python/bagz_reader.cc"
+  "src/python/bagz_writer.cc"
+)
+
+option(ENABLE_CORE "Enable Core bagz extension" ON)
+if(ENABLE_CORE)
+  bagz_nanobind_extension(
+    bagz
+    SOURCES ${bagz_python_sources}
+
+    DEPS
+      absl::flat_hash_map
+      absl::log
+      absl::status
+      absl::statusor
+      absl::strings
+      bagz_core
+  )
+endif()
+
+if(ENABLE_GCS)
+  bagz_nanobind_extension(
+    bagz_gcs
+    SOURCES "src/file/file_systems/gcs/bagz_gcs.cc"
+    DEPS
+      bagz_file_system_gcs
+      bagz_file
+      google-cloud-cpp::storage
+      absl::log
+      absl::status
+  )
+endif()
+
+file(WRITE ${CMAKE_BINARY_DIR}/__init__.py [[# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from bagz.lib.bagz import *
+
+import importlib.metadata
+import logging
+
+def _load_plugins():
+  try:
+    eps = importlib.metadata.entry_points(group='bagz.plugins')
+  except TypeError:
+    eps = importlib.metadata.entry_points().get('bagz.plugins', [])
+
+  for ep in eps:
+    try:
+      ep.load()
+    except Exception as e:
+      logging.warning(f"Failed to load bagz plugin {ep.name}: {e}")
+
+_load_plugins()
+]])
+
+
+if(ENABLE_CORE)
+  install(
+    FILES "${CMAKE_BINARY_DIR}/__init__.py"
+    DESTINATION "bagz"
+  )
+
+  install(
+    TARGETS bagz
+    DESTINATION "bagz/lib"
+  )
+endif()
+
+if(ENABLE_GCS)
+  install(
+    TARGETS bagz_gcs
+    DESTINATION "bagz_gcs"
+  )
+endif()
+
+install(
+  DIRECTORY "beam/"
+  DESTINATION bagz/beam
+  FILES_MATCHING
+  PATTERN "*.py"
+)
+
+install(
+  FILES "src/python/bagz.pyi"
+  DESTINATION "bagz"
+  RENAME "__init__.pyi"
+)
+

bagz-0.3.0/CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# How to contribute
+
+We'd love to accept your patches and contributions to this project.
+
+## Before you begin
+
+### Sign our Contributor License Agreement
+
+Contributions to this project must be accompanied by a
+[Contributor License Agreement](https://cla.developers.google.com/about) (CLA).
+You (or your employer) retain the copyright to your contribution; this simply
+gives us permission to use and redistribute your contributions as part of the
+project.
+
+If you or your current employer have already signed the Google CLA (even if it
+was for a different project), you probably don't need to do it again.
+
+Visit <https://cla.developers.google.com/> to see your current agreements or to
+sign a new one.
+
+### Review our community guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google/conduct/).
+
+## Contribution process
+
+### Code reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.