cf-pipeline-engine 0.1.0__tar.gz

cf_pipeline_engine-0.1.0/CMakeLists.txt

@@ -0,0 +1,140 @@
+cmake_minimum_required(VERSION 3.20)
+project(cf_pipeline_engine LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
+
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+execute_process(
+  COMMAND ${Python3_EXECUTABLE} -c "from pathlib import Path; import cf_ontology.datatypes_mapping as m; print(Path(m.__file__).resolve().with_name('type_registry.v0.json'))"
+  OUTPUT_VARIABLE TYPE_REGISTRY_V0_SOURCE_PATH
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+include(FetchContent)
+FetchContent_Declare(
+  nlohmann_json
+  GIT_REPOSITORY https://github.com/nlohmann/json.git
+  GIT_TAG v3.11.2
+)
+FetchContent_MakeAvailable(nlohmann_json)
+
+add_library(cf_pipeline_v2_utils STATIC
+  src/common/sha256.cpp
+  src/common/type_registry.cc
+  src/compiler/jsonld_loader.cpp
+  src/compiler/signature.cpp
+)
+
+target_include_directories(cf_pipeline_v2_utils PUBLIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+
+set(TYPE_REGISTRY_V0_FILENAME "type_registry.v0.json")
+
+if (TYPE_REGISTRY_V0_SOURCE_PATH STREQUAL "" OR NOT EXISTS "${TYPE_REGISTRY_V0_SOURCE_PATH}")
+  message(FATAL_ERROR "cf_ontology type_registry.v0.json not found in '${TYPE_REGISTRY_V0_SOURCE_PATH}'. Reinstall cf-ontology.")
+endif()
+
+file(TO_CMAKE_PATH "${TYPE_REGISTRY_V0_SOURCE_PATH}" TYPE_REGISTRY_V0_SOURCE_PATH)
+
+target_compile_definitions(cf_pipeline_v2_utils PRIVATE
+  CF_TYPE_REGISTRY_V0_PATH="${TYPE_REGISTRY_V0_SOURCE_PATH}"
+  CF_TYPE_REGISTRY_V0_FILENAME="${TYPE_REGISTRY_V0_FILENAME}"
+)
+
+target_link_libraries(cf_pipeline_v2_utils PUBLIC nlohmann_json::nlohmann_json)
+
+add_executable(cf_siggen tools/cf_siggen_main.cpp)
+target_include_directories(cf_siggen PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+target_link_libraries(cf_siggen PRIVATE cf_pipeline_v2_utils)
+
+set(GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated)
+file(MAKE_DIRECTORY ${GENERATED_DIR})
+
+add_custom_command(
+  OUTPUT ${GENERATED_DIR}/cf_test_signature_hashes.h
+  COMMAND ${Python3_EXECUTABLE} -m cogniflow_pipeline_sdk.siggen
+          --steps ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_steps.jsonld
+          --out ${GENERATED_DIR}/cf_test_signature_hashes.h
+          --scratch
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_steps.jsonld
+  COMMENT "Generating test signature hashes"
+)
+
+add_library(cf_pipeline_v2_core STATIC
+  src/compiler/pipeline_parser.cpp
+  src/compiler/plan_builder.cpp
+  src/plugin_loader/plugin_loader.cpp
+  src/runtime/concurrency_controller.cpp
+  src/scheduler/thread_pool.cpp
+  src/scheduler/scheduler.cpp
+)
+
+target_include_directories(cf_pipeline_v2_core PUBLIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+
+target_link_libraries(cf_pipeline_v2_core PUBLIC cf_pipeline_v2_utils)
+
+if(UNIX AND NOT APPLE)
+  target_link_libraries(cf_pipeline_v2_core PUBLIC dl)
+endif()
+
+add_executable(cf_pipeline_v2 src/main.cpp)
+
+target_link_libraries(cf_pipeline_v2 PRIVATE cf_pipeline_v2_core)
+
+add_custom_command(TARGET cf_pipeline_v2 POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+          ${TYPE_REGISTRY_V0_SOURCE_PATH}
+          $<TARGET_FILE_DIR:cf_pipeline_v2>/${TYPE_REGISTRY_V0_FILENAME}
+)
+
+# Tests
+add_library(cf_pipeline_v2_test_plugin SHARED
+  ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_plugin.cpp
+  ${GENERATED_DIR}/cf_test_signature_hashes.h
+)
+
+target_include_directories(cf_pipeline_v2_test_plugin PUBLIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+  ${GENERATED_DIR}
+)
+
+target_link_libraries(cf_pipeline_v2_test_plugin PUBLIC cf_pipeline_v2_utils)
+target_compile_definitions(cf_pipeline_v2_test_plugin PRIVATE CF_STEP_ABI_EXPORTS)
+
+set_target_properties(cf_pipeline_v2_test_plugin PROPERTIES
+  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests
+  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests
+)
+
+add_executable(cf_pipeline_v2_tests tests/test_main.cpp)
+
+target_include_directories(cf_pipeline_v2_tests PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+
+target_link_libraries(cf_pipeline_v2_tests PRIVATE cf_pipeline_v2_core)
+
+add_custom_command(TARGET cf_pipeline_v2_tests POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+          ${TYPE_REGISTRY_V0_SOURCE_PATH}
+          $<TARGET_FILE_DIR:cf_pipeline_v2_tests>/${TYPE_REGISTRY_V0_FILENAME}
+)
+
+install(TARGETS cf_siggen RUNTIME DESTINATION bin)
+install(TARGETS cf_pipeline_v2 RUNTIME DESTINATION bin)
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION include)
+install(FILES ${TYPE_REGISTRY_V0_SOURCE_PATH} DESTINATION bin)

cf_pipeline_engine-0.1.0/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.2
+Name: cf-pipeline-engine
+Version: 0.1.0
+Summary: Cogniflow pipeline engine (C++ core) with a thin Python package wrapper.
+Keywords: cogniflow,pipeline,engine,cpp
+Author-Email: ODEA Project <info@odea-project.org>
+License: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: C++
+Requires-Python: >=3.10
+Provides-Extra: test
+Requires-Dist: pytest>=8.0; extra == "test"
+Description-Content-Type: text/markdown
+
+# cf-pipeline-engine
+
+The Cogniflow pipeline engine implemented in C++ (compiler, scheduler, runtime).
+
+This folder is structured as a Python package for consistency with the other
+`cf_*` components, even though the core implementation is native C++ and built
+via CMake.
+
+## Build (CMake)
+
+```bash
+cmake -S . -B build
+cmake --build build
+```
+
+## Python package wrapper
+
+The Python package is intentionally thin and provides access to the packaged
+native engine assets:
+
+- `cf_pipeline_engine.cf_pipeline_v2_path()`
+- `cf_pipeline_engine.cf_siggen_path()`
+- `cf_pipeline_engine.cf_engine_include_path()`
+- `cf_pipeline_engine.cf_type_registry_path()`
+
+Published distribution name:
+
+```bash
+pip install cf-pipeline-engine
+```
+
+The published wheel installs:
+
+- `bin/cf_pipeline_v2(.exe)`
+- `bin/cf_siggen(.exe)`
+- `bin/type_registry.v0.json`
+- `include/*.h`
+
+## Publishing
+
+`cf_pipeline_engine` is published with the dedicated Windows workflow:
+
+- Workflow: `.github/workflows/cf_pipeline_engine_windows_publish.yml`
+- Package directory: `sandcastle/cf_pipeline/cf_pipeline_engine`
+- PyPI tag: `cf-pipeline-engine-v<version>`
+- TestPyPI tag: `cf-pipeline-engine-v<version>-test`
+
+Local preflight:
+
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/mimic_windows_python_publish_workflow.ps1 `
+  -WorkflowFile .github/workflows/cf_pipeline_engine_windows_publish.yml `
+  -PackageDir sandcastle/cf_pipeline/cf_pipeline_engine `
+  -PythonExe py `
+  -PythonVersion 3.13
+```
+
+Queue a dry-run dispatch:
+
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/queue_windows_python_publish_workflow.ps1 `
+  -WorkflowFile .github/workflows/cf_pipeline_engine_windows_publish.yml `
+  -PackageDir sandcastle/cf_pipeline/cf_pipeline_engine `
+  -PublishTarget testpypi `
+  -Ref main `
+  -RequireLocalPass `
+  -DryRun
+```
+
+## OPC UA Demo Pipeline Sink
+
+The existing demo pipeline
+`examples/opcua_fifo_avg_to_duckdb_parquet_triggered.jsonld` now uses
+`cfsink:DataHiveParquetSinkStep` from `cf_basic_sinks`.
+
+The sink writes through the C++ gatekeeper library `cf_datahive_cpp`,
+producing one committed data hive run with 20 rows (`cycle_id` 0..19) and no
+`archive.jsonl`.
+
+Run the one-click demo:
+
+```powershell
+.\scripts\fresh_install.ps1 -Clean -RunDemo
+```
+
+Expected output layout after one demo session:
+
+- `workspace/<data_hive>/opcua_fifo_avg/latest.txt`
+- `workspace/<data_hive>/opcua_fifo_avg/runs/<run_id>/manifest.json`
+- `workspace/<data_hive>/opcua_fifo_avg/runs/<run_id>/tables/measurements/part-*.parquet`

cf_pipeline_engine-0.1.0/README.md

@@ -0,0 +1,90 @@
+# cf-pipeline-engine
+
+The Cogniflow pipeline engine implemented in C++ (compiler, scheduler, runtime).
+
+This folder is structured as a Python package for consistency with the other
+`cf_*` components, even though the core implementation is native C++ and built
+via CMake.
+
+## Build (CMake)
+
+```bash
+cmake -S . -B build
+cmake --build build
+```
+
+## Python package wrapper
+
+The Python package is intentionally thin and provides access to the packaged
+native engine assets:
+
+- `cf_pipeline_engine.cf_pipeline_v2_path()`
+- `cf_pipeline_engine.cf_siggen_path()`
+- `cf_pipeline_engine.cf_engine_include_path()`
+- `cf_pipeline_engine.cf_type_registry_path()`
+
+Published distribution name:
+
+```bash
+pip install cf-pipeline-engine
+```
+
+The published wheel installs:
+
+- `bin/cf_pipeline_v2(.exe)`
+- `bin/cf_siggen(.exe)`
+- `bin/type_registry.v0.json`
+- `include/*.h`
+
+## Publishing
+
+`cf_pipeline_engine` is published with the dedicated Windows workflow:
+
+- Workflow: `.github/workflows/cf_pipeline_engine_windows_publish.yml`
+- Package directory: `sandcastle/cf_pipeline/cf_pipeline_engine`
+- PyPI tag: `cf-pipeline-engine-v<version>`
+- TestPyPI tag: `cf-pipeline-engine-v<version>-test`
+
+Local preflight:
+
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/mimic_windows_python_publish_workflow.ps1 `
+  -WorkflowFile .github/workflows/cf_pipeline_engine_windows_publish.yml `
+  -PackageDir sandcastle/cf_pipeline/cf_pipeline_engine `
+  -PythonExe py `
+  -PythonVersion 3.13
+```
+
+Queue a dry-run dispatch:
+
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/queue_windows_python_publish_workflow.ps1 `
+  -WorkflowFile .github/workflows/cf_pipeline_engine_windows_publish.yml `
+  -PackageDir sandcastle/cf_pipeline/cf_pipeline_engine `
+  -PublishTarget testpypi `
+  -Ref main `
+  -RequireLocalPass `
+  -DryRun
+```
+
+## OPC UA Demo Pipeline Sink
+
+The existing demo pipeline
+`examples/opcua_fifo_avg_to_duckdb_parquet_triggered.jsonld` now uses
+`cfsink:DataHiveParquetSinkStep` from `cf_basic_sinks`.
+
+The sink writes through the C++ gatekeeper library `cf_datahive_cpp`,
+producing one committed data hive run with 20 rows (`cycle_id` 0..19) and no
+`archive.jsonl`.
+
+Run the one-click demo:
+
+```powershell
+.\scripts\fresh_install.ps1 -Clean -RunDemo
+```
+
+Expected output layout after one demo session:
+
+- `workspace/<data_hive>/opcua_fifo_avg/latest.txt`
+- `workspace/<data_hive>/opcua_fifo_avg/runs/<run_id>/manifest.json`
+- `workspace/<data_hive>/opcua_fifo_avg/runs/<run_id>/tables/measurements/part-*.parquet`

cf_pipeline_engine-0.1.0/cf-package.yaml

@@ -0,0 +1,154 @@
+schema_version: 1
+package_id: "cf-pipeline-engine"
+defaults:
+  owner: "cf-team"
+  last_reviewed: "2026-02-09"
+files:
+  entries:
+  - path: CMakeLists.txt
+    status: current
+    purpose: build
+  - path: README.md
+    status: current
+    purpose: docs
+  - path: cf-package.yaml
+    status: current
+    purpose: metadata
+  - path: pyproject.toml
+    status: current
+    purpose: build
+  - path: cf_pipeline_engine/__init__.py
+    status: current
+    purpose: source
+  - path: examples/omp_reduction_parallel.jsonld
+    status: current
+    purpose: examples
+  - path: examples/opcua_fifo_avg_to_duckdb_parquet_triggered.jsonld
+    status: current
+    purpose: examples
+  - path: examples/opcua_temp_fifo_avg_to_duckdb_parquet_triggered.jsonld
+    status: current
+    purpose: examples
+  - path: include/cf_datacube.h
+    status: current
+    purpose: source
+  - path: include/cf_plugin_table.h
+    status: current
+    purpose: source
+  - path: include/cf_step_abi.h
+    status: current
+    purpose: source
+  - path: include/cf_step_utils.h
+    status: current
+    purpose: source
+  - path: src/common/sha256.cpp
+    status: current
+    purpose: source
+  - path: src/common/sha256.h
+    status: current
+    purpose: source
+  - path: src/compiler/jsonld_loader.cpp
+    status: current
+    purpose: source
+  - path: src/compiler/jsonld_loader.h
+    status: current
+    purpose: source
+  - path: src/compiler/pipeline_parser.cpp
+    status: current
+    purpose: source
+  - path: src/compiler/pipeline_parser.h
+    status: current
+    purpose: source
+  - path: src/compiler/plan_builder.cpp
+    status: current
+    purpose: source
+  - path: src/compiler/plan_builder.h
+    status: current
+    purpose: source
+  - path: src/compiler/signature.cpp
+    status: current
+    purpose: source
+  - path: src/compiler/signature.h
+    status: current
+    purpose: source
+  - path: src/main.cpp
+    status: current
+    purpose: source
+  - path: src/plugin_loader/plugin_loader.cpp
+    status: current
+    purpose: source
+  - path: src/plugin_loader/plugin_loader.h
+    status: current
+    purpose: source
+  - path: src/runtime/concurrency_controller.cpp
+    status: current
+    purpose: source
+  - path: src/runtime/concurrency_controller.h
+    status: current
+    purpose: source
+  - path: src/runtime/datacube.h
+    status: current
+    purpose: source
+  - path: src/runtime/execution_plan.h
+    status: current
+    purpose: source
+  - path: src/scheduler/scheduler.cpp
+    status: current
+    purpose: source
+  - path: src/scheduler/scheduler.h
+    status: current
+    purpose: source
+  - path: src/scheduler/thread_pool.cpp
+    status: current
+    purpose: source
+  - path: src/scheduler/thread_pool.h
+    status: current
+    purpose: source
+  - path: tests/pipeline_budget_aware_hints.jsonld
+    status: current
+    purpose: tests
+  - path: tests/pipeline_budget_deterministic.jsonld
+    status: current
+    purpose: tests
+  - path: tests/pipeline_budget_weighted.jsonld
+    status: current
+    purpose: tests
+  - path: tests/pipeline_edges.jsonld
+    status: current
+    purpose: tests
+  - path: tests/pipeline_invalid.jsonld
+    status: current
+    purpose: tests
+  - path: tests/pipeline_levels.jsonld
+    status: current
+    purpose: tests
+  - path: tests/pipeline_mismatch.jsonld
+    status: current
+    purpose: tests
+  - path: tests/pipeline_parallel.jsonld
+    status: current
+    purpose: tests
+  - path: tests/test_main.cpp
+    status: current
+    purpose: tests
+  - path: tests/test_plugin.cpp
+    status: current
+    purpose: tests
+  - path: tests/test_opcua_fifo_avg_datahive_integration.py
+    status: current
+    purpose: tests
+  - path: tests/test_install_surface.py
+    status: current
+    purpose: tests
+  - path: tests/test_steps.jsonld
+    status: current
+    purpose: tests
+  - path: tests/test_steps_mismatch.jsonld
+    status: current
+    purpose: tests
+  - path: tools/bench_parallelism.py
+    status: current
+    purpose: tools
+  - path: tools/cf_siggen_main.cpp
+    status: current
+    purpose: tools

cf_pipeline_engine-0.1.0/cf_pipeline_engine/__init__.py

@@ -0,0 +1,45 @@
+"""Python package wrapper for the C++ Cogniflow pipeline engine."""
+
+from __future__ import annotations
+
+from importlib import metadata
+from pathlib import Path
+
+__version__ = "0.1.0"
+
+
+def _locate_distribution_file(*relative_candidates: str) -> Path:
+    distribution = metadata.distribution("cf-pipeline-engine")
+    files = distribution.files or []
+    for candidate in relative_candidates:
+        for entry in files:
+            if entry.as_posix() == candidate:
+                return Path(distribution.locate_file(entry)).resolve()
+    raise FileNotFoundError(
+        f"Could not locate any of {relative_candidates!r} in cf-pipeline-engine distribution files."
+    )
+
+
+def cf_pipeline_v2_path() -> str:
+    return str(_locate_distribution_file("bin/cf_pipeline_v2.exe", "bin/cf_pipeline_v2"))
+
+
+def cf_siggen_path() -> str:
+    return str(_locate_distribution_file("bin/cf_siggen.exe", "bin/cf_siggen"))
+
+
+def cf_engine_include_path() -> str:
+    return str(_locate_distribution_file("include/cf_step_abi.h").parent)
+
+
+def cf_type_registry_path() -> str:
+    return str(_locate_distribution_file("bin/type_registry.v0.json"))
+
+
+__all__ = [
+    "__version__",
+    "cf_pipeline_v2_path",
+    "cf_siggen_path",
+    "cf_engine_include_path",
+    "cf_type_registry_path",
+]

cf_pipeline_engine-0.1.0/examples/invocations/opcua_fifo_avg.endpoint_4840.invocation.jsonld

@@ -0,0 +1,22 @@
+{
+  "@context": {
+    "cf": "https://cogniflow.org/ns#",
+    "ex": "http://cogniflow.org/examples/pipeline#",
+    "schema": "https://schema.org/"
+  },
+  "@id": "ex:opcua_fifo_avg_invocation_endpoint_4840",
+  "@type": "cf:PipelineInvocation",
+  "cf:invokesPipeline": "ex:OpcuaFifoMeanToDuckDb",
+  "cf:overridesEntryPoint": {
+    "@type": "cf:EntryPointOverride",
+    "cf:targetsNode": "ex:n1",
+    "cf:overrideSettings": {
+      "@type": "schema:StructuredValue",
+      "schema:additionalProperty": {
+        "@type": "schema:PropertyValue",
+        "schema:propertyID": "endpoint",
+        "schema:value": "opc.tcp://127.0.0.1:4840/VirtualPhServer"
+      }
+    }
+  }
+}

cf_pipeline_engine-0.1.0/examples/invocations/opcua_fifo_avg.endpoint_4841.invocation.jsonld

@@ -0,0 +1,22 @@
+{
+  "@context": {
+    "cf": "https://cogniflow.org/ns#",
+    "ex": "http://cogniflow.org/examples/pipeline#",
+    "schema": "https://schema.org/"
+  },
+  "@id": "ex:opcua_fifo_avg_invocation_endpoint_4841",
+  "@type": "cf:PipelineInvocation",
+  "cf:invokesPipeline": "ex:OpcuaFifoMeanToDuckDb",
+  "cf:overridesEntryPoint": {
+    "@type": "cf:EntryPointOverride",
+    "cf:targetsNode": "ex:n1",
+    "cf:overrideSettings": {
+      "@type": "schema:StructuredValue",
+      "schema:additionalProperty": {
+        "@type": "schema:PropertyValue",
+        "schema:propertyID": "endpoint",
+        "schema:value": "opc.tcp://127.0.0.1:4841/VirtualPhServer"
+      }
+    }
+  }
+}

cf_pipeline_engine-0.1.0/examples/omp_reduction_parallel.jsonld

@@ -0,0 +1,38 @@
+{
+  "@context": [
+    "../../../cf_ontology/src/cf_ontology/ontology/cf_context.jsonld",
+    {
+      "cfbd": "http://cogniflow.org/basic-dev#"
+    }
+  ],
+  "@id": "https://cogniflow.org/examples/pipeline/omp_reduction_parallel",
+  "@type": "cf:Pipeline",
+  "cf:hasStepsHeader": {
+    "cf:stepsPath": [
+      "../../../cf_basic_steps/cf_basic_dev/src/cf_basic_dev/steps.jsonld"
+    ]
+  },
+  "cf:hasPluginsHeader": {
+    "cf:pluginPath": [
+      "../../../cf_basic_steps/cf_basic_dev/src/cf_basic_dev/bin"
+    ]
+  },
+  "cf:hasNode": [
+    {
+      "@id": "n1",
+      "@type": "cf:PipelineNode",
+      "cf:nodeDefinition": "cfbd:OmpReductionStep",
+      "cf:hasParameter": [
+        {"cf:parameterName": "terms", "cf:parameterValue": 5000000}
+      ]
+    },
+    {
+      "@id": "n2",
+      "@type": "cf:PipelineNode",
+      "cf:nodeDefinition": "cfbd:OmpReductionStep",
+      "cf:hasParameter": [
+        {"cf:parameterName": "terms", "cf:parameterValue": 5000000}
+      ]
+    }
+  ]
+}