textflow-ir 0.1.0__tar.gz

textflow_ir-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+/.DS_Store
+/__pycache__/
+*.pyc
+*.egg-info/
+/.venv/
+/build/
+/dist/
+/wheelhouse/
+/.local/
+/.tmp_bench_corpus/
+/.idea/

textflow_ir-0.1.0/CMakeLists.txt

@@ -0,0 +1,171 @@
+cmake_minimum_required(VERSION 3.24)
+
+project(
+  termflow
+  VERSION 0.1.0
+  DESCRIPTION "English text analysis library for C++"
+  LANGUAGES CXX)
+
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+
+option(TERMFLOW_BUILD_TESTS "Build the termflow test suite" ON)
+option(TERMFLOW_BUILD_EXAMPLES "Build the termflow example programs" ON)
+option(TERMFLOW_BUILD_TOOLS "Build the termflow command-line tools" ON)
+option(TERMFLOW_BUILD_PYTHON "Build the termflow Python bindings" OFF)
+option(TERMFLOW_INSTALL_CPP_ARTIFACTS "Install C++ library and development artifacts" ON)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+find_package(ICU REQUIRED COMPONENTS i18n uc)
+
+if(TERMFLOW_BUILD_PYTHON)
+  find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module)
+  find_package(pybind11 CONFIG REQUIRED)
+endif()
+
+add_library(termflow
+  src/analysis/analyzer.cpp
+  src/analysis/ascii_folding_filter.cpp
+  src/analysis/english_analyzer.cpp
+  src/analysis/english_possessive_filter.cpp
+  src/analysis/lower_case_filter.cpp
+  src/analysis/porter_stem_filter.cpp
+  src/analysis/standard_tokenizer.cpp
+  src/analysis/stop_filter.cpp
+  src/analysis/term_extractor.cpp
+  src/analysis/unicode_normalize_filter.cpp
+  src/query/query_analyzer.cpp
+  src/query/query_parser.cpp
+  src/query/rewrite_loader.cpp
+  src/query/rewrite_validator.cpp
+  src/util/porter_stemmer.cpp
+  src/util/unicode.cpp)
+
+add_library(termflow::termflow ALIAS termflow)
+
+target_compile_features(termflow PUBLIC cxx_std_20)
+target_link_libraries(termflow PUBLIC ICU::i18n ICU::uc)
+target_include_directories(termflow
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/src)
+
+if(MSVC)
+  target_compile_options(termflow PRIVATE /W4 /permissive-)
+else()
+  target_compile_options(termflow PRIVATE -Wall -Wextra -Wpedantic)
+endif()
+
+set_target_properties(termflow PROPERTIES
+  EXPORT_NAME termflow
+  POSITION_INDEPENDENT_CODE ON
+  VERSION ${PROJECT_VERSION}
+  SOVERSION 0)
+
+if(TERMFLOW_BUILD_EXAMPLES)
+  add_executable(termflow_analyze examples/analyze_text.cpp)
+  target_link_libraries(termflow_analyze PRIVATE termflow::termflow)
+
+  add_executable(termflow_extract_terms examples/extract_terms.cpp)
+  target_link_libraries(termflow_extract_terms PRIVATE termflow::termflow)
+
+  add_executable(termflow_custom_analyzer examples/custom_analyzer.cpp)
+  target_link_libraries(termflow_custom_analyzer PRIVATE termflow::termflow)
+
+  add_executable(termflow_analyze_query examples/analyze_query.cpp)
+  target_link_libraries(termflow_analyze_query PRIVATE termflow::termflow)
+endif()
+
+if(TERMFLOW_BUILD_TOOLS)
+endif()
+
+if(TERMFLOW_BUILD_PYTHON)
+  set(TERMFLOW_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/python/termflow)
+  file(MAKE_DIRECTORY ${TERMFLOW_PYTHON_PACKAGE_DIR})
+  configure_file(
+    ${CMAKE_CURRENT_SOURCE_DIR}/python/termflow/__init__.py
+    ${TERMFLOW_PYTHON_PACKAGE_DIR}/__init__.py
+    COPYONLY)
+
+  pybind11_add_module(termflow_python MODULE bindings/python/module.cpp)
+  target_link_libraries(termflow_python PRIVATE termflow::termflow)
+  set_target_properties(termflow_python PROPERTIES
+    OUTPUT_NAME _termflow
+    LIBRARY_OUTPUT_DIRECTORY ${TERMFLOW_PYTHON_PACKAGE_DIR}
+    RUNTIME_OUTPUT_DIRECTORY ${TERMFLOW_PYTHON_PACKAGE_DIR})
+
+  install(
+    TARGETS termflow_python
+    LIBRARY DESTINATION termflow
+    RUNTIME DESTINATION termflow)
+
+  install(
+    FILES python/termflow/__init__.py
+    DESTINATION termflow)
+endif()
+
+if(TERMFLOW_BUILD_TESTS)
+  enable_testing()
+
+  add_executable(termflow_tests
+    tests/english_analyzer_tests.cpp
+    tests/filter_tests.cpp
+    tests/query_tests.cpp
+    tests/standard_tokenizer_tests.cpp
+    tests/term_extractor_tests.cpp
+    tests/test_framework.cpp
+    tests/test_main.cpp
+    tests/test_helpers.hpp)
+
+  target_link_libraries(termflow_tests PRIVATE termflow::termflow)
+
+  add_test(NAME termflow_tests COMMAND termflow_tests)
+
+  if(TERMFLOW_BUILD_PYTHON)
+    add_test(
+      NAME termflow_python_smoke
+      COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/python_smoke_test.py)
+    set_tests_properties(
+      termflow_python_smoke
+      PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/python")
+  endif()
+endif()
+if(TERMFLOW_INSTALL_CPP_ARTIFACTS)
+  install(
+    TARGETS termflow
+    EXPORT termflowTargets
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+  install(
+    DIRECTORY include/
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+  install(
+    EXPORT termflowTargets
+    FILE termflowTargets.cmake
+    NAMESPACE termflow::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/termflow)
+
+  write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/termflowConfigVersion.cmake
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY SameMajorVersion)
+
+  configure_package_config_file(
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/termflowConfig.cmake.in
+    ${CMAKE_CURRENT_BINARY_DIR}/termflowConfig.cmake
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/termflow)
+
+  install(
+    FILES
+      ${CMAKE_CURRENT_BINARY_DIR}/termflowConfig.cmake
+      ${CMAKE_CURRENT_BINARY_DIR}/termflowConfigVersion.cmake
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/termflow)
+endif()

textflow_ir-0.1.0/PKG-INFO

@@ -0,0 +1,283 @@
+Metadata-Version: 2.2
+Name: textflow-ir
+Version: 0.1.0
+Summary: English text analysis for information retrieval
+Keywords: text-analysis,information-retrieval,tokenization,stemming,search
+Author: Mustafa Abualsaud
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: C++
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# termflow
+
+`termflow` is a standalone C++20 library for English text analysis aimed at information retrieval workloads.
+
+## Documentation
+
+- [docs/usage.md](docs/usage.md) for day-to-day analyzer, term extraction, query, and Python usage
+- [docs/customization.md](docs/customization.md) for pipeline tuning, query rewrites, and custom analyzers in C++
+- [docs/installation.md](docs/installation.md) for current C++ and Python installation paths
+- [docs/installation-roadmap.md](docs/installation-roadmap.md) for the prioritized installation improvement plan
+
+## Goals
+
+- Library-first design with a small public API
+- CMake-based build and install flow
+- Clean, testable analysis components
+- English-first analysis for v1
+- Optional Python bindings on top of the same core library
+
+## Current Scope
+
+`termflow` currently includes:
+
+- `Token`
+- `CharFilter`
+- `Tokenizer`
+- `TokenFilter`
+- `Analyzer`
+- `TermExtractor`
+- `termflow::query::QueryParser`
+- `termflow::query::RewriteLoader`
+- `termflow::query::RewriteValidator`
+- `termflow::query::QueryAnalyzer`
+- `StandardTokenizer`
+- `UnicodeNormalizeFilter`
+- `LowerCaseFilter`
+- `EnglishPossessiveFilter`
+- `StopFilter`
+- `PorterStemFilter`
+- `AsciiFoldingFilter`
+- `EnglishAnalyzer`
+
+Core analyzer APIs:
+
+- `analyze(text) -> std::vector<Token>`
+- `analyze_terms(text) -> std::vector<std::string>`
+- `analyze_term(text) -> std::optional<std::string>`
+- `analyze_to_string(text, separator = " ") -> std::string`
+- `normalize(text) -> std::string`
+
+`analyze()` runs the full token pipeline. `normalize()` is deliberately lighter-weight and does not tokenize.
+
+The query module is intentionally text-focused. It can parse lightweight query intent, analyze clauses through an existing analyzer, and attach optional query-time rewrites. It does not score documents or execute retrieval.
+
+## Behavior Contract
+
+For `EnglishAnalyzer`, `analyze()` applies stages in this order:
+
+1. `StandardTokenizer`
+2. optional Unicode normalization
+3. optional English possessive stripping
+4. optional lowercasing
+5. optional stop-word filtering
+6. keyword marking from `stem_exclusion_set`
+7. optional Porter stemming
+8. optional ASCII folding
+
+`normalize()` is intentionally narrower. It honors only:
+
+- `unicode_normalization`
+- `lowercase`
+- `ascii_folding`
+
+It does not tokenize, strip possessives, remove stop words, mark keywords, or stem.
+
+Stage toggles are explicit. Disabling a stage removes only that stage. For example, if `lowercase=false` and `stemming=true`, stemming runs on the token text produced by the earlier enabled stages without forcing lowercase.
+
+Custom `stop_words` and `stem_exclusion_set` are normalized using the enabled pre-stem stages so they match the token text seen by stop-word filtering and keyword marking.
+
+## Term Extraction
+
+`TermExtractor` is the public helper for consumers that want finalized term strings with explicit post-analysis filtering. It can enforce minimum length, numeric-term handling, and character policy filters without changing the analyzer itself.
+
+Example:
+
+```cpp
+termflow::EnglishAnalyzerOptions analyzer_options;
+analyzer_options.use_default_stop_words = false;
+termflow::EnglishAnalyzer analyzer(analyzer_options);
+
+termflow::TermExtractionOptions extraction_options;
+extraction_options.min_term_length = 2;
+extraction_options.keep_numeric_terms = false;
+extraction_options.character_policy = termflow::TermCharacterPolicy::ascii_alphabetic;
+
+termflow::TermExtractor extractor(analyzer, extraction_options);
+auto terms = extractor.extract("Running beaches 2024 e-mail");
+```
+
+A runnable example is included at [examples/extract_terms.cpp](examples/extract_terms.cpp).
+
+## Query Analysis
+
+`termflow::query` provides a small query-intent layer on top of existing analyzers. The current scope is:
+
+- clause parsing for bare terms, `+required`, `-excluded`, and quoted groups
+- analyzed query terms produced by an existing `Analyzer`
+- optional query-time rewrites for canonicalization, equivalent alternatives, and expansions
+- analyzer-aware validation for rewrite packs
+
+This is intentionally not a search engine query model. The query module does not score, rank, filter, or execute queries against an index.
+
+Example:
+
+```cpp
+#include "termflow/analysis/english_analyzer.hpp"
+#include "termflow/query/query_analyzer.hpp"
+#include "termflow/query/rewrite_loader.hpp"
+
+termflow::EnglishAnalyzer analyzer;
+termflow::query::RewriteLoader loader;
+termflow::query::RewriteValidator validator(analyzer);
+
+termflow::query::QueryAnalysisOptions options;
+options.rewrites = loader.load_file("config/query-rewrites.tfq");
+const auto validation = validator.validate(options);
+
+termflow::query::QueryAnalyzer query_analyzer(analyzer, options);
+const auto query = query_analyzer.analyze(R"(tv +"car rentals" -expired)");
+```
+
+In that example:
+
+- `tv` is canonically rewritten before it reaches downstream scoring logic
+- `car rentals` keeps its primary analyzed terms and also receives an equivalent alternative
+- `-expired` is parsed as an excluded clause
+
+Rewrites match exact analyzed term sequences at the clause level. Quoted groups remain grouped in the parsed and analyzed query representation, but `termflow` does not implement phrase execution logic.
+
+`RewriteValidator` reports rewrite rules that analyze to no terms, canonical conflicts and cycles, and alternatives that are duplicated or shadowed by the configured deduplication policy. It is a reporting layer only; it does not rewrite or reorder the supplied rules.
+
+Rewrite files use a small declarative syntax:
+
+```text
+# Canonical rewrites
+lotr -> lord of the rings;
+colour -> color;
+
+# Equivalent alternatives
+car rentals => automobile rentals;
+
+# Expansion alternatives
+car rentals ~> vehicle rentals;
+
+# Includes are resolved relative to the current file
+include "common-rewrites.tfq";
+```
+
+Supported forms are intentionally limited to text rewrites and includes. `termflow` does not interpret field syntax, conditional macros, stop-word directives, boosts, or other higher-level query language constructs in rewrite files.
+
+A runnable example is included at [examples/analyze_query.cpp](examples/analyze_query.cpp).
+
+## API Contracts
+
+- Offsets are represented as UTF-16 code-unit offsets.
+- `Analyzer`, `Tokenizer`, and `TermExtractor` are batch-oriented APIs that materialize results into `std::vector` containers.
+- `QueryParser`, `RewriteLoader`, `RewriteValidator`, and `QueryAnalyzer` are also batch-oriented APIs that materialize parsed clauses, rewrite rules, validation reports, and analyzed term sequences into `std::vector` containers.
+- v1 does not expose a streaming token API.
+- Implementations may throw `std::runtime_error` when ICU cannot be initialized or fails while processing text.
+- Invalid configuration is reported with standard exceptions such as `std::invalid_argument`.
+- Constructed analyzers and filters are safe for concurrent const use. A typical parallel usage pattern is to share one configured analyzer across worker threads and analyze different documents concurrently.
+- `QueryParser`, `RewriteLoader`, `RewriteValidator`, and `QueryAnalyzer` are also safe for concurrent const use after construction.
+- Do not reconfigure mutable objects, such as a tokenizer with a new max token length, concurrently with analysis.
+
+## Design Notes
+
+- Tokenization uses ICU word boundaries.
+- ASCII folding is optional and disabled by default.
+- The Porter stemmer is embedded and scoped for English terms.
+- The built-in English analyzer uses an optimized internal fast path for its pre-stem normalization stages. Tests compare that path against an explicitly composed public-filter pipeline to keep them aligned.
+
+## Custom Analyzers
+
+The public tokenizer and filter interfaces are intended to support explicit analyzer construction. A runnable example is included at [examples/custom_analyzer.cpp](examples/custom_analyzer.cpp).
+
+That example shows how to:
+
+- subclass `Analyzer`
+- reuse `StandardTokenizer`
+- compose public token filters directly
+- define a separate `normalize()` path for the custom analyzer
+
+The query module is designed to compose with custom analyzers as well. `QueryAnalyzer` accepts any configured `Analyzer` and uses that analyzer's `analyze_terms()` behavior when normalizing query clauses and rewrite rules.
+
+## Limitations
+
+This is intentionally a small v1.
+
+- English only
+- No token graphs
+- No token-graph-style synonym processing
+- No phrase execution logic
+- Query parsing is intentionally lightweight: no escaping, precedence rules, boosts, or field syntax
+- No index structures
+- No multilingual analyzers
+- No concrete `CharFilter` implementations yet
+
+## Future Direction
+
+The core abstractions are language-agnostic. The intended direction is:
+
+- keep `termflow` core focused on shared analysis primitives
+- keep query-intent analysis focused on parsing and rewrite preparation rather than scoring
+- add language analyzers as independent modules or packages
+- keep analyzer construction explicit rather than reflection-driven
+- reuse shared Unicode handling where it improves correctness
+
+## Building
+
+Local build:
+
+```bash
+cmake -S . -B build -G Ninja
+cmake --build build
+ctest --test-dir build --output-on-failure
+```
+
+The examples are also built by default:
+
+- `termflow_analyze`
+- `termflow_extract_terms`
+- `termflow_custom_analyzer`
+- `termflow_analyze_query`
+
+## Python Bindings
+
+Optional Python bindings can be built on top of the same core library:
+
+```bash
+cmake -S . -B build -G Ninja -DTERMFLOW_BUILD_PYTHON=ON
+cmake --build build
+PYTHONPATH=build/python python3 -c 'import termflow; print(termflow.EnglishAnalyzer().analyze_terms("Running Cars"))'
+```
+
+Python wheel and sdist build:
+
+```bash
+python3 -m build --sdist --wheel
+python3 -m twine check dist/*
+```
+
+The repository includes [pyproject.toml](pyproject.toml) and two helper scripts:
+
+- [tools/build_python_dist.sh](tools/build_python_dist.sh)
+- [tools/publish_python_dist.sh](tools/publish_python_dist.sh)
+
+The optional Python bindings expose the same query module under `termflow.query`.
+
+## Container Build
+
+The repository also includes a containerized dev/test environment for reproducible builds:
+
+```bash
+docker compose run --rm dev bash -lc './tools/test_in_docker.sh'
+```