nind 2.1.4__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nind-2.2.0/.gitignore +6 -0
- {nind-2.1.4 → nind-2.2.0}/CLAUDE.md +6 -1
- {nind-2.1.4 → nind-2.2.0}/CMakeLists.txt +1 -1
- {nind-2.1.4 → nind-2.2.0}/PKG-INFO +1 -1
- {nind-2.1.4 → nind-2.2.0}/pyproject.toml +2 -2
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindAmose/NindTermAmose.cpp +22 -22
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindPy/bindings.cpp +48 -182
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/NindFile.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/NindIndex.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/NindLexiconindex.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/NindLocalindex.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/NindPadFile.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/NindRetrolexicon.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/NindTermindex.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/Nind_dumpDocument.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/Nind_search.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/Nind_trouveMotsSansOccurrence.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/Nind_trouveNombresPremiers.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/amose/NindIdentifiantsClef.py +1 -1
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/nind_engine.py +51 -33
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindAmose/NindTermAmoseTest.cpp +59 -0
- nind-2.2.0/tst/py/test_native.py +115 -0
- nind-2.1.4/.gitignore +0 -9
- nind-2.1.4/bench/README.md +0 -189
- nind-2.1.4/bench/index_bulk.py +0 -328
- nind-2.1.4/bench/lookup_latency_bench.py +0 -142
- nind-2.1.4/bench/msmarco/convert_to_nind.py +0 -61
- nind-2.1.4/bench/run_bulk_indexing_benchmark.sh +0 -39
- nind-2.1.4/bench/run_indexing_benchmark.sh +0 -89
- nind-2.1.4/tst/py/test_native.py +0 -260
- {nind-2.1.4 → nind-2.2.0}/.github/workflows/build.yml +0 -0
- {nind-2.1.4 → nind-2.2.0}/.github/workflows/docs.yml +0 -0
- {nind-2.1.4 → nind-2.2.0}/.github/workflows/publish-pypi.yml +0 -0
- {nind-2.1.4 → nind-2.2.0}/AGENTS.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/Dockerfile-ubuntu24.04 +0 -0
- {nind-2.1.4 → nind-2.2.0}/LICENCE.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/NindConfig-version-src.cmake +0 -0
- {nind-2.1.4 → nind-2.2.0}/NindConfig.cmake +0 -0
- {nind-2.1.4 → nind-2.2.0}/README.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/SystemSpecificInformations.cmake +0 -0
- {nind-2.1.4 → nind-2.2.0}/cicd/extract_package.sh +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/api/engine.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/api/index.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/api/readers.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/architecture.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/cli.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/conf.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/index.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/quickstart.md +0 -0
- {nind-2.1.4 → nind-2.2.0}/docs/requirements.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/gbuild.sh +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/Doxyfile +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindAmose/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindAmose/NindLexiconAmose.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindAmose/NindLexiconAmose.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindAmose/NindLocalAmose.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindAmose/NindLocalAmose.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindAmose/NindTermAmose.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindBasics/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindBasics/NindFile.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindBasics/NindFile.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindBasics/NindPadFile.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindBasics/NindPadFile.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindBasics/NindSignalCatcher.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindBasics/NindSignalCatcher.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindCommonExport.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindExceptions.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindIndex.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindIndex.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindLexiconIndex.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindLexiconIndex.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindLexiconIndexK.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindLexiconIndexK.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindLocalIndex.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindLocalIndex.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindTermIndex.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindIndex/NindTermIndex.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindLexicon/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindLexicon/NindLexicon.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindLexicon/NindLexicon.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindLexicon/NindLexiconFile.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindLexicon/NindLexiconFile.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindPy/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindRetrolexicon/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindRetrolexicon/NindRetrolexicon.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/cpp/NindRetrolexicon/NindRetrolexicon.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/Nind_checkAndRepair.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/Nind_checkDocument.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/Nind_checkRejtroAndLexicon.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/__init__.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/amose/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/amose/LatXmlParser.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/amose/Nind_convertitLuceneDump.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/amose/Nind_convertitSampleDump.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/src/py/nind/amose/Nind_convertitXmlClef.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/NindAmose_effaceDocument.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/NindAmose_indexeCorpus.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/NindAmose_litTexteAnalysej.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/NindAmose_litTexteAnalysej.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/NindAmose_testLecteur.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/NindAmose_testSynchro.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindAmose/Nind_indexeCorpus.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindBasics/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindBasics/NindBasics_testLecture.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindDate.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindDate.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindFichiers.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindFichiers.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndexTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndexTest.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndex_indexe.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndex_indexe.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndex_litDumpS2.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndex_litDumpS2.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndex_testEcrivain.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndex_testLecteur.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindIndex_testSynchro.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindLexiconIndex_testEcrivain.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindLexiconIndex_testLecteur.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindTermIndex_testEcrivain.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/NindTermIndex_testLecteur.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/Nind_testLateconNumber.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindIndex/Nind_testNumbersInLocalIndexes.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/NindLexicon_testEcrivain.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/NindLexicon_testInter.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/NindLexicon_testLecteur.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/NindLexicon_testLimites.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/NindLexicon_testLimitesB.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/NindLexicon_testLimitesC.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindLexicon/NindLexicon_testMemoire.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindSearch/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindSearch/NindGetDocTerms.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindSearch/NindSearch.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindSearch/NindSearchAmose.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindTests/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindTests/NindTestControlC.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindTests/Nind_test1.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/NindTests/Nind_testSynchro.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindAmose/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindAmose/NindLexiconAmoseTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindAmose/NindLocalAmoseTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindBasics/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindBasics/NindBasicsRobustnessTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindBasics/NindFileTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindBasics/NindPadFileTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindIndex/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindIndex/NindIndexRobustnessTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindIndex/NindLexiconIndexTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindIndex/NindLocalIndexTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindIndex/NindTermIndexTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindLexicon/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindLexicon/NindLexiconFileTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindLexicon/NindLexiconTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindRetrolexicon/CMakeLists.txt +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/NindRetrolexicon/NindRetrolexiconTest.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/TestFileBytes.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/TestTempDir.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/doctest.h +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/cpp/unit/doctest_main.cpp +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/py/test_nind_engine.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/py/test_nindfile.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/py/test_nindlexiconindex.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/py/test_nindlocalindex.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/py/test_nindretrolexicon.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/tst/py/test_nindtermindex.py +0 -0
- {nind-2.1.4 → nind-2.2.0}/uv.lock +0 -0
nind-2.2.0/.gitignore
ADDED
|
@@ -105,7 +105,7 @@ Parallels the C++ layering with lowercase-suffixed module names for the index cl
|
|
|
105
105
|
- `NindFile.py` — low-level binary codec. Historically read-only-oriented (the `litXxx` getters have long existed for every format), but now has a full writer counterpart too: `ejcritNombre1/3/4/5`, `ejcritNombreULat`/`ejcritNombreSLat` (both varint encodings, full tier range matching the readers and the C++ implementation), `ejcritChaine`, `ejcritZejros`.
|
|
106
106
|
- `NindPadFile.py` — the shared "pad file" envelope (header, indirection block(s), specifics, identification trailer); read-only, no writer (mirrors `NindBasics::NindPadFile`, but nothing in Python subclasses it as a writer the way `NindIndex`/`NindRetrolexicon` do in C++).
|
|
107
107
|
- `NindRetrolexicon.py`, `NindLexiconindex.py`, `NindTermindex.py`, `NindLocalindex.py` — ergonomic read-only Python wrappers around the corresponding `NindIndex`-family C++ classes; their hot-path lookups delegate to `nind._native`, as does the shared structural-diagnostic base (`analyseFichierPadFile`/`analyseFichierIndex`, inherited from `NindPadFile.py`/`NindIndex.py`). The remaining, per-format diagnostic/analysis methods (`dumpeFichier`, `debogueIndex`, `donneCollisions`, `donneMax`, `donneClef`, `afficheTerme`, `afficheDocument`, and each class's own `analyseFichierXxx` beyond the shared base) remain hand-rolled pure-Python parsing, since `nind._native` doesn't expose that introspection surface yet (see "Planned follow-up work" below). `NindLexiconindex` is hash-bucketed (`clefB(word) % nombreIndirection`), the other two are directly indexed by id.
|
|
108
|
-
- `nind_engine.py` — `NindIndexer` (writer) and `NindEngine` (BM25-style search) built on top of the above. `NindIndexer` is the *only* Python writer for the index-family formats (`.nindlexiconindex`/`.nindtermindex`/`.nindlocalindex`) — it builds them via `nind._native`'s `is_writer=True` constructors. It does not write `.nindretrolexicon` (not needed by `NindEngine`, which only supports simple, non-compound words).
|
|
108
|
+
- `nind_engine.py` — `NindIndexer` (writer) and `NindEngine` (BM25-style search) built on top of the above. `NindIndexer` is the *only* Python writer for the index-family formats (`.nindlexiconindex`/`.nindtermindex`/`.nindlocalindex`) — it builds them via `nind._native`'s `is_writer=True` constructors. It does not write `.nindretrolexicon` (not needed by `NindEngine`, which only supports simple, non-compound words). Indexing is parallel but every file keeps a single writer: files are tokenized in worker processes (`workers` argument of `index_files`; in-process below `PARALLEL_MIN_FILES` files), then the term and local index files are written concurrently from two threads through the bulk bindings `NindTermIndex.set_postings`/`NindLocalIndex.set_local_def_from_term_ids`, which take plain ints (building `Document`/`Term`/`Localisation` objects in Python cost ~20× the write itself) and release the GIL. `set_local_def_from_term_ids` splits a term's positions into entries of ≤255 (`<nbreLocalisations>` is one byte).
|
|
109
109
|
- `Nind_*.py` at the package root — standalone CLI scripts (dump a document, convert corpora, search, diagnostics)
|
|
110
110
|
- `amose/` — Amose-specific conversion/parsing scripts (Lucene dump, XML-CLEF, sample corpora)
|
|
111
111
|
|
|
@@ -123,3 +123,8 @@ Binary file formats are specified in EBNF grammar (see README). The Python packa
|
|
|
123
123
|
- `NindRetrolexicon`: `analyseFichierRetrolexicon`'s own stats (it isn't a `NindIndex` subclass, so it only ever gets the `analyseFichierPadFile` base, not `analyseFichierIndex`), `dumpeFichier`.
|
|
124
124
|
|
|
125
125
|
Not part of this: `.nindretrolexicon` writing (`NindIndexer` still doesn't write this format — unrelated, no change planned).
|
|
126
|
+
|
|
127
|
+
**Sharded indexing (planned next, after the in-process parallelism above).** Nind files are single-writer by design (no locking, readers resync lazily; see `NindPadFile::getEntryPos`), and every write rewrites the shared specifics/identification tail at EOF, so concurrent writers on one file would serialize anyway — sharing a file between writers is explicitly *not* the plan. Instead, scale writes across N independent shards (each its own lexicon + term + local files, one single-threaded writer each, over a partition of the documents), then either:
|
|
128
|
+
- merge offline: union the lexicons into a new one, remap every shard's term ids, concatenate posting lists with doc ids kept in increasing order (the term index stores relative doc ids), and remap term ids inside each shard's local index; or
|
|
129
|
+
- search across shards: `NindEngine` fans out per shard, but BM25 needs corpus-wide statistics (N, df, avgdl) summed over all shards, not per-shard values.
|
|
130
|
+
Expected cost ~1–2 weeks including tests; zero overhead on the write path.
|
|
@@ -113,7 +113,7 @@ endif ()
|
|
|
113
113
|
|
|
114
114
|
########### setting project verion number ###############
|
|
115
115
|
|
|
116
|
-
set(NIND_VERSION "2.
|
|
116
|
+
set(NIND_VERSION "2.2.0")
|
|
117
117
|
|
|
118
118
|
string(REGEX REPLACE "^([0-9]+)\\.[0-9]+\\.[0-9]+$" "\\1" NIND_VERSION_MAJOR "${NIND_VERSION}")
|
|
119
119
|
string(REGEX REPLACE "^[0-9]+\\.([0-9]+)\\.[0-9]+$" "\\1" NIND_VERSION_MINOR "${NIND_VERSION}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "nind"
|
|
3
|
-
version = "2.
|
|
3
|
+
version = "2.2.0"
|
|
4
4
|
description = "Python frontend for the Nind Indexing Module, with its native C++ index stack built in"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.8"
|
|
@@ -25,7 +25,7 @@ testpaths = ["tst/py"]
|
|
|
25
25
|
pythonpath = ["src/py"]
|
|
26
26
|
|
|
27
27
|
[tool.bumpver]
|
|
28
|
-
current_version = "v2.
|
|
28
|
+
current_version = "v2.2.0"
|
|
29
29
|
version_pattern = "vMAJOR.MINOR.PATCH"
|
|
30
30
|
commit_message = "Bump version {old_version} -> {new_version}"
|
|
31
31
|
commit = true
|
|
@@ -24,6 +24,12 @@ using namespace std;
|
|
|
24
24
|
////////////////////////////////////////////////////////////
|
|
25
25
|
#define NOMBRE_COMPTEURS 8
|
|
26
26
|
////////////////////////////////////////////////////////////
|
|
27
|
+
//ordre des documents par ident
|
|
28
|
+
static bool identInfejrieur(const NindTermIndex::Document &doc1, const NindTermIndex::Document &doc2)
|
|
29
|
+
{
|
|
30
|
+
return doc1.ident < doc2.ident;
|
|
31
|
+
}
|
|
32
|
+
////////////////////////////////////////////////////////////
|
|
27
33
|
//brief Creates NindTermAmose with a specified name associated with.
|
|
28
34
|
//param fileNameExtensionLess absolute path file name without extension
|
|
29
35
|
//param isTermIndexWriter true if termIndex writer, false if termIndex reader */
|
|
@@ -72,28 +78,22 @@ void NindTermAmose::addDocsToTerm(const unsigned int ident,
|
|
|
72
78
|
}
|
|
73
79
|
//travaille sur l'unique ejlejment
|
|
74
80
|
TermCG &termcg = termDef.front();
|
|
75
|
-
list<Document> &documents = termcg.documents; //documents dejjah lah
|
|
81
|
+
list<Document> &documents = termcg.documents; //documents dejjah lah, ordonnejs par ident
|
|
82
|
+
//trie les nouveaux documents pour les fusionner en une seule passe dans la liste ordonneje,
|
|
83
|
+
//au lieu de la reparcourir depuis le dejbut pour chacun (quadratique pour un gros paquet)
|
|
84
|
+
list<Document> sortedDocuments(newDocuments);
|
|
85
|
+
sortedDocuments.sort(identInfejrieur);
|
|
76
86
|
//ajoute tous les documents
|
|
77
|
-
|
|
87
|
+
list<NindTermIndex::Document>::iterator it2 = documents.begin();
|
|
88
|
+
for (list<Document>::const_iterator itdoc = sortedDocuments.begin(); itdoc != sortedDocuments.end(); itdoc++) {
|
|
78
89
|
const Document &document = (*itdoc); //document ah ajouter
|
|
79
90
|
unsigned int frequency = document.frequency; //sa frejquence
|
|
80
|
-
//trouve la place dans la liste ordonnee
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
break;
|
|
87
|
-
}
|
|
88
|
-
//insere a l'interieur de la liste
|
|
89
|
-
if ((*it2).ident > document.ident) {
|
|
90
|
-
documents.insert(it2, document);
|
|
91
|
-
break;
|
|
92
|
-
}
|
|
93
|
-
it2++;
|
|
94
|
-
}
|
|
95
|
-
//si fin de liste, insere en fin
|
|
96
|
-
if (it2 == documents.end()) documents.push_back(document);
|
|
91
|
+
//trouve la place dans la liste ordonnee, ah partir de celle du document prejcejdent
|
|
92
|
+
while (it2 != documents.end() && (*it2).ident < document.ident) it2++;
|
|
93
|
+
//deja dans la liste, met ah jour la frejquence
|
|
94
|
+
if (it2 != documents.end() && (*it2).ident == document.ident) (*it2).frequency += frequency;
|
|
95
|
+
//sinon insere ah cette place (en fin si fin de liste)
|
|
96
|
+
else it2 = documents.insert(it2, document);
|
|
97
97
|
//increjmente les occurrences pour ce type
|
|
98
98
|
m_termOccurrences[type] += frequency;
|
|
99
99
|
m_termOccurrences[ALL] += frequency;
|
|
@@ -132,9 +132,9 @@ void NindTermAmose::removeDocFromTerm(const unsigned int ident,
|
|
|
132
132
|
//dejcrejmente la frejquence globale de ce terme
|
|
133
133
|
termcg.frequency -= document.frequency;
|
|
134
134
|
//enlehve le doc de la liste
|
|
135
|
-
|
|
136
|
-
//si c'ejtait le
|
|
137
|
-
if (
|
|
135
|
+
documents.erase(itdoc);
|
|
136
|
+
//si c'ejtait le seul restant, efface le terme
|
|
137
|
+
if (documents.empty()) {
|
|
138
138
|
//dejcrejmente le nombre de termes pour ce type
|
|
139
139
|
m_uniqueTermCount[type] -=1;
|
|
140
140
|
m_uniqueTermCount[ALL] -=1;
|
|
@@ -19,9 +19,7 @@
|
|
|
19
19
|
#include <sstream>
|
|
20
20
|
#include <map>
|
|
21
21
|
#include <vector>
|
|
22
|
-
#include <
|
|
23
|
-
#include <cstdint>
|
|
24
|
-
#include <limits>
|
|
22
|
+
#include <utility>
|
|
25
23
|
#include "NindBasics/NindPadFile.h"
|
|
26
24
|
#include "NindRetrolexicon/NindRetrolexicon.h"
|
|
27
25
|
#include "NindLexicon/NindLexicon.h"
|
|
@@ -33,88 +31,6 @@
|
|
|
33
31
|
namespace py = pybind11;
|
|
34
32
|
using namespace latecon::nindex;
|
|
35
33
|
////////////////////////////////////////////////////////////
|
|
36
|
-
namespace {
|
|
37
|
-
// 1-D contiguous buffer of 32-bit unsigned ints (array.array('I'), numpy
|
|
38
|
-
// uint32, ...), as used by the *_arrays bulk writers below: they build the
|
|
39
|
-
// C++ definition straight from flat buffers instead of going through one
|
|
40
|
-
// Python Term/Localisation/Document object per entry.
|
|
41
|
-
struct UIntView {
|
|
42
|
-
py::buffer_info info;
|
|
43
|
-
const uint32_t *data;
|
|
44
|
-
size_t size;
|
|
45
|
-
};
|
|
46
|
-
UIntView uintView(const py::buffer &buffer, const char *name)
|
|
47
|
-
{
|
|
48
|
-
py::buffer_info info = buffer.request();
|
|
49
|
-
const std::string &format = info.format;
|
|
50
|
-
const bool isUInt32 = info.itemsize == 4 && !format.empty() &&
|
|
51
|
-
(format.back() == 'I' || format.back() == 'L') &&
|
|
52
|
-
(format.size() == 1 || format[0] == '<' || format[0] == '=' || format[0] == '@');
|
|
53
|
-
if (!isUInt32)
|
|
54
|
-
throw py::value_error(std::string(name) + ": expected a buffer of 32-bit unsigned ints (e.g. array('I')), got format '" + format + "'");
|
|
55
|
-
if (info.ndim != 1 || (info.shape[0] > 1 && info.strides[0] != 4))
|
|
56
|
-
throw py::value_error(std::string(name) + ": expected a 1-D contiguous buffer");
|
|
57
|
-
const uint32_t *data = static_cast<const uint32_t *>(info.ptr);
|
|
58
|
-
const size_t size = static_cast<size_t>(info.shape[0]);
|
|
59
|
-
return UIntView{std::move(info), data, size};
|
|
60
|
-
}
|
|
61
|
-
// validates a batch's end offsets: ascending, the last one == itemsNb
|
|
62
|
-
void checkEnds(const UIntView &ends, const size_t itemsNb, const char *name)
|
|
63
|
-
{
|
|
64
|
-
uint32_t prev = 0;
|
|
65
|
-
for (size_t i = 0; i < ends.size; i++) {
|
|
66
|
-
if (ends.data[i] < prev || ends.data[i] > itemsNb)
|
|
67
|
-
throw py::value_error(std::string(name) + ": end offsets must be ascending and within the data");
|
|
68
|
-
prev = ends.data[i];
|
|
69
|
-
}
|
|
70
|
-
if (prev != itemsNb)
|
|
71
|
-
throw py::value_error(std::string(name) + ": last end offset must equal the data length");
|
|
72
|
-
}
|
|
73
|
-
// one term definition from its [begin, end) slice of (docIds, freqs)
|
|
74
|
-
std::list<NindTermIndex::TermCG> buildTermDef(const uint32_t *docIds, const uint32_t *freqs,
|
|
75
|
-
const size_t begin, const size_t end,
|
|
76
|
-
const unsigned char cg)
|
|
77
|
-
{
|
|
78
|
-
std::vector<std::pair<uint32_t, uint32_t> > postings(end - begin);
|
|
79
|
-
for (size_t i = begin; i < end; i++) postings[i - begin] = std::make_pair(docIds[i], freqs[i]);
|
|
80
|
-
if (!std::is_sorted(postings.begin(), postings.end())) std::sort(postings.begin(), postings.end());
|
|
81
|
-
std::list<NindTermIndex::TermCG> termDef;
|
|
82
|
-
if (postings.empty()) return termDef;
|
|
83
|
-
uint64_t total = 0;
|
|
84
|
-
NindTermIndex::TermCG termCG(cg, 0);
|
|
85
|
-
for (size_t i = 0; i < postings.size(); i++) {
|
|
86
|
-
if (i > 0 && postings[i].first == postings[i - 1].first)
|
|
87
|
-
throw py::value_error("doc_ids: duplicate document id " + std::to_string(postings[i].first));
|
|
88
|
-
total += postings[i].second;
|
|
89
|
-
termCG.documents.push_back(NindTermIndex::Document(postings[i].first, postings[i].second));
|
|
90
|
-
}
|
|
91
|
-
if (total > std::numeric_limits<unsigned int>::max())
|
|
92
|
-
throw py::value_error("frequencies: total term frequency overflows 32 bits");
|
|
93
|
-
termCG.frequency = static_cast<unsigned int>(total);
|
|
94
|
-
termDef.push_back(std::move(termCG));
|
|
95
|
-
return termDef;
|
|
96
|
-
}
|
|
97
|
-
// one local definition from its [begin, end) slice of occurrences: one Term
|
|
98
|
-
// per term id (ascending), localisations kept in occurrence order
|
|
99
|
-
std::list<NindLocalIndex::Term> buildLocalDef(const uint32_t *termIds, const uint32_t *positions,
|
|
100
|
-
const uint32_t *lengths,
|
|
101
|
-
const size_t begin, const size_t end,
|
|
102
|
-
const unsigned char cg)
|
|
103
|
-
{
|
|
104
|
-
std::map<unsigned int, NindLocalIndex::Term> byTerm;
|
|
105
|
-
for (size_t i = begin; i < end; i++) {
|
|
106
|
-
std::map<unsigned int, NindLocalIndex::Term>::iterator it = byTerm.find(termIds[i]);
|
|
107
|
-
if (it == byTerm.end())
|
|
108
|
-
it = byTerm.insert(std::make_pair(termIds[i], NindLocalIndex::Term(termIds[i], cg))).first;
|
|
109
|
-
it->second.localisation.push_back(NindLocalIndex::Localisation(positions[i], lengths[i]));
|
|
110
|
-
}
|
|
111
|
-
std::list<NindLocalIndex::Term> localDef;
|
|
112
|
-
for (std::map<unsigned int, NindLocalIndex::Term>::iterator it = byTerm.begin(); it != byTerm.end(); it++)
|
|
113
|
-
localDef.push_back(std::move(it->second));
|
|
114
|
-
return localDef;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
////////////////////////////////////////////////////////////
|
|
118
34
|
PYBIND11_MODULE(_native, m) {
|
|
119
35
|
m.doc() = "Native (C++) implementation of the nind core index stack";
|
|
120
36
|
|
|
@@ -230,55 +146,30 @@ PYBIND11_MODULE(_native, m) {
|
|
|
230
146
|
return specifics;
|
|
231
147
|
})
|
|
232
148
|
.def("set_term_def", &NindTermIndex::setTermDef,
|
|
233
|
-
py::arg("ident"), py::arg("term_def"), py::arg("file_identification"), py::arg("specifics")
|
|
234
|
-
|
|
235
|
-
//
|
|
236
|
-
//
|
|
237
|
-
//
|
|
238
|
-
//
|
|
239
|
-
.def("
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
const UIntView freqs = uintView(frequencies, "frequencies");
|
|
245
|
-
if (ids.size != freqs.size)
|
|
246
|
-
throw py::value_error("doc_ids and frequencies must have the same length");
|
|
247
|
-
const std::list<NindTermIndex::TermCG> termDef = buildTermDef(ids.data, freqs.data, 0, ids.size, cg);
|
|
149
|
+
py::arg("ident"), py::arg("term_def"), py::arg("file_identification"), py::arg("specifics"),
|
|
150
|
+
py::call_guard<py::gil_scoped_release>())
|
|
151
|
+
// Bulk writer taking plain ints instead of Document/TermCG objects (building
|
|
152
|
+
// millions of those in Python costs far more than the write itself): one
|
|
153
|
+
// single-TermCG definition per term, frequency = sum of its postings.
|
|
154
|
+
// The GIL is released for the whole loop, so another thread can keep working.
|
|
155
|
+
.def("set_postings", [](NindTermIndex &self,
|
|
156
|
+
const std::vector<std::pair<unsigned int, std::vector<std::pair<unsigned int, unsigned int> > > > &termPostings,
|
|
157
|
+
const NindPadFile::Identification &fileIdentification,
|
|
158
|
+
const std::list<unsigned int> &specifics,
|
|
159
|
+
const unsigned char cg) {
|
|
248
160
|
py::gil_scoped_release release;
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
const NindPadFile::Identification &fileIdentification,
|
|
260
|
-
unsigned char cg, const std::list<unsigned int> &specifics) {
|
|
261
|
-
const UIntView terms = uintView(idents, "idents");
|
|
262
|
-
const UIntView termEnds = uintView(ends, "ends");
|
|
263
|
-
const UIntView ids = uintView(docIds, "doc_ids");
|
|
264
|
-
const UIntView freqs = uintView(frequencies, "frequencies");
|
|
265
|
-
if (ids.size != freqs.size)
|
|
266
|
-
throw py::value_error("doc_ids and frequencies must have the same length");
|
|
267
|
-
if (terms.size != termEnds.size)
|
|
268
|
-
throw py::value_error("idents and ends must have the same length");
|
|
269
|
-
checkEnds(termEnds, ids.size, "ends");
|
|
270
|
-
std::vector<std::list<NindTermIndex::TermCG> > termDefs(terms.size);
|
|
271
|
-
size_t begin = 0;
|
|
272
|
-
for (size_t i = 0; i < terms.size; i++) {
|
|
273
|
-
termDefs[i] = buildTermDef(ids.data, freqs.data, begin, termEnds.data[i], cg);
|
|
274
|
-
begin = termEnds.data[i];
|
|
161
|
+
for (size_t i = 0; i < termPostings.size(); i++) {
|
|
162
|
+
const std::vector<std::pair<unsigned int, unsigned int> > &postings = termPostings[i].second;
|
|
163
|
+
std::list<NindTermIndex::TermCG> termDef;
|
|
164
|
+
termDef.push_back(NindTermIndex::TermCG(cg, 0));
|
|
165
|
+
NindTermIndex::TermCG &termCG = termDef.back();
|
|
166
|
+
for (size_t j = 0; j < postings.size(); j++) {
|
|
167
|
+
termCG.documents.push_back(NindTermIndex::Document(postings[j].first, postings[j].second));
|
|
168
|
+
termCG.frequency += postings[j].second;
|
|
169
|
+
}
|
|
170
|
+
self.setTermDef(termPostings[i].first, termDef, fileIdentification, specifics);
|
|
275
171
|
}
|
|
276
|
-
|
|
277
|
-
for (size_t i = 0; i < terms.size; i++)
|
|
278
|
-
self.setTermDef(terms.data[i], termDefs[i], fileIdentification, specifics);
|
|
279
|
-
}, py::arg("idents"), py::arg("ends"), py::arg("doc_ids"), py::arg("frequencies"),
|
|
280
|
-
py::arg("file_identification"), py::arg("cg") = 0,
|
|
281
|
-
py::arg("specifics") = std::list<unsigned int>())
|
|
172
|
+
}, py::arg("term_postings"), py::arg("file_identification"), py::arg("specifics"), py::arg("cg") = 0)
|
|
282
173
|
.def("get_file_name", &NindPadFile::getFileName)
|
|
283
174
|
.def("analyse_pad_file", &NindPadFile::analysePadFile)
|
|
284
175
|
.def("analyse_index", &NindIndex::analyseIndex);
|
|
@@ -317,58 +208,33 @@ PYBIND11_MODULE(_native, m) {
|
|
|
317
208
|
return py::cast(termIdents);
|
|
318
209
|
}, py::arg("ident"))
|
|
319
210
|
.def("set_local_def", &NindLocalIndex::setLocalDef,
|
|
320
|
-
py::arg("ident"), py::arg("local_def"), py::arg("file_identification")
|
|
321
|
-
|
|
322
|
-
//
|
|
323
|
-
//
|
|
324
|
-
//
|
|
325
|
-
//
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
const UIntView pos = uintView(positions, "positions");
|
|
333
|
-
const UIntView lens = uintView(lengths, "lengths");
|
|
334
|
-
if (terms.size != pos.size || terms.size != lens.size)
|
|
335
|
-
throw py::value_error("term_ids, positions and lengths must have the same length");
|
|
336
|
-
const std::list<NindLocalIndex::Term> localDef =
|
|
337
|
-
buildLocalDef(terms.data, pos.data, lens.data, 0, terms.size, cg);
|
|
338
|
-
py::gil_scoped_release release;
|
|
339
|
-
self.setLocalDef(ident, localDef, fileIdentification);
|
|
340
|
-
}, py::arg("ident"), py::arg("term_ids"), py::arg("positions"), py::arg("lengths"),
|
|
341
|
-
py::arg("file_identification"), py::arg("cg") = 0)
|
|
342
|
-
// Whole-batch form of set_local_def_arrays: document i is idents[i],
|
|
343
|
-
// its occurrences are term_ids/positions/lengths[ends[i-1]:ends[i]].
|
|
344
|
-
// Documents are written in the given order, built one at a time with
|
|
345
|
-
// the GIL released; the offsets are validated up front, so a
|
|
346
|
-
// ValueError means nothing was written.
|
|
347
|
-
.def("set_local_defs_arrays", [](NindLocalIndex &self, const py::buffer &idents,
|
|
348
|
-
const py::buffer &ends, const py::buffer &termIds,
|
|
349
|
-
const py::buffer &positions, const py::buffer &lengths,
|
|
350
|
-
const NindPadFile::Identification &fileIdentification,
|
|
351
|
-
unsigned char cg) {
|
|
352
|
-
const UIntView docs = uintView(idents, "idents");
|
|
353
|
-
const UIntView docEnds = uintView(ends, "ends");
|
|
354
|
-
const UIntView terms = uintView(termIds, "term_ids");
|
|
355
|
-
const UIntView pos = uintView(positions, "positions");
|
|
356
|
-
const UIntView lens = uintView(lengths, "lengths");
|
|
357
|
-
if (terms.size != pos.size || terms.size != lens.size)
|
|
358
|
-
throw py::value_error("term_ids, positions and lengths must have the same length");
|
|
359
|
-
if (docs.size != docEnds.size)
|
|
360
|
-
throw py::value_error("idents and ends must have the same length");
|
|
361
|
-
checkEnds(docEnds, terms.size, "ends");
|
|
211
|
+
py::arg("ident"), py::arg("local_def"), py::arg("file_identification"),
|
|
212
|
+
py::call_guard<py::gil_scoped_release>())
|
|
213
|
+
// Writer taking a document as its term ids in token order (position = token
|
|
214
|
+
// index, length = 1) instead of Term/Localisation objects, grouped per term in
|
|
215
|
+
// increasing term id order. <nbreLocalisations> is one byte, so a term occurring
|
|
216
|
+
// more than 255 times is written as several consecutive entries for the same id
|
|
217
|
+
// (readers sum them). The GIL is released while building and writing.
|
|
218
|
+
.def("set_local_def_from_term_ids", [](NindLocalIndex &self,
|
|
219
|
+
const unsigned int ident,
|
|
220
|
+
const std::vector<unsigned int> &termIds,
|
|
221
|
+
const NindPadFile::Identification &fileIdentification,
|
|
222
|
+
const unsigned char cg) {
|
|
362
223
|
py::gil_scoped_release release;
|
|
363
|
-
|
|
364
|
-
for (size_t
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
224
|
+
std::map<unsigned int, std::vector<unsigned int> > positionsById;
|
|
225
|
+
for (size_t position = 0; position < termIds.size(); position++)
|
|
226
|
+
positionsById[termIds[position]].push_back((unsigned int)position);
|
|
227
|
+
std::list<NindLocalIndex::Term> localDef;
|
|
228
|
+
for (std::map<unsigned int, std::vector<unsigned int> >::const_iterator it = positionsById.begin();
|
|
229
|
+
it != positionsById.end(); it++) {
|
|
230
|
+
const std::vector<unsigned int> &positions = it->second;
|
|
231
|
+
for (size_t i = 0; i < positions.size(); i++) {
|
|
232
|
+
if (i % 255 == 0) localDef.push_back(NindLocalIndex::Term(it->first, cg));
|
|
233
|
+
localDef.back().localisation.push_back(NindLocalIndex::Localisation(positions[i], 1));
|
|
234
|
+
}
|
|
369
235
|
}
|
|
370
|
-
|
|
371
|
-
|
|
236
|
+
self.setLocalDef(ident, localDef, fileIdentification);
|
|
237
|
+
}, py::arg("ident"), py::arg("term_ids"), py::arg("file_identification"), py::arg("cg") = 0)
|
|
372
238
|
.def("get_doc_count", &NindLocalIndex::getDocCount)
|
|
373
239
|
.def("get_file_name", &NindPadFile::getFileName)
|
|
374
240
|
.def("analyse_pad_file", &NindPadFile::analysePadFile)
|
|
@@ -17,7 +17,7 @@ the primitive types it implements.
|
|
|
17
17
|
__author__ = "jys"
|
|
18
18
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
19
19
|
__license__ = "GNU LGPL"
|
|
20
|
-
__version__ = "2.
|
|
20
|
+
__version__ = "2.2.0"
|
|
21
21
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
22
22
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
23
23
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -18,7 +18,7 @@ inside a definition.
|
|
|
18
18
|
__author__ = "jys"
|
|
19
19
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
20
20
|
__license__ = "GNU LGPL"
|
|
21
|
-
__version__ = "2.
|
|
21
|
+
__version__ = "2.2.0"
|
|
22
22
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
23
23
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
24
24
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -14,7 +14,7 @@ the identifier used to look it up in :class:`~nind.NindTermindex.NindTermindex`.
|
|
|
14
14
|
__author__ = "jys"
|
|
15
15
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
16
16
|
__license__ = "GNU LGPL"
|
|
17
|
-
__version__ = "2.
|
|
17
|
+
__version__ = "2.2.0"
|
|
18
18
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
19
19
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
20
20
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -14,7 +14,7 @@ document-length calculations.
|
|
|
14
14
|
__author__ = "jys"
|
|
15
15
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
16
16
|
__license__ = "GNU LGPL"
|
|
17
|
-
__version__ = "2.
|
|
17
|
+
__version__ = "2.2.0"
|
|
18
18
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
19
19
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
20
20
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -23,7 +23,7 @@ the meaning of the "en vrac" definitions and the specifics block.
|
|
|
23
23
|
__author__ = "jys"
|
|
24
24
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
25
25
|
__license__ = "GNU LGPL"
|
|
26
|
-
__version__ = "2.
|
|
26
|
+
__version__ = "2.2.0"
|
|
27
27
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
28
28
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
29
29
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -17,7 +17,7 @@ exists to read files produced by the C++ side or by legacy nind tooling.
|
|
|
17
17
|
__author__ = "jys"
|
|
18
18
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
19
19
|
__license__ = "GNU LGPL"
|
|
20
|
-
__version__ = "2.
|
|
20
|
+
__version__ = "2.2.0"
|
|
21
21
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
22
22
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
23
23
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -15,7 +15,7 @@ frequencies for BM25 scoring.
|
|
|
15
15
|
__author__ = "jys"
|
|
16
16
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
17
17
|
__license__ = "GNU LGPL"
|
|
18
|
-
__version__ = "2.
|
|
18
|
+
__version__ = "2.2.0"
|
|
19
19
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
20
20
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
21
21
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
__author__ = "jys"
|
|
4
4
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
5
5
|
__license__ = "GNU LGPL"
|
|
6
|
-
__version__ = "2.
|
|
6
|
+
__version__ = "2.2.0"
|
|
7
7
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
8
8
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
9
9
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
__author__ = "jys"
|
|
4
4
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
5
5
|
__license__ = "GNU LGPL"
|
|
6
|
-
__version__ = "2.
|
|
6
|
+
__version__ = "2.2.0"
|
|
7
7
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
8
8
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
9
9
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
__author__ = "jys"
|
|
4
4
|
__copyright__ = "Copyright (C) 2019 LATEJCON"
|
|
5
5
|
__license__ = "GNU LGPL"
|
|
6
|
-
__version__ = "2.
|
|
6
|
+
__version__ = "2.2.0"
|
|
7
7
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2019
|
|
8
8
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
9
9
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
__author__ = "jys"
|
|
4
4
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
5
5
|
__license__ = "GNU LGPL"
|
|
6
|
-
__version__ = "2.
|
|
6
|
+
__version__ = "2.2.0"
|
|
7
7
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
8
8
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
9
9
|
# This file is part of NIND (as "nouvelle indexation").
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
__author__ = "jys"
|
|
4
4
|
__copyright__ = "Copyright (C) 2017 LATEJCON"
|
|
5
5
|
__license__ = "GNU LGPL"
|
|
6
|
-
__version__ = "2.
|
|
6
|
+
__version__ = "2.2.0"
|
|
7
7
|
# Author: jys <jy.sage@orange.fr>, (C) LATEJCON 2017
|
|
8
8
|
# Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
|
|
9
9
|
# This file is part of NIND (as "nouvelle indexation").
|