nind 2.2.1__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. {nind-2.2.1 → nind-2.4.0}/CLAUDE.md +17 -8
  2. {nind-2.2.1 → nind-2.4.0}/CMakeLists.txt +1 -1
  3. {nind-2.2.1 → nind-2.4.0}/PKG-INFO +15 -1
  4. {nind-2.2.1 → nind-2.4.0}/README.md +14 -0
  5. {nind-2.2.1 → nind-2.4.0}/bench/README.md +3 -2
  6. {nind-2.2.1 → nind-2.4.0}/docs/api/engine.md +7 -0
  7. {nind-2.2.1 → nind-2.4.0}/docs/quickstart.md +59 -5
  8. {nind-2.2.1 → nind-2.4.0}/pyproject.toml +2 -2
  9. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindAmose/CMakeLists.txt +3 -0
  10. nind-2.4.0/src/cpp/NindAmose/NindAmoseMerge.cpp +77 -0
  11. nind-2.4.0/src/cpp/NindAmose/NindAmoseMerge.h +58 -0
  12. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindBasics/NindFile.cpp +77 -4
  13. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindBasics/NindFile.h +50 -2
  14. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindBasics/NindPadFile.cpp +88 -4
  15. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindBasics/NindPadFile.h +25 -0
  16. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/CMakeLists.txt +5 -1
  17. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindIndex.cpp +135 -8
  18. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindIndex.h +28 -0
  19. nind-2.4.0/src/cpp/NindIndex/NindIndexMerge.cpp +308 -0
  20. nind-2.4.0/src/cpp/NindIndex/NindIndexMerge.h +116 -0
  21. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndex.cpp +163 -0
  22. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndex.h +39 -1
  23. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindLocalIndex.cpp +15 -0
  24. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindLocalIndex.h +5 -0
  25. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindTermIndex.cpp +16 -0
  26. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindTermIndex.h +6 -0
  27. nind-2.4.0/src/cpp/NindIndex/NindTokenizer.cpp +175 -0
  28. nind-2.4.0/src/cpp/NindIndex/NindTokenizer.h +100 -0
  29. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindLexicon/NindLexiconFile.cpp +7 -1
  30. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindPy/CMakeLists.txt +2 -0
  31. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindPy/bindings.cpp +301 -0
  32. {nind-2.2.1 → nind-2.4.0}/src/py/nind/NindFile.py +1 -1
  33. {nind-2.2.1 → nind-2.4.0}/src/py/nind/NindIndex.py +1 -1
  34. {nind-2.2.1 → nind-2.4.0}/src/py/nind/NindLexiconindex.py +1 -1
  35. {nind-2.2.1 → nind-2.4.0}/src/py/nind/NindLocalindex.py +1 -1
  36. {nind-2.2.1 → nind-2.4.0}/src/py/nind/NindPadFile.py +1 -1
  37. {nind-2.2.1 → nind-2.4.0}/src/py/nind/NindRetrolexicon.py +1 -1
  38. {nind-2.2.1 → nind-2.4.0}/src/py/nind/NindTermindex.py +1 -1
  39. {nind-2.2.1 → nind-2.4.0}/src/py/nind/Nind_dumpDocument.py +1 -1
  40. {nind-2.2.1 → nind-2.4.0}/src/py/nind/Nind_search.py +1 -1
  41. {nind-2.2.1 → nind-2.4.0}/src/py/nind/Nind_trouveMotsSansOccurrence.py +1 -1
  42. {nind-2.2.1 → nind-2.4.0}/src/py/nind/Nind_trouveNombresPremiers.py +1 -1
  43. nind-2.4.0/src/py/nind/_index_store.py +305 -0
  44. {nind-2.2.1 → nind-2.4.0}/src/py/nind/amose/NindIdentifiantsClef.py +1 -1
  45. nind-2.4.0/src/py/nind/nind_engine.py +1190 -0
  46. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindAmose/CMakeLists.txt +1 -0
  47. nind-2.4.0/tst/cpp/unit/NindAmose/NindAmoseMergeTest.cpp +128 -0
  48. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindIndex/CMakeLists.txt +3 -0
  49. nind-2.4.0/tst/cpp/unit/NindIndex/NindBulkTest.cpp +344 -0
  50. nind-2.4.0/tst/cpp/unit/NindIndex/NindIndexMergeTest.cpp +379 -0
  51. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindIndexRobustnessTest.cpp +4 -0
  52. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindTermIndexTest.cpp +22 -0
  53. nind-2.4.0/tst/cpp/unit/NindIndex/NindTokenizerTest.cpp +88 -0
  54. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindLexicon/NindLexiconFileTest.cpp +11 -0
  55. {nind-2.2.1 → nind-2.4.0}/tst/py/test_native.py +68 -0
  56. nind-2.4.0/tst/py/test_nind_engine.py +775 -0
  57. {nind-2.2.1 → nind-2.4.0}/tst/py/test_nindlexiconindex.py +2 -2
  58. {nind-2.2.1 → nind-2.4.0}/tst/py/test_nindlocalindex.py +1 -1
  59. {nind-2.2.1 → nind-2.4.0}/tst/py/test_nindtermindex.py +1 -1
  60. {nind-2.2.1 → nind-2.4.0}/uv.lock +1 -1
  61. nind-2.2.1/src/py/nind/nind_engine.py +0 -351
  62. nind-2.2.1/tst/py/test_nind_engine.py +0 -142
  63. {nind-2.2.1 → nind-2.4.0}/.github/workflows/build.yml +0 -0
  64. {nind-2.2.1 → nind-2.4.0}/.github/workflows/docs.yml +0 -0
  65. {nind-2.2.1 → nind-2.4.0}/.github/workflows/publish-pypi.yml +0 -0
  66. {nind-2.2.1 → nind-2.4.0}/.gitignore +0 -0
  67. {nind-2.2.1 → nind-2.4.0}/AGENTS.md +0 -0
  68. {nind-2.2.1 → nind-2.4.0}/Dockerfile-ubuntu24.04 +0 -0
  69. {nind-2.2.1 → nind-2.4.0}/LICENCE.md +0 -0
  70. {nind-2.2.1 → nind-2.4.0}/NindConfig-version-src.cmake +0 -0
  71. {nind-2.2.1 → nind-2.4.0}/NindConfig.cmake +0 -0
  72. {nind-2.2.1 → nind-2.4.0}/SystemSpecificInformations.cmake +0 -0
  73. {nind-2.2.1 → nind-2.4.0}/bench/index_bulk.py +0 -0
  74. {nind-2.2.1 → nind-2.4.0}/bench/lookup_latency_bench.py +0 -0
  75. {nind-2.2.1 → nind-2.4.0}/bench/msmarco/convert_to_nind.py +0 -0
  76. {nind-2.2.1 → nind-2.4.0}/bench/run_bulk_indexing_benchmark.sh +0 -0
  77. {nind-2.2.1 → nind-2.4.0}/bench/run_indexing_benchmark.sh +0 -0
  78. {nind-2.2.1 → nind-2.4.0}/cicd/extract_package.sh +0 -0
  79. {nind-2.2.1 → nind-2.4.0}/docs/api/index.md +0 -0
  80. {nind-2.2.1 → nind-2.4.0}/docs/api/readers.md +0 -0
  81. {nind-2.2.1 → nind-2.4.0}/docs/architecture.md +0 -0
  82. {nind-2.2.1 → nind-2.4.0}/docs/cli.md +0 -0
  83. {nind-2.2.1 → nind-2.4.0}/docs/conf.py +0 -0
  84. {nind-2.2.1 → nind-2.4.0}/docs/index.md +0 -0
  85. {nind-2.2.1 → nind-2.4.0}/docs/requirements.txt +0 -0
  86. {nind-2.2.1 → nind-2.4.0}/gbuild.sh +0 -0
  87. {nind-2.2.1 → nind-2.4.0}/src/CMakeLists.txt +0 -0
  88. {nind-2.2.1 → nind-2.4.0}/src/cpp/CMakeLists.txt +0 -0
  89. {nind-2.2.1 → nind-2.4.0}/src/cpp/Doxyfile +0 -0
  90. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindAmose/NindLexiconAmose.cpp +0 -0
  91. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindAmose/NindLexiconAmose.h +0 -0
  92. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindAmose/NindLocalAmose.cpp +0 -0
  93. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindAmose/NindLocalAmose.h +0 -0
  94. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindAmose/NindTermAmose.cpp +0 -0
  95. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindAmose/NindTermAmose.h +0 -0
  96. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindBasics/CMakeLists.txt +0 -0
  97. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindBasics/NindSignalCatcher.cpp +0 -0
  98. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindBasics/NindSignalCatcher.h +0 -0
  99. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindCommonExport.h +0 -0
  100. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindExceptions.h +0 -0
  101. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndexK.cpp +0 -0
  102. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndexK.h +0 -0
  103. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindLexicon/CMakeLists.txt +0 -0
  104. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindLexicon/NindLexicon.cpp +0 -0
  105. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindLexicon/NindLexicon.h +0 -0
  106. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindLexicon/NindLexiconFile.h +0 -0
  107. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindRetrolexicon/CMakeLists.txt +0 -0
  108. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindRetrolexicon/NindRetrolexicon.cpp +0 -0
  109. {nind-2.2.1 → nind-2.4.0}/src/cpp/NindRetrolexicon/NindRetrolexicon.h +0 -0
  110. {nind-2.2.1 → nind-2.4.0}/src/py/CMakeLists.txt +0 -0
  111. {nind-2.2.1 → nind-2.4.0}/src/py/Nind_checkAndRepair.py +0 -0
  112. {nind-2.2.1 → nind-2.4.0}/src/py/Nind_checkDocument.py +0 -0
  113. {nind-2.2.1 → nind-2.4.0}/src/py/Nind_checkRejtroAndLexicon.py +0 -0
  114. {nind-2.2.1 → nind-2.4.0}/src/py/nind/__init__.py +0 -0
  115. {nind-2.2.1 → nind-2.4.0}/src/py/nind/amose/CMakeLists.txt +0 -0
  116. {nind-2.2.1 → nind-2.4.0}/src/py/nind/amose/LatXmlParser.py +0 -0
  117. {nind-2.2.1 → nind-2.4.0}/src/py/nind/amose/Nind_convertitLuceneDump.py +0 -0
  118. {nind-2.2.1 → nind-2.4.0}/src/py/nind/amose/Nind_convertitSampleDump.py +0 -0
  119. {nind-2.2.1 → nind-2.4.0}/src/py/nind/amose/Nind_convertitXmlClef.py +0 -0
  120. {nind-2.2.1 → nind-2.4.0}/tst/CMakeLists.txt +0 -0
  121. {nind-2.2.1 → nind-2.4.0}/tst/cpp/CMakeLists.txt +0 -0
  122. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/CMakeLists.txt +0 -0
  123. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_effaceDocument.cpp +0 -0
  124. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_indexeCorpus.cpp +0 -0
  125. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_litTexteAnalysej.cpp +0 -0
  126. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_litTexteAnalysej.h +0 -0
  127. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_testLecteur.cpp +0 -0
  128. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_testSynchro.cpp +0 -0
  129. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindAmose/Nind_indexeCorpus.cpp +0 -0
  130. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindBasics/CMakeLists.txt +0 -0
  131. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindBasics/NindBasics_testLecture.cpp +0 -0
  132. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/CMakeLists.txt +0 -0
  133. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindDate.cpp +0 -0
  134. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindDate.h +0 -0
  135. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindFichiers.cpp +0 -0
  136. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindFichiers.h +0 -0
  137. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndexTest.cpp +0 -0
  138. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndexTest.h +0 -0
  139. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_indexe.cpp +0 -0
  140. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_indexe.h +0 -0
  141. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_litDumpS2.cpp +0 -0
  142. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_litDumpS2.h +0 -0
  143. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_testEcrivain.cpp +0 -0
  144. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_testLecteur.cpp +0 -0
  145. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_testSynchro.cpp +0 -0
  146. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindLexiconIndex_testEcrivain.cpp +0 -0
  147. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindLexiconIndex_testLecteur.cpp +0 -0
  148. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindTermIndex_testEcrivain.cpp +0 -0
  149. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/NindTermIndex_testLecteur.cpp +0 -0
  150. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/Nind_testLateconNumber.cpp +0 -0
  151. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindIndex/Nind_testNumbersInLocalIndexes.cpp +0 -0
  152. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/CMakeLists.txt +0 -0
  153. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testEcrivain.cpp +0 -0
  154. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testInter.cpp +0 -0
  155. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLecteur.cpp +0 -0
  156. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLimites.cpp +0 -0
  157. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLimitesB.cpp +0 -0
  158. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLimitesC.cpp +0 -0
  159. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testMemoire.cpp +0 -0
  160. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindSearch/CMakeLists.txt +0 -0
  161. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindSearch/NindGetDocTerms.cpp +0 -0
  162. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindSearch/NindSearch.cpp +0 -0
  163. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindSearch/NindSearchAmose.cpp +0 -0
  164. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindTests/CMakeLists.txt +0 -0
  165. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindTests/NindTestControlC.cpp +0 -0
  166. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindTests/Nind_test1.cpp +0 -0
  167. {nind-2.2.1 → nind-2.4.0}/tst/cpp/NindTests/Nind_testSynchro.cpp +0 -0
  168. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/CMakeLists.txt +0 -0
  169. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindAmose/NindLexiconAmoseTest.cpp +0 -0
  170. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindAmose/NindLocalAmoseTest.cpp +0 -0
  171. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindAmose/NindTermAmoseTest.cpp +0 -0
  172. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindBasics/CMakeLists.txt +0 -0
  173. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindBasics/NindBasicsRobustnessTest.cpp +0 -0
  174. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindBasics/NindFileTest.cpp +0 -0
  175. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindBasics/NindPadFileTest.cpp +0 -0
  176. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindLexiconIndexTest.cpp +0 -0
  177. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindLocalIndexTest.cpp +0 -0
  178. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindLexicon/CMakeLists.txt +0 -0
  179. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindLexicon/NindLexiconTest.cpp +0 -0
  180. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindRetrolexicon/CMakeLists.txt +0 -0
  181. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/NindRetrolexicon/NindRetrolexiconTest.cpp +0 -0
  182. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/TestFileBytes.h +0 -0
  183. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/TestTempDir.h +0 -0
  184. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/doctest.h +0 -0
  185. {nind-2.2.1 → nind-2.4.0}/tst/cpp/unit/doctest_main.cpp +0 -0
  186. {nind-2.2.1 → nind-2.4.0}/tst/py/test_nindfile.py +0 -0
  187. {nind-2.2.1 → nind-2.4.0}/tst/py/test_nindretrolexicon.py +0 -0
@@ -33,18 +33,19 @@ To run tests manually instead of via the full `gbuild.sh` cycle: `ctest --output
33
33
 
34
34
  C++ unit tests live in `tst/cpp/unit/<Module>/` (one executable per library, one ctest entry each) and use **doctest**, vendored as the single header `tst/cpp/unit/doctest.h` (v2.4.12) — no system package or download needed, so they are always built. Each executable accepts doctest options, e.g. `NindIndexUnitTests -tc='NindIndexRobustness*' -s`. Corrupt/hostile-input tests live in the `*RobustnessTest.cpp` files: they patch or byte-by-byte corrupt files produced by the writers and require every read to either succeed or throw a `FileException` — run them under `-a ON` so ASan/UBSan/LSan turn any memory error, UB or leak into a failure. (`-a ON` needs the GCC sanitizer runtimes — `libasan`, `libubsan`, `liblsan` packages on RHEL-like systems.)
35
35
 
36
- Memory-safety conventions in the C++ core (keep them when touching it): readers must never trust a length/offset/count read from a file — bound it against the buffer (`NindFile::get*` check before advancing) or the real file size (`NindFile::getCurrentFileSize()`); byte assembly is done in `uint32_t`/`uint64_t` (never `int`, which overflows); chains read from files (indirection blocks, compound words) must be provably finite; `NindFile`/`NindPadFile` are non-copyable; critical sections use the RAII `NindCriticalSection`, never raw `beginCriticalSection`/`endCriticalSection`.
36
+ Memory-safety conventions in the C++ core (keep them when touching it): readers must never trust a length/offset/count read from a file — bound it against the buffer (`NindFile::get*` check before advancing) or the real file size (`NindFile::getCurrentFileSize()`); byte assembly is done in `uint32_t`/`uint64_t` (never `int`, which overflows); chains read from files (indirection blocks, compound words) must be provably finite; `NindFile`/`NindPadFile` are non-copyable; critical sections use the RAII `NindCriticalSection`, never raw `beginCriticalSection`/`endCriticalSection`; a file has at most one writer — `NindPadFile`/`NindLexiconFile` writers take a non-blocking exclusive `flock` (`NindFile::lockExclusive`, no-op on Windows) and create files with `"w+bx"` (never truncating a file another writer just created), so a second writer, even in-process, gets an exception instead of corrupting the file (readers never lock).
37
37
 
38
38
  `-DDEBUG_NIND` is added automatically in `Debug` and `RelWithDebInfo` modes (and in `Release` if configured with `-DWITH_DEBUG_MESSAGES=ON`) to enable conditional debug output.
39
39
 
40
40
  ## Build & test — Python
41
41
 
42
- The Python package (`nind`, in `src/py/nind`) is managed with `pyproject.toml`/`scikit-build-core` (the old `src/py/CMakeLists.txt` build path is kept only for compatibility). `uv sync`/`uv build` compiles `nind._native` (via CMake — see `src/cpp/NindPy/CMakeLists.txt`) and installs the package (editable for `sync`) plus dev dependencies (pytest) into `.venv`. Building requires a C++14 compiler; `scikit-build-core`, `pybind11`, `cmake`, and `ninja` themselves are all pulled automatically as PEP 517 build requirements (the latter two so builds work even in containers with no system CMake/Ninja, e.g. manylinux in CI) — no manual install needed.
42
+ The Python package (`nind`, in `src/py/nind`) is managed with `pyproject.toml`/`scikit-build-core` (the old `src/py/CMakeLists.txt` build path is kept only for compatibility). `uv sync`/`uv build` compiles `nind._native` (via CMake — see `src/cpp/NindPy/CMakeLists.txt`) and installs the package (editable for `sync`: Python edits are live, but the compiled `_native` is only rebuilt by `uv sync --reinstall-package nind`) plus dev dependencies (pytest) into `.venv`. Building requires a C++14 compiler; `scikit-build-core`, `pybind11`, `cmake`, and `ninja` themselves are all pulled automatically as PEP 517 build requirements (the latter two so builds work even in containers with no system CMake/Ninja, e.g. manylinux in CI) — no manual install needed.
43
43
 
44
44
  Tests live in `tst/py/` (one file per module, mirroring `tst/cpp/unit/`), configured via `[tool.pytest.ini_options]` in `pyproject.toml`:
45
45
 
46
46
  ```
47
- uv sync # first time / after touching pyproject.toml or src/cpp/NindPy/**
47
+ uv sync # first time / after touching pyproject.toml
48
+ uv sync --reinstall-package nind # after any C++ change (src/cpp/**, bindings included): plain `uv sync` doesn't rebuild _native
48
49
  uv run pytest tst/py -v
49
50
  uv build # produce the platform wheel (nind-*.whl) + sdist in dist/
50
51
  ```
@@ -80,9 +81,12 @@ NindIndex (deps: NindBasics, NindRetrolexicon)
80
81
  — NindTermIndex: the inverted file itself, as an index file
81
82
  — NindLocalIndex: per-document local index file
82
83
  — NindIndex: ties the above into a full inverted/local index file
84
+ — NindIndexMerge: offline merge of index shards (disjoint documents) into one standard index
85
+ — NindTokenizer: tokenizer for source-code-like text (used by the Python indexer)
83
86
 
84
87
  NindAmose (deps: NindIndex, NindRetrolexicon)
85
88
  — adapts nind to Amose's richer term-type model (see internal doc refs "LAT2015.JYS.448")
89
+ — NindAmoseMerge: NindIndexMerge for Amose indexes (retro lexicon, recomputed term counters)
86
90
 
87
91
  NindPy (deps: NindBasics, NindRetrolexicon, NindLexicon, NindIndex sources, compiled in directly)
88
92
  — pybind11 bindings, built only through the Python wheel (scikit-build-core), not gbuild.sh
@@ -105,7 +109,8 @@ Parallels the C++ layering with lowercase-suffixed module names for the index cl
105
109
  - `NindFile.py` — low-level binary codec. Historically read-only-oriented (the `litXxx` getters have long existed for every format), but now has a full writer counterpart too: `ejcritNombre1/3/4/5`, `ejcritNombreULat`/`ejcritNombreSLat` (both varint encodings, full tier range matching the readers and the C++ implementation), `ejcritChaine`, `ejcritZejros`.
106
110
  - `NindPadFile.py` — the shared "pad file" envelope (header, indirection block(s), specifics, identification trailer); read-only, no writer (mirrors `NindBasics::NindPadFile`, but nothing in Python subclasses it as a writer the way `NindIndex`/`NindRetrolexicon` do in C++).
107
111
  - `NindRetrolexicon.py`, `NindLexiconindex.py`, `NindTermindex.py`, `NindLocalindex.py` — ergonomic read-only Python wrappers around the corresponding `NindIndex`-family C++ classes; their hot-path lookups delegate to `nind._native`, as does the shared structural-diagnostic base (`analyseFichierPadFile`/`analyseFichierIndex`, inherited from `NindPadFile.py`/`NindIndex.py`). The remaining, per-format diagnostic/analysis methods (`dumpeFichier`, `debogueIndex`, `donneCollisions`, `donneMax`, `donneClef`, `afficheTerme`, `afficheDocument`, and each class's own `analyseFichierXxx` beyond the shared base) remain hand-rolled pure-Python parsing, since `nind._native` doesn't expose that introspection surface yet (see "Planned follow-up work" below). `NindLexiconindex` is hash-bucketed (`clefB(word) % nombreIndirection`), the other two are directly indexed by id.
108
- - `nind_engine.py` — `NindIndexer` (writer) and `NindEngine` (BM25-style search) built on top of the above. `NindIndexer` is the *only* Python writer for the index-family formats (`.nindlexiconindex`/`.nindtermindex`/`.nindlocalindex`) — it builds them via `nind._native`'s `is_writer=True` constructors. It does not write `.nindretrolexicon` (not needed by `NindEngine`, which only supports simple, non-compound words). Indexing is parallel but every file keeps a single writer: files are tokenized in worker processes (`workers` argument of `index_files`; in-process below `PARALLEL_MIN_FILES` files), then the term and local index files are written concurrently from two threads through the flat-buffer bulk bindings `NindTermIndex.set_term_defs_arrays`/`NindLocalIndex.set_local_defs_arrays`, which take `array('I')` buffers (building `Document`/`Term`/`Localisation` objects in Python cost ~20× the write itself) and release the GIL. The `*_local_def*_arrays` writers split a term's positions into entries of ≤255 (`<nbreLocalisations>` is one byte).
112
+ - `nind_engine.py` — `NindIndexer` (writer) and `NindEngine` (BM25-style search) built on top of the above. `NindIndexer` is the *only* Python writer for the index-family formats (`.nindlexiconindex`/`.nindtermindex`/`.nindlocalindex`) — it builds them via `nind._native`'s `is_writer=True` constructors. It does not write `.nindretrolexicon` (not needed by `NindEngine`, which only supports simple, non-compound words). `NindIndexer.index_jsonl` reads pyserini's JSON Lines corpus format (`{"id": ..., "contents": ...}` per line; record n = document n, the record ids are returned, not stored) — worker processes each parse a line-aligned byte range of the file. Tokenization is native (`NindTokenizer`, bound as `nind._native.Tokenizer`; options `tokenizer=` of `NindIndexer`, see `TOKENIZER_DEFAULTS`: each identifier split optional, ASCII lowercasing, tokens over 254 bytes — the lexicon's limit, `NindFile::putString` — dropped); a subclass overriding `_default_tokenize` falls back to Python token lists (`_chunk_from_token_lists`). Indexing is parallel but every file keeps a single writer: worker processes tokenize chunks of documents with `Tokenizer.tokenize_batch`, which returns the chunk's vocabulary plus uint32 term ids/ends (cheap to pickle); the parent writes the lexicon with one `add_simple_words` call (`NindLexiconIndex::addWords`: each hash bucket written once), feeds the chunks to `nind._native.IndexBuilder` (remaps chunk ids to term ids), which writes the term and local index files from two threads with the GIL released. Every writer of a new generation runs in bulk mode (`begin_bulk`/`end_bulk`, `NindIndex::beginBulk`: no per-write flush/seek/signal handler, indirections and tail written once — byte-identical files). The local writers split a term's positions into entries of ≤255 (`<nbreLocalisations>` is one byte). `local_index=False` leaves the local index out. Each generation also holds `<prefix>.nindmeta` (JSON: tokenizer options, local index or not) and `<prefix>.ninddoclengths` (binary: `NINDDL01`, count, doc ids, lengths): `NindEngine` reads document lengths from it (no local index needed) and tokenizes queries with the stored options; indexes without them (older versions) are read from their local index and queried/updated with the original regex tokenizer (`LEGACY_TOKENIZER`). Every build goes into a new generation directory published atomically (see `_index_store.py` below), so rebuilding never disturbs readers. With `shards=N` it instead builds N independent indexes over contiguous parts of the files in worker processes (in a temporary directory inside the new generation), then merges them with `nind._native.merge_indexes`; opt-in because the merge costs about as much as writing the index again (see "Sharded indexing" below), so it only pays off when per-document work dominates. Incremental indexing goes through `NindIndexer.update()` → `IndexUpdate` (shortcuts `add_file`/`add_files`/`remove_documents`): a context manager that, under the writer lock, copies the published generation into a new one (keeping its tokenizer options and its local index or not; explicitly different `NindIndexer` arguments are refused), collects adds/replacements/removals in memory, and on exit reopens the native writers on the copy (existing files: `r+b`, bulk mode), adds the new words, rewrites every touched term definition whole (a replaced/removed document's terms come from the local index, or, without one, from a scan of every term) (read with `get_term_def`, minus removed docs, plus added ones; empty = erased) and every touched document (empty = erased), then publishes; an empty or failed update publishes nothing. Term/local writers are opened with the lexicon's *old* identification and written with the new one, so all three files end up paired. The lexicon's hash bucket count is fixed in the file (`NindLexiconIndex::getModulo`, bound as `get_modulo`): past `LEXICON_MAX_LOAD` words per bucket, the update rebuilds the index with `merge_indexes` on itself with twice as many buckets as words (term ids change, document ids don't). Indirection blocks added by an existing writer use the constructor's `indirection_bloc_size` (it must not be 0 for a writer reopening a file), so updates pass at least `INDIRECTION_BLOC_MIN`. Per-update cost is dominated by reopening the native writers (they scan their whole file; see the I/O follow-up below), so batching documents per update matters on large indexes.
113
+ - `_index_store.py` — the on-disk layout that makes every `NindIndexer`/`NindEngine` operation safe without the caller doing anything: each build writes a fresh, never-modified generation directory `<prefix>.gen-<n>-<hex>/`, flushed to disk (`fsync`) then published by atomically replacing the pointer file `<prefix>.CURRENT` (a pointer file rather than a symlink: atomic on Windows too). Readers open every file of the generation `CURRENT` names under a shared `flock` on its `generation.lock` (only while opening: open files stay readable after deletion on POSIX); `NindEngine` checks `CURRENT` (one `stat`) at the start of each public call and switches to a newly published build, each call answering from one snapshot (`_Snapshot`), calls serialized by an `RLock` (the native readers aren't thread-safe). Writers of one prefix are serialized by `<prefix>.lock`; after publishing, the writer deletes older generations nobody is opening, crashed builds' leftovers, and flat files of indexes written before generations existed (still readable: no `CURRENT` = flat layout). A failed build never touches `CURRENT`. Lock fds are closed in forked children (`os.register_at_fork`) — otherwise a worker outliving a killed writer would keep the index locked forever — and `NindIndexer`'s pool workers exit when their parent dies. Windows: `msvcrt` locks are exclusive only, so readers don't lock and rely on retrying; generations still open can't be deleted and are retried at the next build.
109
114
  - `Nind_*.py` at the package root — standalone CLI scripts (dump a document, convert corpora, search, diagnostics)
110
115
  - `amose/` — Amose-specific conversion/parsing scripts (Lucene dump, XML-CLEF, sample corpora)
111
116
 
@@ -124,7 +129,11 @@ Binary file formats are specified in EBNF grammar (see README). The Python packa
124
129
 
125
130
  Not part of this: `.nindretrolexicon` writing (`NindIndexer` still doesn't write this format — unrelated, no change planned).
126
131
 
127
- **Sharded indexing (planned next, after the in-process parallelism above).** Nind files are single-writer by design (no locking, readers resync lazily; see `NindPadFile::getEntryPos`), and every write rewrites the shared specifics/identification tail at EOF, so concurrent writers on one file would serialize anyway — sharing a file between writers is explicitly *not* the plan. Instead, scale writes across N independent shards (each its own lexicon + term + local files, one single-threaded writer each, over a partition of the documents), then either:
128
- - merge offline: union the lexicons into a new one, remap every shard's term ids, concatenate posting lists with doc ids kept in increasing order (the term index stores relative doc ids), and remap term ids inside each shard's local index; or
129
- - search across shards: `NindEngine` fans out per shard, but BM25 needs corpus-wide statistics (N, df, avgdl) summed over all shards, not per-shard values.
130
- Expected cost ~1–2 weeks including tests; zero overhead on the write path.
132
+ **Sharded indexing — offline merge implemented; follow-ups below.** Nind files are single-writer by design (no locking, readers resync lazily; see `NindPadFile::getEntryPos`), and every write rewrites the shared specifics/identification tail at EOF, so concurrent writers on one file would serialize anyway — sharing a file between writers is explicitly *not* the plan. Writes scale instead across N independent shards (each its own lexicon + term + local files, one single-threaded writer each, over a partition of the documents), merged offline into one standard index that every existing reader uses unchanged (search across shards was considered and rejected: every reader, Amose's included, would have to learn about shards, and BM25 would need corpus-wide statistics summed over them).
133
+
134
+ How the merge works (`NindIndexMerge`, `src/cpp/NindIndex/`): shard lexicons are enumerated with `NindLexiconIndex::getEntries` (walks the hash buckets; no retro lexicon needed) and unioned with words canonicalised as (union id of the prefix, last component) — which also gives the exact merged size, used as the lexicon's hash modulo; documents are listed with `NindLocalIndex::getDocIdents` and checked disjoint *before anything is written* (shards without local index — all or none — give a merged index without one; a document in two shards then only shows up as a duplicate in a shared term), merged files are written in bulk mode with a single `addWords`; each merged term's postings are gathered from every shard (per term, so memory stays small) and each document is copied with its term ids remapped (local term order is kept, so Amose's per-occurrence reading order survives). A failed merge removes the target files. `NindAmoseMerge` (`src/cpp/NindAmose/`) forces a retro lexicon and recomputes the 8 term counters, a term's type being derived from its components as in `NindLexiconAmose::getWord` (unique-term counts aren't additive across shards); they're written once at the end with `NindTermIndex::setSpecificWords`. The plain merge refuses shards with term specifics.
135
+
136
+ Follow-ups:
137
+ - **Merge cost is I/O-syscall bound, and so are all writes** — largely addressed for writers of new files by bulk mode (`NindIndex::beginBulk`, used by `NindIndexer` and `NindIndexMerge`; a 10k-document build went from ~1.9M to ~0.2M syscalls, the rest being the input files). Still open: Merging 10k documents / 58k terms from 4 shards makes ~2.3M `lseek`, ~1.5M `read`, ~280k `write` and ~250k `rt_sigaction` calls: `NindIndex::getDefinition` discards the stdio buffer (`flush`) then seeks+reads twice (indirection, then data) on every read, `NindFile::writeBuffer` flushes every write, and every critical section installs/restores a signal handler. A direct build pays the same per definition, so a merge costs about a full rewrite. Reading with `pread` (one syscall per read, no buffer discard) and a bulk mode for a writer creating a fresh file (no per-write flush, tail written once) would speed up merges, direct builds and search alike — mind the reader-resync semantics above when touching this.
138
+ - **CLI for Amose/LIMA** multi-process indexing: a small C++ program (like the legacy `tst/cpp/NindAmose/` tools) running `NindAmoseMerge` over shard prefixes. `NindAmoseMerge` isn't bound in `nind._native` (the wheel doesn't compile `NindAmose`).
139
+ - `NindLexiconAmose::getWord` dereferences past the end of the component list for the lexicon word `§` alone (the first prefix of every named entity): undefined behaviour, seen as a `bad_alloc`. Normal lookups never ask for that id; the merge doesn't call `getWord`.
@@ -113,7 +113,7 @@ endif ()
113
113
 
114
114
  ########### setting project verion number ###############
115
115
 
116
- set(NIND_VERSION "2.2.1")
116
+ set(NIND_VERSION "2.4.0")
117
117
 
118
118
  string(REGEX REPLACE "^([0-9]+)\\.[0-9]+\\.[0-9]+$" "\\1" NIND_VERSION_MAJOR "${NIND_VERSION}")
119
119
  string(REGEX REPLACE "^[0-9]+\\.([0-9]+)\\.[0-9]+$" "\\1" NIND_VERSION_MINOR "${NIND_VERSION}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: nind
3
- Version: 2.2.1
3
+ Version: 2.4.0
4
4
  Summary: Python frontend for the Nind Indexing Module, with its native C++ index stack built in
5
5
  Requires-Python: >=3.8
6
6
  Description-Content-Type: text/markdown
@@ -84,6 +84,20 @@ for doc_id, score in engine.search("ejcrit nombre", top_k=5):
84
84
 
85
85
  `index_dir` must already exist; each input file becomes one document,
86
86
  identified externally by its position in the list (`0`, `1`, `2`, ...).
87
+ A JSON Lines corpus (pyserini's format) is indexed with
88
+ `indexer.index_jsonl("corpus.jsonl")`. Tokenizer options (each identifier
89
+ split optional, lowercasing) and `local_index=False` (a much smaller index,
90
+ enough for `NindEngine`) are `NindIndexer` arguments.
91
+ To add, replace or remove documents in an existing index (or build one
92
+ file by file), use an incremental update instead of rebuilding:
93
+
94
+ ```python
95
+ with indexer.update() as update:
96
+ for path in more_files:
97
+ update.add_file(path) # returns the new document's id
98
+ update.remove(1)
99
+ ```
100
+
87
101
  `NindEngine` also exposes the building blocks it uses internally
88
102
  (`get_term_id`, `get_df`, `get_tf`, `get_doc_len`) for inspecting relevance
89
103
  or a specific document directly.
@@ -77,6 +77,20 @@ for doc_id, score in engine.search("ejcrit nombre", top_k=5):
77
77
 
78
78
  `index_dir` must already exist; each input file becomes one document,
79
79
  identified externally by its position in the list (`0`, `1`, `2`, ...).
80
+ A JSON Lines corpus (pyserini's format) is indexed with
81
+ `indexer.index_jsonl("corpus.jsonl")`. Tokenizer options (each identifier
82
+ split optional, lowercasing) and `local_index=False` (a much smaller index,
83
+ enough for `NindEngine`) are `NindIndexer` arguments.
84
+ To add, replace or remove documents in an existing index (or build one
85
+ file by file), use an incremental update instead of rebuilding:
86
+
87
+ ```python
88
+ with indexer.update() as update:
89
+ for path in more_files:
90
+ update.add_file(path) # returns the new document's id
91
+ update.remove(1)
92
+ ```
93
+
80
94
  `NindEngine` also exposes the building blocks it uses internally
81
95
  (`get_term_id`, `get_df`, `get_tf`, `get_doc_len`) for inspecting relevance
82
96
  or a specific document directly.
@@ -96,8 +96,9 @@ The main process is the bottleneck: at 1M docs, 16 and 48 workers take the
96
96
  same time. Its time goes almost entirely into the C++ writers, which make
97
97
  every entry durable on its own (~13 system calls per definition written:
98
98
  seeks, reads of the indirection entry, `fwrite`+`fflush`, and signal-handler
99
- swaps around each Ctrl-C-safe critical section). A faster bulk load would
100
- need a C++-side bulk mode (e.g. buffered indirection and sequential writes).
99
+ swaps around each Ctrl-C-safe critical section). These measurements predate
100
+ the writers' bulk mode (`begin_bulk`/`end_bulk`, see `NindIndex::beginBulk`),
101
+ which `NindIndexer` now uses and this script doesn't yet.
101
102
  The main process's memory is mostly worker results waiting to be written;
102
103
  at most `2 × workers` chunks (~8 MB of corpus each) are in flight.
103
104
 
@@ -23,3 +23,10 @@ See {doc}`../quickstart` for a walk-through.
23
23
  :members:
24
24
  :show-inheritance:
25
25
  ```
26
+
27
+ ## IndexUpdate
28
+
29
+ ```{eval-rst}
30
+ .. autoclass:: nind.nind_engine.IndexUpdate
31
+ :members: add_file, add_text, add_tokens, remove, LEXICON_MIN_BUCKETS, LEXICON_MAX_LOAD, INDIRECTION_BLOC_MIN
32
+ ```
@@ -18,11 +18,65 @@ indexer.index_files([
18
18
  ])
19
19
  ```
20
20
 
21
- This writes three files into `indices/`:
22
- `corpus.nindlexiconindex`, `corpus.nindtermindex` and
23
- `corpus.nindlocalindex`. `index_dir` must already exist; each input file
24
- becomes one document, identified externally by its position in the list
25
- (`0`, `1`, `2`, ...).
21
+ This writes the index into a new generation directory inside `indices/`
22
+ (`corpus.gen-<n>-<hex>/`: `corpus.nindlexiconindex`, `corpus.nindtermindex`,
23
+ `corpus.nindlocalindex`, plus `corpus.nindmeta` - the tokenizer options - and
24
+ `corpus.ninddoclengths`), published atomically through `corpus.CURRENT`.
25
+ `index_dir` must already exist; each input file becomes one document,
26
+ identified externally by its position in the list (`0`, `1`, `2`, ...).
27
+
28
+ A corpus in pyserini's JSON Lines format (one `{"id": ..., "contents": ...}`
29
+ object per line) is indexed directly, record `n` becoming document `n`:
30
+
31
+ ```python
32
+ record_ids = indexer.index_jsonl("corpus.jsonl") # record_ids[n]: the "id" of document n
33
+ ```
34
+
35
+ ## Tokenizer and index options
36
+
37
+ Texts are tokenized by a native tokenizer for source-code-like text: words
38
+ are runs of letters and digits, split at camelCase (`fooBar` -> `foo` `Bar`,
39
+ `HTTPServer` -> `HTTP` `Server`), underscore (`foo_bar` -> `foo` `bar`) and
40
+ letter/digit (`utf8` -> `utf` `8`) boundaries. Each split is optional, and
41
+ tokens can be lowercased:
42
+
43
+ ```python
44
+ indexer = NindIndexer("indices", prefix="corpus",
45
+ tokenizer={"lowercase": True, "split_digits": False})
46
+ ```
47
+
48
+ The options are stored with the index: `NindEngine` tokenizes queries the
49
+ same way, and updates keep them. Tokens longer than 254 bytes (the longest
50
+ word a nind lexicon stores) are dropped.
51
+
52
+ `local_index=False` leaves out the local index (each document's terms and
53
+ their positions). `NindEngine` doesn't need it - document lengths are stored
54
+ apart - and it makes up most of an index's size and a good part of its build
55
+ time; other readers of nind indexes (Amose...) may need it:
56
+
57
+ ```python
58
+ indexer = NindIndexer("indices", prefix="corpus", local_index=False)
59
+ ```
60
+
61
+ ## Index incrementally
62
+
63
+ `index_files` rebuilds the whole index. To add, replace or remove documents
64
+ in an existing index instead - or to build one file by file - group the
65
+ changes in an update; they are published together when the `with` block
66
+ ends (nothing changes if it raises):
67
+
68
+ ```python
69
+ with indexer.update() as update:
70
+ for path in new_files:
71
+ doc_id = update.add_file(path) # new id: one more than the highest
72
+ update.add_file("changed.py", doc_id=1) # replaces document 1
73
+ update.remove(2)
74
+ ```
75
+
76
+ `add_file(path)`, `add_files(paths)` and `remove_documents(ids)` on the
77
+ indexer are one-update shortcuts. Each update copies the index and reopens
78
+ its files, so on a large index group many documents per update rather than
79
+ calling `add_file` once per file.
26
80
 
27
81
  ## Search it
28
82
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "nind"
3
- version = "2.2.1"
3
+ version = "2.4.0"
4
4
  description = "Python frontend for the Nind Indexing Module, with its native C++ index stack built in"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.8"
@@ -25,7 +25,7 @@ testpaths = ["tst/py"]
25
25
  pythonpath = ["src/py"]
26
26
 
27
27
  [tool.bumpver]
28
- current_version = "v2.2.1"
28
+ current_version = "v2.4.0"
29
29
  version_pattern = "vMAJOR.MINOR.PATCH"
30
30
  commit_message = "Bump version {old_version} -> {new_version}"
31
31
  commit = true
@@ -8,6 +8,8 @@ SET(NindAmose_LIB_SRCS
8
8
  NindTermAmose.cpp
9
9
  NindLocalAmose.h
10
10
  NindLocalAmose.cpp
11
+ NindAmoseMerge.h
12
+ NindAmoseMerge.cpp
11
13
  )
12
14
 
13
15
  add_library(NindAmose SHARED ${NindAmose_LIB_SRCS})
@@ -22,4 +24,5 @@ install(FILES
22
24
  NindTermAmose.h
23
25
  NindLocalAmose.h
24
26
  NindLexiconAmose.h
27
+ NindAmoseMerge.h
25
28
  DESTINATION include/NindAmose)
@@ -0,0 +1,77 @@
1
+ //
2
+ // C++ Implementation: NindAmoseMerge
3
+ //
4
+ // Description: Fusion hors ligne d'index Amose, voir NindAmoseMerge.h
5
+ //
6
+ // Author: Claude <noreply@anthropic.com>
7
+ //
8
+ // Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
9
+ // This file is part of NIND (as "nouvelle indexation").
10
+ // NIND is free software: you can redistribute it and/or modify it under the terms of the
11
+ // GNU Less General Public License (LGPL) as published by the Free Software Foundation,
12
+ // (see <http://www.gnu.org/licenses/>), either version 3 of the License, or any later version.
13
+ // NIND is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
14
+ // even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ // GNU Less General Public License for more details.
16
+ ////////////////////////////////////////////////////////////
17
+ #include "NindAmoseMerge.h"
18
+ using namespace latecon::nindex;
19
+ using namespace std;
20
+ ////////////////////////////////////////////////////////////
21
+ //mesme nombre et mesme ordre que NindTermAmose
22
+ #define NOMBRE_COMPTEURS 8
23
+ ////////////////////////////////////////////////////////////
24
+ static NindIndexMerge::Options forceRetrolexicon(const NindIndexMerge::Options &options)
25
+ {
26
+ NindIndexMerge::Options result = options;
27
+ result.withRetrolexicon = true; //NindLexiconAmose a toujours un retro lexique
28
+ return result;
29
+ }
30
+ ////////////////////////////////////////////////////////////
31
+ //brief Prepare the merge of the specified shards
32
+ //param shardNames absolute path file names without extension of the shards
33
+ //param options indirection block sizes of the merged files (withRetrolexicon is forced) */
34
+ NindAmoseMerge::NindAmoseMerge(const vector<string> &shardNames,
35
+ const Options &options):
36
+ NindIndexMerge(shardNames, forceRetrolexicon(options)),
37
+ m_uniqueTermCount(4, 0),
38
+ m_termOccurrences(4, 0)
39
+ {
40
+ }
41
+ ////////////////////////////////////////////////////////////
42
+ NindAmoseMerge::~NindAmoseMerge()
43
+ {
44
+ }
45
+ ////////////////////////////////////////////////////////////
46
+ unsigned int NindAmoseMerge::getTermSpecificsNumber() const
47
+ {
48
+ return NOMBRE_COMPTEURS;
49
+ }
50
+ ////////////////////////////////////////////////////////////
51
+ //compte le terme selon son type, dejduit de ses composants comme NindLexiconAmose::getWord
52
+ void NindAmoseMerge::countTerm(const list<string> &components,
53
+ const list<NindTermIndex::TermCG> &termDef)
54
+ {
55
+ AmoseTypes type = (components.size() == 1) ? SIMPLE_TERM : MULTI_TERM;
56
+ if (!components.empty() && components.front() == "§") type = NAMED_ENTITY;
57
+ unsigned int occurrences = 0;
58
+ for (list<NindTermIndex::TermCG>::const_iterator it = termDef.begin(); it != termDef.end(); it++)
59
+ occurrences += (*it).frequency;
60
+ m_uniqueTermCount[type] += 1;
61
+ m_uniqueTermCount[ALL] += 1;
62
+ m_termOccurrences[type] += occurrences;
63
+ m_termOccurrences[ALL] += occurrences;
64
+ }
65
+ ////////////////////////////////////////////////////////////
66
+ //mesme ordre que NindTermAmose::setCountsAsList
67
+ list<unsigned int> NindAmoseMerge::getTermSpecifics() const
68
+ {
69
+ list<unsigned int> result;
70
+ const AmoseTypes types[] = {ALL, SIMPLE_TERM, MULTI_TERM, NAMED_ENTITY};
71
+ for (unsigned int i = 0; i < 4; i++) {
72
+ result.push_back(m_uniqueTermCount[types[i]]);
73
+ result.push_back(m_termOccurrences[types[i]]);
74
+ }
75
+ return result;
76
+ }
77
+ ////////////////////////////////////////////////////////////
@@ -0,0 +1,58 @@
1
+ //
2
+ // C++ Interface: NindAmoseMerge
3
+ //
4
+ // Description: Fusion hors ligne d'index Amose (voir NindIndexMerge) : le lexique fusionnej a
5
+ // toujours un retro lexique, et les compteurs du fichier inverse (termes uniques et occurrences
6
+ // par type) sont recalculejs sur l'index fusionnej, le type d'un terme ejtant dejduit de ses
7
+ // composants comme le fait NindLexiconAmose::getWord.
8
+ //
9
+ // Author: Claude <noreply@anthropic.com>
10
+ //
11
+ // Copyright: 2014-2017 LATEJCON. See LICENCE.md file that comes with this distribution
12
+ // This file is part of NIND (as "nouvelle indexation").
13
+ // NIND is free software: you can redistribute it and/or modify it under the terms of the
14
+ // GNU Less General Public License (LGPL) as published by the Free Software Foundation,
15
+ // (see <http://www.gnu.org/licenses/>), either version 3 of the License, or any later version.
16
+ // NIND is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
17
+ // even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18
+ // GNU Less General Public License for more details.
19
+ ////////////////////////////////////////////////////////////
20
+ #ifndef NindAmoseMerge_H
21
+ #define NindAmoseMerge_H
22
+ ////////////////////////////////////////////////////////////
23
+ #include "NindIndex/NindIndexMerge.h"
24
+ #include "NindLexiconAmose.h"
25
+ #include "NindCommonExport.h"
26
+ #include <string>
27
+ #include <list>
28
+ #include <vector>
29
+ ////////////////////////////////////////////////////////////
30
+ namespace latecon {
31
+ namespace nindex {
32
+ ////////////////////////////////////////////////////////////
33
+ /**\brief Merges several Amose indexes (shards over disjoint sets of documents) into one */
34
+ class DLLExportLexicon NindAmoseMerge : public NindIndexMerge {
35
+ public:
36
+ /**\brief Prepare the merge of the specified shards
37
+ *\param shardNames absolute path file names without extension of the shards
38
+ *\param options indirection block sizes of the merged files (withRetrolexicon is forced) */
39
+ NindAmoseMerge(const std::vector<std::string> &shardNames,
40
+ const Options &options = Options());
41
+
42
+ virtual ~NindAmoseMerge();
43
+
44
+ protected:
45
+ virtual unsigned int getTermSpecificsNumber() const;
46
+ virtual void countTerm(const std::list<std::string> &components,
47
+ const std::list<NindTermIndex::TermCG> &termDef);
48
+ virtual std::list<unsigned int> getTermSpecifics() const;
49
+
50
+ private:
51
+ std::vector<unsigned int> m_uniqueTermCount; //par AmoseTypes
52
+ std::vector<unsigned int> m_termOccurrences; //par AmoseTypes
53
+ };
54
+ ////////////////////////////////////////////////////////////
55
+ } // end namespace
56
+ } // end namespace
57
+ #endif
58
+ ////////////////////////////////////////////////////////////
@@ -18,6 +18,9 @@
18
18
  #include "NindFile.h"
19
19
  #include <string.h>
20
20
  #include <sys/stat.h>
21
+ #ifndef _WIN32
22
+ #include <sys/file.h>
23
+ #endif
21
24
  //#include <iostream>
22
25
  using namespace latecon::nindex;
23
26
  using namespace std;
@@ -79,6 +82,7 @@ NindFile::NindFile(const string &fileName):
79
82
  m_rPtr(0),
80
83
  m_fileSize(0),
81
84
  m_isLittleEndian(false),
85
+ m_isBulk(false),
82
86
  m_nindSignalCatcher(NindSignalCatcher::Instance())
83
87
  {
84
88
  //teste si la plateforme est little endian ou big endian
@@ -124,6 +128,18 @@ bool NindFile::open(const string &accessMode)
124
128
  return true;
125
129
  }
126
130
  ////////////////////////////////////////////////////////////
131
+ //brief Take an advisory exclusive lock on the open file, without waiting
132
+ //return true if the lock was taken, false if another open of the file holds it */
133
+ bool NindFile::lockExclusive()
134
+ {
135
+ #ifdef _WIN32
136
+ return true;
137
+ #else
138
+ //flock est par description de fichier ouvert : refuse aussi un 2e ejcrivain du mesme processus
139
+ return flock(fileno(m_file), LOCK_EX | LOCK_NB) == 0;
140
+ #endif
141
+ }
142
+ ////////////////////////////////////////////////////////////
127
143
  //brief Close file associated with. Success in any case. */
128
144
  void NindFile::close()
129
145
  {
@@ -641,10 +657,12 @@ void NindFile::writeBuffer(const unsigned int position,
641
657
  if ((uint64_t)position + size > (uint64_t)(m_wbufferEnd - m_wbuffer))
642
658
  throw OutWriteBufferException("in write buffer (W) " + m_fileName);
643
659
  const unsigned int writeSize = fwrite(m_wbuffer + position, 1, size, m_file);
644
- fflush(m_file);
645
- if (writeSize != size) throw WriteFileException(m_fileName);
646
660
  //remet le pointeur d'ejcriture en teste du buffer pour une hypothejtique rejutilisation
647
661
  m_wPtr = m_wbuffer;
662
+ if (writeSize != size) throw WriteFileException(m_fileName);
663
+ //en mode massif, ni vidange ni taille : l'appelant sait oug il ejcrit
664
+ if (m_isBulk) return;
665
+ fflush(m_file);
648
666
  //memorise la taille du fichier
649
667
  const long int current = ftell(m_file);
650
668
  if (current > m_fileSize) m_fileSize = current;
@@ -660,9 +678,10 @@ void NindFile::writeValue(const unsigned char value,
660
678
  if (buffer == 0) throw BadAllocException("in write value buffer " + m_fileName);
661
679
  for (unsigned int it = 0; it != count; it++) buffer[it] = value;
662
680
  const unsigned int writeSize = fwrite(buffer, 1, count, m_file);
663
- fflush(m_file);
664
681
  delete [] buffer;
665
682
  if (writeSize != count) throw WriteFileException(m_fileName);
683
+ if (m_isBulk) return;
684
+ fflush(m_file);
666
685
  //memorise la taille du fichier
667
686
  const long int current = ftell(m_file);
668
687
  if (current > m_fileSize) m_fileSize = current;
@@ -680,4 +699,58 @@ void NindFile::readBytes(unsigned char* bytes,
680
699
  }
681
700
  }
682
701
  ////////////////////////////////////////////////////////////
683
-
702
+ //brief Switch bulk mode on or off
703
+ //param bulk true to enter bulk mode, false to leave it */
704
+ void NindFile::setBulk(const bool bulk)
705
+ {
706
+ if (bulk == m_isBulk) return;
707
+ if (fflush(m_file) != 0) throw WriteFileException(m_fileName);
708
+ m_isBulk = bulk;
709
+ if (bulk) return;
710
+ //en sortie, la taille du fichier est rejtablie
711
+ const long int current = ftell(m_file);
712
+ if (fseek(m_file, 0, SEEK_END) != 0) throw WriteFileException(m_fileName);
713
+ m_fileSize = ftell(m_file);
714
+ fseek(m_file, current, SEEK_SET);
715
+ }
716
+ ////////////////////////////////////////////////////////////
717
+ //brief Replace the read buffer with bytes from memory
718
+ //param bytes bytes to read next
719
+ //param bytesNb number of bytes */
720
+ void NindFile::loadInBuffer(const unsigned char *bytes,
721
+ const unsigned int bytesNb)
722
+ {
723
+ delete [] m_rbuffer;
724
+ m_rbuffer = m_rbufferAbsEnd = m_rbufferEnd = m_rPtr = 0;
725
+ m_rbuffer = new (nothrow) unsigned char[bytesNb == 0 ? 1 : bytesNb];
726
+ if (m_rbuffer == 0) throw BadAllocException("in read buffer " + m_fileName);
727
+ if (bytesNb != 0) memcpy(m_rbuffer, bytes, bytesNb);
728
+ m_rbufferAbsEnd = m_rbuffer + bytesNb;
729
+ m_rbufferEnd = m_rbufferAbsEnd;
730
+ m_rPtr = m_rbuffer;
731
+ }
732
+ ////////////////////////////////////////////////////////////
733
+ //brief Copy part of the write buffer
734
+ //param position offset into write buffer of the first byte to copy
735
+ //param length number of bytes to copy
736
+ //param bytes receives the copied bytes */
737
+ void NindFile::copyOutBuffer(const unsigned int position,
738
+ const unsigned int length,
739
+ std::vector<unsigned char> &bytes) const
740
+ {
741
+ if ((uint64_t)position + length > (uint64_t)(m_wPtr - m_wbuffer))
742
+ throw OutWriteBufferException("in write buffer (C) " + m_fileName);
743
+ bytes.assign(m_wbuffer + position, m_wbuffer + position + length);
744
+ }
745
+ ////////////////////////////////////////////////////////////
746
+ //brief Put bytes into the intermediate buffer
747
+ //param bytes bytes to put
748
+ //param bytesNb number of bytes */
749
+ void NindFile::putBytes(const unsigned char *bytes,
750
+ const unsigned int bytesNb)
751
+ {
752
+ if ((size_t)(m_wbufferEnd - m_wPtr) < bytesNb) throw OutWriteBufferException("in write buffer (B) " + m_fileName);
753
+ if (bytesNb != 0) memcpy(m_wPtr, bytes, bytesNb);
754
+ m_wPtr += bytesNb;
755
+ }
756
+ ////////////////////////////////////////////////////////////