nind 2.3.0__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. {nind-2.3.0 → nind-2.4.0}/CLAUDE.md +7 -5
  2. {nind-2.3.0 → nind-2.4.0}/CMakeLists.txt +1 -1
  3. {nind-2.3.0 → nind-2.4.0}/PKG-INFO +15 -1
  4. {nind-2.3.0 → nind-2.4.0}/README.md +14 -0
  5. {nind-2.3.0 → nind-2.4.0}/bench/README.md +3 -2
  6. {nind-2.3.0 → nind-2.4.0}/docs/api/engine.md +7 -0
  7. {nind-2.3.0 → nind-2.4.0}/docs/quickstart.md +59 -5
  8. {nind-2.3.0 → nind-2.4.0}/pyproject.toml +2 -2
  9. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindBasics/NindFile.cpp +62 -4
  10. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindBasics/NindFile.h +44 -2
  11. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindBasics/NindPadFile.cpp +73 -3
  12. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindBasics/NindPadFile.h +21 -0
  13. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/CMakeLists.txt +3 -1
  14. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindIndex.cpp +135 -8
  15. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindIndex.h +28 -0
  16. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindIndexMerge.cpp +30 -9
  17. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindIndexMerge.h +4 -2
  18. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndex.cpp +108 -0
  19. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndex.h +18 -1
  20. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindTermIndex.cpp +1 -2
  21. nind-2.4.0/src/cpp/NindIndex/NindTokenizer.cpp +175 -0
  22. nind-2.4.0/src/cpp/NindIndex/NindTokenizer.h +100 -0
  23. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindPy/CMakeLists.txt +1 -0
  24. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindPy/bindings.cpp +255 -0
  25. {nind-2.3.0 → nind-2.4.0}/src/py/nind/NindFile.py +1 -1
  26. {nind-2.3.0 → nind-2.4.0}/src/py/nind/NindIndex.py +1 -1
  27. {nind-2.3.0 → nind-2.4.0}/src/py/nind/NindLexiconindex.py +1 -1
  28. {nind-2.3.0 → nind-2.4.0}/src/py/nind/NindLocalindex.py +1 -1
  29. {nind-2.3.0 → nind-2.4.0}/src/py/nind/NindPadFile.py +1 -1
  30. {nind-2.3.0 → nind-2.4.0}/src/py/nind/NindRetrolexicon.py +1 -1
  31. {nind-2.3.0 → nind-2.4.0}/src/py/nind/NindTermindex.py +1 -1
  32. {nind-2.3.0 → nind-2.4.0}/src/py/nind/Nind_dumpDocument.py +1 -1
  33. {nind-2.3.0 → nind-2.4.0}/src/py/nind/Nind_search.py +1 -1
  34. {nind-2.3.0 → nind-2.4.0}/src/py/nind/Nind_trouveMotsSansOccurrence.py +1 -1
  35. {nind-2.3.0 → nind-2.4.0}/src/py/nind/Nind_trouveNombresPremiers.py +1 -1
  36. {nind-2.3.0 → nind-2.4.0}/src/py/nind/_index_store.py +28 -4
  37. {nind-2.3.0 → nind-2.4.0}/src/py/nind/amose/NindIdentifiantsClef.py +1 -1
  38. nind-2.4.0/src/py/nind/nind_engine.py +1190 -0
  39. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindIndex/CMakeLists.txt +2 -0
  40. nind-2.4.0/tst/cpp/unit/NindIndex/NindBulkTest.cpp +344 -0
  41. nind-2.4.0/tst/cpp/unit/NindIndex/NindTokenizerTest.cpp +88 -0
  42. nind-2.4.0/tst/py/test_nind_engine.py +775 -0
  43. {nind-2.3.0 → nind-2.4.0}/uv.lock +1 -1
  44. nind-2.3.0/src/py/nind/nind_engine.py +0 -466
  45. nind-2.3.0/tst/py/test_nind_engine.py +0 -382
  46. {nind-2.3.0 → nind-2.4.0}/.github/workflows/build.yml +0 -0
  47. {nind-2.3.0 → nind-2.4.0}/.github/workflows/docs.yml +0 -0
  48. {nind-2.3.0 → nind-2.4.0}/.github/workflows/publish-pypi.yml +0 -0
  49. {nind-2.3.0 → nind-2.4.0}/.gitignore +0 -0
  50. {nind-2.3.0 → nind-2.4.0}/AGENTS.md +0 -0
  51. {nind-2.3.0 → nind-2.4.0}/Dockerfile-ubuntu24.04 +0 -0
  52. {nind-2.3.0 → nind-2.4.0}/LICENCE.md +0 -0
  53. {nind-2.3.0 → nind-2.4.0}/NindConfig-version-src.cmake +0 -0
  54. {nind-2.3.0 → nind-2.4.0}/NindConfig.cmake +0 -0
  55. {nind-2.3.0 → nind-2.4.0}/SystemSpecificInformations.cmake +0 -0
  56. {nind-2.3.0 → nind-2.4.0}/bench/index_bulk.py +0 -0
  57. {nind-2.3.0 → nind-2.4.0}/bench/lookup_latency_bench.py +0 -0
  58. {nind-2.3.0 → nind-2.4.0}/bench/msmarco/convert_to_nind.py +0 -0
  59. {nind-2.3.0 → nind-2.4.0}/bench/run_bulk_indexing_benchmark.sh +0 -0
  60. {nind-2.3.0 → nind-2.4.0}/bench/run_indexing_benchmark.sh +0 -0
  61. {nind-2.3.0 → nind-2.4.0}/cicd/extract_package.sh +0 -0
  62. {nind-2.3.0 → nind-2.4.0}/docs/api/index.md +0 -0
  63. {nind-2.3.0 → nind-2.4.0}/docs/api/readers.md +0 -0
  64. {nind-2.3.0 → nind-2.4.0}/docs/architecture.md +0 -0
  65. {nind-2.3.0 → nind-2.4.0}/docs/cli.md +0 -0
  66. {nind-2.3.0 → nind-2.4.0}/docs/conf.py +0 -0
  67. {nind-2.3.0 → nind-2.4.0}/docs/index.md +0 -0
  68. {nind-2.3.0 → nind-2.4.0}/docs/requirements.txt +0 -0
  69. {nind-2.3.0 → nind-2.4.0}/gbuild.sh +0 -0
  70. {nind-2.3.0 → nind-2.4.0}/src/CMakeLists.txt +0 -0
  71. {nind-2.3.0 → nind-2.4.0}/src/cpp/CMakeLists.txt +0 -0
  72. {nind-2.3.0 → nind-2.4.0}/src/cpp/Doxyfile +0 -0
  73. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/CMakeLists.txt +0 -0
  74. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindAmoseMerge.cpp +0 -0
  75. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindAmoseMerge.h +0 -0
  76. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindLexiconAmose.cpp +0 -0
  77. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindLexiconAmose.h +0 -0
  78. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindLocalAmose.cpp +0 -0
  79. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindLocalAmose.h +0 -0
  80. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindTermAmose.cpp +0 -0
  81. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindAmose/NindTermAmose.h +0 -0
  82. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindBasics/CMakeLists.txt +0 -0
  83. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindBasics/NindSignalCatcher.cpp +0 -0
  84. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindBasics/NindSignalCatcher.h +0 -0
  85. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindCommonExport.h +0 -0
  86. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindExceptions.h +0 -0
  87. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndexK.cpp +0 -0
  88. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindLexiconIndexK.h +0 -0
  89. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindLocalIndex.cpp +0 -0
  90. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindLocalIndex.h +0 -0
  91. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindIndex/NindTermIndex.h +0 -0
  92. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindLexicon/CMakeLists.txt +0 -0
  93. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindLexicon/NindLexicon.cpp +0 -0
  94. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindLexicon/NindLexicon.h +0 -0
  95. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindLexicon/NindLexiconFile.cpp +0 -0
  96. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindLexicon/NindLexiconFile.h +0 -0
  97. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindRetrolexicon/CMakeLists.txt +0 -0
  98. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindRetrolexicon/NindRetrolexicon.cpp +0 -0
  99. {nind-2.3.0 → nind-2.4.0}/src/cpp/NindRetrolexicon/NindRetrolexicon.h +0 -0
  100. {nind-2.3.0 → nind-2.4.0}/src/py/CMakeLists.txt +0 -0
  101. {nind-2.3.0 → nind-2.4.0}/src/py/Nind_checkAndRepair.py +0 -0
  102. {nind-2.3.0 → nind-2.4.0}/src/py/Nind_checkDocument.py +0 -0
  103. {nind-2.3.0 → nind-2.4.0}/src/py/Nind_checkRejtroAndLexicon.py +0 -0
  104. {nind-2.3.0 → nind-2.4.0}/src/py/nind/__init__.py +0 -0
  105. {nind-2.3.0 → nind-2.4.0}/src/py/nind/amose/CMakeLists.txt +0 -0
  106. {nind-2.3.0 → nind-2.4.0}/src/py/nind/amose/LatXmlParser.py +0 -0
  107. {nind-2.3.0 → nind-2.4.0}/src/py/nind/amose/Nind_convertitLuceneDump.py +0 -0
  108. {nind-2.3.0 → nind-2.4.0}/src/py/nind/amose/Nind_convertitSampleDump.py +0 -0
  109. {nind-2.3.0 → nind-2.4.0}/src/py/nind/amose/Nind_convertitXmlClef.py +0 -0
  110. {nind-2.3.0 → nind-2.4.0}/tst/CMakeLists.txt +0 -0
  111. {nind-2.3.0 → nind-2.4.0}/tst/cpp/CMakeLists.txt +0 -0
  112. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/CMakeLists.txt +0 -0
  113. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_effaceDocument.cpp +0 -0
  114. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_indexeCorpus.cpp +0 -0
  115. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_litTexteAnalysej.cpp +0 -0
  116. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_litTexteAnalysej.h +0 -0
  117. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_testLecteur.cpp +0 -0
  118. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/NindAmose_testSynchro.cpp +0 -0
  119. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindAmose/Nind_indexeCorpus.cpp +0 -0
  120. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindBasics/CMakeLists.txt +0 -0
  121. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindBasics/NindBasics_testLecture.cpp +0 -0
  122. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/CMakeLists.txt +0 -0
  123. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindDate.cpp +0 -0
  124. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindDate.h +0 -0
  125. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindFichiers.cpp +0 -0
  126. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindFichiers.h +0 -0
  127. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndexTest.cpp +0 -0
  128. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndexTest.h +0 -0
  129. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_indexe.cpp +0 -0
  130. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_indexe.h +0 -0
  131. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_litDumpS2.cpp +0 -0
  132. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_litDumpS2.h +0 -0
  133. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_testEcrivain.cpp +0 -0
  134. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_testLecteur.cpp +0 -0
  135. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindIndex_testSynchro.cpp +0 -0
  136. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindLexiconIndex_testEcrivain.cpp +0 -0
  137. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindLexiconIndex_testLecteur.cpp +0 -0
  138. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindTermIndex_testEcrivain.cpp +0 -0
  139. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/NindTermIndex_testLecteur.cpp +0 -0
  140. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/Nind_testLateconNumber.cpp +0 -0
  141. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindIndex/Nind_testNumbersInLocalIndexes.cpp +0 -0
  142. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/CMakeLists.txt +0 -0
  143. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testEcrivain.cpp +0 -0
  144. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testInter.cpp +0 -0
  145. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLecteur.cpp +0 -0
  146. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLimites.cpp +0 -0
  147. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLimitesB.cpp +0 -0
  148. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testLimitesC.cpp +0 -0
  149. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindLexicon/NindLexicon_testMemoire.cpp +0 -0
  150. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindSearch/CMakeLists.txt +0 -0
  151. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindSearch/NindGetDocTerms.cpp +0 -0
  152. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindSearch/NindSearch.cpp +0 -0
  153. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindSearch/NindSearchAmose.cpp +0 -0
  154. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindTests/CMakeLists.txt +0 -0
  155. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindTests/NindTestControlC.cpp +0 -0
  156. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindTests/Nind_test1.cpp +0 -0
  157. {nind-2.3.0 → nind-2.4.0}/tst/cpp/NindTests/Nind_testSynchro.cpp +0 -0
  158. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/CMakeLists.txt +0 -0
  159. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindAmose/CMakeLists.txt +0 -0
  160. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindAmose/NindAmoseMergeTest.cpp +0 -0
  161. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindAmose/NindLexiconAmoseTest.cpp +0 -0
  162. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindAmose/NindLocalAmoseTest.cpp +0 -0
  163. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindAmose/NindTermAmoseTest.cpp +0 -0
  164. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindBasics/CMakeLists.txt +0 -0
  165. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindBasics/NindBasicsRobustnessTest.cpp +0 -0
  166. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindBasics/NindFileTest.cpp +0 -0
  167. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindBasics/NindPadFileTest.cpp +0 -0
  168. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindIndexMergeTest.cpp +0 -0
  169. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindIndexRobustnessTest.cpp +0 -0
  170. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindLexiconIndexTest.cpp +0 -0
  171. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindLocalIndexTest.cpp +0 -0
  172. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindIndex/NindTermIndexTest.cpp +0 -0
  173. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindLexicon/CMakeLists.txt +0 -0
  174. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindLexicon/NindLexiconFileTest.cpp +0 -0
  175. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindLexicon/NindLexiconTest.cpp +0 -0
  176. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindRetrolexicon/CMakeLists.txt +0 -0
  177. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/NindRetrolexicon/NindRetrolexiconTest.cpp +0 -0
  178. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/TestFileBytes.h +0 -0
  179. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/TestTempDir.h +0 -0
  180. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/doctest.h +0 -0
  181. {nind-2.3.0 → nind-2.4.0}/tst/cpp/unit/doctest_main.cpp +0 -0
  182. {nind-2.3.0 → nind-2.4.0}/tst/py/test_native.py +0 -0
  183. {nind-2.3.0 → nind-2.4.0}/tst/py/test_nindfile.py +0 -0
  184. {nind-2.3.0 → nind-2.4.0}/tst/py/test_nindlexiconindex.py +0 -0
  185. {nind-2.3.0 → nind-2.4.0}/tst/py/test_nindlocalindex.py +0 -0
  186. {nind-2.3.0 → nind-2.4.0}/tst/py/test_nindretrolexicon.py +0 -0
  187. {nind-2.3.0 → nind-2.4.0}/tst/py/test_nindtermindex.py +0 -0
@@ -39,12 +39,13 @@ Memory-safety conventions in the C++ core (keep them when touching it): readers
39
39
 
40
40
  ## Build & test — Python
41
41
 
42
- The Python package (`nind`, in `src/py/nind`) is managed with `pyproject.toml`/`scikit-build-core` (the old `src/py/CMakeLists.txt` build path is kept only for compatibility). `uv sync`/`uv build` compiles `nind._native` (via CMake — see `src/cpp/NindPy/CMakeLists.txt`) and installs the package (editable for `sync`) plus dev dependencies (pytest) into `.venv`. Building requires a C++14 compiler; `scikit-build-core`, `pybind11`, `cmake`, and `ninja` themselves are all pulled automatically as PEP 517 build requirements (the latter two so builds work even in containers with no system CMake/Ninja, e.g. manylinux in CI) — no manual install needed.
42
+ The Python package (`nind`, in `src/py/nind`) is managed with `pyproject.toml`/`scikit-build-core` (the old `src/py/CMakeLists.txt` build path is kept only for compatibility). `uv sync`/`uv build` compiles `nind._native` (via CMake — see `src/cpp/NindPy/CMakeLists.txt`) and installs the package (editable for `sync`: Python edits are live, but the compiled `_native` is only rebuilt by `uv sync --reinstall-package nind`) plus dev dependencies (pytest) into `.venv`. Building requires a C++14 compiler; `scikit-build-core`, `pybind11`, `cmake`, and `ninja` themselves are all pulled automatically as PEP 517 build requirements (the latter two so builds work even in containers with no system CMake/Ninja, e.g. manylinux in CI) — no manual install needed.
43
43
 
44
44
  Tests live in `tst/py/` (one file per module, mirroring `tst/cpp/unit/`), configured via `[tool.pytest.ini_options]` in `pyproject.toml`:
45
45
 
46
46
  ```
47
- uv sync # first time / after touching pyproject.toml or src/cpp/NindPy/**
47
+ uv sync # first time / after touching pyproject.toml
48
+ uv sync --reinstall-package nind # after any C++ change (src/cpp/**, bindings included): plain `uv sync` doesn't rebuild _native
48
49
  uv run pytest tst/py -v
49
50
  uv build # produce the platform wheel (nind-*.whl) + sdist in dist/
50
51
  ```
@@ -81,6 +82,7 @@ NindIndex (deps: NindBasics, NindRetrolexicon)
81
82
  — NindLocalIndex: per-document local index file
82
83
  — NindIndex: ties the above into a full inverted/local index file
83
84
  — NindIndexMerge: offline merge of index shards (disjoint documents) into one standard index
85
+ — NindTokenizer: tokenizer for source-code-like text (used by the Python indexer)
84
86
 
85
87
  NindAmose (deps: NindIndex, NindRetrolexicon)
86
88
  — adapts nind to Amose's richer term-type model (see internal doc refs "LAT2015.JYS.448")
@@ -107,7 +109,7 @@ Parallels the C++ layering with lowercase-suffixed module names for the index cl
107
109
  - `NindFile.py` — low-level binary codec. Historically read-only-oriented (the `litXxx` getters have long existed for every format), but now has a full writer counterpart too: `ejcritNombre1/3/4/5`, `ejcritNombreULat`/`ejcritNombreSLat` (both varint encodings, full tier range matching the readers and the C++ implementation), `ejcritChaine`, `ejcritZejros`.
108
110
  - `NindPadFile.py` — the shared "pad file" envelope (header, indirection block(s), specifics, identification trailer); read-only, no writer (mirrors `NindBasics::NindPadFile`, but nothing in Python subclasses it as a writer the way `NindIndex`/`NindRetrolexicon` do in C++).
109
111
  - `NindRetrolexicon.py`, `NindLexiconindex.py`, `NindTermindex.py`, `NindLocalindex.py` — ergonomic read-only Python wrappers around the corresponding `NindIndex`-family C++ classes; their hot-path lookups delegate to `nind._native`, as does the shared structural-diagnostic base (`analyseFichierPadFile`/`analyseFichierIndex`, inherited from `NindPadFile.py`/`NindIndex.py`). The remaining, per-format diagnostic/analysis methods (`dumpeFichier`, `debogueIndex`, `donneCollisions`, `donneMax`, `donneClef`, `afficheTerme`, `afficheDocument`, and each class's own `analyseFichierXxx` beyond the shared base) remain hand-rolled pure-Python parsing, since `nind._native` doesn't expose that introspection surface yet (see "Planned follow-up work" below). `NindLexiconindex` is hash-bucketed (`clefB(word) % nombreIndirection`), the other two are directly indexed by id.
110
- - `nind_engine.py` — `NindIndexer` (writer) and `NindEngine` (BM25-style search) built on top of the above. `NindIndexer` is the *only* Python writer for the index-family formats (`.nindlexiconindex`/`.nindtermindex`/`.nindlocalindex`) — it builds them via `nind._native`'s `is_writer=True` constructors. It does not write `.nindretrolexicon` (not needed by `NindEngine`, which only supports simple, non-compound words). Indexing is parallel but every file keeps a single writer: files are tokenized in worker processes (`workers` argument of `index_files`; in-process below `PARALLEL_MIN_FILES` files), then the term and local index files are written concurrently from two threads through the flat-buffer bulk bindings `NindTermIndex.set_term_defs_arrays`/`NindLocalIndex.set_local_defs_arrays`, which take `array('I')` buffers (building `Document`/`Term`/`Localisation` objects in Python cost ~20× the write itself) and release the GIL. The `*_local_def*_arrays` writers split a term's positions into entries of ≤255 (`<nbreLocalisations>` is one byte). Every build goes into a new generation directory published atomically (see `_index_store.py` below), so rebuilding never disturbs readers. With `shards=N` it instead builds N independent indexes over contiguous parts of the files in worker processes (in a temporary directory inside the new generation), then merges them with `nind._native.merge_indexes`; opt-in because the merge costs about as much as writing the index again (see "Sharded indexing" below), so it only pays off when per-document work dominates.
112
+ - `nind_engine.py` — `NindIndexer` (writer) and `NindEngine` (BM25-style search) built on top of the above. `NindIndexer` is the *only* Python writer for the index-family formats (`.nindlexiconindex`/`.nindtermindex`/`.nindlocalindex`) — it builds them via `nind._native`'s `is_writer=True` constructors. It does not write `.nindretrolexicon` (not needed by `NindEngine`, which only supports simple, non-compound words). `NindIndexer.index_jsonl` reads pyserini's JSON Lines corpus format (`{"id": ..., "contents": ...}` per line; record n = document n, the record ids are returned, not stored) — worker processes each parse a line-aligned byte range of the file. Tokenization is native (`NindTokenizer`, bound as `nind._native.Tokenizer`; options `tokenizer=` of `NindIndexer`, see `TOKENIZER_DEFAULTS`: each identifier split optional, ASCII lowercasing, tokens over 254 bytes — the lexicon's limit, `NindFile::putString` — dropped); a subclass overriding `_default_tokenize` falls back to Python token lists (`_chunk_from_token_lists`). Indexing is parallel but every file keeps a single writer: worker processes tokenize chunks of documents with `Tokenizer.tokenize_batch`, which returns the chunk's vocabulary plus uint32 term ids/ends (cheap to pickle); the parent writes the lexicon with one `add_simple_words` call (`NindLexiconIndex::addWords`: each hash bucket written once), feeds the chunks to `nind._native.IndexBuilder` (remaps chunk ids to term ids), which writes the term and local index files from two threads with the GIL released. Every writer of a new generation runs in bulk mode (`begin_bulk`/`end_bulk`, `NindIndex::beginBulk`: no per-write flush/seek/signal handler, indirections and tail written once — byte-identical files). The local writers split a term's positions into entries of ≤255 (`<nbreLocalisations>` is one byte). `local_index=False` leaves the local index out. Each generation also holds `<prefix>.nindmeta` (JSON: tokenizer options, local index or not) and `<prefix>.ninddoclengths` (binary: `NINDDL01`, count, doc ids, lengths): `NindEngine` reads document lengths from it (no local index needed) and tokenizes queries with the stored options; indexes without them (older versions) are read from their local index and queried/updated with the original regex tokenizer (`LEGACY_TOKENIZER`). Every build goes into a new generation directory published atomically (see `_index_store.py` below), so rebuilding never disturbs readers. With `shards=N` it instead builds N independent indexes over contiguous parts of the files in worker processes (in a temporary directory inside the new generation), then merges them with `nind._native.merge_indexes`; opt-in because the merge costs about as much as writing the index again (see "Sharded indexing" below), so it only pays off when per-document work dominates. Incremental indexing goes through `NindIndexer.update()` → `IndexUpdate` (shortcuts `add_file`/`add_files`/`remove_documents`): a context manager that, under the writer lock, copies the published generation into a new one (keeping its tokenizer options and its local index or not; explicitly different `NindIndexer` arguments are refused), collects adds/replacements/removals in memory, and on exit reopens the native writers on the copy (existing files: `r+b`, bulk mode), adds the new words, rewrites every touched term definition whole (a replaced/removed document's terms come from the local index, or, without one, from a scan of every term) (read with `get_term_def`, minus removed docs, plus added ones; empty = erased) and every touched document (empty = erased), then publishes; an empty or failed update publishes nothing. Term/local writers are opened with the lexicon's *old* identification and written with the new one, so all three files end up paired. The lexicon's hash bucket count is fixed in the file (`NindLexiconIndex::getModulo`, bound as `get_modulo`): past `LEXICON_MAX_LOAD` words per bucket, the update rebuilds the index with `merge_indexes` on itself with twice as many buckets as words (term ids change, document ids don't). Indirection blocks added by an existing writer use the constructor's `indirection_bloc_size` (it must not be 0 for a writer reopening a file), so updates pass at least `INDIRECTION_BLOC_MIN`. Per-update cost is dominated by reopening the native writers (they scan their whole file; see the I/O follow-up below), so batching documents per update matters on large indexes.
111
113
  - `_index_store.py` — the on-disk layout that makes every `NindIndexer`/`NindEngine` operation safe without the caller doing anything: each build writes a fresh, never-modified generation directory `<prefix>.gen-<n>-<hex>/`, flushed to disk (`fsync`) then published by atomically replacing the pointer file `<prefix>.CURRENT` (a pointer file rather than a symlink: atomic on Windows too). Readers open every file of the generation `CURRENT` names under a shared `flock` on its `generation.lock` (only while opening: open files stay readable after deletion on POSIX); `NindEngine` checks `CURRENT` (one `stat`) at the start of each public call and switches to a newly published build, each call answering from one snapshot (`_Snapshot`), calls serialized by an `RLock` (the native readers aren't thread-safe). Writers of one prefix are serialized by `<prefix>.lock`; after publishing, the writer deletes older generations nobody is opening, crashed builds' leftovers, and flat files of indexes written before generations existed (still readable: no `CURRENT` = flat layout). A failed build never touches `CURRENT`. Lock fds are closed in forked children (`os.register_at_fork`) — otherwise a worker outliving a killed writer would keep the index locked forever — and `NindIndexer`'s pool workers exit when their parent dies. Windows: `msvcrt` locks are exclusive only, so readers don't lock and rely on retrying; generations still open can't be deleted and are retried at the next build.
112
114
  - `Nind_*.py` at the package root — standalone CLI scripts (dump a document, convert corpora, search, diagnostics)
113
115
  - `amose/` — Amose-specific conversion/parsing scripts (Lucene dump, XML-CLEF, sample corpora)
@@ -129,9 +131,9 @@ Not part of this: `.nindretrolexicon` writing (`NindIndexer` still doesn't write
129
131
 
130
132
  **Sharded indexing — offline merge implemented; follow-ups below.** Nind files are single-writer by design (no locking, readers resync lazily; see `NindPadFile::getEntryPos`), and every write rewrites the shared specifics/identification tail at EOF, so concurrent writers on one file would serialize anyway — sharing a file between writers is explicitly *not* the plan. Writes scale instead across N independent shards (each its own lexicon + term + local files, one single-threaded writer each, over a partition of the documents), merged offline into one standard index that every existing reader uses unchanged (search across shards was considered and rejected: every reader, Amose's included, would have to learn about shards, and BM25 would need corpus-wide statistics summed over them).
131
133
 
132
- How the merge works (`NindIndexMerge`, `src/cpp/NindIndex/`): shard lexicons are enumerated with `NindLexiconIndex::getEntries` (walks the hash buckets; no retro lexicon needed) and unioned with words canonicalised as (union id of the prefix, last component) — which also gives the exact merged size, used as the lexicon's hash modulo; documents are listed with `NindLocalIndex::getDocIdents` and checked disjoint *before anything is written*; each merged term's postings are gathered from every shard (per term, so memory stays small) and each document is copied with its term ids remapped (local term order is kept, so Amose's per-occurrence reading order survives). A failed merge removes the target files. `NindAmoseMerge` (`src/cpp/NindAmose/`) forces a retro lexicon and recomputes the 8 term counters, a term's type being derived from its components as in `NindLexiconAmose::getWord` (unique-term counts aren't additive across shards); they're written once at the end with `NindTermIndex::setSpecificWords`. The plain merge refuses shards with term specifics.
134
+ How the merge works (`NindIndexMerge`, `src/cpp/NindIndex/`): shard lexicons are enumerated with `NindLexiconIndex::getEntries` (walks the hash buckets; no retro lexicon needed) and unioned with words canonicalised as (union id of the prefix, last component) — which also gives the exact merged size, used as the lexicon's hash modulo; documents are listed with `NindLocalIndex::getDocIdents` and checked disjoint *before anything is written* (shards without local index — all or none — give a merged index without one; a document in two shards then only shows up as a duplicate in a shared term), merged files are written in bulk mode with a single `addWords`; each merged term's postings are gathered from every shard (per term, so memory stays small) and each document is copied with its term ids remapped (local term order is kept, so Amose's per-occurrence reading order survives). A failed merge removes the target files. `NindAmoseMerge` (`src/cpp/NindAmose/`) forces a retro lexicon and recomputes the 8 term counters, a term's type being derived from its components as in `NindLexiconAmose::getWord` (unique-term counts aren't additive across shards); they're written once at the end with `NindTermIndex::setSpecificWords`. The plain merge refuses shards with term specifics.
133
135
 
134
136
  Follow-ups:
135
- - **Merge cost is I/O-syscall bound, and so are all writes.** Merging 10k documents / 58k terms from 4 shards makes ~2.3M `lseek`, ~1.5M `read`, ~280k `write` and ~250k `rt_sigaction` calls: `NindIndex::getDefinition` discards the stdio buffer (`flush`) then seeks+reads twice (indirection, then data) on every read, `NindFile::writeBuffer` flushes every write, and every critical section installs/restores a signal handler. A direct build pays the same per definition, so a merge costs about a full rewrite. Reading with `pread` (one syscall per read, no buffer discard) and a bulk mode for a writer creating a fresh file (no per-write flush, tail written once) would speed up merges, direct builds and search alike — mind the reader-resync semantics above when touching this.
137
+ - **Merge cost is I/O-syscall bound, and so are all writes** — largely addressed for writers of new files by bulk mode (`NindIndex::beginBulk`, used by `NindIndexer` and `NindIndexMerge`; a 10k-document build went from ~1.9M to ~0.2M syscalls, the rest being the input files). Still open: Merging 10k documents / 58k terms from 4 shards makes ~2.3M `lseek`, ~1.5M `read`, ~280k `write` and ~250k `rt_sigaction` calls: `NindIndex::getDefinition` discards the stdio buffer (`flush`) then seeks+reads twice (indirection, then data) on every read, `NindFile::writeBuffer` flushes every write, and every critical section installs/restores a signal handler. A direct build pays the same per definition, so a merge costs about a full rewrite. Reading with `pread` (one syscall per read, no buffer discard) and a bulk mode for a writer creating a fresh file (no per-write flush, tail written once) would speed up merges, direct builds and search alike — mind the reader-resync semantics above when touching this.
136
138
  - **CLI for Amose/LIMA** multi-process indexing: a small C++ program (like the legacy `tst/cpp/NindAmose/` tools) running `NindAmoseMerge` over shard prefixes. `NindAmoseMerge` isn't bound in `nind._native` (the wheel doesn't compile `NindAmose`).
137
139
  - `NindLexiconAmose::getWord` dereferences past the end of the component list for the lexicon word `§` alone (the first prefix of every named entity): undefined behaviour, seen as a `bad_alloc`. Normal lookups never ask for that id; the merge doesn't call `getWord`.
@@ -113,7 +113,7 @@ endif ()
113
113
 
114
114
  ########### setting project verion number ###############
115
115
 
116
- set(NIND_VERSION "2.3.0")
116
+ set(NIND_VERSION "2.4.0")
117
117
 
118
118
  string(REGEX REPLACE "^([0-9]+)\\.[0-9]+\\.[0-9]+$" "\\1" NIND_VERSION_MAJOR "${NIND_VERSION}")
119
119
  string(REGEX REPLACE "^[0-9]+\\.([0-9]+)\\.[0-9]+$" "\\1" NIND_VERSION_MINOR "${NIND_VERSION}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: nind
3
- Version: 2.3.0
3
+ Version: 2.4.0
4
4
  Summary: Python frontend for the Nind Indexing Module, with its native C++ index stack built in
5
5
  Requires-Python: >=3.8
6
6
  Description-Content-Type: text/markdown
@@ -84,6 +84,20 @@ for doc_id, score in engine.search("ejcrit nombre", top_k=5):
84
84
 
85
85
  `index_dir` must already exist; each input file becomes one document,
86
86
  identified externally by its position in the list (`0`, `1`, `2`, ...).
87
+ A JSON Lines corpus (pyserini's format) is indexed with
88
+ `indexer.index_jsonl("corpus.jsonl")`. Tokenizer options (each identifier
89
+ split optional, lowercasing) and `local_index=False` (a much smaller index,
90
+ enough for `NindEngine`) are `NindIndexer` arguments.
91
+ To add, replace or remove documents in an existing index (or build one
92
+ file by file), use an incremental update instead of rebuilding:
93
+
94
+ ```python
95
+ with indexer.update() as update:
96
+ for path in more_files:
97
+ update.add_file(path) # returns the new document's id
98
+ update.remove(1)
99
+ ```
100
+
87
101
  `NindEngine` also exposes the building blocks it uses internally
88
102
  (`get_term_id`, `get_df`, `get_tf`, `get_doc_len`) for inspecting relevance
89
103
  or a specific document directly.
@@ -77,6 +77,20 @@ for doc_id, score in engine.search("ejcrit nombre", top_k=5):
77
77
 
78
78
  `index_dir` must already exist; each input file becomes one document,
79
79
  identified externally by its position in the list (`0`, `1`, `2`, ...).
80
+ A JSON Lines corpus (pyserini's format) is indexed with
81
+ `indexer.index_jsonl("corpus.jsonl")`. Tokenizer options (each identifier
82
+ split optional, lowercasing) and `local_index=False` (a much smaller index,
83
+ enough for `NindEngine`) are `NindIndexer` arguments.
84
+ To add, replace or remove documents in an existing index (or build one
85
+ file by file), use an incremental update instead of rebuilding:
86
+
87
+ ```python
88
+ with indexer.update() as update:
89
+ for path in more_files:
90
+ update.add_file(path) # returns the new document's id
91
+ update.remove(1)
92
+ ```
93
+
80
94
  `NindEngine` also exposes the building blocks it uses internally
81
95
  (`get_term_id`, `get_df`, `get_tf`, `get_doc_len`) for inspecting relevance
82
96
  or a specific document directly.
@@ -96,8 +96,9 @@ The main process is the bottleneck: at 1M docs, 16 and 48 workers take the
96
96
  same time. Its time goes almost entirely into the C++ writers, which make
97
97
  every entry durable on its own (~13 system calls per definition written:
98
98
  seeks, reads of the indirection entry, `fwrite`+`fflush`, and signal-handler
99
- swaps around each Ctrl-C-safe critical section). A faster bulk load would
100
- need a C++-side bulk mode (e.g. buffered indirection and sequential writes).
99
+ swaps around each Ctrl-C-safe critical section). These measurements predate
100
+ the writers' bulk mode (`begin_bulk`/`end_bulk`, see `NindIndex::beginBulk`),
101
+ which `NindIndexer` now uses and this script doesn't yet.
101
102
  The main process's memory is mostly worker results waiting to be written;
102
103
  at most `2 × workers` chunks (~8 MB of corpus each) are in flight.
103
104
 
@@ -23,3 +23,10 @@ See {doc}`../quickstart` for a walk-through.
23
23
  :members:
24
24
  :show-inheritance:
25
25
  ```
26
+
27
+ ## IndexUpdate
28
+
29
+ ```{eval-rst}
30
+ .. autoclass:: nind.nind_engine.IndexUpdate
31
+ :members: add_file, add_text, add_tokens, remove, LEXICON_MIN_BUCKETS, LEXICON_MAX_LOAD, INDIRECTION_BLOC_MIN
32
+ ```
@@ -18,11 +18,65 @@ indexer.index_files([
18
18
  ])
19
19
  ```
20
20
 
21
- This writes three files into `indices/`:
22
- `corpus.nindlexiconindex`, `corpus.nindtermindex` and
23
- `corpus.nindlocalindex`. `index_dir` must already exist; each input file
24
- becomes one document, identified externally by its position in the list
25
- (`0`, `1`, `2`, ...).
21
+ This writes the index into a new generation directory inside `indices/`
22
+ (`corpus.gen-<n>-<hex>/`: `corpus.nindlexiconindex`, `corpus.nindtermindex`,
23
+ `corpus.nindlocalindex`, plus `corpus.nindmeta` - the tokenizer options - and
24
+ `corpus.ninddoclengths`), published atomically through `corpus.CURRENT`.
25
+ `index_dir` must already exist; each input file becomes one document,
26
+ identified externally by its position in the list (`0`, `1`, `2`, ...).
27
+
28
+ A corpus in pyserini's JSON Lines format (one `{"id": ..., "contents": ...}`
29
+ object per line) is indexed directly, record `n` becoming document `n`:
30
+
31
+ ```python
32
+ record_ids = indexer.index_jsonl("corpus.jsonl") # record_ids[n]: the "id" of document n
33
+ ```
34
+
35
+ ## Tokenizer and index options
36
+
37
+ Texts are tokenized by a native tokenizer for source-code-like text: words
38
+ are runs of letters and digits, split at camelCase (`fooBar` -> `foo` `Bar`,
39
+ `HTTPServer` -> `HTTP` `Server`), underscore (`foo_bar` -> `foo` `bar`) and
40
+ letter/digit (`utf8` -> `utf` `8`) boundaries. Each split is optional, and
41
+ tokens can be lowercased:
42
+
43
+ ```python
44
+ indexer = NindIndexer("indices", prefix="corpus",
45
+ tokenizer={"lowercase": True, "split_digits": False})
46
+ ```
47
+
48
+ The options are stored with the index: `NindEngine` tokenizes queries the
49
+ same way, and updates keep them. Tokens longer than 254 bytes (the longest
50
+ word a nind lexicon stores) are dropped.
51
+
52
+ `local_index=False` leaves out the local index (each document's terms and
53
+ their positions). `NindEngine` doesn't need it - document lengths are stored
54
+ apart - and it makes up most of an index's size and a good part of its build
55
+ time; other readers of nind indexes (Amose...) may need it:
56
+
57
+ ```python
58
+ indexer = NindIndexer("indices", prefix="corpus", local_index=False)
59
+ ```
60
+
61
+ ## Index incrementally
62
+
63
+ `index_files` rebuilds the whole index. To add, replace or remove documents
64
+ in an existing index instead - or to build one file by file - group the
65
+ changes in an update; they are published together when the `with` block
66
+ ends (nothing changes if it raises):
67
+
68
+ ```python
69
+ with indexer.update() as update:
70
+ for path in new_files:
71
+ doc_id = update.add_file(path) # new id: one more than the highest
72
+ update.add_file("changed.py", doc_id=1) # replaces document 1
73
+ update.remove(2)
74
+ ```
75
+
76
+ `add_file(path)`, `add_files(paths)` and `remove_documents(ids)` on the
77
+ indexer are one-update shortcuts. Each update copies the index and reopens
78
+ its files, so on a large index group many documents per update rather than
79
+ calling `add_file` once per file.
26
80
 
27
81
  ## Search it
28
82
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "nind"
3
- version = "2.3.0"
3
+ version = "2.4.0"
4
4
  description = "Python frontend for the Nind Indexing Module, with its native C++ index stack built in"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.8"
@@ -25,7 +25,7 @@ testpaths = ["tst/py"]
25
25
  pythonpath = ["src/py"]
26
26
 
27
27
  [tool.bumpver]
28
- current_version = "v2.3.0"
28
+ current_version = "v2.4.0"
29
29
  version_pattern = "vMAJOR.MINOR.PATCH"
30
30
  commit_message = "Bump version {old_version} -> {new_version}"
31
31
  commit = true
@@ -82,6 +82,7 @@ NindFile::NindFile(const string &fileName):
82
82
  m_rPtr(0),
83
83
  m_fileSize(0),
84
84
  m_isLittleEndian(false),
85
+ m_isBulk(false),
85
86
  m_nindSignalCatcher(NindSignalCatcher::Instance())
86
87
  {
87
88
  //teste si la plateforme est little endian ou big endian
@@ -656,10 +657,12 @@ void NindFile::writeBuffer(const unsigned int position,
656
657
  if ((uint64_t)position + size > (uint64_t)(m_wbufferEnd - m_wbuffer))
657
658
  throw OutWriteBufferException("in write buffer (W) " + m_fileName);
658
659
  const unsigned int writeSize = fwrite(m_wbuffer + position, 1, size, m_file);
659
- fflush(m_file);
660
- if (writeSize != size) throw WriteFileException(m_fileName);
661
660
  //remet le pointeur d'ejcriture en teste du buffer pour une hypothejtique rejutilisation
662
661
  m_wPtr = m_wbuffer;
662
+ if (writeSize != size) throw WriteFileException(m_fileName);
663
+ //en mode massif, ni vidange ni taille : l'appelant sait oug il ejcrit
664
+ if (m_isBulk) return;
665
+ fflush(m_file);
663
666
  //memorise la taille du fichier
664
667
  const long int current = ftell(m_file);
665
668
  if (current > m_fileSize) m_fileSize = current;
@@ -675,9 +678,10 @@ void NindFile::writeValue(const unsigned char value,
675
678
  if (buffer == 0) throw BadAllocException("in write value buffer " + m_fileName);
676
679
  for (unsigned int it = 0; it != count; it++) buffer[it] = value;
677
680
  const unsigned int writeSize = fwrite(buffer, 1, count, m_file);
678
- fflush(m_file);
679
681
  delete [] buffer;
680
682
  if (writeSize != count) throw WriteFileException(m_fileName);
683
+ if (m_isBulk) return;
684
+ fflush(m_file);
681
685
  //memorise la taille du fichier
682
686
  const long int current = ftell(m_file);
683
687
  if (current > m_fileSize) m_fileSize = current;
@@ -695,4 +699,58 @@ void NindFile::readBytes(unsigned char* bytes,
695
699
  }
696
700
  }
697
701
  ////////////////////////////////////////////////////////////
698
-
702
+ //brief Switch bulk mode on or off
703
+ //param bulk true to enter bulk mode, false to leave it */
704
+ void NindFile::setBulk(const bool bulk)
705
+ {
706
+ if (bulk == m_isBulk) return;
707
+ if (fflush(m_file) != 0) throw WriteFileException(m_fileName);
708
+ m_isBulk = bulk;
709
+ if (bulk) return;
710
+ //en sortie, la taille du fichier est rejtablie
711
+ const long int current = ftell(m_file);
712
+ if (fseek(m_file, 0, SEEK_END) != 0) throw WriteFileException(m_fileName);
713
+ m_fileSize = ftell(m_file);
714
+ fseek(m_file, current, SEEK_SET);
715
+ }
716
+ ////////////////////////////////////////////////////////////
717
+ //brief Replace the read buffer with bytes from memory
718
+ //param bytes bytes to read next
719
+ //param bytesNb number of bytes */
720
+ void NindFile::loadInBuffer(const unsigned char *bytes,
721
+ const unsigned int bytesNb)
722
+ {
723
+ delete [] m_rbuffer;
724
+ m_rbuffer = m_rbufferAbsEnd = m_rbufferEnd = m_rPtr = 0;
725
+ m_rbuffer = new (nothrow) unsigned char[bytesNb == 0 ? 1 : bytesNb];
726
+ if (m_rbuffer == 0) throw BadAllocException("in read buffer " + m_fileName);
727
+ if (bytesNb != 0) memcpy(m_rbuffer, bytes, bytesNb);
728
+ m_rbufferAbsEnd = m_rbuffer + bytesNb;
729
+ m_rbufferEnd = m_rbufferAbsEnd;
730
+ m_rPtr = m_rbuffer;
731
+ }
732
+ ////////////////////////////////////////////////////////////
733
+ //brief Copy part of the write buffer
734
+ //param position offset into write buffer of the first byte to copy
735
+ //param length number of bytes to copy
736
+ //param bytes receives the copied bytes */
737
+ void NindFile::copyOutBuffer(const unsigned int position,
738
+ const unsigned int length,
739
+ std::vector<unsigned char> &bytes) const
740
+ {
741
+ if ((uint64_t)position + length > (uint64_t)(m_wPtr - m_wbuffer))
742
+ throw OutWriteBufferException("in write buffer (C) " + m_fileName);
743
+ bytes.assign(m_wbuffer + position, m_wbuffer + position + length);
744
+ }
745
+ ////////////////////////////////////////////////////////////
746
+ //brief Put bytes into the intermediate buffer
747
+ //param bytes bytes to put
748
+ //param bytesNb number of bytes */
749
+ void NindFile::putBytes(const unsigned char *bytes,
750
+ const unsigned int bytesNb)
751
+ {
752
+ if ((size_t)(m_wbufferEnd - m_wPtr) < bytesNb) throw OutWriteBufferException("in write buffer (B) " + m_fileName);
753
+ if (bytesNb != 0) memcpy(m_wPtr, bytes, bytesNb);
754
+ m_wPtr += bytesNb;
755
+ }
756
+ ////////////////////////////////////////////////////////////
@@ -23,6 +23,7 @@
23
23
  #include "NindSignalCatcher.h"
24
24
  #include <stdio.h>
25
25
  #include <string>
26
+ #include <vector>
26
27
  ////////////////////////////////////////////////////////////
27
28
  namespace latecon {
28
29
  namespace nindex {
@@ -245,6 +246,36 @@ public:
245
246
  *\param count number of bytes to write */
246
247
  void writeValue(const unsigned char value,
247
248
  const unsigned int count);
249
+
250
+ /**\brief Switch bulk mode on or off. In bulk mode, writes are not flushed one by one
251
+ * (stdio buffers them) and critical sections don't install a signal handler: for a file
252
+ * nobody reads while it is written. Leaving bulk mode flushes the file.
253
+ *\param bulk true to enter bulk mode, false to leave it */
254
+ void setBulk(const bool bulk);
255
+
256
+ /**\brief Test if bulk mode is on
257
+ *\return true if bulk mode is on */
258
+ inline bool isBulk() const;
259
+
260
+ /**\brief Replace the read buffer with bytes from memory (instead of the file)
261
+ *\param bytes bytes to read next
262
+ *\param bytesNb number of bytes */
263
+ void loadInBuffer(const unsigned char *bytes,
264
+ const unsigned int bytesNb);
265
+
266
+ /**\brief Copy part of the write buffer
267
+ *\param position offset into write buffer of the first byte to copy
268
+ *\param length number of bytes to copy
269
+ *\param bytes receives the copied bytes */
270
+ void copyOutBuffer(const unsigned int position,
271
+ const unsigned int length,
272
+ std::vector<unsigned char> &bytes) const;
273
+
274
+ /**\brief Put bytes into the intermediate buffer
275
+ *\param bytes bytes to put
276
+ *\param bytesNb number of bytes */
277
+ void putBytes(const unsigned char *bytes,
278
+ const unsigned int bytesNb);
248
279
 
249
280
  private:
250
281
  //non copiable : possehde un FILE* et deux buffers allouejs (une copie provoquerait double fclose/delete[])
@@ -266,6 +297,7 @@ private:
266
297
  unsigned char *m_rPtr; //pointeur buffer de lecture
267
298
  long int m_fileSize; //taille du fichier
268
299
  bool m_isLittleEndian;
300
+ bool m_isBulk; //vrai en mode massif (ejcritures non vidanges une ah une)
269
301
  NindSignalCatcher *m_nindSignalCatcher;
270
302
  };
271
303
  ////////////////////////////////////////////////////////////
@@ -273,14 +305,24 @@ private:
273
305
  * The section is closed on every exit path, including early returns and exceptions. */
274
306
  class NindCriticalSection {
275
307
  public:
276
- explicit NindCriticalSection(NindFile &file): m_file(file) { m_file.beginCriticalSection(); }
277
- ~NindCriticalSection() { m_file.endCriticalSection(); }
308
+ //en mode massif, le fichier n'est cohejrent qu'ah la fin : rien ah protejger
309
+ explicit NindCriticalSection(NindFile &file): m_file(file), m_isActive(!file.isBulk()) {
310
+ if (m_isActive) m_file.beginCriticalSection(); }
311
+ ~NindCriticalSection() { if (m_isActive) m_file.endCriticalSection(); }
278
312
  private:
279
313
  NindCriticalSection(const NindCriticalSection &);
280
314
  NindCriticalSection &operator=(const NindCriticalSection &);
281
315
  NindFile &m_file;
316
+ const bool m_isActive;
282
317
  };
283
318
  ////////////////////////////////////////////////////////////
319
+ //brief Test if bulk mode is on
320
+ //return true if bulk mode is on */
321
+ inline bool NindFile::isBulk() const
322
+ {
323
+ return m_isBulk;
324
+ }
325
+ ////////////////////////////////////////////////////////////
284
326
  //brief get current position in file
285
327
  //return current position in file */
286
328
  inline long int NindFile::getPos() const
@@ -76,6 +76,10 @@ NindPadFile::NindPadFile(const string &fileName,
76
76
  m_fileName(fileName),
77
77
  m_isWriter(isWriter),
78
78
  m_isExistingWriter(false),
79
+ m_isBulk(false),
80
+ m_bulkTail(),
81
+ m_bulkEnd(0),
82
+ m_bulkPos(UINT64_MAX),
79
83
  m_entriesBlocksMap(),
80
84
  m_specificsSize(specificsSize),
81
85
  m_dataEntrySize(dataEntrySize),
@@ -178,6 +182,25 @@ uint64_t NindPadFile::getEntryPos(const unsigned int ident)
178
182
  //param fileIdentification unique identification of file */
179
183
  void NindPadFile::addEntriesBlock(const Identification &fileIdentification)
180
184
  {
185
+ if (m_isBulk) {
186
+ //en mode massif, le bloc est ajoutej ah la fin des donnejes, la queue reste en mejmoire
187
+ const uint64_t entriesBlock = m_bulkEnd;
188
+ m_file.createBuffer(TAILLE_TETE_INDEX);
189
+ m_file.putInt1(FLAG_INDEXEJ);
190
+ m_file.putInt5(0);
191
+ m_file.putInt3(m_dataEntriesBlocSize);
192
+ bulkWrite(entriesBlock, 0, TAILLE_TETE_INDEX);
193
+ m_file.writeValue(0, m_dataEntriesBlocSize*m_dataEntrySize);
194
+ m_bulkPos += (uint64_t)m_dataEntriesBlocSize*m_dataEntrySize;
195
+ m_bulkEnd = m_bulkPos;
196
+ if (m_entriesBlocksMap.size() != 0) {
197
+ m_file.createBuffer(5);
198
+ m_file.putInt5(entriesBlock);
199
+ bulkWrite(m_entriesBlocksMap.back().first - 8, 0, 5);
200
+ }
201
+ m_entriesBlocksMap.push_back(pair<uint64_t, unsigned int>(entriesBlock + TAILLE_TETE_INDEX, m_dataEntriesBlocSize));
202
+ return;
203
+ }
181
204
  //se positionne sur les spejcifiques
182
205
  const int offset = TAILLE_ENTETE_SPEJCIFIQUE + TAILLE_IDENTIFICATION + m_specificsSize;
183
206
  m_file.setPos(-offset, SEEK_END);
@@ -215,6 +238,7 @@ void NindPadFile::addEntriesBlock(const Identification &fileIdentification)
215
238
  //return size of 1rst entries block */
216
239
  unsigned int NindPadFile::getFirstEntriesBlockSize()
217
240
  {
241
+ if (m_isBulk) { m_file.flush(); bulkForgetPosition(); }
218
242
  m_file.setPos(TAILLE_ENTETE_FIXE, SEEK_SET); //positionne aprehs l'en-teste fixe
219
243
  //<flagIndexej=47> <addrBlocSuivant> <nombreIndex>
220
244
  m_file.readBuffer(TAILLE_TETE_INDEX);
@@ -237,6 +261,12 @@ unsigned int NindPadFile::getMaxIdent() const
237
261
  //brief Read from file specific datas and leave result into read buffer */
238
262
  void NindPadFile::getSpecifics()
239
263
  {
264
+ if (m_isBulk) {
265
+ m_file.loadInBuffer(&m_bulkTail[0], m_specificsSize + TAILLE_ENTETE_SPEJCIFIQUE);
266
+ if (m_file.getInt1() != FLAG_SPEJCIFIQUE)
267
+ throw NindPadFileException("NindPadFile::getSpecifics : " + m_fileName);
268
+ return;
269
+ }
240
270
  m_file.flush();
241
271
  //se positionne sur la teste des spejcifiques
242
272
  const int offset = TAILLE_ENTETE_SPEJCIFIQUE + TAILLE_IDENTIFICATION + m_specificsSize;
@@ -251,9 +281,14 @@ void NindPadFile::getSpecifics()
251
281
  //param identification where unique identification of file is returned */
252
282
  void NindPadFile::getFileIdentification(Identification &identification)
253
283
  {
254
- m_file.setPos(-TAILLE_IDENTIFICATION, SEEK_END); //se positionne sur l'identification
255
- //<flagIdentification=53> <maxIdentifiant> <identifieurUnique>
256
- m_file.readBuffer(TAILLE_IDENTIFICATION);
284
+ if (m_isBulk) {
285
+ m_file.loadInBuffer(&m_bulkTail[m_bulkTail.size() - TAILLE_IDENTIFICATION], TAILLE_IDENTIFICATION);
286
+ }
287
+ else {
288
+ m_file.setPos(-TAILLE_IDENTIFICATION, SEEK_END); //se positionne sur l'identification
289
+ //<flagIdentification=53> <maxIdentifiant> <identifieurUnique>
290
+ m_file.readBuffer(TAILLE_IDENTIFICATION);
291
+ }
257
292
  if (m_file.getInt1() != FLAG_IDENTIFICATION)
258
293
  throw NindPadFileException("NindPadFile::getFileIdentification : " + m_fileName);
259
294
  const unsigned int wordsNb = m_file.getInt4();
@@ -299,6 +334,40 @@ void NindPadFile::writeIdentification(const Identification &fileIdentification)
299
334
  m_file.putInt4(fileIdentification.lexiconTime);
300
335
  }
301
336
  ////////////////////////////////////////////////////////////
337
+ //brief write the specifics and identification (the file's tail) from the write buffer
338
+ //param position offset into write buffer of the tail
339
+ //param length length of the tail */
340
+ void NindPadFile::writeTail(const unsigned int position,
341
+ const unsigned int length)
342
+ {
343
+ if (length != getSpecificsAndIdentificationSize())
344
+ throw NindPadFileException("NindPadFile::writeTail bad tail size " + m_fileName);
345
+ if (m_isBulk) {
346
+ m_file.copyOutBuffer(position, length, m_bulkTail);
347
+ return;
348
+ }
349
+ m_file.setPos(-(long int)length, SEEK_END); //se positionne sur les spejcifiques
350
+ m_file.writeBuffer(position, length);
351
+ }
352
+ ////////////////////////////////////////////////////////////
353
+ //ejcrit [position, position + length) du buffer d'ejcriture ah l'adresse spejcifieje du fichier
354
+ void NindPadFile::bulkWrite(const uint64_t address,
355
+ const unsigned int position,
356
+ const unsigned int length)
357
+ {
358
+ if (address != m_bulkPos) {
359
+ if (m_file.setPos(address, SEEK_SET) != 0) throw WriteFileException(m_fileName);
360
+ }
361
+ m_file.writeBuffer(position, length);
362
+ m_bulkPos = address + length;
363
+ }
364
+ ////////////////////////////////////////////////////////////
365
+ //ah appeler aprehs toute lecture du fichier en mode massif
366
+ void NindPadFile::bulkForgetPosition()
367
+ {
368
+ m_bulkPos = UINT64_MAX;
369
+ }
370
+ ////////////////////////////////////////////////////////////
302
371
  //retourne la position d'une entreje
303
372
  uint64_t NindPadFile::getJustEntryPos(const unsigned int ident)
304
373
  {
@@ -456,6 +525,7 @@ NindPadFile::HoleStats NindPadFile::chercheVides(list<pair<uint64_t, unsigned in
456
525
  //throws NindPadFileException if the file's structure is invalid */
457
526
  NindPadFile::PadFileStats NindPadFile::analysePadFile()
458
527
  {
528
+ if (m_isBulk) throw NindPadFileException("NindPadFile::analysePadFile : file in bulk mode " + m_fileName);
459
529
  PadFileStats stats;
460
530
  stats.dataEntrySize = m_dataEntrySize;
461
531
  stats.specificsSize = m_specificsSize;
@@ -165,6 +165,27 @@ protected:
165
165
  * \param fileIdentification unique identification of file */
166
166
  void writeIdentification(const Identification &fileIdentification);
167
167
 
168
+ /**\brief write the specifics and identification (the file's tail) from the write buffer
169
+ * In bulk mode, the tail is only kept in memory until bulk mode ends.
170
+ *\param position offset into write buffer of the tail
171
+ *\param length length of the tail (getSpecificsAndIdentificationSize()) */
172
+ void writeTail(const unsigned int position,
173
+ const unsigned int length);
174
+
175
+ //--- mode massif (voir NindIndex::beginBulk) ---
176
+ //ejcrit [position, position + length) du buffer d'ejcriture ah l'adresse spejcifieje du fichier,
177
+ //sans repositionnement si l'ejcriture prejcejdente s'est arresteje lah
178
+ void bulkWrite(const uint64_t address,
179
+ const unsigned int position,
180
+ const unsigned int length);
181
+ //ah appeler aprehs toute lecture du fichier en mode massif (la position courante est perdue)
182
+ void bulkForgetPosition();
183
+
184
+ bool m_isBulk; //vrai en mode massif
185
+ std::vector<unsigned char> m_bulkTail; //spejcifiques + identification, pas encore ejcrits
186
+ uint64_t m_bulkEnd; //fin des donnejes (lah oug ira la queue)
187
+ uint64_t m_bulkPos; //position courante connue du fichier (sinon BULK_POS_UNKNOWN)
188
+
168
189
  public:
169
190
  /**\brief get file name
170
191
  * \return file name */
@@ -13,6 +13,8 @@ SET(NindIndex_LIB_SRCS
13
13
  NindLocalIndex.cpp
14
14
  NindIndexMerge.h
15
15
  NindIndexMerge.cpp
16
+ NindTokenizer.h
17
+ NindTokenizer.cpp
16
18
  )
17
19
 
18
20
  add_library(NindIndex SHARED ${NindIndex_LIB_SRCS})
@@ -23,4 +25,4 @@ set_target_properties(NindIndex PROPERTIES VERSION ${NIND_LIB_VERSION} SOVERSION
23
25
  install(TARGETS NindIndex DESTINATION lib)
24
26
 
25
27
  ########### install files ###############
26
- install(FILES NindIndex.h NindLexiconIndex.h NindTermIndex.h NindLocalIndex.h NindIndexMerge.h DESTINATION include/NindIndex)
28
+ install(FILES NindIndex.h NindLexiconIndex.h NindTermIndex.h NindLocalIndex.h NindIndexMerge.h NindTokenizer.h DESTINATION include/NindIndex)