vruksha 0.0.2__tar.gz

vruksha-0.0.2/.gitignore

@@ -0,0 +1,164 @@
+_docs/
+_proc/
+
+*.bak
+.gitattributes
+.last_checked
+.gitconfig
+*.bak
+*.log
+*~
+~*
+_tmp*
+tmp*
+tags
+*.pkg
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+conda/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.vscode
+*.swp
+
+# osx generated files
+.DS_Store
+.DS_Store?
+.Trashes
+ehthumbs.db
+Thumbs.db
+.idea
+
+# pytest
+.pytest_cache
+
+# tools/trust-doc-nbs
+docs_src/.last_checked
+
+# symlinks to fastai
+docs_src/fastai
+tools/fastai
+
+# link checker
+checklink/cookies.txt
+
+# .gitconfig is now autogenerated
+.gitconfig
+
+# Quarto installer
+.deb
+.pkg
+
+# Quarto
+.quarto
+
+# local vault + extracted assets
+*.db
+*.usearch
+# ...except the recorded model replies the notebook tests replay in CI. The sqlite side files are
+# transient, so they stay ignored.
+!nbs/chatcache/cache.db
+nbs/chatcache/*.db-wal
+nbs/chatcache/*.db-shm
+assets/
+.venv/
+.kosha/

vruksha-0.0.2/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022, fastai
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vruksha-0.0.2/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.5
+Name: vruksha
+Version: 0.0.2
+Summary: an entity graph over a litesearch store: extraction, resolution, and a PageRank search leg
+Project-URL: Repository, https://github.com/vedicreader/vruksha
+Project-URL: Documentation, https://vedicreader.github.io/vruksha/
+Author-email: Karthik <karthik.rajgopal@hotmail.com>
+License: Apache-2.0
+License-File: LICENSE
+Keywords: entity resolution,knowledge graph,nbdev,pagerank,search,sqlite
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.10
+Requires-Dist: fastcore>=2.2.15
+Requires-Dist: litesearch>=0.1.33
+Requires-Dist: yake>=0.7.3
+Description-Content-Type: text/markdown
+
+# vruksha
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Four problems, and none of them needs a model.
+
+| problem | how it is solved |
+|----|----|
+| what the entities are | AST symbols for code, yake keyphrases for prose |
+| which mentions are the same thing | embeddings propose, a lexical guard decides |
+| what connects to what | PMI over co-occurrence windows |
+| what to do with the graph | personalised PageRank, fused as a third search leg |
+
+``` python
+from litesearch import database
+from vruksha import build_graph, resolve_entities
+
+db = database('corpus.db')
+build_graph(db, rows, emb_fn=enc)     # entities and co-occurrence edges
+resolve_entities(db)                  # `hnsw` and `HNSW index` become one node
+db.graph_search('how does resolution work', qemb, graph_w=0.5)
+```
+
+## The lexical guard
+
+Embedding similarity alone merges `python 3.11` into `python 3.12`. [`_lex_ok`](https://vedicreader.github.io/vruksha/entities.html#_lex_ok) requires token
+overlap, matching digits and a matching acronym before a merge goes through.
+
+``` python
+from vruksha.entities import _lex_ok
+
+_lex_ok('usearch', 'usearch index'), _lex_ok('python 3.11', 'python 3.12')
+```
+
+## When to turn the search leg on
+
+Measured against plain hybrid search:
+
+- **Regulation and legal text: a loss.** p_mrr 0.8170 for plain hybrid against 0.7395, 0.6859 and
+  0.6463 at `graph_w` 0.25, 0.5 and 1.0, at two to four times the latency.
+- **Papers and prose: a win.** Better in seven of nine paired-bootstrap comparisons, +0.0387
+  target MRR on arXiv at `graph_w=1.0`.
+
+So `graph_search` is opt-in by name and off by default. Turn it on for a corpus whose entities
+carry meaning, and raise `graph_w` towards 1.0 when you do.
+
+## Install
+
+``` sh
+pip install vruksha
+```

vruksha-0.0.2/README.md

@@ -0,0 +1,52 @@
+# vruksha
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+Four problems, and none of them needs a model.
+
+| problem | how it is solved |
+|----|----|
+| what the entities are | AST symbols for code, yake keyphrases for prose |
+| which mentions are the same thing | embeddings propose, a lexical guard decides |
+| what connects to what | PMI over co-occurrence windows |
+| what to do with the graph | personalised PageRank, fused as a third search leg |
+
+``` python
+from litesearch import database
+from vruksha import build_graph, resolve_entities
+
+db = database('corpus.db')
+build_graph(db, rows, emb_fn=enc)     # entities and co-occurrence edges
+resolve_entities(db)                  # `hnsw` and `HNSW index` become one node
+db.graph_search('how does resolution work', qemb, graph_w=0.5)
+```
+
+## The lexical guard
+
+Embedding similarity alone merges `python 3.11` into `python 3.12`. [`_lex_ok`](https://vedicreader.github.io/vruksha/entities.html#_lex_ok) requires token
+overlap, matching digits and a matching acronym before a merge goes through.
+
+``` python
+from vruksha.entities import _lex_ok
+
+_lex_ok('usearch', 'usearch index'), _lex_ok('python 3.11', 'python 3.12')
+```
+
+## When to turn the search leg on
+
+Measured against plain hybrid search:
+
+- **Regulation and legal text: a loss.** p_mrr 0.8170 for plain hybrid against 0.7395, 0.6859 and
+  0.6463 at `graph_w` 0.25, 0.5 and 1.0, at two to four times the latency.
+- **Papers and prose: a win.** Better in seven of nine paired-bootstrap comparisons, +0.0387
+  target MRR on arXiv at `graph_w=1.0`.
+
+So `graph_search` is opt-in by name and off by default. Turn it on for a corpus whose entities
+carry meaning, and raise `graph_w` towards 1.0 when you do.
+
+## Install
+
+``` sh
+pip install vruksha
+```

vruksha-0.0.2/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vruksha"
+dynamic = ["version"]
+description = "an entity graph over a litesearch store: extraction, resolution, and a PageRank search leg"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "Apache-2.0"}
+authors = [{name = "Karthik", email = "karthik.rajgopal@hotmail.com"}]
+keywords = ['nbdev', 'knowledge graph', 'entity resolution', 'pagerank', 'search', 'sqlite']
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Text Processing :: Indexing",
+]
+dependencies = [
+    "fastcore>=2.2.15",
+    "litesearch>=0.1.33",
+    "yake>=0.7.3",
+]
+
+[project.urls]
+Repository = "https://github.com/vedicreader/vruksha"
+Documentation = "https://vedicreader.github.io/vruksha/"
+
+[project.entry-points.nbdev]
+vruksha = "vruksha._modidx:d"
+
+[tool.nbdev]
+
+[tool.hatch.build.targets.wheel]
+packages = ["vruksha"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/vruksha", "/README.md", "/pyproject.toml"]
+
+[tool.hatch.version]
+path = "vruksha/__init__.py"
+
+[dependency-groups]
+dev = ["ipykernel>=7.3.0", "nbdev>=3.3.4", "notebook>=7.6.1"]

vruksha-0.0.2/vruksha/__init__.py

@@ -0,0 +1,4 @@
+__version__ = "0.0.2"
+from .entities import *
+from .build import *
+from .search import *

vruksha-0.0.2/vruksha/_modidx.py

@@ -0,0 +1,47 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'main',
+                'doc_baseurl': '/vruksha',
+                'doc_host': 'https://vedicreader.github.io',
+                'git_url': 'https://github.com/vedicreader/vruksha',
+                'lib_path': 'vruksha'},
+  'syms': { 'vruksha.build': { 'vruksha.build._WindowStore': ('build.html#_windowstore', 'vruksha/build.py'),
+                               'vruksha.build._WindowStore.__init__': ('build.html#_windowstore.__init__', 'vruksha/build.py'),
+                               'vruksha.build._WindowStore.__iter__': ('build.html#_windowstore.__iter__', 'vruksha/build.py'),
+                               'vruksha.build._WindowStore.__len__': ('build.html#_windowstore.__len__', 'vruksha/build.py'),
+                               'vruksha.build._WindowStore.drop': ('build.html#_windowstore.drop', 'vruksha/build.py'),
+                               'vruksha.build._WindowStore.entity_counts': ('build.html#_windowstore.entity_counts', 'vruksha/build.py'),
+                               'vruksha.build._WindowStore.extend': ('build.html#_windowstore.extend', 'vruksha/build.py'),
+                               'vruksha.build._WindowStore.pair_counts': ('build.html#_windowstore.pair_counts', 'vruksha/build.py'),
+                               'vruksha.build._ann_pairs': ('build.html#_ann_pairs', 'vruksha/build.py'),
+                               'vruksha.build._collapse_edges': ('build.html#_collapse_edges', 'vruksha/build.py'),
+                               'vruksha.build._is_code': ('build.html#_is_code', 'vruksha/build.py'),
+                               'vruksha.build._lexical_pairs': ('build.html#_lexical_pairs', 'vruksha/build.py'),
+                               'vruksha.build._n_workers': ('build.html#_n_workers', 'vruksha/build.py'),
+                               'vruksha.build._pmi_edges': ('build.html#_pmi_edges', 'vruksha/build.py'),
+                               'vruksha.build._pool': ('build.html#_pool', 'vruksha/build.py'),
+                               'vruksha.build._prose_job': ('build.html#_prose_job', 'vruksha/build.py'),
+                               'vruksha.build._uf_find': ('build.html#_uf_find', 'vruksha/build.py'),
+                               'vruksha.build._uf_union': ('build.html#_uf_union', 'vruksha/build.py'),
+                               'vruksha.build.build_graph': ('build.html#build_graph', 'vruksha/build.py'),
+                               'vruksha.build.cooccur_edges': ('build.html#cooccur_edges', 'vruksha/build.py'),
+                               'vruksha.build.resolve_entities': ('build.html#resolve_entities', 'vruksha/build.py')},
+            'vruksha.entities': { 'vruksha.entities._acr': ('entities.html#_acr', 'vruksha/entities.py'),
+                                  'vruksha.entities._def_name': ('entities.html#_def_name', 'vruksha/entities.py'),
+                                  'vruksha.entities._jac': ('entities.html#_jac', 'vruksha/entities.py'),
+                                  'vruksha.entities._lex_ok': ('entities.html#_lex_ok', 'vruksha/entities.py'),
+                                  'vruksha.entities._norm': ('entities.html#_norm', 'vruksha/entities.py'),
+                                  'vruksha.entities._nums': ('entities.html#_nums', 'vruksha/entities.py'),
+                                  'vruksha.entities._sentences': ('entities.html#_sentences', 'vruksha/entities.py'),
+                                  'vruksha.entities._toks': ('entities.html#_toks', 'vruksha/entities.py'),
+                                  'vruksha.entities._yake_terms': ('entities.html#_yake_terms', 'vruksha/entities.py'),
+                                  'vruksha.entities.code_entities': ('entities.html#code_entities', 'vruksha/entities.py'),
+                                  'vruksha.entities.prose_windows': ('entities.html#prose_windows', 'vruksha/entities.py'),
+                                  'vruksha.entities.text_entities': ('entities.html#text_entities', 'vruksha/entities.py')},
+            'vruksha.search': { 'vruksha.search.Database.graph_search': ('search.html#database.graph_search', 'vruksha/search.py'),
+                                'vruksha.search._adjacency': ('search.html#_adjacency', 'vruksha/search.py'),
+                                'vruksha.search._canon_mentions': ('search.html#_canon_mentions', 'vruksha/search.py'),
+                                'vruksha.search._csr': ('search.html#_csr', 'vruksha/search.py'),
+                                'vruksha.search._leg': ('search.html#_leg', 'vruksha/search.py'),
+                                'vruksha.search._ppr': ('search.html#_ppr', 'vruksha/search.py'),
+                                'vruksha.search.graph_stats': ('search.html#graph_stats', 'vruksha/search.py')}}}

vruksha-0.0.2/vruksha/build.py

@@ -0,0 +1,438 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_build.ipynb.
+
+# %% auto #0
+__all__ = ['MIN_PARALLEL_CHUNKS', 'build_graph', 'resolve_entities', 'cooccur_edges']
+
+# %% ../nbs/01_build.ipynb #ab82e9d12af8898f
+import ast, math, re, sys
+from functools import lru_cache
+import numpy as np
+from fastcore.all import AttrDict, L, Path, chunked, defaults, first, ifnone, merge, patch, store_attr
+from fastcore.parallel import ProcessPoolExecutor
+from fastlite import Database
+from apswutils.db import Table
+from multiprocessing import get_context
+from litesearch.core import (sql_in, rowid_sel, content_id, NP_DTYPE, process_content, write_txn, db_lock,
+                             upsert_all, rrf_all)
+from litesearch.topics import get_graph
+from litesearch.utils import hash_embed
+from .entities import (_acr, _jac, _lex_ok, _norm, _nums, _sentences, _toks, _yake_terms,
+                              code_entities, prose_windows, text_entities)
+
+# %% ../nbs/01_build.ipynb #8e1af70983a03a3f
+_CODE_TYPES = {'FunctionDef','AsyncFunctionDef','ClassDef'}
+
+def _is_code(chunk):
+    md = chunk.get('metadata') or {}
+    if not isinstance(md, dict): return False
+    return md.get('lang') == '.py' or md.get('type') in _CODE_TYPES
+
+def _pmi_edges(wins,              # list of entity-id sets (one per co-occurrence window)
+               min_n=2,           # min co-occurrence count
+               min_npmi=0.15,     # min normalized PMI
+               max_df=0.4,        # drop entities present in more than this fraction of windows
+               max_degree=48,     # keep only the strongest edges per node
+               rel='cooc'):
+    'Normalized-PMI co-occurrence edges over windows. max_df is what kills hub terms like "section".'
+    n = len(wins)
+    if n < 2: return []
+    if hasattr(wins, 'pair_counts'):                 # a _WindowStore counts on disk, same numbers
+        cnt = wins.entity_counts()
+        hub = {e for e, c in cnt.items() if c / n > max_df}
+        pairs = ((a, b, c) for a, b, c in wins.pair_counts(hub, min_n))
+    else:
+        cnt = {}
+        for w in wins:
+            for e in w: cnt[e] = cnt.get(e, 0) + 1
+        hub = {e for e, c in cnt.items() if c / n > max_df}
+        pair = {}
+        for w in wins:
+            es = sorted(e for e in w if e not in hub)
+            for i, a in enumerate(es):
+                for b in es[i+1:]: pair[(a, b)] = pair.get((a, b), 0) + 1
+        pairs = ((a, b, c) for (a, b), c in pair.items())
+    out = []
+    for a, b, c in pairs:
+        if c < min_n: continue
+        pa, pb, pab = cnt[a]/n, cnt[b]/n, c/n
+        npmi = math.log(pab/(pa*pb)) / (-math.log(pab)) if 0 < pab < 1 else 0.0
+        if npmi < min_npmi: continue
+        out.append(dict(src=a, dst=b, rel=rel, weight=round(npmi, 5), n=c))
+    if max_degree:
+        # (src, dst) breaks weight ties explicitly. Without it the survivors of the degree cap depend
+        # on the order pairs were counted in, which differs between the in-memory dict and SQLite's
+        # group-by — 30 of 2,707 edges on a 500-chunk corpus, all of them ties, none of them wrong.
+        deg, kept = {}, []
+        for e in sorted(out, key=lambda r: (-r['weight'], r['src'], r['dst'])):
+            if deg.get(e['src'], 0) < max_degree and deg.get(e['dst'], 0) < max_degree:
+                deg[e['src']] = deg.get(e['src'], 0)+1; deg[e['dst']] = deg.get(e['dst'], 0)+1
+                kept.append(e)
+        out = kept
+    return out
+
+# Below this many prose chunks in one drain, a process pool costs more to start than it saves.
+MIN_PARALLEL_CHUNKS = 200
+
+def _prose_job(txt):
+    '''Windows for one prose chunk, as plain tuples. Module level so a process pool can pickle it.
+
+    Only text crosses the boundary and only tuples come back — no database handle, nothing that a
+    fork would have to keep consistent.'''
+    return [[(s, k) for s, k in w] for w in prose_windows(txt)]
+
+def _pool(nw):
+    '''A `ProcessPoolExecutor` started the way `fastcore.parallel` starts one.
+
+    fork, not spawn, on darwin. A spawned worker unpickles the job by module and name, and under
+    nbdev `_prose_job` is defined in the notebook's `__main__`, which the worker has no copy of —
+    so every worker died on unpickle. Only text crosses the boundary and only tuples come back,
+    so there is nothing here a fork has to keep consistent.'''
+    kw = dict(mp_context=get_context('fork')) if sys.platform == 'darwin' else {}
+    return ProcessPoolExecutor(nw, **kw)
+
+def _n_workers(n, n_workers):
+    'Resolve `n_workers`: None picks by size, 0 stays serial, anything else is taken literally.'
+    if n_workers is not None: return n_workers
+    return 0 if n < MIN_PARALLEL_CHUNKS else defaults.cpus
+
+# %% ../nbs/01_build.ipynb #bd12970f219d8213
+class _WindowStore:
+    '''Co-occurrence windows kept in a SQLite table instead of a python list.
+    The table is TEMP: scratch for one build does not belong in the shared schema, where creating
+    and dropping it bumps the schema cookie and forces every other connection to re-prepare.'''
+    PAGE = 10_000   # windows per read; keeps the cursor closed between yields
+    def __init__(self, db, name):
+        store_attr(); self.n, self._w = 0, 0
+        with write_txn(db):
+            db.conn.execute(f'drop table if exists main.{name}')   # left behind by older versions
+            db.conn.execute(f'create temp table if not exists {name} (w integer, e text)')
+            db.conn.execute(f'delete from {name}')
+    def extend(self, wins):
+        rows = [(self._w + i, e) for i, w in enumerate(wins) for e in w if e]
+        if not rows: self._w += len(wins); self.n += len(wins); return
+        with write_txn(self.db): self.db.conn.executemany(f'insert into {self.name} values (?,?)', rows)
+        self._w += len(wins); self.n += len(wins)
+    def entity_counts(self):
+        'entity -> windows containing it. Bounded by vocabulary, which saturates; safe in memory.'
+        with db_lock(self.db):
+            return dict(self.db.conn.execute(f'select e, count(*) from {self.name} group by e'))
+    def pair_counts(self, hub, min_n):
+        """Co-occurrence count per unordered pair, aggregated on disk. Materialised: the caller
+        builds edges between rows, and an open cursor makes the connection busy for that whole walk."""
+        c = self.db.conn
+        with write_txn(self.db):
+            c.execute(f'create index if not exists {self.name}_w on {self.name}(w)')
+            c.execute(f'create temp table if not exists {self.name}_hub (e text primary key)')
+            c.execute(f'delete from {self.name}_hub')
+            if hub: c.executemany(f'insert or ignore into {self.name}_hub values (?)', [(e,) for e in hub])
+        with db_lock(self.db): return c.execute(f'''select a.e, b.e, count(*) c from {self.name} a
+              join {self.name} b on a.w = b.w and a.e < b.e
+              where a.e not in (select e from {self.name}_hub)
+                and b.e not in (select e from {self.name}_hub)
+              group by a.e, b.e having c >= {int(min_n)}''').fetchall()
+    def __len__(self): return self.n
+    def __iter__(self):
+        'One page of whole windows at a time: a cursor left open across a yield holds the connection.'
+        last = -1
+        while True:
+            with db_lock(self.db):
+                ws = [r[0] for r in self.db.conn.execute(
+                    f'select distinct w from {self.name} where w > ? order by w limit {self.PAGE}', (last,)).fetchall()]
+                if not ws: return
+                rows = self.db.conn.execute(f'select w, e from {self.name} where w between ? and ? order by w',
+                                            (ws[0], ws[-1])).fetchall()
+            w, acc = None, set()
+            for wi, e in rows:
+                if wi != w:
+                    if w is not None: yield acc
+                    w, acc = wi, set()
+                acc.add(e)
+            if w is not None: yield acc
+            last = ws[-1]
+    def drop(self):
+        with write_txn(self.db):
+            self.db.conn.execute(f'drop table if exists temp.{self.name}')
+            self.db.conn.execute(f'drop table if exists temp.{self.name}_hub')
+
+def build_graph(db,                  # Database with a chunk store
+                chunks,              # chunk dicts ({'content','metadata'}) as returned by dir2chunks/pkg2chunks
+                store='store',       # chunk store name
+                prefix=None,         # graph table prefix
+                terms_fn=None,       # (text, topk) -> terms, replacing yake (see `sanskrit_terms`)
+                emb_fn=None,         # embedder for entity names (required for resolve_entities)
+                code=True,           # extract AST symbols from code chunks
+                prose=True,          # extract surfaces from prose chunks
+                cooc=True,           # also write PMI co-occurrence edges over sentence windows
+                min_n=2,             # min co-occurrence count
+                min_npmi=0.15,       # min normalized PMI
+                max_df=0.4,          # drop entities present in >max_df of windows
+                max_degree=48,       # max cooc edges kept per node
+                batch:int=None,      # chunks per flush; keeps windows on disk instead of in memory
+                n_workers:int=None): # extraction workers; 0 is serial, None picks by queue size
+    '''Extract entities + mentions + edges (exact for code, PMI co-occurrence for prose) from chunks.
+
+    `batch` is what makes a large corpus finish. Left as None everything accumulates in memory for
+    the whole call, which is fine for a package or a few thousand pages and is not fine for a
+    corpus: windows are one per sentence and never stop arriving, so memory tracks the corpus with
+    no bound. Set it and mentions are flushed every `batch` chunks and windows go to a scratch
+    table, which `_pmi_edges` reads back twice exactly as it would a list. The edges are identical
+    either way — the batched path is a different place to keep the same numbers.
+
+    `n_workers` spreads extraction across processes, which is where a build spends 88% of its time.
+    Use it with `batch` large enough to be worth a pool: the queue is drained once per batch, so a
+    small `batch` pays pool startup repeatedly for very little work.'''
+    g = db.get_graph(store, prefix)
+    ents, mens, edges = {}, {}, {}
+    wins = _WindowStore(db, f'{g.prefix}_win_scratch') if batch else []
+    def ent(name, kind):
+        'Register an entity by canonical name, returning its hash id.'
+        n = _norm(name)
+        if not n: return None
+        i = content_id(n)
+        e = ents.setdefault(i, dict(content=n, kind=kind, freq=0, canon=i))
+        e['freq'] += 1
+        return i
+    def men(cid, eid, surface):
+        if not (cid and eid): return
+        m = mens.setdefault((cid, eid), dict(chunk_id=cid, entity_id=eid, surface=surface, n=0))
+        m['n'] += 1
+    def edge(s, d, rel, w=1.0):
+        if not (s and d) or s == d: return
+        e = edges.setdefault((s, d, rel), dict(src=s, dst=d, rel=rel, weight=0.0, n=0))
+        e['weight'] += w; e['n'] += 1
+    n_mens = 0
+    def flush_mentions():
+        'Mentions are complete once their chunk is processed, so they need not wait for the corpus.'
+        nonlocal n_mens
+        if not mens: return
+        upsert_all(g.mentions, mens.values(), ('chunk_id','entity_id'))
+        n_mens += len(mens); mens.clear()
+
+    pend = []                                   # windows waiting for the next flush
+    def add_wins(ws):
+        if batch: pend.extend(ws)
+        else: wins.extend(ws)
+
+    prose_q, pool = [], None
+    def prose_wins():
+        '''Windows per queued prose chunk, over a process pool once there are enough to pay for one.
+        Order is preserved by both pools, and the reduce depends on it: `ents.setdefault` keeps the
+        `kind` of an entity's *first* mention, so a reordered stream would relabel entities.'''
+        nonlocal pool
+        nw = 0 if terms_fn is not None else _n_workers(len(prose_q), n_workers)
+        if nw and nw > 1:
+            if pool is None: pool = _pool(nw)
+            # both sides materialised: `drain_prose` clears the queue these are drawn from
+            cids, txts = [c for c, _ in prose_q], [t for _, t in prose_q]
+            return zip(cids, pool.map(_prose_job, txts))
+        return ((cid, prose_windows(txt, terms_fn=terms_fn)) for cid, txt in prose_q)
+
+    def drain_prose():
+        'Turn the queued prose chunks into entities, mentions and windows, then clear the queue.'
+        for cid, wl in prose_wins():
+            for win in wl:
+                w = set()
+                for surf, kind in win:
+                    i = ent(surf, kind)
+                    if i: men(cid, i, surf); w.add(i)
+                if len(w) > 1: add_wins([w])
+        prose_q.clear()
+
+    n_seen = 0
+    try:
+        for c in ([chunks] if isinstance(chunks, dict) else chunks):
+            txt = c.get('content')
+            if not (txt and txt.strip()): continue
+            cid = c.get('id') or content_id(txt)
+            if code and _is_code(c):
+                dname, calls, imps = code_entities(c)
+                did = ent(dname, 'symbol') if dname else None
+                if did: men(cid, did, dname)
+                w = {did} if did else set()
+                for nm in calls:
+                    i = ent(nm, 'symbol')
+                    men(cid, i, nm)
+                    edge(did, i, 'calls')
+                    w.add(i)
+                for nm in imps:
+                    i = ent(nm, 'module')
+                    men(cid, i, nm)
+                    edge(did, i, 'imports')
+                    w.add(i)
+                if len(w) > 1: add_wins([w - {None}])
+            elif prose: prose_q.append((cid, txt))
+            n_seen += 1
+            if batch and n_seen % batch == 0:
+                drain_prose()
+                wins.extend(pend)
+                pend.clear()
+                flush_mentions()
+        drain_prose()
+    finally:
+        if pool is not None: pool.shutdown()
+    if batch: wins.extend(pend); pend.clear()
+    rows = list(ents.values())
+    if cooc and len(wins):
+        for e in _pmi_edges(wins, min_n, min_npmi, max_df, max_degree): edges[(e['src'], e['dst'], e['rel'])] = e
+    n_wins = len(wins)
+    if batch: wins.drop()
+    with write_txn(db):
+        if rows:
+            if emb_fn: process_content(g.entities, rows, embed=True, emb_fn=emb_fn)
+            else: g.entities.insert_all(rows, upsert=True, hash_id='id', hash_id_columns=['content'])
+        if mens:
+            upsert_all(g.mentions, mens.values(), ('chunk_id','entity_id')); n_mens += len(mens)
+        if edges: upsert_all(g.edges, edges.values(), ('src','dst','rel'))
+    if emb_fn and rows: g.entities.rebuild_index()
+    return dict(entities=len(rows), mentions=n_mens, edges=len(edges), windows=n_wins)
+
+
+# %% ../nbs/01_build.ipynb #f1e0bc8fd1ac5d91
+_EXACT_KINDS = ('symbol', 'module', 'topic')   # names that are already canonical — never merge these
+
+def _uf_find(par, x):
+    while par[x] != x: par[x] = par[par[x]]; x = par[x]
+    return x
+
+def _uf_union(par, rank, a, b, name=None, ok=None, members=None, max_check=32):
+    'Merge two groups. With `name`/`ok`, only if the merged group stays a **clique** under `ok`.'
+    ra, rb = _uf_find(par, a), _uf_find(par, b)
+    if ra == rb: return False
+    if name is not None and ok is not None:
+        A = (members.get(ra, [ra]) if members else [ra])[:max_check]
+        B = (members.get(rb, [rb]) if members else [rb])[:max_check]
+        if not all(ok(name[x], name[y]) for x in A for y in B): return False
+    if rank[ra] < rank[rb]: ra, rb = rb, ra
+    par[rb] = ra
+    if members is not None: members.setdefault(ra, [ra]).extend(members.pop(rb, [rb]))
+    return True
+
+def _ann_pairs(tbl, rows, k=8, dtype=np.float16):
+    '''`(id, neighbour_id, distance)` for every embedded row, from one batched HNSW probe.
+
+    usearch searches a matrix of queries across its own thread pool, so asking it once for 8,487
+    vectors is not the same work as asking it 8,487 times: the per-entity loop also ran one
+    `rowid IN (...)` query per entity, and the two together were 10.3s of a 23.0s resolve. The
+    candidate set is unchanged — same k, same neighbours, same distances.'''
+    idx_ = tbl.db.get_index(tbl.name)
+    emb = [r for r in rows if r['embedding']]
+    if not emb or not idx_.size: return
+    key = {r['rowid']: r for r in tbl.db.q(f'select {rowid_sel()}, id, content from {tbl.name}')}
+    res = idx_.search(np.stack([np.frombuffer(r['embedding'], dtype=dtype) for r in emb]),
+                      count=min(k, idx_.size))
+    ks, ds = np.atleast_2d(res.keys), np.atleast_2d(res.distances)
+    for r, kr, dr in zip(emb, ks, ds):
+        for kk, dd in zip(np.atleast_1d(kr).tolist(), np.atleast_1d(dr).tolist()):
+            if (o := key.get(int(kk))): yield r, o, float(dd)
+
+def _lexical_pairs(name, max_group=60):
+    '''Candidate merge pairs by shared-token blocking — catches containment variants
+
+    Tokens are walked in sorted order, which is what makes a resolve reproducible. `_toks` returns
+    a frozenset, so its iteration order follows string hashes and therefore `PYTHONHASHSEED`; that
+    decided the insertion order of `inv`, which decided the order pairs were proposed in, and
+    `_uf_union` only accepts a merge that keeps the group a clique — an order-dependent test. Two
+    resolves of the *same* database in two processes came back with different partitions and merge
+    counts drifting over a range of three, which looked like HNSW noise and was not: hold the seed
+    still and both the old code and the new one are exactly reproducible, and `verify_ann_probe`
+    says the ANN candidates were stable the whole time.'''
+    inv = {}
+    for i, s in name.items():
+        for t in sorted(_toks(s)):
+            if len(t) > 2: inv.setdefault(t, []).append(i)
+    seen = set()
+    for ids in inv.values():
+        if len(ids) < 2: continue
+        if len(ids) <= max_group: pairs = ((ids[a], ids[b]) for a in range(len(ids)) for b in range(a+1, len(ids)))
+        else:
+            srt = sorted(ids, key=lambda i: name[i])
+            pairs = ((srt[a], srt[b]) for a in range(len(srt)) for b in range(a+1, min(a+max_group, len(srt))))
+        for x, y in pairs:
+            p = (x, y) if x < y else (y, x)
+            if p not in seen: seen.add(p); yield p
+
+def resolve_entities(db,                # Database
+                     store='store',     # chunk store the graph belongs to
+                     prefix=None,       # graph table prefix
+                     thresh=0.18,       # max ANN distance for a merge candidate
+                     lex=0.34,          # min token Jaccard for the lexical guard
+                     k=8,               # ANN neighbours considered per entity
+                     lexical=True,      # also block on shared tokens (works without embeddings)
+                     max_group=60,      # skip token groups bigger than this in the lexical pass
+                     skip_kinds=_EXACT_KINDS,  # kinds whose names are already canonical
+                     dtype=np.float16):
+    'Merge near-duplicate entities: ANN + shared-token candidates, both gated by the lexical guard.'
+    g = db.get_graph(store, prefix)
+    allr = L(g.entities(select=f'{rowid_sel()}, id, content, freq, embedding, kind, canon'))
+    rows = allr.filter(lambda r: r['kind'] not in set(skip_kinds or ()))
+    if len(rows) < 2:
+        return dict(merged=0, by_ann=0, by_lexical=0, edges=len(list(g.edges())),
+                    entities=len(allr), resolvable=len(rows), canonical=len(allr))
+    par  = {r['id']: r['id'] for r in rows}
+    rank = {r['id']: (r['freq'] or 0) for r in rows}
+    name = {r['id']: r['content'] for r in rows}
+    ann_m = lex_m = 0
+    guard, members = (lambda x, y: _lex_ok(x, y, lex)), {}
+    for r, h, dist in _ann_pairs(g.entities, rows, k, dtype):
+        oid = h.get('id')
+        if not oid or oid == r['id'] or oid not in par: continue
+        if dist > thresh: continue
+        if not _lex_ok(r['content'], h['content'], lex): continue
+        if _uf_union(par, rank, r['id'], oid, name, guard, members): ann_m += 1
+    if lexical:
+        for a, b in _lexical_pairs(name, max_group):
+            if _lex_ok(name[a], name[b], lex) and _uf_union(par, rank, a, b, name, guard, members): lex_m += 1
+    upd = [(_uf_find(par, i), i) for i in par]
+    was = {r['id']: r['canon'] for r in rows}
+    chg = [(c, i) for c, i in upd if c != was.get(i)]
+    if chg:
+        with write_txn(db):
+            db.conn.cursor().executemany(f'update {g.entities.name} set canon=? where id=?', chg)
+    canon = {i: c for c, i in upd}
+    n_edges = _collapse_edges(db, g, canon)
+    skipped = len(allr) - len(rows)
+    return dict(merged=ann_m+lex_m, by_ann=ann_m, by_lexical=lex_m, edges=n_edges,
+                entities=len(allr), resolvable=len(rows),
+                canonical=len({c for c, _ in upd}) + skipped)
+
+def _collapse_edges(db, g, canon):
+    'Rewrite edge endpoints onto canonical ids. Traversal reads src/dst straight from the table,'
+    rows = list(g.edges())
+    if not rows: return 0
+    agg = {}
+    for r in rows:
+        s, d = canon.get(r['src'], r['src']), canon.get(r['dst'], r['dst'])
+        if s == d: continue
+        if s > d and r['rel'] == 'cooc': s, d = d, s     # cooc is symmetric; keep one direction
+        k = (s, d, r['rel'])
+        a = agg.setdefault(k, dict(src=s, dst=d, rel=r['rel'], weight=0.0, n=0))
+        a['weight'] = max(a['weight'], r['weight'] or 0.0); a['n'] += (r['n'] or 0)
+    with write_txn(db):
+        g.edges.delete_where()
+        upsert_all(g.edges, agg.values(), ('src','dst','rel'))
+    return len(agg)
+
+
+# %% ../nbs/01_build.ipynb #dd02940510f58745
+def cooccur_edges(db,                 # Database
+                  store='store',      # chunk store
+                  prefix=None,        # graph table prefix
+                  min_n=2,            # min co-occurrence count
+                  min_npmi=0.15,      # min normalized PMI (prunes stopword-ish hub nodes)
+                  max_df=0.4,         # drop entities present in >max_df of chunks
+                  max_degree=48,      # keep only the strongest edges per node
+                  rel='cooc',
+                  use_canon=True):    # collapse to canonical ids from resolve_entities
+    'Rebuild co-occurrence edges from the stored mentions, using the chunk as the window.'
+    g = db.get_graph(store, prefix)
+    canon = {}
+    if use_canon:
+        canon = {r['id']: (r['canon'] or r['id']) for r in g.entities(select='id, canon')}
+    cid_ents = {}
+    for m in g.mentions(select='chunk_id, entity_id'):
+        e = canon.get(m['entity_id'], m['entity_id'])
+        cid_ents.setdefault(m['chunk_id'], set()).add(e)
+    out = _pmi_edges(list(cid_ents.values()), min_n, min_npmi, max_df, max_degree, rel)
+    if out: upsert_all(g.edges, out, ('src','dst','rel'))
+    return len(out)
+

vruksha-0.0.2/vruksha/entities.py

@@ -0,0 +1,141 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_entities.ipynb.
+
+# %% auto #0
+__all__ = ['code_entities', 'prose_windows', 'text_entities']
+
+# %% ../nbs/00_entities.ipynb #ab82e9d12af8898f
+import ast, math, re, sys
+from functools import lru_cache
+import numpy as np
+from fastcore.all import AttrDict, L, Path, chunked, defaults, first, ifnone, merge, patch, store_attr
+from fastcore.parallel import ProcessPoolExecutor
+from fastlite import Database
+from apswutils.db import Table
+from multiprocessing import get_context
+from litesearch.core import (sql_in, rowid_sel, content_id, NP_DTYPE, process_content, write_txn, db_lock,
+                             upsert_all, rrf_all)
+from litesearch.topics import get_graph
+from litesearch.utils import hash_embed
+
+# %% ../nbs/00_entities.ipynb #673ffc3a5da7b038
+_DET = re.compile(r'^(the|a|an|this|that|these|those|its|their|our|your|his|her)\s+', re.I)
+_WS  = re.compile(r'\s+')
+_PRON = {'it','its','we','our','they','their','he','she','you','i','me','us','them','this','that',
+         'these','those','which','who','what','there','here','one','ones','something','anything'}
+_TOK = re.compile(r'[a-z0-9]+')
+_NUM = re.compile(r'\d+')
+
+# %% ../nbs/00_entities.ipynb #109cd07acfcb6b94
+def _norm(s):
+    'Canonical surface form for a mention; None when the phrase is not entity-like.'
+    if not s: return None
+    s = _WS.sub(' ', s).strip().strip('.,;:!?()[]{}"\'`')
+    s = _DET.sub('', s).strip()
+    s = re.sub(r"'s$", '', s).strip()
+    if not (2 <= len(s) <= 60): return None
+    if len(s.split()) > 5: return None
+    if s.lower() in _PRON: return None
+    if not any(c.isalpha() for c in s): return None
+    return s.lower()
+
+@lru_cache(maxsize=1<<17)
+def _toks(s):
+    'Tokens for the lexical guard. UAX#29 treats `_` as a word joiner, so `fts_search` stays one'
+    try: from apsw.unicode import word_iter, casefold
+    except ImportError: return frozenset(_TOK.findall(s.lower()))
+    return frozenset(casefold(t) for t in word_iter(s or '') if any(c.isalnum() for c in t))
+
+@lru_cache(maxsize=1<<17)
+def _acr(s):  return ''.join(w[0] for w in s.split() if w)
+
+@lru_cache(maxsize=1<<17)
+def _nums(s): return frozenset(_NUM.findall(s))
+
+def _sentences(text):
+    'UAX#29 sentence split via apsw; falls back to the whole text as one window.'
+    try: from apsw.unicode import sentence_iter
+    except ImportError: return [text]
+    return [s for s in (x.strip() for x in sentence_iter(text)) if s]
+
+def _jac(a, b):
+    A, B = _toks(a), _toks(b)
+    u = len(A) + len(B) - len(A & B)
+    return len(A & B) / u if u else 0.0
+
+# %% ../nbs/00_entities.ipynb #7710055a5e8346fe
+def _lex_ok(a, b, lex=0.34, cover=0.5):
+    '''Lexical guard on a proposed merge.'''
+    if a == b: return True
+    A, B = _toks(a), _toks(b)
+    ok = False
+    if A and B:
+        inter = len(A & B)
+        ok = ((inter == min(len(A), len(B)) and inter/max(len(A), len(B)) >= cover)
+              or inter/(len(A) + len(B) - inter) >= lex)
+    if not ok and not (_acr(a) == b.lower() or _acr(b) == a.lower()): return False
+    return _nums(a) == _nums(b)
+
+# %% ../nbs/00_entities.ipynb #5952ad1336601711
+_PY_SKIP = {'self','cls','super','print','len','str','int','float','bool','list','dict','set','tuple',
+            'range','enumerate','zip','map','filter','isinstance','getattr','setattr','hasattr','type',
+            'format','join','append','get','items','keys','values','open','sorted','sum','min','max'}
+
+def _def_name(chunk):
+    'Symbol defined by a code chunk, from pyparse metadata or the chunk source itself.'
+    md = chunk.get('metadata') or {}
+    if isinstance(md, dict) and md.get('name'): return md['name']
+    try: tree = ast.parse(chunk['content'])
+    except SyntaxError: return None
+    n = first(tree.body, lambda x: isinstance(x, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)))
+    return getattr(n, 'name', None)
+
+def code_entities(chunk):
+    'Exact symbols for a python chunk: (defined, called, imported). AST-derived, no model.'
+    try: tree = ast.parse(chunk['content'])
+    except (SyntaxError, ValueError): return None, L(), L()
+    calls, imps = L(), L()
+    for n in ast.walk(tree):
+        if isinstance(n, ast.Call):
+            f = n.func
+            if   isinstance(f, ast.Name):      calls.append(f.id)
+            elif isinstance(f, ast.Attribute): calls.append(f.attr)
+        elif isinstance(n, ast.Import):
+            imps += [a.name.split('.')[0] for a in n.names]
+        elif isinstance(n, ast.ImportFrom):
+            if n.module: imps.append(n.module.split('.')[0])
+    keep = lambda s: s and s not in _PY_SKIP and not s.startswith('_') and len(s) > 2
+    return _def_name(chunk), calls.filter(keep).unique(), imps.filter(keep).unique()
+
+# %% ../nbs/00_entities.ipynb #d337238a913ea5d3
+def _yake_terms(text, topk=12):
+    'Keyphrases via yake — zero model, zero labels, and the default prose extractor.'
+    try: from yake import KeywordExtractor
+    except ImportError: return L()
+    try: return L(KeywordExtractor(n=3, top=topk).extract_keywords(text)).map(lambda kv: kv[0])
+    except Exception: return L()
+
+def prose_windows(text,             # chunk text
+                  topk=12,          # keyphrase count
+                  terms_fn=None):   # (text, topk) -> terms; None -> yake
+    'Entity surfaces grouped into co-occurrence windows — one per sentence.'
+    terms = (terms_fn or _yake_terms)(text, topk)
+    if not terms: return L()
+    # the term list is fixed for the whole chunk, so it is lowercased once here rather than once
+    # per sentence — `t.lower()` sat in the inner loop and ran topk times for every sentence
+    low = [(t, t.lower()) for t in terms]
+    wins = []
+    for s in _sentences(text):
+        sl = s.lower()
+        hit = L([(t, 'keyphrase') for t, tl in low if tl in sl])
+        if hit: wins.append(hit)
+    return L(wins)
+
+def text_entities(text, **kw):
+    'Entity surfaces for a prose chunk, flattened across windows. Returns L of (surface, kind).'
+    seen, out = set(), L()
+    for w in prose_windows(text, **kw):
+        for s, k in w:
+            n = _norm(s)
+            if n and n not in seen: seen.add(n); out.append((s, k))
+    return out
+

vruksha-0.0.2/vruksha/search.py

@@ -0,0 +1,164 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_search.ipynb.
+
+# %% auto #0
+__all__ = ['graph_stats']
+
+# %% ../nbs/02_search.ipynb #ab82e9d12af8898f
+import ast, math, re, sys
+from functools import lru_cache
+import numpy as np
+from fastcore.all import AttrDict, L, Path, chunked, defaults, first, ifnone, merge, patch, store_attr
+from fastcore.parallel import ProcessPoolExecutor
+from fastlite import Database
+from apswutils.db import Table
+from multiprocessing import get_context
+from litesearch.core import (sql_in, rowid_sel, content_id, NP_DTYPE, process_content, write_txn, db_lock,
+                             upsert_all, rrf_all)
+from litesearch.topics import get_graph
+from litesearch.utils import hash_embed
+from .build import build_graph, resolve_entities, cooccur_edges
+
+# %% ../nbs/02_search.ipynb #a380070d2dd9d9da
+def _adjacency(g, nodes, hops=2, max_nodes=4000):
+    '''BFS the edge table out to `hops` from `nodes`; returns an undirected dict-of-dict adjacency.
+
+    Read as raw tuples rather than through the table wrapper. A two-hop walk off twelve seeds pulls
+    ~16k edges, and fastlite turns each one into a dict with a description lookup per row — 326k
+    dicts over twenty queries, of which this needs three columns and none of the keys.'''
+    adj, seen, frontier = {}, set(nodes), set(nodes)
+    con, tbl = g.edges.db.conn, g.edges.name
+    for _ in range(max(hops, 0)):
+        if not frontier or len(seen) >= max_nodes: break
+        fl = list(frontier)
+        rows = []
+        for b in chunked(fl, 400):
+            rows += con.execute(f"select src, dst, weight from {tbl} "
+                                f"where {sql_in('src', b)} OR {sql_in('dst', b)}").fetchall()
+        nxt = set()
+        for s, d, w in rows:
+            w = w or 1.0
+            adj.setdefault(s, {})[d] = max(adj.setdefault(s, {}).get(d, 0.0), w)
+            adj.setdefault(d, {})[s] = max(adj.setdefault(d, {}).get(s, 0.0), w)
+            for x in (s, d):
+                if x not in seen: nxt.add(x)
+        seen |= nxt
+        frontier = nxt
+    return adj
+
+def _csr(adj, order):
+    '''`(src, dst, w)` index arrays for `adj`, row-normalised — the transition matrix as three arrays.
+
+    Built once and reused by every power iteration, which is the whole point: the walk itself is
+    then twelve numpy passes instead of twelve nested python loops over the same unchanging edges.'''
+    src, dst, wt = [], [], []
+    for u, nb in adj.items():
+        iu, s = order[u], (sum(nb.values()) or 1.0)
+        for v, w in nb.items(): src.append(iu); dst.append(order[v]); wt.append(w/s)
+    return (np.array(src, dtype=np.intp), np.array(dst, dtype=np.intp),
+            np.array(wt, dtype=np.float64))
+
+def _ppr(adj, seeds, damping=0.85, iters=12):
+    '''Personalized PageRank over a dict-of-dict adjacency.
+
+    The iteration is `r <- d * Wᵀr + (1-d) * p0` and nothing but `r` changes between rounds, so the
+    edges are flattened into index arrays once and each round becomes a gather plus a `bincount` —
+    the same sum, done by numpy instead of by a dict lookup per edge per round. Dangling nodes still
+    drop their mass rather than redistributing it, exactly as the dict version did.'''
+    if not seeds: return {}
+    order = {}
+    for u, nb in adj.items():
+        if u not in order: order[u] = len(order)
+        for v in nb:
+            if v not in order: order[v] = len(order)
+    for k in seeds:
+        if k not in order: order[k] = len(order)
+    n = len(order)
+    p0 = np.zeros(n)
+    tot = sum(seeds.values()) or 1.0
+    for k, v in seeds.items(): p0[order[k]] = v/tot
+    src, dst, wt = _csr(adj, order)
+    wt = wt * damping
+    r, rest = p0, (1-damping)*p0
+    for _ in range(iters):
+        r = np.bincount(dst, weights=r[src]*wt, minlength=n) + rest
+    return {k: float(r[i]) for k, i in order.items() if r[i]}
+
+# %% ../nbs/02_search.ipynb #66f98425ed3e448
+def _canon_mentions(g,                # graph tables from `get_graph`
+                    col,              # 'chunk_id' or 'entity_id' — the side being filtered
+                    vals,             # values to filter on
+                    use_canon=True,   # resolve entity_id to its canonical id
+                    batch=400):
+    '''`(chunk_id, entity_id)` for the matching mentions, with `entity_id` already canonicalised.'''
+    con, mn, en = g.mentions.db.conn, g.mentions.name, g.entities.name
+    sel = (f'select m.chunk_id, coalesce(nullif(e.canon, \'\'), m.entity_id) from {mn} m '
+           f'left join {en} e on e.id = m.entity_id') if use_canon else \
+          f'select m.chunk_id, m.entity_id from {mn} m'
+    out = []
+    for b in chunked(vals, batch):
+        out += con.execute(f"{sel} where {sql_in('m.'+col, b)}").fetchall()
+    return out
+
+# %% ../nbs/02_search.ipynb #fba9515fca9c5b56
+def _leg(db, g, seeds_from, cols, table_name, limit, hops, damping, iters, use_canon):
+    'Chunks the walk reaches, ranked by PPR mass. Empty at every dead end, which RRF then ignores.'
+    if not seeds_from: return []
+    # canon is resolved by joining the mentions being read rather than by loading the whole entity
+    # table into a dict. Every query paid O(entities) for a map it used O(mentions-of-12-chunks) of,
+    # which is the one cost here that grew with the corpus rather than with the query.
+    seeds = {}
+    for _, e in _canon_mentions(g, 'chunk_id', seeds_from, use_canon): seeds[e] = seeds.get(e, 0.0) + 1.0
+    if not seeds: return []
+    mass = _ppr(_adjacency(g, set(seeds), hops), seeds, damping, iters)
+    eids = [e for e, m in sorted(mass.items(), key=lambda kv: -kv[1])[:200] if m > 0]
+    if not eids: return []
+    inv = {}
+    for cid, e in _canon_mentions(g, 'entity_id', eids, use_canon):
+        inv[cid] = inv.get(cid, 0.0) + mass.get(e, 0.0)
+    if not inv: return []
+    ranked = sorted(inv.items(), key=lambda kv: -kv[1])[:limit*3]
+    sel = ','.join([rowid_sel() if c == 'rowid' else c for c in cols])
+    rows = {r['id']: r for r in db.t[table_name](select=sel, where=sql_in('id', [c for c, _ in ranked]))}
+    return [rows[c] for c, _ in ranked if c in rows]
+
+@patch
+def graph_search(self:Database,
+                 q:str,                 # query string
+                 emb:bytes,             # query embedding
+                 columns:list=None,     # columns to return
+                 limit:int=20,          # max results
+                 table_name='store',    # chunk store
+                 prefix=None,           # graph table prefix
+                 seed_n:int=12,         # hybrid hits used to seed the graph walk
+                 hops:int=2,            # edge-table BFS depth
+                 damping:float=0.85,    # PPR damping
+                 iters:int=10,          # PPR iterations
+                 graph_w:float=0.5,     # weight of the graph leg, low by default. See the docstring.
+                 rrf_k:int=60,
+                 use_canon=True,
+                 **kw):                 # forwarded to Database.search
+    'Hybrid search plus a graph leg: PPR over the entity graph seeded by the top hybrid hits.'
+    g = self.get_graph(table_name, prefix)
+    cols = list(columns or [])
+    if 'rowid' not in cols: cols = ['rowid'] + cols
+    if 'id' not in cols: cols = cols + ['id']
+    base = self.search(q, emb, columns=cols, limit=max(seed_n*3, limit), table_name=table_name,
+                       rrf=False, **kw)
+    if not base: return []
+    fts, vec = base['fts'], base['vec']
+    seeds = [r['id'] for r in rrf_all([fts, vec], rrf_k, seed_n) if r.get('id')]
+    leg = _leg(self, g, seeds, cols, table_name, limit, hops, damping, iters, use_canon)
+    return rrf_all([fts, vec, leg], rrf_k, limit, weights=[1.0, 1.0, graph_w])
+
+
+# %% ../nbs/02_search.ipynb #96b2124dafe08d01
+def graph_stats(db, store='store', prefix=None):
+    'Row counts and top-degree nodes for a built graph.'
+    g = db.get_graph(store, prefix)
+    ne = first(db.q(f'select count(*) c from {g.entities.name}'))['c']
+    nm = first(db.q(f'select count(*) c from {g.mentions.name}'))['c']
+    ng = first(db.q(f'select count(*) c from {g.edges.name}'))['c']
+    nc = first(db.q(f'select count(distinct canon) c from {g.entities.name}'))['c']
+    top = db.q(f'''select e.content, e.kind, count(*) d from {g.edges.name} g
+                   join {g.entities.name} e on e.id=g.src group by g.src order by d desc limit 10''')
+    return dict(entities=ne, canonical=nc, mentions=nm, edges=ng, top_degree=top)