docforge-cli 0.4.1__tar.gz → 0.5.1__tar.gz

{docforge_cli-0.4.1 → docforge_cli-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docforge-cli
-Version: 0.4.1
+Version: 0.5.1
 Summary: Forge searchable context from Confluence and git repos for AI coding assistants
 License: MIT
 Project-URL: Homepage, https://GranatenUdo.github.io/docforge/

{docforge_cli-0.4.1 → docforge_cli-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docforge-cli"
-version = "0.4.1"
+version = "0.5.1"
 description = "Forge searchable context from Confluence and git repos for AI coding assistants"
 readme = "README.md"
 license = {text = "MIT"}

{docforge_cli-0.4.1 → docforge_cli-0.5.1}/src/docforge/api.py

@@ -209,31 +209,65 @@ async def search(
         async with pool.acquire() as conn:
             rows = await conn.fetch(
                 """
-                SELECT
-                    c.text,
-                    c.section_title,
-                    s.title AS source_title,
-                    s.url AS source_url,
-                    s.tags AS source_tags,
-                    1 - (c.embedding <=> $1::vector) AS similarity,
-                    (1 - (c.embedding <=> $1::vector)) *
-                        (1
-                         + $2::float * cardinality(
-                             ARRAY(SELECT unnest(s.tags) INTERSECT SELECT unnest($3::text[]))
-                           )
-                         + $4::float * (CASE WHEN 'org' = ANY(s.tags) THEN 1 ELSE 0 END)
-                        ) AS boosted_score
-                FROM chunks c
-                JOIN sources s ON c.source_id = s.id
-                WHERE s.status = 'active'
+                WITH q_tsq AS (SELECT websearch_to_tsquery($8::regconfig, $2::text) AS q),
+                     dense AS (
+                         SELECT id, source_id, text, section_title,
+                                ROW_NUMBER() OVER (ORDER BY dist) AS rank
+                         FROM (
+                             SELECT c.id, c.source_id, c.text, c.section_title,
+                                    c.embedding <=> $1::vector AS dist
+                             FROM chunks c JOIN sources s ON c.source_id = s.id
+                             WHERE s.status = 'active'
+                             ORDER BY c.embedding <=> $1::vector
+                             LIMIT $3
+                         ) AS t
+                     ),
+                     sparse AS (
+                         SELECT id, source_id, text, section_title,
+                                ROW_NUMBER() OVER (ORDER BY rk DESC) AS rank
+                         FROM (
+                             SELECT c.id, c.source_id, c.text, c.section_title,
+                                    ts_rank_cd(c.text_tsv, (SELECT q FROM q_tsq)) AS rk
+                             FROM chunks c JOIN sources s ON c.source_id = s.id
+                             WHERE s.status = 'active'
+                               AND c.text_tsv @@ (SELECT q FROM q_tsq)
+                             ORDER BY ts_rank_cd(c.text_tsv, (SELECT q FROM q_tsq)) DESC
+                             LIMIT $3
+                         ) AS t
+                     ),
+                     fused AS (
+                         SELECT COALESCE(d.id, sp.id) AS id,
+                                COALESCE(d.source_id, sp.source_id) AS source_id,
+                                COALESCE(d.text, sp.text) AS text,
+                                COALESCE(d.section_title, sp.section_title) AS section_title,
+                                COALESCE($10::float / ($9 + d.rank), 0)
+                                  + COALESCE($11::float / ($9 + sp.rank), 0) AS rrf
+                         FROM dense d FULL OUTER JOIN sparse sp ON d.id = sp.id
+                     )
+                SELECT f.text, f.section_title,
+                       s.title AS source_title, s.url AS source_url, s.tags AS source_tags,
+                       f.rrf AS similarity,
+                       f.rrf * (1
+                                + $4::float * cardinality(
+                                    ARRAY(SELECT unnest(s.tags) INTERSECT SELECT unnest($5::text[]))
+                                  )
+                                + $6::float * (CASE WHEN 'org' = ANY(s.tags) THEN 1 ELSE 0 END)
+                       ) AS boosted_score
+                FROM fused f JOIN sources s ON f.source_id = s.id
                 ORDER BY boosted_score DESC
-                LIMIT $5
+                LIMIT $7
                 """,
-                np.array(query_vector, dtype=np.float32),
-                settings.tag_match_weight,
-                user_tags,
-                settings.org_tag_weight,
-                req.limit,
+                np.array(query_vector, dtype=np.float32),  # $1
+                req.query,  # $2
+                settings.hybrid_pool_size,  # $3
+                settings.tag_match_weight,  # $4
+                user_tags,  # $5
+                settings.org_tag_weight,  # $6
+                req.limit,  # $7
+                settings.fts_language,  # $8
+                settings.rrf_k,  # $9
+                settings.dense_weight,  # $10
+                settings.sparse_weight,  # $11
             )
     except Exception as e:
         logger.error("Database error during search: %s", e)

{docforge_cli-0.4.1 → docforge_cli-0.5.1}/src/docforge/config.py

@@ -66,6 +66,23 @@ class Settings(BaseSettings):
     tag_match_weight: float = 0.1
     org_tag_weight: float = 0.05
 
+    # Hybrid retrieval (RRF over dense + sparse). rrf_k=60 matches the universal
+    # default (Azure AI Search, Elasticsearch, OpenSearch); higher k flattens
+    # the rank distribution, lower amplifies. hybrid_pool_size is the top-N
+    # from each retriever feeding RRF — 4-10x req.limit is the standard rule,
+    # and req.limit caps at 50 so 100 covers under-recalled queries with margin.
+    # fts_language is the Postgres text-search config; switch to 'simple' if
+    # non-English content appears in the corpus.
+    rrf_k: int = 60
+    hybrid_pool_size: int = 100
+    fts_language: str = "english"
+
+    # Weighted RRF — multipliers on each retriever's reciprocal-rank contribution.
+    # Defaults at 1.0 = classic RRF (the v0.5.0 default). Tune via env var
+    # (DENSE_WEIGHT / SPARSE_WEIGHT) or docforge.yml; eval-driven.
+    dense_weight: float = 1.0
+    sparse_weight: float = 1.0
+
     # Default identity (used as CLI flag defaults when set via env/yml)
     default_user_name: str = ""
     default_team_name: str = ""

docforge_cli-0.5.1/src/docforge/sql/migrations/007_add_chunks_text_tsv.sql

@@ -0,0 +1,16 @@
+-- Migration 007: add tsvector column and GIN index for hybrid retrieval.
+--
+-- text_tsv is GENERATED ALWAYS AS STORED, so Postgres backfills existing
+-- rows as part of the ALTER TABLE and auto-populates on every INSERT.
+-- No application changes required for ingest.
+--
+-- The GIN index is built non-concurrently. For the current chunk count
+-- (~tens of thousands) this is sub-second. If chunks grows past ~1M
+-- rows, switch a future migration to CREATE INDEX CONCURRENTLY (which
+-- requires running outside a transaction).
+
+ALTER TABLE chunks
+    ADD COLUMN IF NOT EXISTS text_tsv tsvector
+        GENERATED ALWAYS AS (to_tsvector('english', text)) STORED;
+
+CREATE INDEX IF NOT EXISTS chunks_text_tsv_idx ON chunks USING GIN (text_tsv);

{docforge_cli-0.4.1 → docforge_cli-0.5.1}/src/docforge_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docforge-cli
-Version: 0.4.1
+Version: 0.5.1
 Summary: Forge searchable context from Confluence and git repos for AI coding assistants
 License: MIT
 Project-URL: Homepage, https://GranatenUdo.github.io/docforge/

{docforge_cli-0.4.1 → docforge_cli-0.5.1}/src/docforge_cli.egg-info/SOURCES.txt

@@ -32,6 +32,7 @@ src/docforge/sql/migrations/003_add_source_tags.sql
 src/docforge/sql/migrations/004_add_query_log.sql
 src/docforge/sql/migrations/005_add_query_log_user_oid.sql
 src/docforge/sql/migrations/006_add_query_log_request_ms.sql
+src/docforge/sql/migrations/007_add_chunks_text_tsv.sql
 src/docforge/templates/docforge.yml
 src/docforge/templates/docker-compose.yml
 src/docforge/templates/mcp_client.py