world-reference-mcp 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. world_reference_mcp-0.3.1/LICENSE +20 -0
  2. world_reference_mcp-0.3.1/PKG-INFO +57 -0
  3. world_reference_mcp-0.3.1/README.md +34 -0
  4. world_reference_mcp-0.3.1/pyproject.toml +84 -0
  5. world_reference_mcp-0.3.1/scripts/compute_tool_fingerprints.py +79 -0
  6. world_reference_mcp-0.3.1/scripts/security_sanitizer.py +318 -0
  7. world_reference_mcp-0.3.1/setup.cfg +4 -0
  8. world_reference_mcp-0.3.1/tests/conftest.py +34 -0
  9. world_reference_mcp-0.3.1/tests/test_adapter_conformance.py +164 -0
  10. world_reference_mcp-0.3.1/tests/test_api_fdc.py +29 -0
  11. world_reference_mcp-0.3.1/tests/test_api_gbif.py +45 -0
  12. world_reference_mcp-0.3.1/tests/test_api_inaturalist.py +28 -0
  13. world_reference_mcp-0.3.1/tests/test_api_ncbi_taxonomy.py +47 -0
  14. world_reference_mcp-0.3.1/tests/test_api_noaa.py +37 -0
  15. world_reference_mcp-0.3.1/tests/test_api_ols4.py +25 -0
  16. world_reference_mcp-0.3.1/tests/test_api_open_meteo.py +48 -0
  17. world_reference_mcp-0.3.1/tests/test_api_wikidata.py +41 -0
  18. world_reference_mcp-0.3.1/tests/test_concept_parity.py +87 -0
  19. world_reference_mcp-0.3.1/tests/test_init_dynamics.py +66 -0
  20. world_reference_mcp-0.3.1/tests/test_presets_and_adapters.py +67 -0
  21. world_reference_mcp-0.3.1/tests/test_sdk_rate_limiter.py +26 -0
  22. world_reference_mcp-0.3.1/tests/test_startup.py +28 -0
  23. world_reference_mcp-0.3.1/world_reference_mcp/__init__.py +19 -0
  24. world_reference_mcp-0.3.1/world_reference_mcp/__main__.py +8 -0
  25. world_reference_mcp-0.3.1/world_reference_mcp/_version.py +5 -0
  26. world_reference_mcp-0.3.1/world_reference_mcp/api/__init__.py +11 -0
  27. world_reference_mcp-0.3.1/world_reference_mcp/api/fdc.py +68 -0
  28. world_reference_mcp-0.3.1/world_reference_mcp/api/gbif.py +84 -0
  29. world_reference_mcp-0.3.1/world_reference_mcp/api/inaturalist.py +61 -0
  30. world_reference_mcp-0.3.1/world_reference_mcp/api/ncbi_taxonomy.py +74 -0
  31. world_reference_mcp-0.3.1/world_reference_mcp/api/noaa.py +71 -0
  32. world_reference_mcp-0.3.1/world_reference_mcp/api/ols4.py +52 -0
  33. world_reference_mcp-0.3.1/world_reference_mcp/api/open_meteo.py +83 -0
  34. world_reference_mcp-0.3.1/world_reference_mcp/api/wikidata.py +64 -0
  35. world_reference_mcp-0.3.1/world_reference_mcp/config.py +126 -0
  36. world_reference_mcp-0.3.1/world_reference_mcp/connectors/__init__.py +14 -0
  37. world_reference_mcp-0.3.1/world_reference_mcp/connectors/adapters.py +81 -0
  38. world_reference_mcp-0.3.1/world_reference_mcp/connectors/mcp_source_presets.json +312 -0
  39. world_reference_mcp-0.3.1/world_reference_mcp/connectors/tool_schema_fingerprints.json +14 -0
  40. world_reference_mcp-0.3.1/world_reference_mcp/credentials.py +58 -0
  41. world_reference_mcp-0.3.1/world_reference_mcp/errors.py +13 -0
  42. world_reference_mcp-0.3.1/world_reference_mcp/http_clients.py +28 -0
  43. world_reference_mcp-0.3.1/world_reference_mcp/mcp/__init__.py +8 -0
  44. world_reference_mcp-0.3.1/world_reference_mcp/mcp/clients.py +78 -0
  45. world_reference_mcp-0.3.1/world_reference_mcp/mcp/server.py +60 -0
  46. world_reference_mcp-0.3.1/world_reference_mcp/mcp/tools_nutrition.py +37 -0
  47. world_reference_mcp-0.3.1/world_reference_mcp/mcp/tools_observations.py +88 -0
  48. world_reference_mcp-0.3.1/world_reference_mcp/mcp/tools_taxonomy.py +42 -0
  49. world_reference_mcp-0.3.1/world_reference_mcp/mcp/tools_terms.py +33 -0
  50. world_reference_mcp-0.3.1/world_reference_mcp/mcp/tools_wikidata.py +37 -0
  51. world_reference_mcp-0.3.1/world_reference_mcp/mcp_config.json +3 -0
  52. world_reference_mcp-0.3.1/world_reference_mcp/mcp_server.py +26 -0
  53. world_reference_mcp-0.3.1/world_reference_mcp/ontology/__init__.py +13 -0
  54. world_reference_mcp-0.3.1/world_reference_mcp/ontology/certification.json +36 -0
  55. world_reference_mcp-0.3.1/world_reference_mcp/ontology/fixtures/records.json +226 -0
  56. world_reference_mcp-0.3.1/world_reference_mcp/ontology/mappings/source.yaml +196 -0
  57. world_reference_mcp-0.3.1/world_reference_mcp/ontology/migrations/manifest.json +6 -0
  58. world_reference_mcp-0.3.1/world_reference_mcp/ontology/shapes/connector.shacl.ttl +24 -0
  59. world_reference_mcp-0.3.1/world_reference_mcp/ontology/world_reference.ttl +42 -0
  60. world_reference_mcp-0.3.1/world_reference_mcp/prompts/__init__.py +0 -0
  61. world_reference_mcp-0.3.1/world_reference_mcp/prompts/main_agent.json +20 -0
  62. world_reference_mcp-0.3.1/world_reference_mcp/prompts/world_reference_specialist.json +60 -0
  63. world_reference_mcp-0.3.1/world_reference_mcp/skills/__init__.py +0 -0
  64. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-alignment/SKILL.md +80 -0
  65. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-alignment/WORKFLOW.md +63 -0
  66. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-mcp-operations/SKILL.md +37 -0
  67. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-mcp-operations/agents/openai.yaml +4 -0
  68. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-mcp-operations/references/catalog.md +9 -0
  69. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-nutrition/SKILL.md +71 -0
  70. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-nutrition/WORKFLOW.md +56 -0
  71. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-observations/SKILL.md +100 -0
  72. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-observations/WORKFLOW.md +82 -0
  73. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-taxonomy/SKILL.md +93 -0
  74. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-taxonomy/WORKFLOW.md +76 -0
  75. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-terms/SKILL.md +79 -0
  76. world_reference_mcp-0.3.1/world_reference_mcp/skills/world-reference-terms/WORKFLOW.md +63 -0
  77. world_reference_mcp-0.3.1/world_reference_mcp.egg-info/PKG-INFO +57 -0
  78. world_reference_mcp-0.3.1/world_reference_mcp.egg-info/SOURCES.txt +80 -0
  79. world_reference_mcp-0.3.1/world_reference_mcp.egg-info/dependency_links.txt +1 -0
  80. world_reference_mcp-0.3.1/world_reference_mcp.egg-info/entry_points.txt +14 -0
  81. world_reference_mcp-0.3.1/world_reference_mcp.egg-info/requires.txt +8 -0
  82. world_reference_mcp-0.3.1/world_reference_mcp.egg-info/top_level.txt +9 -0
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2012-2023 Project Maintainers
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,57 @@
1
+ Metadata-Version: 2.4
2
+ Name: world-reference-mcp
3
+ Version: 0.3.1
4
+ Summary: World-reference data (taxonomy, OBO terms, nutrition, observations, Wikidata alignment) MCP server and agent-connector-sdk source adapters
5
+ Author-email: Project Maintainers <maintainers@users.noreply.github.com>
6
+ License: MIT
7
+ Classifier: Development Status :: 3 - Alpha
8
+ Classifier: License :: OSI Approved :: MIT License
9
+ Classifier: Environment :: Console
10
+ Classifier: Operating System :: POSIX :: Linux
11
+ Classifier: Programming Language :: Python :: 3
12
+ Requires-Python: <3.15,>=3.12
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.13
16
+ Requires-Dist: httpx>=0.28.1
17
+ Provides-Extra: test
18
+ Requires-Dist: pytest-xdist>=3.8.0; extra == "test"
19
+ Requires-Dist: pytest>=9.1.1; extra == "test"
20
+ Requires-Dist: pytest-asyncio>=1.4.0; extra == "test"
21
+ Requires-Dist: pytest-cov>=7.1.0; extra == "test"
22
+ Dynamic: license-file
23
+
24
+ # world-reference-mcp
25
+
26
+ *Version: 0.3.1*
27
+
28
+ World-reference data — organism taxonomy, OBO reference terms, food/nutrition
29
+ composition, organism and weather observations, and Wikidata alignment — as an
30
+ MCP tool surface and, through `agent-connector-sdk`'s declarative `mcp_tool`
31
+ source adapter, as bounded, checkpointed ingestion streams onto the
32
+ epistemic-graph `world_model` ontology (`feat/world-model-ontology`).
33
+
34
+ ## Sources
35
+
36
+ | Ledger row | Sources | Target class |
37
+ |---|---|---|
38
+ | EH-358 (taxonomy) | NCBI Taxonomy (E-utilities), GBIF backbone | `Taxon` |
39
+ | EH-359 (reference terms) | EBI OLS4 (GO, UBERON, ENVO, PATO, ChEBI, FoodOn) | `BiologicalProcessTerm`, `AnatomyTerm`, `EnvironmentTerm`, `QualityTerm`, `ChemicalEntityTerm`, `FoodTerm` |
40
+ | EH-360 (nutrition) | USDA FoodData Central | `FoodCompositionRecord` |
41
+ | EH-361 (observations) | GBIF occurrences, iNaturalist, Open-Meteo, NOAA GHCND | `OrganismObservation`, `WeatherObservation` |
42
+ | EH-362 (alignment) | Wikidata SPARQL | `wikidataId` / `skos:exactMatch` keys for `Taxon`, `ChemicalEntityTerm`, `Country` |
43
+
44
+ ## Tools
45
+
46
+ Nine MCP tools, one FastMCP server (`world-reference-mcp` console script):
47
+ `taxonomy_ncbi_search`, `taxonomy_gbif_search`, `reference_term_list`,
48
+ `fdc_food_search`, `gbif_occurrence_search`, `inaturalist_observation_search`,
49
+ `open_meteo_weather_observation`, `noaa_ghcnd_daily`, `wikidata_alignment_search`.
50
+
51
+ Every tool is keyless except `fdc_food_search` and `noaa_ghcnd_daily`, whose
52
+ credentials are resolved server-side through `agent_connector_sdk.credentials`
53
+ (`WORLD_REFERENCE_FDC_API_KEY`, `WORLD_REFERENCE_NOAA_TOKEN`) — never accepted
54
+ as a tool argument.
55
+
56
+ See [`AGENTS.md`](AGENTS.md) for architecture, the ontology-mapping
57
+ authority, rate limits, and commands.
@@ -0,0 +1,34 @@
1
+ # world-reference-mcp
2
+
3
+ *Version: 0.3.1*
4
+
5
+ World-reference data — organism taxonomy, OBO reference terms, food/nutrition
6
+ composition, organism and weather observations, and Wikidata alignment — as an
7
+ MCP tool surface and, through `agent-connector-sdk`'s declarative `mcp_tool`
8
+ source adapter, as bounded, checkpointed ingestion streams onto the
9
+ epistemic-graph `world_model` ontology (`feat/world-model-ontology`).
10
+
11
+ ## Sources
12
+
13
+ | Ledger row | Sources | Target class |
14
+ |---|---|---|
15
+ | EH-358 (taxonomy) | NCBI Taxonomy (E-utilities), GBIF backbone | `Taxon` |
16
+ | EH-359 (reference terms) | EBI OLS4 (GO, UBERON, ENVO, PATO, ChEBI, FoodOn) | `BiologicalProcessTerm`, `AnatomyTerm`, `EnvironmentTerm`, `QualityTerm`, `ChemicalEntityTerm`, `FoodTerm` |
17
+ | EH-360 (nutrition) | USDA FoodData Central | `FoodCompositionRecord` |
18
+ | EH-361 (observations) | GBIF occurrences, iNaturalist, Open-Meteo, NOAA GHCND | `OrganismObservation`, `WeatherObservation` |
19
+ | EH-362 (alignment) | Wikidata SPARQL | `wikidataId` / `skos:exactMatch` keys for `Taxon`, `ChemicalEntityTerm`, `Country` |
20
+
21
+ ## Tools
22
+
23
+ Nine MCP tools, one FastMCP server (`world-reference-mcp` console script):
24
+ `taxonomy_ncbi_search`, `taxonomy_gbif_search`, `reference_term_list`,
25
+ `fdc_food_search`, `gbif_occurrence_search`, `inaturalist_observation_search`,
26
+ `open_meteo_weather_observation`, `noaa_ghcnd_daily`, `wikidata_alignment_search`.
27
+
28
+ Every tool is keyless except `fdc_food_search` and `noaa_ghcnd_daily`, whose
29
+ credentials are resolved server-side through `agent_connector_sdk.credentials`
30
+ (`WORLD_REFERENCE_FDC_API_KEY`, `WORLD_REFERENCE_NOAA_TOKEN`) — never accepted
31
+ as a tool argument.
32
+
33
+ See [`AGENTS.md`](AGENTS.md) for architecture, the ontology-mapping
34
+ authority, rate limits, and commands.
@@ -0,0 +1,84 @@
1
+ [build-system]
2
+ requires = ["setuptools>=80.9.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "world-reference-mcp"
7
+ version = "0.3.1"
8
+ description = "World-reference data (taxonomy, OBO terms, nutrition, observations, Wikidata alignment) MCP server and agent-connector-sdk source adapters"
9
+ readme = "README.md"
10
+ classifiers = [
11
+ "Development Status :: 3 - Alpha",
12
+ "License :: OSI Approved :: MIT License",
13
+ "Environment :: Console",
14
+ "Operating System :: POSIX :: Linux",
15
+ "Programming Language :: Python :: 3",
16
+ ]
17
+ requires-python = ">=3.12, <3.15"
18
+ dependencies = [
19
+ "agent-connector-sdk[mcp]>=0.3.13,<1.0.0",
20
+ "httpx>=0.28.1",
21
+ ]
22
+
23
+ [[project.authors]]
24
+ name = "Project Maintainers"
25
+ email = "maintainers@users.noreply.github.com"
26
+
27
+ [project.license]
28
+ text = "MIT"
29
+
30
+ [project.optional-dependencies]
31
+ test = ["pytest-xdist>=3.8.0", "pytest>=9.1.1", "pytest-asyncio>=1.4.0", "pytest-cov>=7.1.0"]
32
+
33
+ [project.scripts]
34
+ world-reference-mcp = "world_reference_mcp.mcp_server:mcp_server"
35
+
36
+ [project.entry-points."agent_connector_sdk.skill_providers"]
37
+ world-reference-mcp = "world_reference_mcp.skills"
38
+
39
+ [project.entry-points."agent_connector_sdk.ontology_providers"]
40
+ world-reference-mcp = "world_reference_mcp.ontology"
41
+
42
+ [project.entry-points."agent_connector_sdk.prompt_providers"]
43
+ world-reference-mcp = "world_reference_mcp.prompts"
44
+
45
+ [project.entry-points."agent_connector_sdk.source_connector_providers"]
46
+ world-reference-mcp = "world_reference_mcp.connectors"
47
+
48
+ [tool.uv.workspace]
49
+ members = ["."]
50
+
51
+ [tool.setuptools]
52
+ include-package-data = true
53
+
54
+ [tool.ruff]
55
+ line-length = 88
56
+ target-version = "py312"
57
+
58
+ [tool.mypy]
59
+ python_version = "3.12"
60
+ ignore_missing_imports = true
61
+ check_untyped_defs = true
62
+
63
+ [dependency-groups]
64
+ dev = ["pytest-timeout>=2.4.0", "mypy>=1.20.2", "ruff>=0.16.0"]
65
+
66
+ [tool.setuptools.package-data]
67
+ world_reference_mcp = ["mcp_config.json", "prompts/**", "skills/**", "ontology/**", "connectors/**"]
68
+
69
+ [tool.ruff.lint]
70
+ select = ["E", "F", "I", "UP", "B"]
71
+ ignore = ["E402", "E501", "B008"]
72
+
73
+ [tool.setuptools.packages.find]
74
+ where = ["."]
75
+
76
+
77
+ [tool.pipelines_hooks]
78
+ packages = ["world_reference_mcp"]
79
+
80
+ [tool.pipelines_hooks.stdout_writes]
81
+ served_paths = ["world_reference_mcp"]
82
+
83
+ [tool.pipelines_hooks.ste]
84
+ exempt = ["specs/*", "specs/**/*"]
@@ -0,0 +1,79 @@
1
+ #!/usr/bin/env python3
2
+ """Regenerate ``connectors/tool_schema_fingerprints.json`` from the live tool surface.
3
+
4
+ Connects an in-process FastMCP client to a plain server carrying this
5
+ package's nine tools (no auth/network/fleet-registration wiring — those are
6
+ concerns of ``agent_connector_sdk.mcp.server.create_mcp_server``, irrelevant
7
+ to a tool's client-visible schema) and fingerprints exactly what a real
8
+ client's ``tools/list`` would see, via
9
+ ``agent_connector_sdk.certify.fingerprints.tool_fingerprint`` — the same
10
+ function ``discover()`` verifies against at extraction time.
11
+
12
+ Run after changing any tool's signature or docstring-derived schema:
13
+
14
+ uv run --frozen python scripts/compute_tool_fingerprints.py --apply
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import argparse
20
+ import asyncio
21
+ import json
22
+ from pathlib import Path
23
+
24
+ from agent_connector_sdk.certify.fingerprints import tool_fingerprint
25
+ from fastmcp import Client, FastMCP
26
+
27
+ from world_reference_mcp.mcp.clients import Clients
28
+ from world_reference_mcp.mcp.tools_nutrition import register_nutrition_tools
29
+ from world_reference_mcp.mcp.tools_observations import register_observation_tools
30
+ from world_reference_mcp.mcp.tools_taxonomy import register_taxonomy_tools
31
+ from world_reference_mcp.mcp.tools_terms import register_term_tools
32
+ from world_reference_mcp.mcp.tools_wikidata import register_wikidata_tools
33
+
34
+ _OUTPUT = Path(__file__).resolve().parent.parent / (
35
+ "world_reference_mcp/connectors/tool_schema_fingerprints.json"
36
+ )
37
+
38
+
39
+ def _bare_server() -> FastMCP:
40
+ mcp: FastMCP = FastMCP("world-reference-mcp")
41
+ clients = Clients()
42
+ register_taxonomy_tools(mcp, clients)
43
+ register_term_tools(mcp, clients)
44
+ register_nutrition_tools(mcp, clients)
45
+ register_observation_tools(mcp, clients)
46
+ register_wikidata_tools(mcp, clients)
47
+ return mcp
48
+
49
+
50
+ async def _fingerprints() -> dict[str, str]:
51
+ mcp = _bare_server()
52
+ async with Client(mcp) as client:
53
+ tools = await client.list_tools()
54
+ return {tool.name: tool_fingerprint(tool) for tool in tools}
55
+
56
+
57
+ def main() -> int:
58
+ parser = argparse.ArgumentParser(description=__doc__)
59
+ parser.add_argument("--apply", action="store_true", help="write the output file")
60
+ args = parser.parse_args()
61
+ fingerprints = asyncio.run(_fingerprints())
62
+ document = {
63
+ "_comment": (
64
+ "Generated by scripts/compute_tool_fingerprints.py from the live "
65
+ "tools/list schema. Never hand-edit."
66
+ ),
67
+ "tools": dict(sorted(fingerprints.items())),
68
+ }
69
+ text = json.dumps(document, indent=2, sort_keys=False) + "\n"
70
+ if args.apply:
71
+ _OUTPUT.write_text(text)
72
+ print(f"wrote {_OUTPUT}")
73
+ else:
74
+ print(text)
75
+ return 0
76
+
77
+
78
+ if __name__ == "__main__":
79
+ raise SystemExit(main())
@@ -0,0 +1,318 @@
1
+ #!/usr/bin/env python3
2
+ import os
3
+ import re
4
+ import subprocess
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ MAX_SCAN_BYTES = 8 * 1024 * 1024
9
+
10
+ # Config
11
+ # llms.txt is the deliberate root-level AI entry index (llms-txt convention,
12
+ # like robots.txt) shipped by the docs/day-0 work — not garbage.
13
+ # overrides.txt is the sanctioned uv dependency-override file (UV_OVERRIDE in
14
+ # docker/Dockerfile, mirroring [tool.uv] override-dependencies) shipped by the
15
+ # pydantic-ai v2 migration — a canonical packaging input, not scratch.
16
+ # .security-audit-allow.txt is the OSV dependency-audit risk-acceptance ledger
17
+ # (scripts/audit_dependencies.py, wired into the dependency-audit pre-commit hook)
18
+ # — a committed, actively-read governance input, not scratch/garbage.
19
+ ALLOWED_TXT_NAMES = {
20
+ "requirements.txt",
21
+ "requirements-dev.txt",
22
+ "llms.txt",
23
+ "overrides.txt",
24
+ ".security-audit-allow.txt",
25
+ }
26
+ TRANSIENT_PY_PATTERNS = [
27
+ re.compile(r"^test_.*\.py$"),
28
+ re.compile(r"^fix_.*\.py$"),
29
+ re.compile(r"^debug_.*\.py$"),
30
+ re.compile(r"^scratch_.*\.py$"),
31
+ re.compile(r"^temp_.*\.py$"),
32
+ ]
33
+
34
+ SECRET_PATTERNS = [
35
+ ("GitHub PAT", re.compile(r"ghp_[A-Za-z0-9_]{36,255}")),
36
+ ("GitHub Fine-grained PAT", re.compile(r"github_pat_[A-Za-z0-9_]{82,255}")),
37
+ ("GitLab PAT", re.compile(r"glpat-[A-Za-z0-9\-]{20,255}")),
38
+ ("Private key", re.compile(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----")),
39
+ ("AWS access key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
40
+ ("Slack token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b")),
41
+ ("Langfuse secret", re.compile(r"\bsk-lf-[A-Za-z0-9_-]{16,}\b")),
42
+ (
43
+ "Generic Secret Assignment",
44
+ re.compile(
45
+ r"secret[A-Za-z0-9_]*\s*[:=]\s*['\"][A-Za-z0-9_\-\.\~\*]{16,255}['\"]",
46
+ re.IGNORECASE,
47
+ ),
48
+ ),
49
+ (
50
+ "Generic Token Assignment",
51
+ re.compile(
52
+ r"token\s*[:=]\s*['\"][A-Za-z0-9_\-\.\~\*]{16,255}['\"]", re.IGNORECASE
53
+ ),
54
+ ),
55
+ ]
56
+
57
+ EXCLUDED_DIRS = {
58
+ ".git",
59
+ ".venv",
60
+ "venv",
61
+ "node_modules",
62
+ "build",
63
+ "dist",
64
+ "__pycache__",
65
+ ".tox",
66
+ ".specify",
67
+ ".mypy_cache",
68
+ ".pytest_cache",
69
+ ".ruff_cache",
70
+ ".cache",
71
+ }
72
+ EXCLUDED_EXTENSIONS = {
73
+ ".png",
74
+ ".jpg",
75
+ ".jpeg",
76
+ ".gif",
77
+ ".webp",
78
+ ".ico",
79
+ ".pyc",
80
+ ".db",
81
+ ".kuzu",
82
+ ".sqlite",
83
+ ".sqlite3",
84
+ ".zip",
85
+ ".tar.gz",
86
+ ".tgz",
87
+ ".bz2",
88
+ ".xz",
89
+ ".pdf",
90
+ ".bin",
91
+ ".exe",
92
+ ".dll",
93
+ ".so",
94
+ ".dylib",
95
+ ".woff",
96
+ ".woff2",
97
+ ".eot",
98
+ ".ttf",
99
+ ".mp4",
100
+ ".mp3",
101
+ ".wav",
102
+ ".lock",
103
+ ".svg",
104
+ }
105
+
106
+ # Placeholder / Mock indicators
107
+ PLACEHOLDER_VALUES = {
108
+ "1234567890",
109
+ "abcdef12345",
110
+ "abc123youandme",
111
+ "askdfalskdvjas",
112
+ "test_token",
113
+ "test_secret",
114
+ "glpat-askdfalskdvjas",
115
+ "github_pat_12345",
116
+ "glpat-abc123youandme",
117
+ "github_pat_...",
118
+ "glpat-*************",
119
+ "ghp_*************",
120
+ "github_pat_*************",
121
+ "token_*************",
122
+ "secret_*************",
123
+ "glpat-abc",
124
+ "ghp_abc",
125
+ "github_pat_abc",
126
+ "${env:",
127
+ }
128
+ PLACEHOLDER_PREFIXES = ("your_", "your-", "dummy", "example", "mock")
129
+
130
+
131
+ def is_placeholder(match_str: str) -> bool:
132
+ # Generic assignment patterns include the variable name. Judge only the
133
+ # quoted value so names such as ``secret_example`` cannot suppress a real
134
+ # credential finding.
135
+ assignment = re.search(r"['\"]([^'\"]+)['\"]\s*$", match_str)
136
+ candidate = assignment.group(1) if assignment else match_str
137
+ candidate_lower = candidate.strip().lower()
138
+ if candidate_lower in PLACEHOLDER_VALUES or candidate_lower.startswith(
139
+ PLACEHOLDER_PREFIXES
140
+ ):
141
+ return True
142
+
143
+ # Check if match is mostly asterisks or single repeated char
144
+ cleaned = candidate.replace("'", "").replace('"', "").strip()
145
+ if not cleaned:
146
+ return True
147
+
148
+ # Check if there are sequences of asterisks indicating masked values
149
+ if "*" in cleaned:
150
+ # e.g., glpat-*************
151
+ return True
152
+
153
+ compact = re.sub(r"[^A-Za-z0-9]", "", cleaned).lower()
154
+ if len(compact) >= 8 and len(set(compact)) == 1:
155
+ return True
156
+
157
+ return False
158
+
159
+
160
+ def _is_excluded_relative_path(relative: Path) -> bool:
161
+ if relative.is_absolute() or ".." in relative.parts:
162
+ raise ValueError("unsafe source inventory path")
163
+ return any(part in EXCLUDED_DIRS for part in relative.parts)
164
+
165
+
166
+ def _git_tracked_files(repo_path: Path):
167
+ result = subprocess.run(
168
+ ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
169
+ cwd=str(repo_path),
170
+ capture_output=True,
171
+ text=True,
172
+ check=True,
173
+ timeout=30,
174
+ )
175
+ files = []
176
+ for line in result.stdout.splitlines():
177
+ if not line.strip():
178
+ continue
179
+ relative = Path(line.strip())
180
+ # Avoid files inside excluded directories
181
+ if not _is_excluded_relative_path(relative):
182
+ files.append(repo_path / relative)
183
+ return files
184
+
185
+
186
+ def _walk_tracked_files(repo_path: Path):
187
+ # Fallback to manual recursive scan
188
+ files = []
189
+ for root, dirs, walk_files in os.walk(str(repo_path)):
190
+ # Hidden source/config directories (for example ``.github``) can
191
+ # contain credentials and must not disappear merely because Git
192
+ # inventory was unavailable. Exclude only known generated trees.
193
+ dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS]
194
+ for file in walk_files:
195
+ files.append(Path(root) / file)
196
+ return files
197
+
198
+
199
+ def get_repo_files(repo_path: Path):
200
+ try:
201
+ return _git_tracked_files(repo_path)
202
+ except (OSError, subprocess.SubprocessError, ValueError):
203
+ return _walk_tracked_files(repo_path)
204
+
205
+
206
+ def _root_naming_violations(file_path: Path):
207
+ violations = []
208
+ if file_path.suffix == ".txt":
209
+ if file_path.name.lower() not in ALLOWED_TXT_NAMES:
210
+ violations.append(
211
+ "Non-standard root-level text file detected: "
212
+ f"'{file_path.name}'. Only {sorted(ALLOWED_TXT_NAMES)} "
213
+ "are allowed."
214
+ )
215
+ elif file_path.suffix == ".py":
216
+ for pattern in TRANSIENT_PY_PATTERNS:
217
+ if pattern.match(file_path.name):
218
+ violations.append(
219
+ "Transient/temporary script detected in root: "
220
+ f"'{file_path.name}'. Please move it to a subfolder "
221
+ "or delete it."
222
+ )
223
+ break
224
+ return violations
225
+
226
+
227
+ def _should_secret_scan(file_path: Path) -> bool:
228
+ if file_path.suffix.lower() in EXCLUDED_EXTENSIONS:
229
+ return False
230
+ if file_path.name == "security_sanitizer.py":
231
+ return False
232
+ return True
233
+
234
+
235
+ def _secret_violations_in_line(line: str, rel_path: Path, idx: int):
236
+ violations = []
237
+ for label, pattern in SECRET_PATTERNS:
238
+ for match in pattern.findall(line):
239
+ match_str = match[0] if isinstance(match, tuple) else match
240
+ if not is_placeholder(match_str):
241
+ violations.append(
242
+ f"Potential unmasked secret ({label}) detected in "
243
+ f"{rel_path}:{idx}"
244
+ )
245
+ return violations
246
+
247
+
248
+ def _secret_scan_violations(file_path: Path, repo_path: Path):
249
+ rel_path = file_path.relative_to(repo_path)
250
+ if file_path.stat().st_size > MAX_SCAN_BYTES:
251
+ return [
252
+ f"Source file exceeds security scan boundary: '{rel_path}'",
253
+ ]
254
+
255
+ # Decode strictly. Silently discarding invalid bytes can splice a
256
+ # credential around the discarded byte and make a fail-open scan.
257
+ content = file_path.read_text(encoding="utf-8")
258
+ violations = []
259
+ for idx, line in enumerate(content.splitlines(), 1):
260
+ violations.extend(_secret_violations_in_line(line, rel_path, idx))
261
+ return violations
262
+
263
+
264
+ def _scan_one_file(file_path: Path, repo_path: Path):
265
+ violations = []
266
+
267
+ # 1. Check root level naming constraints
268
+ if file_path.parent == repo_path:
269
+ violations.extend(_root_naming_violations(file_path))
270
+
271
+ # 2. Check for secrets
272
+ if not _should_secret_scan(file_path):
273
+ return violations
274
+
275
+ try:
276
+ violations.extend(_secret_scan_violations(file_path, repo_path))
277
+ except (OSError, UnicodeError):
278
+ violations.append(
279
+ f"Source file could not be inspected: "
280
+ f"'{file_path.relative_to(repo_path)}'"
281
+ )
282
+ return violations
283
+
284
+
285
+ def scan_repository(repo_path: Path):
286
+ violations = []
287
+
288
+ for file_path in get_repo_files(repo_path):
289
+ if not file_path.is_file():
290
+ continue
291
+ if file_path.is_symlink():
292
+ # Git tracks the link target text, not the target contents. Never
293
+ # follow a repository symlink into machine-local material.
294
+ continue
295
+ violations.extend(_scan_one_file(file_path, repo_path))
296
+
297
+ return violations
298
+
299
+
300
+ def main():
301
+ repo_path = Path.cwd()
302
+
303
+ print("šŸ”’ Running Security and Garbage Sanitizer...")
304
+ violations = scan_repository(repo_path)
305
+
306
+ if violations:
307
+ print("\nāŒ SECURITY AND GARBAGE VALIDATION FAILED!")
308
+ print("Please correct the following issues before committing:")
309
+ for idx, violation in enumerate(violations, 1):
310
+ print(f"\n[{idx}] {violation}")
311
+ sys.exit(1)
312
+
313
+ print("āœ… All checks passed! No root garbage or unmasked secrets detected.")
314
+ sys.exit(0)
315
+
316
+
317
+ if __name__ == "__main__":
318
+ main()
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,34 @@
1
+ """Shared test fixtures. No live network: every HTTP call is mocked."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Callable
7
+ from pathlib import Path
8
+
9
+ import httpx
10
+ import pytest
11
+
12
+ FIXTURES = Path(__file__).parent / "fixtures"
13
+
14
+
15
+ def load_fixture(name: str) -> dict:
16
+ """Read one recorded or synthetic fixture JSON file by name."""
17
+ return json.loads((FIXTURES / name).read_text())
18
+
19
+
20
+ def mock_client(
21
+ base_url: str, handler: Callable[[httpx.Request], httpx.Response]
22
+ ) -> httpx.AsyncClient:
23
+ """An ``httpx.AsyncClient`` whose transport is a synchronous fixture handler."""
24
+ return httpx.AsyncClient(base_url=base_url, transport=httpx.MockTransport(handler))
25
+
26
+
27
+ def json_response(payload: dict) -> httpx.Response:
28
+ """A 200 JSON response carrying ``payload``."""
29
+ return httpx.Response(200, json=payload)
30
+
31
+
32
+ @pytest.fixture
33
+ def fixture() -> Callable[[str], dict]:
34
+ return load_fixture