aiset-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aiset/__init__.py +3 -0
- aiset/archmap.py +916 -0
- aiset/brain.py +490 -0
- aiset/brief_index.py +78 -0
- aiset/core.py +1003 -0
- aiset/init_team.py +439 -0
- aiset/registry.py +122 -0
- aiset/research.py +1340 -0
- aiset/roster.json +95 -0
- aiset/server.py +275 -0
- aiset/settings.py +154 -0
- aiset/static/aiset-icon.png +0 -0
- aiset/static/apple-touch-icon.png +0 -0
- aiset/static/architecture.html +405 -0
- aiset/static/brain.html +787 -0
- aiset/static/favicon-16.png +0 -0
- aiset/static/favicon-32.png +0 -0
- aiset/static/favicon-48.png +0 -0
- aiset/static/favicon.ico +0 -0
- aiset/static/icon-192.png +0 -0
- aiset/static/icon-512.png +0 -0
- aiset/static/index.html +1298 -0
- aiset/static/settings.html +384 -0
- aiset/static/site.webmanifest +12 -0
- aiset/sync_names.py +88 -0
- aiset/teammap.py +897 -0
- aiset/teammap_ts.js +135 -0
- aiset/templates/engineer.md +75 -0
- aiset/templates/integration-checker.md +31 -0
- aiset/templates/pm.md +81 -0
- aiset/templates/tester.md +26 -0
- aiset/workspace.py +108 -0
- aiset_cli-0.1.0.dist-info/METADATA +301 -0
- aiset_cli-0.1.0.dist-info/RECORD +38 -0
- aiset_cli-0.1.0.dist-info/WHEEL +5 -0
- aiset_cli-0.1.0.dist-info/entry_points.txt +4 -0
- aiset_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- aiset_cli-0.1.0.dist-info/top_level.txt +1 -0
aiset/archmap.py
ADDED
|
@@ -0,0 +1,916 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""archmap: what each repository (or monorepo service) is built with and what it connects to.
|
|
3
|
+
|
|
4
|
+
Static and evidence-based: every fact carries the file and line it came from.
|
|
5
|
+
|
|
6
|
+
python archmap.py print the map as text (same as `teammap arch`)
|
|
7
|
+
python archmap.py <repo> one repository in detail
|
|
8
|
+
python archmap.py --json the whole map as JSON (what /api/arch serves)
|
|
9
|
+
python archmap.py --mermaid the system map as a Mermaid diagram
|
|
10
|
+
python archmap.py <name> a repo/service in detail, or a component (rabbitmq, mongodb…): who uses it
|
|
11
|
+
python archmap.py impact <svc> who calls it, and who calls those (what a change to it can break)
|
|
12
|
+
python archmap.py path <a> <b> how a call gets from a to b (the HTTP hops, with evidence)
|
|
13
|
+
Add --json for machine-readable output. Results are cached in <team>/graph/index/arch.json until a file changes.
|
|
14
|
+
|
|
15
|
+
Units: the git repositories directly under the workspace or, when the workspace is itself one git
|
|
16
|
+
repository (a monorepo), every top-level folder with a build file (pom.xml, package.json, …).
|
|
17
|
+
|
|
18
|
+
What it reads: dependency files (requirements, pyproject, package.json, pom.xml, Gradle), code
|
|
19
|
+
(.py .java .kt .ts .js …), Spring config (application*.yml / .properties, bootstrap.yml),
|
|
20
|
+
Dockerfiles, compose files, CI files and .env.example files. It never opens .env files or anything
|
|
21
|
+
under a secrets/ folder, keeps only the *names* of environment variables, and masks credentials
|
|
22
|
+
in the evidence lines it shows.
|
|
23
|
+
|
|
24
|
+
Confidence of a component: "declared" (only a dependency), "used" (the code uses it),
|
|
25
|
+
"wired" (also configured: an env var, a property, a compose service, a connection URL).
|
|
26
|
+
Links between services: "exact" (a Feign client, a gateway route or a URL naming the service),
|
|
27
|
+
"inferred" (a client class or env var whose name looks like the service's).
|
|
28
|
+
"""
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import re
|
|
32
|
+
import sys
|
|
33
|
+
import time
|
|
34
|
+
|
|
35
|
+
if __name__ == "__main__" and not __package__: # run as a file: python src/aiset/archmap.py
|
|
36
|
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
|
37
|
+
__package__ = "aiset"
|
|
38
|
+
from .workspace import CODE_ROOT, GRAPHS, norm # noqa: E402
|
|
39
|
+
|
|
40
|
+
HERE = os.path.dirname(os.path.realpath(__file__))
|
|
41
|
+
|
|
42
|
+
SKIP_DIRS = {"node_modules", ".git", "venv", ".venv", "env", "fastapi-env", "__pycache__", "dist", "build", ".next",
|
|
43
|
+
"logs", "output", "dumps", "site-packages", ".mypy_cache", ".pytest_cache", ".ruff_cache", "coverage",
|
|
44
|
+
"secrets", ".idea", ".vscode", "vendor", "target", ".gradle", "bin", "obj", "out"}
|
|
45
|
+
UNIT_FILES = ("pom.xml", "build.gradle", "build.gradle.kts", "package.json", "requirements.txt", "pyproject.toml",
|
|
46
|
+
"setup.py", "go.mod", "Cargo.toml", "composer.json", "Gemfile", "Dockerfile")
|
|
47
|
+
CODE_EXT = {".py": "Python", ".ts": "TypeScript", ".tsx": "TypeScript", ".js": "JavaScript", ".jsx": "JavaScript",
|
|
48
|
+
".mjs": "JavaScript", ".java": "Java", ".kt": "Kotlin", ".go": "Go", ".rb": "Ruby", ".php": "PHP",
|
|
49
|
+
".cs": "C#", ".rs": "Rust", ".scala": "Scala", ".sql": "SQL", ".sh": "Shell"}
|
|
50
|
+
MAX_FILE = 512 * 1024
|
|
51
|
+
MAX_EVIDENCE = 6
|
|
52
|
+
|
|
53
|
+
# ---- detectors ---------------------------------------------------------------------------------
|
|
54
|
+
# py / npm / jvm: dependency names (jvm = Maven/Gradle artifactId); code: regexes over code lines;
|
|
55
|
+
# prop: regexes over Spring-style "key=value" config entries; env: regexes over env var names;
|
|
56
|
+
# image: regexes over compose/k8s image names. kind decides the column on the map.
|
|
57
|
+
DETECTORS = [
|
|
58
|
+
dict(id="postgres", label="PostgreSQL", kind="database",
|
|
59
|
+
py=["psycopg2", "psycopg2-binary", "psycopg", "asyncpg", "pg8000"], npm=["pg", "postgres", "pg-promise"], jvm=["postgresql", "r2dbc-postgresql"],
|
|
60
|
+
code=[r"postgres(?:ql)?(?:\+\w+)?://", r"\basyncpg\.(?:connect|create_pool)", r"\bpsycopg2?\.connect", r"jdbc:postgresql:"],
|
|
61
|
+
prop=[r"jdbc:postgresql:", r"database\s*=\s*postgres", r"PostgreSQL\w*Dialect"], env=[r"^(POSTGRES|PG)_?[A-Z_]*$", r"DATABASE_URL$"], image=[r"postgres|postgis|timescale"]),
|
|
62
|
+
dict(id="mysql", label="MySQL", kind="database", py=["pymysql", "mysqlclient", "aiomysql"], npm=["mysql", "mysql2"],
|
|
63
|
+
jvm=["mysql-connector-java", "mysql-connector-j", "mariadb-java-client"],
|
|
64
|
+
code=[r"mysql(?:\+\w+)?://", r"jdbc:mysql:"], prop=[r"jdbc:mysql:", r"database\s*=\s*mysql", r"MySQL\w*Dialect", r"com\.mysql\."],
|
|
65
|
+
env=[r"^MYSQL_"], image=[r"mysql|mariadb"]),
|
|
66
|
+
dict(id="mongodb", label="MongoDB", kind="database", py=["pymongo", "motor", "mongoengine", "flask-pymongo", "mongoalchemy", "flask-mongoalchemy"],
|
|
67
|
+
npm=["mongodb", "mongoose", "@nestjs/mongoose", "winston-mongodb"], jvm=["spring-boot-starter-data-mongodb", "mongodb-driver", "mongodb-driver-sync", "mongo-java-driver"],
|
|
68
|
+
code=[r"mongodb(?:\+srv)?://", r"\bMongo(?:Template|Repository|Client)\b", r"\bmongoose\.(?:connect|model|Schema)\b"],
|
|
69
|
+
prop=[r"^spring\.data\.mongodb\.", r"mongodb://"], env=[r"^MONGO"], image=[r"mongo"]),
|
|
70
|
+
dict(id="cassandra", label="Cassandra", kind="database", py=["cassandra-driver"], npm=["cassandra-driver"],
|
|
71
|
+
jvm=["spring-boot-starter-data-cassandra", "cassandra-driver-core", "java-driver-core"],
|
|
72
|
+
code=[r"^\s*(?:from|import)\s+cassandra\b", r"\b(?:CassandraTemplate|CqlSession)\b"], prop=[r"^spring\.(?:data\.)?cassandra\."],
|
|
73
|
+
env=[r"^CASSANDRA"], image=[r"cassandra|scylla"]),
|
|
74
|
+
dict(id="sqlite", label="SQLite", kind="database", py=["aiosqlite"], npm=["sqlite3", "better-sqlite3"], jvm=["sqlite-jdbc"],
|
|
75
|
+
code=[r"sqlite(?:\+\w+)?:///", r"\bsqlite3\.connect"], prop=[r"jdbc:sqlite:"], env=[], image=[]),
|
|
76
|
+
dict(id="redis", label="Redis", kind="cache", py=["redis", "aioredis", "redis-py-cluster", "flask-redis"], npm=["redis", "ioredis"],
|
|
77
|
+
jvm=["spring-boot-starter-data-redis", "spring-data-redis", "jedis", "lettuce-core", "redisson"],
|
|
78
|
+
code=[r"redis(?:s)?://", r"^\s*(?:import|from)\s+(?:redis|aioredis)\b", r"\bRedis\(", r"\b(?:RedisTemplate|StringRedisTemplate|Jedis(?:Pool)?)\b"],
|
|
79
|
+
prop=[r"^spring\.(?:data\.)?redis\."], env=[r"^REDIS"], image=[r"redis|valkey"]),
|
|
80
|
+
dict(id="celery", label="Celery", kind="worker", py=["celery"], npm=[], jvm=[], only=(".py",),
|
|
81
|
+
code=[r"\bCelery\(", r"@(?:shared_task|\w+\.task)\b", r"\.delay\(|\.apply_async\("], prop=[], env=[r"^CELERY"], image=[]),
|
|
82
|
+
dict(id="rabbitmq", label="RabbitMQ", kind="broker", py=["pika", "aio-pika", "kombu", "amqp", "aiormq"],
|
|
83
|
+
npm=["amqplib", "amqp-connection-manager", "amqplib-as-promised", "@golevelup/nestjs-rabbitmq"], jvm=["spring-rabbit", "spring-boot-starter-amqp", "amqp-client"],
|
|
84
|
+
code=[r"amqps?://", r"^\s*(?:import|from)\s+(?:pika|aio_pika|kombu)\b", r"\bbasic_(?:publish|consume)\b", r"\b(?:RabbitTemplate|AmqpTemplate)\b|@RabbitListener",
|
|
85
|
+
r"\bamqp\.connect\(|require\(['\"]amqplib"],
|
|
86
|
+
prop=[r"^spring\.rabbitmq\."], env=[r"^RABBIT", r"^AMQP"], image=[r"rabbitmq"]),
|
|
87
|
+
dict(id="kafka", label="Kafka", kind="broker", py=["kafka-python", "aiokafka", "confluent-kafka"], npm=["kafkajs", "node-rdkafka"],
|
|
88
|
+
jvm=["spring-kafka", "kafka-clients", "kafka-streams"],
|
|
89
|
+
code=[r"^\s*(?:import|from)\s+(?:kafka|aiokafka|confluent_kafka)\b", r"\bKafka(?:Producer|Consumer|Template)\b|@KafkaListener", r"new Kafka\("],
|
|
90
|
+
prop=[r"^spring\.kafka\.", r"bootstrap[.-]servers"], env=[r"^KAFKA"], image=[r"kafka|redpanda"]),
|
|
91
|
+
dict(id="mqtt", label="MQTT", kind="broker", py=["paho-mqtt", "asyncio-mqtt", "gmqtt"], npm=["mqtt", "async-mqtt"],
|
|
92
|
+
jvm=["org.eclipse.paho.client.mqttv3", "spring-integration-mqtt", "hivemq-mqtt-client"],
|
|
93
|
+
code=[r"mqtts?://|tcp://[\w.-]+:1883", r"\bMqtt(?:Client|AsyncClient)\b", r"\bmqtt\.connect\("], prop=[r"^\w*\.?mqtt\.|mqtt"],
|
|
94
|
+
env=[r"^MQTT"], image=[r"mosquitto|emqx|hivemq|vernemq"]),
|
|
95
|
+
dict(id="keycloak", label="Keycloak", kind="auth", py=["python-keycloak"], npm=["keycloak-js", "@react-keycloak/web", "keycloak-connect"],
|
|
96
|
+
jvm=["keycloak-spring-boot-starter", "keycloak-spring-security-adapter", "keycloak-admin-client"],
|
|
97
|
+
code=[r"\bKeycloak(?:OpenID)?\(", r"/realms/"], prop=[r"^keycloak\.", r"/realms/"], env=[r"^KEYCLOAK", r"(?:JWT_ISSUER|OIDC|AUTH)_[A-Z_]*URL$"], image=[r"keycloak"]),
|
|
98
|
+
dict(id="jwt", label="OAuth2 / JWT", kind="auth", py=["pyjwt", "python-jose", "authlib"], npm=["jsonwebtoken", "jose", "jwt-decode", "passport-jwt"],
|
|
99
|
+
jvm=["spring-cloud-starter-oauth2", "spring-security-oauth2", "spring-boot-starter-oauth2-resource-server", "spring-security-jwt", "jjwt", "jjwt-api", "java-jwt", "nimbus-jose-jwt"],
|
|
100
|
+
code=[r"^\s*(?:import|from)\s+(?:jwt|jose)\b", r"\bjwt\.(?:decode|verify|sign)\(", r"@EnableResourceServer|@EnableAuthorizationServer|\bJwtDecoder\b"],
|
|
101
|
+
prop=[r"^security\.oauth2\.", r"^spring\.security\.oauth2\.", r"issuer-uri|jwk-set-uri"], env=[r"^JWT_"], image=[]),
|
|
102
|
+
dict(id="s3", label="Amazon S3", kind="storage", py=["boto3", "aioboto3"], npm=["@aws-sdk/client-s3", "aws-sdk"], jvm=["aws-java-sdk", "aws-java-sdk-s3", "s3"],
|
|
103
|
+
code=[r"boto3\.(?:client|resource)\(['\"]s3", r"\bAmazonS3(?:Client)?\b|\bS3Client\b"], prop=[r"^(?:cloud\.)?aws\.s3|amazonaws\.com"], env=[r"^S3_", r"^AWS_S3"], image=[]),
|
|
104
|
+
dict(id="gcs", label="Google Cloud Storage", kind="storage", py=["google-cloud-storage"], npm=["@google-cloud/storage"], jvm=["google-cloud-storage"],
|
|
105
|
+
code=[r"storage\.Client\(", r"\bStorageOptions\b"], prop=[], env=[r"^GCS_", r"^GOOGLE_CLOUD_STORAGE"], image=[]),
|
|
106
|
+
dict(id="minio", label="MinIO", kind="storage", py=["minio"], npm=["minio"], jvm=["minio"], code=[r"\bMinio(?:Client)?\(|new Minio\.Client|MinioClient\.builder"],
|
|
107
|
+
prop=[r"^minio\."], env=[r"^MINIO"], image=[r"minio"]),
|
|
108
|
+
dict(id="bigquery", label="BigQuery", kind="database", py=["google-cloud-bigquery"], npm=["@google-cloud/bigquery"], jvm=["google-cloud-bigquery"],
|
|
109
|
+
code=[r"bigquery\.Client\("], prop=[], env=[r"^BIGQUERY"], image=[]),
|
|
110
|
+
dict(id="elasticsearch", label="Elasticsearch", kind="search", py=["elasticsearch", "opensearch-py"], npm=["@elastic/elasticsearch"],
|
|
111
|
+
jvm=["spring-boot-starter-data-elasticsearch", "elasticsearch-java", "elasticsearch-rest-high-level-client"],
|
|
112
|
+
code=[r"\bElasticsearch(?:Client)?\(|RestHighLevelClient"], prop=[r"^spring\.elasticsearch\.|^spring\.data\.elasticsearch\."], env=[r"^(ELASTIC|OPENSEARCH)"], image=[r"elasticsearch|opensearch"]),
|
|
113
|
+
dict(id="sentry", label="Sentry", kind="observability", py=["sentry-sdk"], npm=["@sentry/react", "@sentry/node", "@sentry/browser"], jvm=["sentry-spring-boot-starter", "sentry"],
|
|
114
|
+
code=[r"sentry_sdk\.init\(|Sentry\.init\("], prop=[r"^sentry\."], env=[r"^SENTRY"], image=[]),
|
|
115
|
+
dict(id="prometheus", label="Prometheus metrics", kind="observability", py=["prometheus-client", "prometheus-flask-exporter", "prometheus-fastapi-instrumentator"],
|
|
116
|
+
npm=["prom-client", "express-prom-bundle"], jvm=["micrometer-registry-prometheus", "simpleclient", "simpleclient_spring_boot", "simpleclient_hotspot", "simpleclient_servlet"],
|
|
117
|
+
code=[r"PrometheusMetrics\(|@EnablePrometheusEndpoint|collectDefaultMetrics\("], prop=[r"prometheus"], env=[], image=[r"prometheus"]),
|
|
118
|
+
dict(id="airflow", label="Airflow", kind="worker", py=["apache-airflow"], npm=[], jvm=[], only=(".py",),
|
|
119
|
+
code=[r"^\s*(?:from|import)\s+airflow\b", r"\bDAG\("], prop=[], env=[r"^AIRFLOW"], image=[r"airflow"]),
|
|
120
|
+
dict(id="scheduler", label="Scheduled jobs", kind="worker", py=["apscheduler", "schedule", "rq-scheduler", "flask-apscheduler", "fastapi-scheduler"],
|
|
121
|
+
npm=["node-cron", "cron", "agenda", "@nestjs/schedule", "node-schedule"], jvm=["quartz", "spring-boot-starter-quartz", "shedlock-spring"],
|
|
122
|
+
code=[r"\bBackgroundScheduler\(|\bAsyncIOScheduler\(|cron\.schedule\(", r"@Scheduled\b|@EnableScheduling", r"new Agenda\(|agenda\.(?:every|define)\(", r"@Cron\("],
|
|
123
|
+
prop=[], env=[], image=[]),
|
|
124
|
+
dict(id="smtp", label="Email (SMTP)", kind="external", py=["aiosmtplib", "fastapi-mail", "flask-mail"], npm=["nodemailer"], jvm=["spring-boot-starter-mail", "javax.mail", "jakarta.mail"],
|
|
125
|
+
code=[r"\bsmtplib\.SMTP", r"\bJavaMailSender\b", r"createTransport\("], prop=[r"^spring\.mail\."], env=[r"^(SMTP|MAIL)_"], image=[]),
|
|
126
|
+
dict(id="websocket", label="WebSocket", kind="external", py=["websockets", "python-socketio", "flask-socketio"], npm=["socket.io-client", "socket.io", "ws", "@nestjs/websockets"],
|
|
127
|
+
jvm=["spring-boot-starter-websocket"], code=[r"@\w+\.websocket\(|new WebSocket\(|io\(['\"]", r"@ServerEndpoint|@MessageMapping|SimpMessagingTemplate|@WebSocketGateway"],
|
|
128
|
+
prop=[], env=[], image=[]),
|
|
129
|
+
]
|
|
130
|
+
FRAMEWORKS_PY = [("fastapi", "FastAPI", "api"), ("django", "Django", "api"), ("flask", "Flask", "api"), ("cherrypy", "CherryPy", "api"),
|
|
131
|
+
("starlette", "Starlette", "api"), ("apache-airflow", "Airflow", "pipeline"), ("streamlit", "Streamlit", "frontend")]
|
|
132
|
+
FRAMEWORKS_NPM = [("next", "Next.js", "frontend"), ("react", "React", "frontend"), ("vue", "Vue", "frontend"),
|
|
133
|
+
("@angular/core", "Angular", "frontend"), ("svelte", "Svelte", "frontend"),
|
|
134
|
+
("@nestjs/core", "NestJS", "api"), ("express", "Express", "api"), ("fastify", "Fastify", "api"), ("koa", "Koa", "api")]
|
|
135
|
+
# order matters: the most specific role wins (a gateway is also a Spring Boot web app)
|
|
136
|
+
FRAMEWORKS_JVM = [("spring-cloud-starter-zuul", "Zuul gateway", "gateway"), ("spring-cloud-starter-netflix-zuul", "Zuul gateway", "gateway"),
|
|
137
|
+
("spring-cloud-starter-gateway", "Spring Cloud Gateway", "gateway"),
|
|
138
|
+
("spring-cloud-starter-eureka-server", "Eureka server", "registry"), ("spring-cloud-starter-netflix-eureka-server", "Eureka server", "registry"),
|
|
139
|
+
("spring-cloud-config-server", "Config server", "registry"),
|
|
140
|
+
("spring-boot-starter-web", "Spring Boot", "api"), ("spring-boot-starter-webflux", "Spring Boot (WebFlux)", "api"),
|
|
141
|
+
("ktor-server-core", "Ktor", "api"), ("quarkus-core", "Quarkus", "api"), ("micronaut-http-server", "Micronaut", "api"),
|
|
142
|
+
("spring-boot-starter", "Spring Boot", "service")]
|
|
143
|
+
CODE_ROLE = [(r"@EnableZuulProxy", "Zuul gateway", "gateway"), (r"@EnableEurekaServer", "Eureka server", "registry"),
|
|
144
|
+
(r"@EnableConfigServer", "Config server", "registry")]
|
|
145
|
+
# libraries worth naming on a repo card
|
|
146
|
+
NOTABLE = {"sqlalchemy": "SQLAlchemy", "sqlmodel": "SQLModel", "alembic": "Alembic", "pydantic": "Pydantic",
|
|
147
|
+
"pandas": "pandas", "numpy": "NumPy", "ortools": "OR-Tools", "pulp": "PuLP", "scikit-learn": "scikit-learn",
|
|
148
|
+
"torch": "PyTorch", "openpyxl": "openpyxl", "uvicorn": "Uvicorn", "gunicorn": "Gunicorn", "httpx": "httpx",
|
|
149
|
+
"requests": "requests", "aiohttp": "aiohttp", "vite": "Vite", "axios": "axios", "@mui/material": "MUI",
|
|
150
|
+
"antd": "Ant Design", "tailwindcss": "Tailwind", "@tanstack/react-query": "React Query", "redux": "Redux",
|
|
151
|
+
"@reduxjs/toolkit": "Redux Toolkit", "zustand": "Zustand", "i18next": "i18next", "leaflet": "Leaflet",
|
|
152
|
+
"maplibre-gl": "MapLibre", "@deck.gl/core": "deck.gl", "typescript": "TypeScript", "prisma": "Prisma", "typeorm": "TypeORM",
|
|
153
|
+
"mongoose": "Mongoose", "inversify": "InversifyJS", "swagger-ui-express": "Swagger", "@nestjs/swagger": "Swagger",
|
|
154
|
+
"spring-boot-starter-data-jpa": "Spring Data JPA", "hibernate-core": "Hibernate", "spring-cloud-starter-feign": "Feign",
|
|
155
|
+
"spring-cloud-starter-openfeign": "OpenFeign", "spring-cloud-starter-eureka": "Eureka client",
|
|
156
|
+
"spring-cloud-starter-netflix-eureka-client": "Eureka client", "spring-cloud-starter-hystrix": "Hystrix",
|
|
157
|
+
"resilience4j-spring-boot2": "Resilience4j", "springfox-swagger2": "Swagger", "springdoc-openapi-ui": "OpenAPI",
|
|
158
|
+
"lombok": "Lombok", "spring-boot-starter-actuator": "Actuator", "spring-cloud-starter-config": "Config client",
|
|
159
|
+
"eureka-js-client": "Eureka client", "flask-marshmallow": "marshmallow", "shapely": "Shapely", "jts": "JTS"}
|
|
160
|
+
HTTP_LIBS_PY, HTTP_LIBS_NPM = {"requests", "httpx", "aiohttp"}, {"axios", "ky", "got", "node-fetch", "request", "request-promise"}
|
|
161
|
+
HTTP_LIBS_JVM = {"spring-cloud-starter-feign", "spring-cloud-starter-openfeign", "okhttp", "httpclient", "spring-boot-starter-webflux", "retrofit"}
|
|
162
|
+
EUREKA_CLIENT = {"spring-cloud-starter-eureka", "spring-cloud-starter-netflix-eureka-client", "eureka-js-client", "py-eureka-client"}
|
|
163
|
+
INFRA_ENV = re.compile(r"DATABASE|DB_|POSTGRES|^PG|REDIS|CELERY|RABBIT|AMQP|KAFKA|BROKER|SENTRY|SMTP|MAIL|S3|GCS|MINIO|MONGO|MYSQL|ELASTIC|SWAGGER|REDOC|DOCS_|STATIC|CALLBACK|REDIRECT|LOG|CORS|FRONTEND|EUREKA|MQTT|CASSANDRA")
|
|
164
|
+
URL_ENV = re.compile(r"(?:_URL|_URI|_HOST|_BASE_URL|_ENDPOINT|_API|_ADDR)$")
|
|
165
|
+
# client classes for infrastructure (RabbitMQClient, MinioClient…) are components, not services
|
|
166
|
+
INFRA_WORDS = {"rabbit", "rabbitmq", "mq", "mqclient", "amqp", "pika", "minio", "redis", "kafka", "s3", "gcs", "storage", "db",
|
|
167
|
+
"database", "postgres", "sql", "mongo", "keycloak", "sentry", "celery", "smtp", "mail", "email", "http", "base",
|
|
168
|
+
"abstract", "generic", "mock", "fake", "test", "websocket", "socket", "auth", "jwt", "cache", "logger", "airflow",
|
|
169
|
+
"mqtt", "cassandra", "eureka", "rest", "web", "feign", "null", "noop", "dummy", "default", "impl", "stub", "local"}
|
|
170
|
+
GENERIC = {"service", "services", "backend", "frontend", "web", "api", "app", "new", "server", "client", "the", "and", "base", "url", "uri",
|
|
171
|
+
"host", "endpoint", "addr", "microservice", "manager", "management"}
|
|
172
|
+
NOT_HOSTS = {"localhost", "host", "example", "www", "127", "0", "your", "server", "api", "app", "schemas", "json", "xml"}
|
|
173
|
+
|
|
174
|
+
SECRET_RE = re.compile(r"((?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret)\w*[\"']?\s*[:=]\s*)(['\"]?)[^\s'\",)]+", re.I)
|
|
175
|
+
CRED_URL_RE = re.compile(r"(\w+://)[^/\s:@'\"]+:[^@\s'\"]+@")
|
|
176
|
+
COMMENT_RE = re.compile(r"^\s*(?://|#|\*|/\*|<!--)")
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def mask(line):
|
|
180
|
+
line = CRED_URL_RE.sub(r"\1***:***@", line)
|
|
181
|
+
return SECRET_RE.sub(r"\1\2***", line).strip()[:180]
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _read(path):
|
|
185
|
+
try:
|
|
186
|
+
if os.path.getsize(path) > MAX_FILE:
|
|
187
|
+
return ""
|
|
188
|
+
with open(path, encoding="utf-8", errors="replace") as f:
|
|
189
|
+
return f.read()
|
|
190
|
+
except OSError:
|
|
191
|
+
return ""
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _tokens(name):
|
|
195
|
+
"""BackendServiceClient / BACKEND_URL / backend -> {'backend', …}"""
|
|
196
|
+
s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
|
|
197
|
+
s = re.sub(r"([A-Za-z])(V\d+)\b", r"\1_\2", s)
|
|
198
|
+
return {t for t in re.split(r"[^a-z0-9]+", s.lower()) if t}
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _key(name):
|
|
202
|
+
"""Service names compare without case, dashes or underscores: Traffic_Telemetry == traffic-telemetry."""
|
|
203
|
+
return re.sub(r"[^a-z0-9]", "", name.lower())
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def is_monorepo():
|
|
207
|
+
try:
|
|
208
|
+
names = os.listdir(CODE_ROOT)
|
|
209
|
+
except OSError:
|
|
210
|
+
return False
|
|
211
|
+
return os.path.isdir(os.path.join(CODE_ROOT, ".git")) and not any(
|
|
212
|
+
os.path.isdir(os.path.join(CODE_ROOT, n, ".git")) for n in names if not n.startswith("."))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def repo_dirs():
|
|
216
|
+
"""The units on the map: git repos under the code root, or a monorepo's service folders."""
|
|
217
|
+
try:
|
|
218
|
+
names = sorted(os.listdir(CODE_ROOT))
|
|
219
|
+
except OSError:
|
|
220
|
+
return []
|
|
221
|
+
if not is_monorepo():
|
|
222
|
+
return [n for n in names if os.path.isdir(os.path.join(CODE_ROOT, n, ".git"))]
|
|
223
|
+
return [n for n in names if not n.startswith(".") and n not in SKIP_DIRS and n != "docs"
|
|
224
|
+
and os.path.isdir(os.path.join(CODE_ROOT, n)) and any(os.path.exists(os.path.join(CODE_ROOT, n, f)) for f in UNIT_FILES)]
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _walk(repo_path):
|
|
228
|
+
for root, dirs, files in os.walk(repo_path):
|
|
229
|
+
dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".") or d == ".github"]
|
|
230
|
+
for f in files:
|
|
231
|
+
yield os.path.join(root, f), f
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _deps(repo_path, files):
|
|
235
|
+
"""Python, npm and JVM dependency names, each with where it was declared."""
|
|
236
|
+
py, npm, jvm = {}, {}, {}
|
|
237
|
+
for path, f in files:
|
|
238
|
+
rel = os.path.relpath(path, repo_path)
|
|
239
|
+
if re.match(r"requirements.*\.txt$", f):
|
|
240
|
+
for i, line in enumerate(_read(path).splitlines(), 1):
|
|
241
|
+
m = re.match(r"\s*([A-Za-z0-9_.\-\[\]]+)", line)
|
|
242
|
+
if m and not line.lstrip().startswith(("#", "-")):
|
|
243
|
+
py.setdefault(re.sub(r"\[.*\]", "", m.group(1)).lower().replace("_", "-"), (rel, i, line.strip()))
|
|
244
|
+
elif f in ("pyproject.toml", "environment.yml", "Pipfile"):
|
|
245
|
+
for i, line in enumerate(_read(path).splitlines(), 1):
|
|
246
|
+
for m in re.finditer(r"(?:^|[\"'\s-])([A-Za-z][A-Za-z0-9_.\-]+)\s*(?:[<>=~^!]=?|\s*=\s*[\"'{]|[\"'])", line):
|
|
247
|
+
name = m.group(1).lower().replace("_", "-")
|
|
248
|
+
if len(name) > 1 and name not in ("python", "name", "version", "description", "readme", "license", "requires"):
|
|
249
|
+
py.setdefault(name, (rel, i, line.strip()))
|
|
250
|
+
elif f == "package.json":
|
|
251
|
+
text = _read(path)
|
|
252
|
+
try:
|
|
253
|
+
p = json.loads(text)
|
|
254
|
+
except ValueError:
|
|
255
|
+
continue
|
|
256
|
+
lines = text.splitlines()
|
|
257
|
+
for sect in ("dependencies", "devDependencies", "peerDependencies"):
|
|
258
|
+
for name in p.get(sect, {}) or {}:
|
|
259
|
+
ln = next((i for i, l in enumerate(lines, 1) if f'"{name}"' in l), 0)
|
|
260
|
+
npm.setdefault(name, (rel, ln, f'"{name}": "{p[sect][name]}"'))
|
|
261
|
+
elif f == "pom.xml":
|
|
262
|
+
lines = _read(path).splitlines()
|
|
263
|
+
for i, line in enumerate(lines, 1):
|
|
264
|
+
m = re.search(r"<artifactId>\s*([\w.\-]+)\s*</artifactId>", line)
|
|
265
|
+
if m and not any("<plugin>" in l for l in lines[max(0, i - 3):i]):
|
|
266
|
+
jvm.setdefault(m.group(1), (rel, i, line.strip()))
|
|
267
|
+
elif f in ("build.gradle", "build.gradle.kts"):
|
|
268
|
+
for i, line in enumerate(_read(path).splitlines(), 1):
|
|
269
|
+
for m in re.finditer(r"['\"]([\w.\-]+):([\w.\-]+)(?::[^'\"]*)?['\"]", line):
|
|
270
|
+
jvm.setdefault(m.group(2), (rel, i, line.strip()))
|
|
271
|
+
return py, npm, jvm
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _spring_props(text, yml):
|
|
275
|
+
"""(key, value, line) from a Spring application.yml / .properties; YAML is flattened by indentation."""
|
|
276
|
+
out, stack = [], []
|
|
277
|
+
for i, raw in enumerate(text.splitlines(), 1):
|
|
278
|
+
if not raw.strip() or raw.lstrip().startswith("#"):
|
|
279
|
+
continue
|
|
280
|
+
if not yml:
|
|
281
|
+
m = re.match(r"\s*([\w.\-\[\]]+)\s*[=:]\s*(.*)", raw)
|
|
282
|
+
if m:
|
|
283
|
+
out.append((m.group(1), m.group(2).strip(), i))
|
|
284
|
+
continue
|
|
285
|
+
m = re.match(r"^(\s*)(?:-\s+)?([\w.\-\"'\[\]/*]+)\s*:\s*(.*?)\s*$", raw)
|
|
286
|
+
if not m:
|
|
287
|
+
continue
|
|
288
|
+
indent = len(m.group(1)) + (2 if raw.lstrip().startswith("- ") else 0)
|
|
289
|
+
key, value = m.group(2).strip("\"'"), m.group(3)
|
|
290
|
+
while stack and stack[-1][0] >= indent:
|
|
291
|
+
stack.pop()
|
|
292
|
+
full = ".".join([k for _, k in stack] + [key])
|
|
293
|
+
if value and not value.startswith(("|", ">")):
|
|
294
|
+
out.append((full, value.split(" #")[0].strip().strip("\"'"), i))
|
|
295
|
+
else:
|
|
296
|
+
stack.append((indent, key))
|
|
297
|
+
return out
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
ENV_NAME_RES = [re.compile(r"os\.(?:environ\.get|getenv|environ\[)\(?\s*['\"]([A-Z][A-Z0-9_]+)"),
|
|
301
|
+
re.compile(r"process\.env\.([A-Z][A-Z0-9_]+)"), re.compile(r"process\.env\[['\"]([A-Z][A-Z0-9_]+)"),
|
|
302
|
+
re.compile(r"import\.meta\.env\.([A-Z][A-Z0-9_]+)"), re.compile(r"System\.getenv\(\s*\"([A-Z][A-Z0-9_]+)"),
|
|
303
|
+
re.compile(r"\$\{([A-Z][A-Z0-9_]+)(?::[^}]*)?\}"),
|
|
304
|
+
re.compile(r"^\s{2,}([A-Z][A-Z0-9_]{2,})\s*:\s*(?:str|int|bool|float|Optional|AnyUrl|HttpUrl|PostgresDsn|RedisDsn|SecretStr|list)")]
|
|
305
|
+
ENV_HINT = re.compile(r"environ|getenv|process\.env|import\.meta\.env|\$\{[A-Z]|^\s{2,}[A-Z][A-Z0-9_]{2,}\s*:")
|
|
306
|
+
CLIENT_RE = re.compile(r"^\s*(?:export\s+)?(?:public\s+|abstract\s+|final\s+)*(?:class|interface)\s+(\w*(?:Client|Api|API|Gateway|Integration|Proxy))\b")
|
|
307
|
+
FEIGN_RE = re.compile(r"@FeignClient\s*\((?P<args>[^)]*)\)")
|
|
308
|
+
URL_HOST_RE = re.compile(r"\b(?:https?|lb|ws)://([A-Za-z][\w-]*)(?::\d+)?(?=[/\"'\s,;)}]|$)")
|
|
309
|
+
# one pass over a line decides whether any detector, client or service-link rule could match it
|
|
310
|
+
ANY_CODE = re.compile("|".join(f"(?:{p})" for d in DETECTORS for p in d["code"])
|
|
311
|
+
+ r"|(?:class|interface)\s+\w*(?:Client|Api|API|Gateway|Integration|Proxy)\b|@FeignClient|://|@Enable(?:Zuul|Eureka|Config)", re.M)
|
|
312
|
+
SPRING_CONF = re.compile(r"^(?:application|bootstrap)(?:-[\w.-]+)?\.(ya?ml|properties)$")
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def scan_repo(name):
|
|
316
|
+
path = os.path.join(CODE_ROOT, name)
|
|
317
|
+
files = list(_walk(path))
|
|
318
|
+
py, npm, jvm = _deps(path, files)
|
|
319
|
+
langs, envs, code_hits, entry, deploy, images, clients, props = {}, {}, {}, [], [], {}, {}, []
|
|
320
|
+
links, aliases, role = [], {name}, None # links: (kind, target name, rel, line, text)
|
|
321
|
+
detectors = [(d, [re.compile(p, re.M) for p in d["code"]]) for d in DETECTORS]
|
|
322
|
+
has_dags = False
|
|
323
|
+
for full, f in files:
|
|
324
|
+
rel = os.path.relpath(full, path)
|
|
325
|
+
ext = os.path.splitext(f)[1].lower()
|
|
326
|
+
if f.startswith(".env") and not f.endswith((".example", ".sample")) or "/secrets/" in "/" + rel:
|
|
327
|
+
continue
|
|
328
|
+
if ext in CODE_EXT:
|
|
329
|
+
try:
|
|
330
|
+
size = os.path.getsize(full)
|
|
331
|
+
except OSError:
|
|
332
|
+
continue
|
|
333
|
+
lang = CODE_EXT[ext]
|
|
334
|
+
l = langs.setdefault(lang, {"lang": lang, "files": 0, "bytes": 0})
|
|
335
|
+
l["files"] += 1
|
|
336
|
+
l["bytes"] += size
|
|
337
|
+
if "/dags/" in "/" + rel and ext == ".py":
|
|
338
|
+
has_dags = True
|
|
339
|
+
if ext in (".sql", ".sh"):
|
|
340
|
+
continue
|
|
341
|
+
text = _read(full)
|
|
342
|
+
is_test = bool(re.search(r"(^|/)(tests?|__tests__|spec)/|\.(test|spec)\.[jt]sx?$|(^|/)test_[^/]+\.py$|Test\.(java|kt)$|/src/test/", rel))
|
|
343
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
344
|
+
if len(line) > 400:
|
|
345
|
+
continue
|
|
346
|
+
if ENV_HINT.search(line):
|
|
347
|
+
for rx in ENV_NAME_RES:
|
|
348
|
+
for m in rx.finditer(line):
|
|
349
|
+
envs.setdefault(m.group(1), (rel, i, line))
|
|
350
|
+
if is_test or not ANY_CODE.search(line) or COMMENT_RE.match(line):
|
|
351
|
+
continue
|
|
352
|
+
for d, rxs in detectors:
|
|
353
|
+
if d.get("only") and ext not in d["only"]: # e.g. jQuery's .delay() isn't Celery
|
|
354
|
+
continue
|
|
355
|
+
for rx in rxs:
|
|
356
|
+
if rx.search(line):
|
|
357
|
+
hits = code_hits.setdefault(d["id"], [])
|
|
358
|
+
if len(hits) < 40:
|
|
359
|
+
hits.append((rel, i, line))
|
|
360
|
+
break
|
|
361
|
+
m = CLIENT_RE.match(line)
|
|
362
|
+
if m and (" interface " not in f" {line} " or ext in (".java", ".kt")): # a TS interface is a type, not a client
|
|
363
|
+
clients.setdefault(m.group(1), (rel, i, line))
|
|
364
|
+
m = FEIGN_RE.search(line)
|
|
365
|
+
if m:
|
|
366
|
+
args = m.group("args")
|
|
367
|
+
url = re.search(r"url\s*=\s*\"([^\"]+)\"", args)
|
|
368
|
+
host = URL_HOST_RE.search(url.group(1)) if url else None
|
|
369
|
+
nm = re.search(r"(?:name|value|serviceId)\s*=\s*\"([^\"]+)\"", args) or re.match(r"\s*\"([^\"]+)\"", args)
|
|
370
|
+
target = (host.group(1) if host else None) or (nm.group(1) if nm else None)
|
|
371
|
+
if target:
|
|
372
|
+
links.append(("feign", target, rel, i, line))
|
|
373
|
+
continue
|
|
374
|
+
for h in URL_HOST_RE.findall(line):
|
|
375
|
+
links.append(("url", h, rel, i, line))
|
|
376
|
+
for rx, label, kind in CODE_ROLE:
|
|
377
|
+
if re.search(rx, line):
|
|
378
|
+
role = (label, kind)
|
|
379
|
+
m = re.search(r"new Eureka\(", line)
|
|
380
|
+
if m:
|
|
381
|
+
aliases.update(re.findall(r"app\s*:\s*['\"]([\w-]+)['\"]", text))
|
|
382
|
+
elif SPRING_CONF.match(f):
|
|
383
|
+
for key, value, i in _spring_props(_read(full), f.endswith(("yml", "yaml"))):
|
|
384
|
+
props.append((key, value, rel, i))
|
|
385
|
+
for e in re.findall(r"\$\{([A-Z][A-Z0-9_]+)", value):
|
|
386
|
+
envs.setdefault(e, (rel, i, f"{key} = ${{{e}}}"))
|
|
387
|
+
if key == "spring.application.name" and re.match(r"^[\w-]+$", value):
|
|
388
|
+
aliases.add(value)
|
|
389
|
+
low = key.lower()
|
|
390
|
+
if low.startswith("eureka."): # registering with the registry is its own link, not an HTTP call
|
|
391
|
+
continue
|
|
392
|
+
if re.search(r"zuul\.routes\.[\w-]+\.(url|serviceid|service-id)$", low) or re.search(r"gateway\.routes\.(\w+\.)?uri$", low) \
|
|
393
|
+
or re.search(r"(service|server|client|api)[\w-]*\.(url|uri|host|base-?url)$", low):
|
|
394
|
+
target = (URL_HOST_RE.search(value) or re.match(r"^([A-Za-z][\w-]*)$", value))
|
|
395
|
+
if target:
|
|
396
|
+
links.append(("route" if "routes" in low else "config", target.group(1), rel, i, f"{key}: {value}"))
|
|
397
|
+
elif "://" in value:
|
|
398
|
+
for h in URL_HOST_RE.findall(value):
|
|
399
|
+
links.append(("config", h, rel, i, f"{key}: {value}"))
|
|
400
|
+
elif re.match(r"\.?env\.example$|env\.example$|\.env\.sample$|example\.env$", f):
|
|
401
|
+
for i, line in enumerate(_read(full).splitlines(), 1):
|
|
402
|
+
m = re.match(r"\s*(?:export\s+)?([A-Z][A-Z0-9_]+)\s*=", line)
|
|
403
|
+
if m:
|
|
404
|
+
envs.setdefault(m.group(1), (rel, i, m.group(1) + "=…"))
|
|
405
|
+
elif re.match(r"(docker-)?compose.*\.ya?ml$", f) or f.startswith("Dockerfile") or f in ("Makefile", "Procfile") \
|
|
406
|
+
or (ext in (".yml", ".yaml") and re.search(r"k8s|kube|deploy|helm|cloudbuild", rel, re.I)):
|
|
407
|
+
text = _read(full)
|
|
408
|
+
for i, line in enumerate(text.splitlines(), 1):
|
|
409
|
+
m = re.match(r"\s*image:\s*['\"]?([\w./:-]+)", line)
|
|
410
|
+
if m:
|
|
411
|
+
images.setdefault(m.group(1), (rel, i, line))
|
|
412
|
+
m = re.match(r"\s*-?\s*([A-Z][A-Z0-9_]{2,})\s*[:=]", line)
|
|
413
|
+
if m and ("compose" in f or ext in (".yml", ".yaml")):
|
|
414
|
+
envs.setdefault(m.group(1), (rel, i, m.group(1)))
|
|
415
|
+
cmd = None
|
|
416
|
+
if f.startswith("Dockerfile"):
|
|
417
|
+
m = re.match(r"\s*(CMD|ENTRYPOINT)\s+(.+)", line)
|
|
418
|
+
cmd = m and m.group(2)
|
|
419
|
+
elif "compose" in f:
|
|
420
|
+
m = re.match(r"\s*command:\s*(.+)", line)
|
|
421
|
+
cmd = m and m.group(1)
|
|
422
|
+
elif f in ("Makefile", "Procfile"):
|
|
423
|
+
m = re.search(r"((?:uvicorn|gunicorn|celery|airflow|python -m|node|npm run|java -jar|mvn spring-boot:run)\b.+)", line)
|
|
424
|
+
cmd = m and m.group(1)
|
|
425
|
+
if cmd:
|
|
426
|
+
entry.append((rel, i, cmd.strip().strip("[]")))
|
|
427
|
+
if re.search(r"cloudbuild", f):
|
|
428
|
+
deploy.append(("Google Cloud Build", rel))
|
|
429
|
+
elif "compose" in f:
|
|
430
|
+
deploy.append(("Docker Compose", rel))
|
|
431
|
+
elif f.startswith("Dockerfile"):
|
|
432
|
+
deploy.append(("Docker image", rel))
|
|
433
|
+
elif re.search(r"^\s*kind:\s*(Deployment|StatefulSet|Service|CronJob)", text, re.M):
|
|
434
|
+
deploy.append(("Kubernetes", rel))
|
|
435
|
+
elif rel.startswith(".github/workflows/"):
|
|
436
|
+
deploy.append(("GitHub Actions", rel))
|
|
437
|
+
elif f in (".gitlab-ci.yml", "Jenkinsfile"):
|
|
438
|
+
deploy.append(("GitLab CI" if f.startswith(".gitlab") else "Jenkins", rel))
|
|
439
|
+
elif f == "package.json":
|
|
440
|
+
try:
|
|
441
|
+
p = json.loads(_read(full))
|
|
442
|
+
except ValueError:
|
|
443
|
+
p = {}
|
|
444
|
+
if p.get("name") and rel == "package.json":
|
|
445
|
+
aliases.add(str(p["name"]).split("/")[-1])
|
|
446
|
+
scripts = p.get("scripts", {}) or {}
|
|
447
|
+
for k in ("start", "dev", "serve"):
|
|
448
|
+
if k in scripts:
|
|
449
|
+
entry.append((rel, 0, f"npm run {k}: {scripts[k]}"))
|
|
450
|
+
elif re.match(r"nginx.*\.conf$|.*\.nginx$", f) or (f.endswith(".conf") and "nginx" in rel):
|
|
451
|
+
deploy.append(("nginx", rel))
|
|
452
|
+
|
|
453
|
+
# ---- profile ----
|
|
454
|
+
fw = next(((label, kind) for key, label, kind in FRAMEWORKS_NPM if key in npm), None) or \
|
|
455
|
+
next(((label, kind) for key, label, kind in FRAMEWORKS_JVM if key in jvm), None) or \
|
|
456
|
+
next(((label, kind) for key, label, kind in FRAMEWORKS_PY if key in py), None)
|
|
457
|
+
if role: # @EnableZuulProxy / @EnableEurekaServer say more than the starters
|
|
458
|
+
fw = role
|
|
459
|
+
total = sum(l["bytes"] for l in langs.values()) or 1
|
|
460
|
+
languages = sorted(({**l, "share": round(l["bytes"] / total, 3)} for l in langs.values() if l["lang"] not in ("SQL", "Shell") or len(langs) == 1),
|
|
461
|
+
key=lambda l: -l["bytes"])
|
|
462
|
+
main_lang = languages[0]["lang"] if languages else None
|
|
463
|
+
prop_lines = [(f"{k}={v}", k, v, rel, i) for k, v, rel, i in props]
|
|
464
|
+
components = []
|
|
465
|
+
for d in DETECTORS:
|
|
466
|
+
ev = []
|
|
467
|
+
dep = next(((n, py[n]) for n in d["py"] if n in py), None) or next(((n, npm[n]) for n in d["npm"] if n in npm), None) \
|
|
468
|
+
or next(((n, jvm[n]) for n in d.get("jvm", []) if n in jvm), None)
|
|
469
|
+
if dep:
|
|
470
|
+
ev.append({"type": "dependency", "file": dep[1][0], "line": dep[1][1], "text": mask(dep[1][2])})
|
|
471
|
+
for rel, i, line in code_hits.get(d["id"], [])[:MAX_EVIDENCE]:
|
|
472
|
+
ev.append({"type": "code", "file": rel, "line": i, "text": mask(line)})
|
|
473
|
+
prop_hits = [(k, v, rel, i) for kv, k, v, rel, i in prop_lines if any(re.search(p, kv, re.I) for p in d.get("prop", []))]
|
|
474
|
+
for k, v, rel, i in prop_hits[:3]:
|
|
475
|
+
ev.append({"type": "config", "file": rel, "line": i, "text": mask(f"{k}: {v}")})
|
|
476
|
+
env_hits = [(k, v) for k, v in envs.items() if any(re.search(p, k) for p in d["env"])]
|
|
477
|
+
for k, (rel, i, _) in env_hits[:4]:
|
|
478
|
+
ev.append({"type": "config", "file": rel, "line": i, "text": f"env {k}"})
|
|
479
|
+
img = [(k, v) for k, v in images.items() if any(re.search(p, k, re.I) for p in d["image"])]
|
|
480
|
+
for k, (rel, i, _) in img[:2]:
|
|
481
|
+
ev.append({"type": "config", "file": rel, "line": i, "text": f"image {k}"})
|
|
482
|
+
if not ev:
|
|
483
|
+
continue
|
|
484
|
+
# configuration alone counts only when it's specific: a compose image, a Spring property, or Keycloak's issuer
|
|
485
|
+
kinds = {e["type"] for e in ev}
|
|
486
|
+
if kinds == {"config"} and not img and not prop_hits and not (d["id"] == "keycloak" and env_hits):
|
|
487
|
+
continue
|
|
488
|
+
conf = "wired" if "config" in kinds and ("code" in kinds or "dependency" in kinds) else \
|
|
489
|
+
"used" if "code" in kinds else "declared" if "dependency" in kinds else "configured"
|
|
490
|
+
detail = {}
|
|
491
|
+
if d["id"] == "celery":
|
|
492
|
+
hits = code_hits.get("celery", [])
|
|
493
|
+
detail = {"tasks": sum(1 for h in hits if re.search(r"@(?:shared_task|\w+\.task)\b", h[2])),
|
|
494
|
+
"producers": sum(1 for h in hits if re.search(r"\.delay\(|\.apply_async\(", h[2])),
|
|
495
|
+
"worker": any(re.search(r"celery\b.*\bworker", e[2]) for e in entry)}
|
|
496
|
+
components.append({"id": d["id"], "label": d["label"], "kind": d["kind"], "confidence": conf,
|
|
497
|
+
"evidence": ev, "detail": detail, "env": sorted(k for k, _ in env_hits)})
|
|
498
|
+
kind = fw[1] if fw else None
|
|
499
|
+
comp_ids = {c["id"] for c in components}
|
|
500
|
+
if has_dags or "airflow" in comp_ids and kind is None:
|
|
501
|
+
kind = "pipeline"
|
|
502
|
+
if kind is None:
|
|
503
|
+
if "celery" in comp_ids:
|
|
504
|
+
kind = "worker"
|
|
505
|
+
elif npm and not py and not jvm:
|
|
506
|
+
kind = "service" # a Node process without a web framework: a consumer, a worker, a socket server
|
|
507
|
+
elif any(d[0] == "Docker image" for d in deploy) and languages:
|
|
508
|
+
kind = "service"
|
|
509
|
+
elif os.path.exists(os.path.join(path, "pyproject.toml")) or os.path.exists(os.path.join(path, "setup.py")):
|
|
510
|
+
kind = "library"
|
|
511
|
+
elif languages:
|
|
512
|
+
kind = "scripts"
|
|
513
|
+
else:
|
|
514
|
+
kind = "config"
|
|
515
|
+
notable = []
|
|
516
|
+
for key, label in NOTABLE.items():
|
|
517
|
+
if (key in py or key in npm or key in jvm) and label not in notable:
|
|
518
|
+
notable.append(label)
|
|
519
|
+
# outbound HTTP by name: client classes and URL-ish env names that aren't infrastructure
|
|
520
|
+
outbound = []
|
|
521
|
+
http_lib = sorted((HTTP_LIBS_PY & set(py)) | (HTTP_LIBS_NPM & set(npm)) | (HTTP_LIBS_JVM & set(jvm)))
|
|
522
|
+
own = _tokens(name) - GENERIC
|
|
523
|
+
for cname, (rel, i, line) in clients.items():
|
|
524
|
+
toks = _tokens(cname) - {"client", "api", "gateway", "integration", "service", "proxy"}
|
|
525
|
+
if not toks or toks & INFRA_WORDS or cname.startswith(("Base", "Abstract", "Mock", "Fake")) or (toks - {t for t in toks if re.fullmatch(r"v\d+", t)}) <= own:
|
|
526
|
+
continue
|
|
527
|
+
outbound.append({"name": cname, "via": "client class", "file": rel, "line": i, "tokens": sorted(toks)})
|
|
528
|
+
for k, (rel, i, _) in envs.items():
|
|
529
|
+
if URL_ENV.search(k) and not INFRA_ENV.search(k) and not any(any(re.search(p, k) for p in d["env"]) for d in DETECTORS):
|
|
530
|
+
outbound.append({"name": k, "via": "env", "file": rel, "line": i, "tokens": sorted(_tokens(k) - GENERIC - {"vite", "react", "next", "public", "url"})})
|
|
531
|
+
seen_e, entrypoints = set(), []
|
|
532
|
+
for rel, i, cmd in entry:
|
|
533
|
+
label = ("Celery worker" if re.search(r"celery\b.*\bworker", cmd) else "Celery beat" if re.search(r"celery\b.*\bbeat", cmd)
|
|
534
|
+
else "Airflow" if "airflow" in cmd else "API server" if re.search(r"uvicorn|gunicorn|hypercorn|manage\.py runserver|flask run", cmd)
|
|
535
|
+
else "JVM app" if re.search(r"\bjava\b.*-jar|spring-boot:run", cmd)
|
|
536
|
+
else "Web dev server" if re.search(r"\bvite\b|next dev|react-scripts start", cmd)
|
|
537
|
+
else "Node process" if re.search(r"\b(node|nodemon|ts-node|nest start|pm2)\b", cmd) else "Process")
|
|
538
|
+
key = (label, rel)
|
|
539
|
+
if key in seen_e:
|
|
540
|
+
continue
|
|
541
|
+
seen_e.add(key)
|
|
542
|
+
entrypoints.append({"label": label, "command": mask(cmd)[:140], "file": rel, "line": i})
|
|
543
|
+
dep_kinds = []
|
|
544
|
+
for k, rel in deploy:
|
|
545
|
+
if k not in [d["kind"] for d in dep_kinds]:
|
|
546
|
+
dep_kinds.append({"kind": k, "file": rel})
|
|
547
|
+
eureka = next(((n, jvm.get(n) or npm.get(n) or py.get(n)) for n in EUREKA_CLIENT if n in jvm or n in npm or n in py), None)
|
|
548
|
+
return {"id": name, "name": name, "kind": kind, "framework": fw[0] if fw else None, "language": main_lang,
|
|
549
|
+
"languages": languages, "notable": notable, "components": components, "entrypoints": entrypoints[:8],
|
|
550
|
+
"deploy": dep_kinds, "http": http_lib, "outbound": outbound, "env_count": len(envs),
|
|
551
|
+
"files": sum(l["files"] for l in langs.values()), "aliases": sorted(aliases),
|
|
552
|
+
"links": [{"via": v, "target": t, "file": rel, "line": i, "text": mask(txt)} for v, t, rel, i, txt in links],
|
|
553
|
+
"registry_client": {"file": eureka[1][0], "line": eureka[1][1], "text": mask(eureka[1][2])} if eureka else None}
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _frontend_links(repos):
|
|
557
|
+
"""Frontend → backend calls matched by URL, from teammap's index (exact, with counts)."""
|
|
558
|
+
try:
|
|
559
|
+
import io
|
|
560
|
+
import contextlib
|
|
561
|
+
from . import teammap
|
|
562
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
563
|
+
loaded = [teammap.load_repo(n) for n in teammap.REPOS if os.path.isdir(os.path.join(CODE_ROOT, n))]
|
|
564
|
+
L = teammap.link(loaded)
|
|
565
|
+
except Exception:
|
|
566
|
+
return []
|
|
567
|
+
counts = {}
|
|
568
|
+
for a in L.get("apifns", []):
|
|
569
|
+
ep = a.get("endpoint")
|
|
570
|
+
if not ep:
|
|
571
|
+
continue
|
|
572
|
+
key = (a["repo"], ep["repo"])
|
|
573
|
+
c = counts.setdefault(key, {"n": 0, "example": None})
|
|
574
|
+
c["n"] += 1
|
|
575
|
+
c["example"] = c["example"] or {"file": a.get("file"), "line": a.get("line"), "text": f"{ep.get('method', '').upper()} {ep.get('path', '')}"}
|
|
576
|
+
return [{"from": f, "to": t, "kind": "http", "label": f"HTTP · {c['n']} API calls", "confidence": "exact",
|
|
577
|
+
"evidence": [{"type": "code", "file": c["example"]["file"], "line": c["example"]["line"] or 0, "text": c["example"]["text"]}] if c["example"] else []}
|
|
578
|
+
for (f, t), c in counts.items() if f in repos and t in repos and f != t]
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _match_repo(tokens, repos, self_id):
|
|
582
|
+
"""The repo an outbound name points at: all of the repo's distinctive words appear in the name."""
|
|
583
|
+
best, best_score = None, 0
|
|
584
|
+
tok = set(tokens)
|
|
585
|
+
ver = next((t for t in tok if re.fullmatch(r"v\d+", t)), None)
|
|
586
|
+
for r in repos:
|
|
587
|
+
if r == self_id:
|
|
588
|
+
continue
|
|
589
|
+
rt = _tokens(r) - GENERIC
|
|
590
|
+
rver = next((t for t in rt if re.fullmatch(r"v\d+", t)), None)
|
|
591
|
+
words = rt - ({rver} if rver else set())
|
|
592
|
+
if not words or not words <= tok:
|
|
593
|
+
continue
|
|
594
|
+
if ver and rver and ver != rver or ver and ver != "v1" and not rver or rver and not ver:
|
|
595
|
+
continue
|
|
596
|
+
score = len(words) + (1 if ver and ver == rver else 0)
|
|
597
|
+
if score > best_score:
|
|
598
|
+
best, best_score = r, score
|
|
599
|
+
return best
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def build():
|
|
603
|
+
t0 = time.time()
|
|
604
|
+
names = repo_dirs()
|
|
605
|
+
repos = [scan_repo(n) for n in names]
|
|
606
|
+
ids = [r["id"] for r in repos]
|
|
607
|
+
by_alias = {}
|
|
608
|
+
for r in repos:
|
|
609
|
+
for a in r["aliases"]:
|
|
610
|
+
by_alias.setdefault(_key(a), r["id"])
|
|
611
|
+
registry = next((r["id"] for r in repos if r["kind"] == "registry" and (r["framework"] or "").startswith("Eureka")), None)
|
|
612
|
+
edges, nodes = [], {}
|
|
613
|
+
for r in repos:
|
|
614
|
+
for c in r["components"]:
|
|
615
|
+
nid = "c:" + c["id"]
|
|
616
|
+
nodes.setdefault(nid, {"id": nid, "label": c["label"], "kind": c["kind"], "repos": []})["repos"].append(r["id"])
|
|
617
|
+
verb = {"database": "SQL" if c["id"] in ("postgres", "mysql", "sqlite") else "data", "cache": "cache",
|
|
618
|
+
"broker": {"rabbitmq": "AMQP", "mqtt": "MQTT", "kafka": "Kafka"}.get(c["id"], "events"), "worker": "jobs",
|
|
619
|
+
"auth": "auth", "storage": "files", "observability": "metrics" if c["id"] == "prometheus" else "errors",
|
|
620
|
+
"search": "search", "external": ""}.get(c["kind"], "")
|
|
621
|
+
edges.append({"from": r["id"], "to": nid, "kind": c["kind"], "label": verb, "confidence": c["confidence"], "evidence": c["evidence"][:4]})
|
|
622
|
+
# service-to-service links that name the service: Feign clients, gateway routes, URLs with a service host
|
|
623
|
+
exact = {}
|
|
624
|
+
for l in r["links"]:
|
|
625
|
+
to = by_alias.get(_key(l["target"]))
|
|
626
|
+
if not to or to == r["id"]:
|
|
627
|
+
if l["via"] in ("feign", "route") and l["target"].lower() not in NOT_HOSTS and not to:
|
|
628
|
+
xid = "x:" + l["target"].lower()
|
|
629
|
+
nodes.setdefault(xid, {"id": xid, "label": l["target"], "kind": "external", "repos": []})["repos"].append(r["id"])
|
|
630
|
+
exact.setdefault(xid, []).append(l)
|
|
631
|
+
continue
|
|
632
|
+
exact.setdefault(to, []).append(l)
|
|
633
|
+
for to, ls in exact.items():
|
|
634
|
+
vias = sorted({l["via"] for l in ls})
|
|
635
|
+
label = {"feign": "Feign", "route": "route", "config": "HTTP", "url": "HTTP"}
|
|
636
|
+
edges.append({"from": r["id"], "to": to, "kind": "http", "label": " · ".join(label[v] for v in vias) + (f" · {len(ls)}" if len(ls) > 1 else ""),
|
|
637
|
+
"confidence": "exact", "evidence": [{"type": "code" if l["via"] in ("feign", "url") else "config", "file": l["file"], "line": l["line"], "text": l["text"]} for l in ls[:4]]})
|
|
638
|
+
if r["registry_client"] and registry and registry != r["id"]:
|
|
639
|
+
edges.append({"from": r["id"], "to": registry, "kind": "discovery", "label": "registers", "confidence": "declared",
|
|
640
|
+
"evidence": [{"type": "dependency", **r["registry_client"]}]})
|
|
641
|
+
targets = {}
|
|
642
|
+
for o in r["outbound"]:
|
|
643
|
+
to = _match_repo(o["tokens"], ids, r["id"])
|
|
644
|
+
o["target"] = to
|
|
645
|
+
if to and to not in exact:
|
|
646
|
+
t = targets.setdefault(to, {"names": [], "evidence": []})
|
|
647
|
+
t["names"].append(o["name"])
|
|
648
|
+
t["evidence"].append({"type": "code" if o["via"] == "client class" else "config", "file": o["file"], "line": o["line"], "text": o["name"]})
|
|
649
|
+
elif not to and o["via"] == "client class" and o["tokens"] and r["kind"] != "frontend": # a frontend's API wrappers aren't outside services
|
|
650
|
+
xid = "x:" + "-".join(o["tokens"])
|
|
651
|
+
nodes.setdefault(xid, {"id": xid, "label": " ".join(o["tokens"]).title() + " service", "kind": "external", "repos": []})["repos"].append(r["id"])
|
|
652
|
+
edges.append({"from": r["id"], "to": xid, "kind": "http", "label": "HTTP", "confidence": "inferred",
|
|
653
|
+
"evidence": [{"type": "code", "file": o["file"], "line": o["line"], "text": o["name"]}]})
|
|
654
|
+
for to, t in targets.items():
|
|
655
|
+
edges.append({"from": r["id"], "to": to, "kind": "http", "label": "HTTP · " + ", ".join(t["names"][:2]),
|
|
656
|
+
"confidence": "inferred", "evidence": t["evidence"][:4]})
|
|
657
|
+
exact_fe = _frontend_links(set(ids))
|
|
658
|
+
for e in exact_fe: # an exact URL match replaces any other link between the same two repos
|
|
659
|
+
edges = [x for x in edges if not (x["from"] == e["from"] and x["to"] == e["to"])]
|
|
660
|
+
edges += exact_fe
|
|
661
|
+
for n in nodes.values():
|
|
662
|
+
n["repos"] = sorted(set(n["repos"]))
|
|
663
|
+
return {"generated": time.time(), "seconds": round(time.time() - t0, 2), "root": norm(CODE_ROOT), "monorepo": is_monorepo(),
|
|
664
|
+
"repos": repos, "nodes": list(nodes.values()), "edges": edges}
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
_cache = {"at": 0, "data": None}
|
|
668
|
+
CACHE_FILE = os.path.join(GRAPHS, "index", "arch.json") # not graph/*.json: the Brain merges those as code graphs
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def fingerprint():
|
|
672
|
+
"""Changes when any scanned file (or this code) changes; walking is fast, reading is what's slow."""
|
|
673
|
+
import hashlib
|
|
674
|
+
h = hashlib.sha1()
|
|
675
|
+
for mod in ("archmap.py", "teammap.py"):
|
|
676
|
+
try:
|
|
677
|
+
h.update(str(os.path.getmtime(os.path.join(HERE, mod))).encode())
|
|
678
|
+
except OSError:
|
|
679
|
+
pass
|
|
680
|
+
for name in repo_dirs():
|
|
681
|
+
for full, _ in _walk(os.path.join(CODE_ROOT, name)):
|
|
682
|
+
try:
|
|
683
|
+
st = os.stat(full)
|
|
684
|
+
except OSError:
|
|
685
|
+
continue
|
|
686
|
+
h.update(f"{full}\0{st.st_mtime_ns}\0{st.st_size}\n".encode())
|
|
687
|
+
return h.hexdigest()
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def cached(force=False):
|
|
691
|
+
"""The map from <team>/graph/arch.json when nothing changed since it was built, else a fresh scan."""
|
|
692
|
+
fp = fingerprint()
|
|
693
|
+
if not force:
|
|
694
|
+
try:
|
|
695
|
+
with open(CACHE_FILE, encoding="utf-8") as f:
|
|
696
|
+
data = json.load(f)
|
|
697
|
+
if data.get("fingerprint") == fp:
|
|
698
|
+
return data
|
|
699
|
+
except (OSError, ValueError):
|
|
700
|
+
pass
|
|
701
|
+
data = build()
|
|
702
|
+
data["fingerprint"] = fp
|
|
703
|
+
try:
|
|
704
|
+
os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True)
|
|
705
|
+
with open(CACHE_FILE, "w", encoding="utf-8") as f:
|
|
706
|
+
json.dump(data, f)
|
|
707
|
+
except OSError:
|
|
708
|
+
pass
|
|
709
|
+
return data
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def get(max_age=60):
|
|
713
|
+
if not _cache["data"] or time.time() - _cache["at"] > max_age:
|
|
714
|
+
_cache["data"], _cache["at"] = cached(force=max_age == 0), time.time()
|
|
715
|
+
return _cache["data"]
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
# ---- queries (for agents: `teammap arch …`) ---------------------------------------------------------
|
|
719
|
+
|
|
720
|
+
def resolve(data, q):
|
|
721
|
+
"""A repo/service or a component by id, name, alias or a unique part of one: ("repo"|"node", item) or (None, candidates)."""
|
|
722
|
+
k = _key(q)
|
|
723
|
+
for r in data["repos"]:
|
|
724
|
+
if _key(r["id"]) == k or any(_key(a) == k for a in r.get("aliases", [])):
|
|
725
|
+
return "repo", r
|
|
726
|
+
for n in data["nodes"]:
|
|
727
|
+
if _key(n["id"].split(":", 1)[1]) == k or _key(n["label"]) == k:
|
|
728
|
+
return "node", n
|
|
729
|
+
reps = [r for r in data["repos"] if k in _key(r["id"])]
|
|
730
|
+
nds = [n for n in data["nodes"] if k in _key(n["label"]) or k in _key(n["id"])]
|
|
731
|
+
if len(reps) + len(nds) == 1:
|
|
732
|
+
return ("repo", reps[0]) if reps else ("node", nds[0])
|
|
733
|
+
return None, [r["id"] for r in reps] + [n["label"] for n in nds]
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _calls(data):
|
|
737
|
+
"""Service-to-service links only (no components, no registry)."""
|
|
738
|
+
return [e for e in data["edges"] if e["kind"] == "http" and not e["to"].startswith("c:")]
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def impact(data, q, depth=3):
|
|
742
|
+
kind, item = resolve(data, q)
|
|
743
|
+
if kind != "repo":
|
|
744
|
+
return None, item
|
|
745
|
+
levels, seen, frontier = [], {item["id"]}, [item["id"]]
|
|
746
|
+
for _ in range(depth):
|
|
747
|
+
nxt = []
|
|
748
|
+
for e in _calls(data):
|
|
749
|
+
if e["to"] in frontier and e["from"] not in seen:
|
|
750
|
+
seen.add(e["from"])
|
|
751
|
+
nxt.append(e)
|
|
752
|
+
if not nxt:
|
|
753
|
+
break
|
|
754
|
+
levels.append(nxt)
|
|
755
|
+
frontier = [e["from"] for e in nxt]
|
|
756
|
+
return item, levels
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def path(data, a, b):
|
|
760
|
+
ka, ra = resolve(data, a)
|
|
761
|
+
kb, rb = resolve(data, b)
|
|
762
|
+
if not ka or not kb:
|
|
763
|
+
return None, (ra if not ka else rb)
|
|
764
|
+
src, dst = (ra["id"] if ka == "repo" else ra["id"]), (rb["id"] if kb == "repo" else rb["id"])
|
|
765
|
+
edges = _calls(data) + [e for e in data["edges"] if e["to"] == dst] # a component can be the destination
|
|
766
|
+
prev, q = {src: None}, [src]
|
|
767
|
+
while q:
|
|
768
|
+
cur = q.pop(0)
|
|
769
|
+
if cur == dst:
|
|
770
|
+
break
|
|
771
|
+
for e in edges:
|
|
772
|
+
if e["from"] == cur and e["to"] not in prev:
|
|
773
|
+
prev[e["to"]] = e
|
|
774
|
+
q.append(e["to"])
|
|
775
|
+
if dst not in prev:
|
|
776
|
+
return (src, dst), None
|
|
777
|
+
hops, cur = [], dst
|
|
778
|
+
while prev[cur]:
|
|
779
|
+
hops.append(prev[cur])
|
|
780
|
+
cur = prev[cur]["from"]
|
|
781
|
+
return (src, dst), list(reversed(hops))
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def _where(e):
|
|
785
|
+
ev = e["evidence"][0] if e.get("evidence") else None
|
|
786
|
+
return f" {e['from']}/{ev['file']}:{ev['line']}" if ev else ""
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def _name(data, nid):
|
|
790
|
+
n = next((n for n in data["nodes"] if n["id"] == nid), None)
|
|
791
|
+
return n["label"] if n else nid
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def node_text(data, n):
|
|
795
|
+
users = [e for e in data["edges"] if e["to"] == n["id"]]
|
|
796
|
+
lines = [f"{n['label']} [{n['kind']}] used by {len(users)}"]
|
|
797
|
+
for e in sorted(users, key=lambda e: e["from"]):
|
|
798
|
+
r = next((x for x in data["repos"] if x["id"] == e["from"]), None)
|
|
799
|
+
c = next((c for c in (r or {}).get("components", []) if "c:" + c["id"] == n["id"]), None)
|
|
800
|
+
env = f" · env {', '.join(c['env'][:3])}" if c and c["env"] else ""
|
|
801
|
+
ev = next((x for x in (c or e)["evidence"] if x["type"] == "code"), None) or ((c or e)["evidence"] or [None])[0]
|
|
802
|
+
where = f" {e['from']}/{ev['file']}:{ev['line']}" if ev else ""
|
|
803
|
+
lines.append(f" {e['from']:32} {e['confidence']}{env}{where}")
|
|
804
|
+
return "\n".join(lines)
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
# ---- text and Mermaid ------------------------------------------------------------------------------
|
|
808
|
+
|
|
809
|
+
MAIN_KINDS = ("frontend", "gateway", "api", "service", "worker", "pipeline", "registry")
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def mermaid(data, include_minor=False):
|
|
813
|
+
main = [r for r in data["repos"] if include_minor or r["kind"] in MAIN_KINDS]
|
|
814
|
+
ids = {r["id"] for r in main}
|
|
815
|
+
safe = lambda s: re.sub(r"[^A-Za-z0-9_]", "_", s)
|
|
816
|
+
out = ["flowchart LR"]
|
|
817
|
+
for r in main:
|
|
818
|
+
sub = " · ".join(x for x in (r["framework"], r["language"]) if x)
|
|
819
|
+
out.append(f' {safe(r["id"])}["{r["name"]}<br/><small>{sub}</small>"]')
|
|
820
|
+
used = {e["to"] for e in data["edges"] if e["from"] in ids and e["to"].startswith(("c:", "x:"))}
|
|
821
|
+
shapes = {"database": '[("{}")]', "cache": '[("{}")]', "broker": '{{"{}"}}', "worker": '[["{}"]]', "auth": '(["{}"])'}
|
|
822
|
+
for n in data["nodes"]:
|
|
823
|
+
if n["id"] in used:
|
|
824
|
+
out.append(f' {safe(n["id"])}' + shapes.get(n["kind"], '("{}")').format(n["label"]))
|
|
825
|
+
for e in data["edges"]:
|
|
826
|
+
if e["from"] in ids and (e["to"] in ids or e["to"] in used) and e["kind"] != "discovery":
|
|
827
|
+
arrow = "-.->" if e["confidence"] in ("inferred", "declared") else "-->"
|
|
828
|
+
label = f'|{e["label"]}|' if e["label"] else ""
|
|
829
|
+
out.append(f' {safe(e["from"])} {arrow}{label} {safe(e["to"])}')
|
|
830
|
+
return "\n".join(out)
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def text(data, repo=None):
|
|
834
|
+
if repo:
|
|
835
|
+
kind, item = resolve(data, repo)
|
|
836
|
+
if kind == "node":
|
|
837
|
+
return node_text(data, item)
|
|
838
|
+
if kind == "repo":
|
|
839
|
+
repo = item["id"]
|
|
840
|
+
elif item:
|
|
841
|
+
return f"{repo!r} matches several: " + ", ".join(item[:20]) + " — be more specific"
|
|
842
|
+
lines = []
|
|
843
|
+
repos = [r for r in data["repos"] if not repo or r["id"] == repo]
|
|
844
|
+
if not repos:
|
|
845
|
+
return f"no repository matches {repo!r}; repos: " + ", ".join(r["id"] for r in data["repos"])
|
|
846
|
+
detail = bool(repo) or len(repos) == 1
|
|
847
|
+
for r in repos:
|
|
848
|
+
langs = ", ".join(f"{l['lang']} {round(l['share'] * 100)}%" for l in r["languages"][:3])
|
|
849
|
+
lines.append(f"{r['id']} [{r['kind']}] {r['framework'] or '-'} ({langs or 'no code'})")
|
|
850
|
+
if detail:
|
|
851
|
+
if r["notable"]:
|
|
852
|
+
lines.append(" stack " + ", ".join(r["notable"]))
|
|
853
|
+
for e in r["entrypoints"]:
|
|
854
|
+
lines.append(f" runs {e['label']}: {e['command']} ({e['file']})")
|
|
855
|
+
if r["deploy"]:
|
|
856
|
+
lines.append(" deploy " + ", ".join(d["kind"] for d in r["deploy"]))
|
|
857
|
+
for c in r["components"]:
|
|
858
|
+
ev = next((e for e in c["evidence"] if e["type"] == "code"), None) or next((e for e in c["evidence"] if e["type"] == "config"), c["evidence"][0])
|
|
859
|
+
extra = f" · {c['detail']['tasks']} tasks, {c['detail']['producers']} enqueue sites" if c["detail"] else ""
|
|
860
|
+
lines.append(f" uses {c['label']} ({c['confidence']}{extra}) {ev['file']}:{ev['line']}")
|
|
861
|
+
for e in data["edges"]:
|
|
862
|
+
if e["from"] == r["id"] and not e["to"].startswith("c:") and (detail or e["kind"] != "discovery"):
|
|
863
|
+
ev = e["evidence"][0] if e["evidence"] else None
|
|
864
|
+
where = f" {ev['file']}:{ev['line']}" if ev and detail else ""
|
|
865
|
+
lines.append(f" calls {e['to'].removeprefix('x:')} · {e['label']} ({e['confidence']}){where}")
|
|
866
|
+
if e["to"] == r["id"] and (detail or e["kind"] != "discovery"):
|
|
867
|
+
lines.append(f" called by {e['from']} · {e['label']} ({e['confidence']})")
|
|
868
|
+
lines.append("")
|
|
869
|
+
return "\n".join(lines).rstrip()
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def main(argv):
|
|
873
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
874
|
+
data = cached(force="--rescan" in argv)
|
|
875
|
+
as_json = "--json" in argv
|
|
876
|
+
rest = [a for a in argv if not a.startswith("--")]
|
|
877
|
+
if "--mermaid" in argv:
|
|
878
|
+
print(mermaid(data, "--all" in argv))
|
|
879
|
+
elif rest[:1] == ["impact"] and len(rest) > 1:
|
|
880
|
+
item, levels = impact(data, rest[1])
|
|
881
|
+
if item is None:
|
|
882
|
+
print(f"no service matches {rest[1]!r}" + (": " + ", ".join(levels[:20]) if levels else ""))
|
|
883
|
+
elif as_json:
|
|
884
|
+
print(json.dumps({"service": item["id"], "levels": levels}, indent=1))
|
|
885
|
+
else:
|
|
886
|
+
print(f"{item['id']}: who calls it" + ("" if levels else " — nothing in the workspace calls it over HTTP"))
|
|
887
|
+
for i, lv in enumerate(levels, 1):
|
|
888
|
+
for e in lv:
|
|
889
|
+
print(f" {' ' * (i - 1)}{'←' if i == 1 else '↖'} {e['from']} → {e['to']} · {e['label']} ({e['confidence']}){_where(e)}")
|
|
890
|
+
elif rest[:1] == ["path"] and len(rest) > 2:
|
|
891
|
+
ends, hops = path(data, rest[1], rest[2])
|
|
892
|
+
if ends is None:
|
|
893
|
+
print(f"no match for one of {rest[1]!r}, {rest[2]!r}" + (": " + ", ".join(hops[:20]) if hops else ""))
|
|
894
|
+
elif hops is None:
|
|
895
|
+
back = path(data, rest[2], rest[1])[1]
|
|
896
|
+
print(f"no call path {_name(data, ends[0])} → {_name(data, ends[1])}" + (f"; the other way: {' → '.join([back[0]['from']] + [h['to'] for h in back])}" if back else ""))
|
|
897
|
+
elif as_json:
|
|
898
|
+
print(json.dumps(hops, indent=1))
|
|
899
|
+
else:
|
|
900
|
+
print(" → ".join([hops[0]["from"]] + [_name(data, h["to"]) for h in hops]) if hops else ends[0])
|
|
901
|
+
for h in hops:
|
|
902
|
+
print(f" {h['from']} → {_name(data, h['to'])} · {h['label']} ({h['confidence']}){_where(h)}")
|
|
903
|
+
elif as_json:
|
|
904
|
+
if rest:
|
|
905
|
+
kind, item = resolve(data, rest[0])
|
|
906
|
+
edges = [e for e in data["edges"] if item and kind and (e["from"] == item["id"] or e["to"] == item["id"])]
|
|
907
|
+
print(json.dumps({"kind": kind, "item": item, "edges": edges}, indent=1))
|
|
908
|
+
else:
|
|
909
|
+
print(json.dumps(data, indent=1))
|
|
910
|
+
else:
|
|
911
|
+
print(text(data, rest[0] if rest else None))
|
|
912
|
+
return 0
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
if __name__ == "__main__":
|
|
916
|
+
sys.exit(main(sys.argv[1:]))
|