hinge 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hinge/__init__.py +3 -0
- hinge/config/__init__.py +0 -0
- hinge/config/logging_setup.py +63 -0
- hinge/config/settings.py +23 -0
- hinge/config/types.yaml +42 -0
- hinge/dbt/.user.yml +1 -0
- hinge/dbt/dbt_project.yml +18 -0
- hinge/dbt/macros/apply_time_window.sql +11 -0
- hinge/dbt/macros/assert_recipe_supported.sql +59 -0
- hinge/dbt/macros/build_incidence.sql +46 -0
- hinge/dbt/macros/collapse_path.sql +26 -0
- hinge/dbt/macros/filter_bots.sql +23 -0
- hinge/dbt/macros/network_edges.sql +39 -0
- hinge/dbt/macros/project_bipartite.sql +33 -0
- hinge/dbt/macros/slice_edges.sql +41 -0
- hinge/dbt/models/hin/hin_accounts.sql +14 -0
- hinge/dbt/models/hin/hin_artifacts.sql +23 -0
- hinge/dbt/models/hin/hin_capabilities.sql +42 -0
- hinge/dbt/models/hin/hin_edges.sql +57 -0
- hinge/dbt/models/hin/hin_nodes.sql +44 -0
- hinge/dbt/models/hin/hin_repositories.sql +15 -0
- hinge/dbt/models/intermediate/int_developer_repo_affiliation.sql +27 -0
- hinge/dbt/models/intermediate/int_user_artifact_incidence.sql +21 -0
- hinge/dbt/models/networks/artifact_reference.sql +27 -0
- hinge/dbt/models/networks/co_commit_user_user.sql +48 -0
- hinge/dbt/models/networks/co_edit_file_user_user.sql +48 -0
- hinge/dbt/models/networks/co_edit_line_user_user.sql +48 -0
- hinge/dbt/models/networks/dev_interaction.sql +98 -0
- hinge/dbt/models/networks/developer_repo_affiliation.sql +20 -0
- hinge/dbt/models/networks/follow_user_user.sql +27 -0
- hinge/dbt/models/networks/fork_repo_repo.sql +27 -0
- hinge/dbt/models/networks/issue_co_participation.sql +50 -0
- hinge/dbt/models/networks/issue_participation.sql +40 -0
- hinge/dbt/models/networks/pr_author_reviewer.sql +66 -0
- hinge/dbt/models/networks/pr_participation.sql +40 -0
- hinge/dbt/models/networks/pr_reviewer_coreview.sql +48 -0
- hinge/dbt/models/networks/repo_shared_contributors.sql +41 -0
- hinge/dbt/models/networks/star_user_repo.sql +27 -0
- hinge/dbt/models/networks/user_mention_user.sql +60 -0
- hinge/dbt/models/networks/watch_user_repo.sql +27 -0
- hinge/dbt/models/sources/sources.yml +63 -0
- hinge/dbt/models/staging/stg_accounts.sql +4 -0
- hinge/dbt/models/staging/stg_adapter_manifest.sql +4 -0
- hinge/dbt/models/staging/stg_artifacts.sql +4 -0
- hinge/dbt/models/staging/stg_relations.sql +4 -0
- hinge/dbt/models/staging/stg_repositories.sql +4 -0
- hinge/dbt/profiles.yml +9 -0
- hinge/dbt/seeds/ref_recipe_requirements.csv +35 -0
- hinge/frontends/__init__.py +0 -0
- hinge/frontends/cli/__init__.py +3 -0
- hinge/frontends/cli/app.py +189 -0
- hinge/frontends/lib.py +148 -0
- hinge/frontends/tui/__init__.py +7 -0
- hinge/frontends/web/__init__.py +6 -0
- hinge/kernel/__init__.py +0 -0
- hinge/kernel/projection/__init__.py +9 -0
- hinge/kernel/projection/projected_graph.py +32 -0
- hinge/kernel/projection/projection_spec.py +35 -0
- hinge/kernel/protocols/__init__.py +24 -0
- hinge/kernel/protocols/dataset_view.py +30 -0
- hinge/kernel/protocols/exporter_stage.py +24 -0
- hinge/kernel/protocols/projection_stage.py +36 -0
- hinge/kernel/protocols/reader_stage.py +52 -0
- hinge/kernel/protocols/store_stage.py +71 -0
- hinge/kernel/registry.py +71 -0
- hinge/kernel/runner.py +236 -0
- hinge/kernel/schema/__init__.py +12 -0
- hinge/kernel/schema/hin_schema.py +38 -0
- hinge/kernel/schema/schema_violation.py +21 -0
- hinge/kernel/schema/typed_edge.py +20 -0
- hinge/kernel/schema/typed_node.py +17 -0
- hinge/stages/__init__.py +0 -0
- hinge/stages/exporters/__init__.py +0 -0
- hinge/stages/exporters/csv_exporter.py +77 -0
- hinge/stages/exporters/dot_exporter.py +13 -0
- hinge/stages/exporters/gexf_exporter.py +13 -0
- hinge/stages/exporters/gml_exporter.py +79 -0
- hinge/stages/exporters/graphml_exporter.py +13 -0
- hinge/stages/exporters/jsonld_exporter.py +13 -0
- hinge/stages/exporters/networkx_exporter.py +57 -0
- hinge/stages/exporters/parquet_exporter.py +13 -0
- hinge/stages/projection/__init__.py +0 -0
- hinge/stages/projection/dbt_projection.py +342 -0
- hinge/stages/projection/specs/__init__.py +0 -0
- hinge/stages/projection/specs/artifact_reference.py +14 -0
- hinge/stages/projection/specs/co_commit_user_user.py +14 -0
- hinge/stages/projection/specs/co_edit_file_user_user.py +14 -0
- hinge/stages/projection/specs/co_edit_line_user_user.py +14 -0
- hinge/stages/projection/specs/dev_interaction.py +26 -0
- hinge/stages/projection/specs/developer_repo_affiliation.py +15 -0
- hinge/stages/projection/specs/follow_user_user.py +14 -0
- hinge/stages/projection/specs/fork_repo_repo.py +14 -0
- hinge/stages/projection/specs/issue_co_participation.py +14 -0
- hinge/stages/projection/specs/issue_participation.py +14 -0
- hinge/stages/projection/specs/pr_author_reviewer.py +14 -0
- hinge/stages/projection/specs/pr_participation.py +14 -0
- hinge/stages/projection/specs/pr_reviewer_coreview.py +14 -0
- hinge/stages/projection/specs/repo_shared_contributors.py +14 -0
- hinge/stages/projection/specs/star_user_repo.py +15 -0
- hinge/stages/projection/specs/user_mention_user.py +14 -0
- hinge/stages/projection/specs/watch_user_repo.py +14 -0
- hinge/stages/readers/__init__.py +0 -0
- hinge/stages/readers/_numfocus_contract_adapter.py +314 -0
- hinge/stages/readers/numfocus_reader.py +69 -0
- hinge/stages/store/__init__.py +0 -0
- hinge/stages/store/duckdb_store.py +466 -0
- hinge-0.1.0.dist-info/METADATA +309 -0
- hinge-0.1.0.dist-info/RECORD +111 -0
- hinge-0.1.0.dist-info/WHEEL +4 -0
- hinge-0.1.0.dist-info/entry_points.txt +35 -0
- hinge-0.1.0.dist-info/licenses/LICENSE +21 -0
hinge/__init__.py
ADDED
hinge/config/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Central logging configuration for hinge.
|
|
2
|
+
|
|
3
|
+
Call ``setup_logging()`` once at process start (CLI entry-point or lib import).
|
|
4
|
+
Every module then does ``logger = logging.getLogger(__name__)`` and logs normally.
|
|
5
|
+
|
|
6
|
+
Environment variables
|
|
7
|
+
---------------------
|
|
8
|
+
HINGE_LOG_LEVEL Verbosity for the hinge.* namespace. Default: INFO.
|
|
9
|
+
Values: DEBUG, INFO, WARNING, ERROR
|
|
10
|
+
HINGE_LOG_FILE If set, logs are also written to this path in addition to
|
|
11
|
+
stderr. Useful for CI or long ingest runs.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
import logging.handlers
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
_FMT = "%(asctime)s %(levelname)-8s %(name)s — %(message)s"
|
|
23
|
+
_DATE_FMT = "%Y-%m-%d %H:%M:%S"
|
|
24
|
+
_configured = False
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def setup_logging() -> None:
|
|
28
|
+
global _configured
|
|
29
|
+
if _configured:
|
|
30
|
+
return
|
|
31
|
+
_configured = True
|
|
32
|
+
|
|
33
|
+
level_name = os.environ.get("HINGE_LOG_LEVEL", "INFO").upper()
|
|
34
|
+
level = getattr(logging, level_name, logging.INFO)
|
|
35
|
+
|
|
36
|
+
formatter = logging.Formatter(_FMT, datefmt=_DATE_FMT)
|
|
37
|
+
|
|
38
|
+
# stderr handler — always on
|
|
39
|
+
stderr_handler = logging.StreamHandler(sys.stderr)
|
|
40
|
+
stderr_handler.setFormatter(formatter)
|
|
41
|
+
stderr_handler.setLevel(level)
|
|
42
|
+
|
|
43
|
+
root = logging.getLogger("hinge")
|
|
44
|
+
root.setLevel(level)
|
|
45
|
+
root.addHandler(stderr_handler)
|
|
46
|
+
|
|
47
|
+
# optional file handler
|
|
48
|
+
log_file = os.environ.get("HINGE_LOG_FILE")
|
|
49
|
+
if log_file:
|
|
50
|
+
path = Path(log_file)
|
|
51
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
file_handler = logging.handlers.RotatingFileHandler(
|
|
53
|
+
path, maxBytes=10 * 1024 * 1024, backupCount=3, encoding="utf-8"
|
|
54
|
+
)
|
|
55
|
+
file_handler.setFormatter(formatter)
|
|
56
|
+
file_handler.setLevel(logging.DEBUG) # always full detail in the file
|
|
57
|
+
root.addHandler(file_handler)
|
|
58
|
+
|
|
59
|
+
# silence noisy third-party loggers unless user explicitly requests DEBUG
|
|
60
|
+
if level > logging.DEBUG:
|
|
61
|
+
logging.getLogger("dbt").setLevel(logging.WARNING)
|
|
62
|
+
|
|
63
|
+
root.debug("logging initialised (level=%s)", level_name)
|
hinge/config/settings.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def store_path() -> Path:
|
|
8
|
+
return Path(os.environ.get("HINGE_STORE_PATH", "./network.duckdb"))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def log_level() -> str:
|
|
12
|
+
"""Verbosity for hinge.* loggers. Values: DEBUG, INFO, WARNING, ERROR."""
|
|
13
|
+
return os.environ.get("HINGE_LOG_LEVEL", "INFO")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def log_file() -> Path | None:
|
|
17
|
+
"""Optional persistent log file. None means stderr only."""
|
|
18
|
+
val = os.environ.get("HINGE_LOG_FILE")
|
|
19
|
+
return Path(val) if val else None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def types_yaml_path() -> Path:
|
|
23
|
+
return Path(__file__).with_name("types.yaml")
|
hinge/config/types.yaml
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# ─── HIN typed vocabulary — the single source of truth ────────────────────
|
|
2
|
+
#
|
|
3
|
+
# Editing this file is how you add or remove an allowed type label.
|
|
4
|
+
# - Bump ``schema_version`` whenever you change the vocabulary; existing
|
|
5
|
+
# DuckDB stores will refuse to open until re-ingested.
|
|
6
|
+
# - Stage code (readers, projections, exporters) treats labels as plain
|
|
7
|
+
# strings — there are no Python enums to update.
|
|
8
|
+
# - Projections may emit labels not listed here; only ingest-time validation
|
|
9
|
+
# is gated by this file. If a projection's new label should become a
|
|
10
|
+
# first-class part of the vocabulary, add it here and bump the version.
|
|
11
|
+
|
|
12
|
+
schema_version: 1
|
|
13
|
+
|
|
14
|
+
node_types:
|
|
15
|
+
- user
|
|
16
|
+
- repo
|
|
17
|
+
- artifact
|
|
18
|
+
|
|
19
|
+
artifact_subtypes:
|
|
20
|
+
- pull_request
|
|
21
|
+
- issue
|
|
22
|
+
- push
|
|
23
|
+
- release
|
|
24
|
+
- wiki_page
|
|
25
|
+
- branch
|
|
26
|
+
|
|
27
|
+
edge_types:
|
|
28
|
+
- opened
|
|
29
|
+
- reviewed
|
|
30
|
+
- commented_on
|
|
31
|
+
- review_commented_on
|
|
32
|
+
- merged
|
|
33
|
+
- closed
|
|
34
|
+
- reopened
|
|
35
|
+
- pushed
|
|
36
|
+
- commented_commits
|
|
37
|
+
- created_fork
|
|
38
|
+
- starred
|
|
39
|
+
- released
|
|
40
|
+
- wiki_edited
|
|
41
|
+
- member_of
|
|
42
|
+
- contains # Repo -> Artifact structural edge
|
hinge/dbt/.user.yml
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
id: e7925ad0-fe3a-4d78-8279-1724d2204227
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
name: hin_projections
|
|
2
|
+
version: "0.1.0"
|
|
3
|
+
config-version: 2
|
|
4
|
+
|
|
5
|
+
profile: hinge
|
|
6
|
+
|
|
7
|
+
model-paths: ["models"]
|
|
8
|
+
seed-paths: ["seeds"]
|
|
9
|
+
target-path: "target"
|
|
10
|
+
clean-targets: ["target", "dbt_packages"]
|
|
11
|
+
|
|
12
|
+
models:
|
|
13
|
+
hin_projections:
|
|
14
|
+
+materialized: table
|
|
15
|
+
|
|
16
|
+
seeds:
|
|
17
|
+
hin_projections:
|
|
18
|
+
+quote_columns: false
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{% macro apply_time_window(timestamp_col='occurred_at', start_date=None, end_date=None) %}
|
|
2
|
+
(
|
|
3
|
+
true
|
|
4
|
+
{% if start_date is not none %}
|
|
5
|
+
AND {{ timestamp_col }} >= CAST('{{ start_date }}' AS TIMESTAMP)
|
|
6
|
+
{% endif %}
|
|
7
|
+
{% if end_date is not none %}
|
|
8
|
+
AND {{ timestamp_col }} < CAST('{{ end_date }}' AS TIMESTAMP)
|
|
9
|
+
{% endif %}
|
|
10
|
+
)
|
|
11
|
+
{% endmacro %}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{% macro assert_recipe_supported(recipe_name, required_capabilities=none) %}
|
|
2
|
+
{% if execute %}
|
|
3
|
+
{% set missing_query %}
|
|
4
|
+
WITH required AS (
|
|
5
|
+
{% if required_capabilities is none %}
|
|
6
|
+
SELECT required_capability AS capability
|
|
7
|
+
FROM ref_recipe_requirements
|
|
8
|
+
WHERE recipe_name = '{{ recipe_name }}'
|
|
9
|
+
{% else %}
|
|
10
|
+
SELECT capability
|
|
11
|
+
FROM (
|
|
12
|
+
VALUES
|
|
13
|
+
{%- for capability in required_capabilities %}
|
|
14
|
+
('{{ capability }}'){% if not loop.last %},{% endif %}
|
|
15
|
+
{%- endfor %}
|
|
16
|
+
) AS values_table(capability)
|
|
17
|
+
{% endif %}
|
|
18
|
+
),
|
|
19
|
+
|
|
20
|
+
manifest AS (
|
|
21
|
+
SELECT *
|
|
22
|
+
FROM {{ source('hin', 'active_contract_adapter_manifest') }}
|
|
23
|
+
),
|
|
24
|
+
|
|
25
|
+
available AS (
|
|
26
|
+
SELECT 'has_accounts' AS capability, bool_or(coalesce(has_accounts, false)) AS is_available FROM manifest
|
|
27
|
+
UNION ALL SELECT 'has_repositories', bool_or(coalesce(has_repositories, false)) FROM manifest
|
|
28
|
+
UNION ALL SELECT 'has_commits', bool_or(coalesce(has_commits, false)) FROM manifest
|
|
29
|
+
UNION ALL SELECT 'has_file_touches', bool_or(coalesce(has_file_touches, false)) FROM manifest
|
|
30
|
+
UNION ALL SELECT 'has_line_touches', bool_or(coalesce(has_line_touches, false)) FROM manifest
|
|
31
|
+
UNION ALL SELECT 'has_pull_requests', bool_or(coalesce(has_pull_requests, false)) FROM manifest
|
|
32
|
+
UNION ALL SELECT 'has_pr_reviews', bool_or(coalesce(has_pr_reviews, false)) FROM manifest
|
|
33
|
+
UNION ALL SELECT 'has_issues', bool_or(coalesce(has_issues, false)) FROM manifest
|
|
34
|
+
UNION ALL SELECT 'has_comments', bool_or(coalesce(has_comments, false)) FROM manifest
|
|
35
|
+
UNION ALL SELECT 'has_stars', bool_or(coalesce(has_stars, false)) FROM manifest
|
|
36
|
+
UNION ALL SELECT 'has_watches', bool_or(coalesce(has_watches, false)) FROM manifest
|
|
37
|
+
UNION ALL SELECT 'has_forks', bool_or(coalesce(has_forks, false)) FROM manifest
|
|
38
|
+
UNION ALL SELECT 'has_follows', bool_or(coalesce(has_follows, false)) FROM manifest
|
|
39
|
+
UNION ALL SELECT 'has_mentions', bool_or(coalesce(has_mentions, false)) FROM manifest
|
|
40
|
+
UNION ALL SELECT 'has_artifact_refs', bool_or(coalesce(has_artifact_refs, false)) FROM manifest
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
SELECT required.capability
|
|
44
|
+
FROM required
|
|
45
|
+
LEFT JOIN available USING (capability)
|
|
46
|
+
WHERE coalesce(available.is_available, false) = false
|
|
47
|
+
{% endset %}
|
|
48
|
+
{% set missing = run_query(missing_query) %}
|
|
49
|
+
{% if missing is not none and missing.rows | length > 0 %}
|
|
50
|
+
{% set missing_names = [] %}
|
|
51
|
+
{% for row in missing.rows %}
|
|
52
|
+
{% do missing_names.append(row[0]) %}
|
|
53
|
+
{% endfor %}
|
|
54
|
+
{{ exceptions.raise_compiler_error(
|
|
55
|
+
'Cannot build `' ~ recipe_name ~ '` from this adapter run. Missing capabilities: ' ~ (missing_names | join(', '))
|
|
56
|
+
) }}
|
|
57
|
+
{% endif %}
|
|
58
|
+
{% endif %}
|
|
59
|
+
{% endmacro %}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{% macro build_incidence(
|
|
2
|
+
left_type,
|
|
3
|
+
right_type,
|
|
4
|
+
edge_types=None,
|
|
5
|
+
left_subtypes=None,
|
|
6
|
+
right_subtypes=None,
|
|
7
|
+
bot_policy='include_bots',
|
|
8
|
+
weight_mode='event_count',
|
|
9
|
+
start_date=None,
|
|
10
|
+
end_date=None
|
|
11
|
+
) %}
|
|
12
|
+
WITH sliced_edges AS (
|
|
13
|
+
{{ slice_edges(
|
|
14
|
+
edge_types=edge_types,
|
|
15
|
+
source_types=[left_type],
|
|
16
|
+
target_types=[right_type],
|
|
17
|
+
source_subtypes=left_subtypes,
|
|
18
|
+
target_subtypes=right_subtypes,
|
|
19
|
+
bot_policy=bot_policy,
|
|
20
|
+
start_date=start_date,
|
|
21
|
+
end_date=end_date
|
|
22
|
+
) }}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
SELECT
|
|
26
|
+
source_node_id AS left_node_id,
|
|
27
|
+
source_node_type AS left_node_type,
|
|
28
|
+
source_node_subtype AS left_node_subtype,
|
|
29
|
+
target_node_id AS right_node_id,
|
|
30
|
+
target_node_type AS right_node_type,
|
|
31
|
+
target_node_subtype AS right_node_subtype,
|
|
32
|
+
edge_id AS evidence_edge_id,
|
|
33
|
+
edge_type AS role,
|
|
34
|
+
occurred_at,
|
|
35
|
+
{% if weight_mode == 'event_count' %}
|
|
36
|
+
coalesce(event_count, 1)
|
|
37
|
+
{% elif weight_mode == 'weight' %}
|
|
38
|
+
coalesce(weight, 1.0)
|
|
39
|
+
{% elif weight_mode == 'binary' %}
|
|
40
|
+
1
|
|
41
|
+
{% else %}
|
|
42
|
+
{{ exceptions.raise_compiler_error('Unknown incidence weight_mode: ' ~ weight_mode) }}
|
|
43
|
+
{% endif %} AS weight,
|
|
44
|
+
'{{ weight_mode }}' AS weight_kind
|
|
45
|
+
FROM sliced_edges
|
|
46
|
+
{% endmacro %}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
{% macro collapse_two_hop_path(
|
|
2
|
+
first_relation,
|
|
3
|
+
second_relation,
|
|
4
|
+
source_col,
|
|
5
|
+
mediator_col,
|
|
6
|
+
second_source_col,
|
|
7
|
+
target_col,
|
|
8
|
+
directed=true
|
|
9
|
+
) %}
|
|
10
|
+
SELECT
|
|
11
|
+
first_hop.{{ source_col }} AS source_node_id,
|
|
12
|
+
second_hop.{{ target_col }} AS target_node_id,
|
|
13
|
+
count(DISTINCT first_hop.{{ mediator_col }}) AS n_contexts,
|
|
14
|
+
list_distinct(list(first_hop.{{ mediator_col }})) AS context_node_ids,
|
|
15
|
+
min(least(first_hop.first_seen_at, second_hop.first_seen_at)) AS first_seen_at,
|
|
16
|
+
max(greatest(first_hop.last_seen_at, second_hop.last_seen_at)) AS last_seen_at
|
|
17
|
+
FROM {{ first_relation }} AS first_hop
|
|
18
|
+
JOIN {{ second_relation }} AS second_hop
|
|
19
|
+
ON first_hop.{{ mediator_col }} = second_hop.{{ second_source_col }}
|
|
20
|
+
{% if directed %}
|
|
21
|
+
WHERE first_hop.{{ source_col }} != second_hop.{{ target_col }}
|
|
22
|
+
{% else %}
|
|
23
|
+
WHERE first_hop.{{ source_col }} < second_hop.{{ target_col }}
|
|
24
|
+
{% endif %}
|
|
25
|
+
GROUP BY first_hop.{{ source_col }}, second_hop.{{ target_col }}
|
|
26
|
+
{% endmacro %}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{% macro filter_bots(source_alias='source_node', target_alias='target_node', policy='include_bots') %}
|
|
2
|
+
(
|
|
3
|
+
{% if policy == 'include_bots' %}
|
|
4
|
+
true
|
|
5
|
+
{% elif policy == 'exclude_bots' %}
|
|
6
|
+
coalesce({{ source_alias }}.node_subtype, '') != 'bot'
|
|
7
|
+
AND coalesce({{ target_alias }}.node_subtype, '') != 'bot'
|
|
8
|
+
{% elif policy == 'only_bots' %}
|
|
9
|
+
(
|
|
10
|
+
coalesce({{ source_alias }}.node_subtype, '') = 'bot'
|
|
11
|
+
OR coalesce({{ target_alias }}.node_subtype, '') = 'bot'
|
|
12
|
+
)
|
|
13
|
+
{% elif policy == 'human_to_bot' %}
|
|
14
|
+
coalesce({{ source_alias }}.node_subtype, '') != 'bot'
|
|
15
|
+
AND coalesce({{ target_alias }}.node_subtype, '') = 'bot'
|
|
16
|
+
{% elif policy == 'bot_to_human' %}
|
|
17
|
+
coalesce({{ source_alias }}.node_subtype, '') = 'bot'
|
|
18
|
+
AND coalesce({{ target_alias }}.node_subtype, '') != 'bot'
|
|
19
|
+
{% else %}
|
|
20
|
+
{{ exceptions.raise_compiler_error('Unknown bot_policy: ' ~ policy) }}
|
|
21
|
+
{% endif %}
|
|
22
|
+
)
|
|
23
|
+
{% endmacro %}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{% macro network_edges(
|
|
2
|
+
relation,
|
|
3
|
+
recipe_name,
|
|
4
|
+
source_node_id,
|
|
5
|
+
source_node_type,
|
|
6
|
+
target_node_id,
|
|
7
|
+
target_node_type,
|
|
8
|
+
edge_type,
|
|
9
|
+
directed,
|
|
10
|
+
weight='NULL',
|
|
11
|
+
weight_kind='NULL',
|
|
12
|
+
n_contexts='NULL',
|
|
13
|
+
n_events='NULL',
|
|
14
|
+
first_seen_at='NULL',
|
|
15
|
+
last_seen_at='NULL',
|
|
16
|
+
time_bin='NULL',
|
|
17
|
+
bot_policy="'include'",
|
|
18
|
+
properties="'{}'"
|
|
19
|
+
) %}
|
|
20
|
+
SELECT
|
|
21
|
+
'{{ recipe_name }}'::TEXT AS recipe_name,
|
|
22
|
+
'1'::TEXT AS recipe_version,
|
|
23
|
+
{{ source_node_id }}::TEXT AS source_node_id,
|
|
24
|
+
{{ source_node_type }}::TEXT AS source_node_type,
|
|
25
|
+
{{ target_node_id }}::TEXT AS target_node_id,
|
|
26
|
+
{{ target_node_type }}::TEXT AS target_node_type,
|
|
27
|
+
{{ directed }}::BOOLEAN AS directed,
|
|
28
|
+
{{ edge_type }}::TEXT AS edge_type,
|
|
29
|
+
{{ weight }}::DOUBLE AS weight,
|
|
30
|
+
{{ weight_kind }}::TEXT AS weight_kind,
|
|
31
|
+
{{ n_contexts }}::INTEGER AS n_contexts,
|
|
32
|
+
{{ n_events }}::INTEGER AS n_events,
|
|
33
|
+
{{ first_seen_at }}::TIMESTAMP AS first_seen_at,
|
|
34
|
+
{{ last_seen_at }}::TIMESTAMP AS last_seen_at,
|
|
35
|
+
{{ time_bin }}::TEXT AS time_bin,
|
|
36
|
+
{{ bot_policy }}::TEXT AS bot_policy,
|
|
37
|
+
{{ properties }} AS properties
|
|
38
|
+
FROM {{ relation }}
|
|
39
|
+
{% endmacro %}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{% macro project_bipartite(
|
|
2
|
+
incidence_relation,
|
|
3
|
+
left_col,
|
|
4
|
+
right_col,
|
|
5
|
+
directed=false,
|
|
6
|
+
weight_mode='shared_count'
|
|
7
|
+
) %}
|
|
8
|
+
SELECT
|
|
9
|
+
a.{{ left_col }} AS source_node_id,
|
|
10
|
+
b.{{ left_col }} AS target_node_id,
|
|
11
|
+
{% if weight_mode == 'shared_count' %}
|
|
12
|
+
count(DISTINCT a.{{ right_col }})::DOUBLE AS weight,
|
|
13
|
+
'shared_count' AS weight_kind,
|
|
14
|
+
{% elif weight_mode == 'event_weighted' %}
|
|
15
|
+
sum(coalesce(a.weight, 1.0) * coalesce(b.weight, 1.0))::DOUBLE AS weight,
|
|
16
|
+
'event_weighted' AS weight_kind,
|
|
17
|
+
{% else %}
|
|
18
|
+
{{ exceptions.raise_compiler_error('Unknown projection weight_mode: ' ~ weight_mode) }}
|
|
19
|
+
{% endif %}
|
|
20
|
+
count(DISTINCT a.{{ right_col }}) AS n_contexts,
|
|
21
|
+
list_distinct(list(a.{{ right_col }})) AS context_node_ids,
|
|
22
|
+
min(least(a.first_seen_at, b.first_seen_at)) AS first_seen_at,
|
|
23
|
+
max(greatest(a.last_seen_at, b.last_seen_at)) AS last_seen_at
|
|
24
|
+
FROM {{ incidence_relation }} AS a
|
|
25
|
+
JOIN {{ incidence_relation }} AS b
|
|
26
|
+
ON a.{{ right_col }} = b.{{ right_col }}
|
|
27
|
+
{% if directed %}
|
|
28
|
+
AND a.{{ left_col }} != b.{{ left_col }}
|
|
29
|
+
{% else %}
|
|
30
|
+
AND a.{{ left_col }} < b.{{ left_col }}
|
|
31
|
+
{% endif %}
|
|
32
|
+
GROUP BY a.{{ left_col }}, b.{{ left_col }}
|
|
33
|
+
{% endmacro %}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{% macro _sql_string_list(values) %}
|
|
2
|
+
{%- for value in values -%}
|
|
3
|
+
'{{ value }}'{% if not loop.last %}, {% endif %}
|
|
4
|
+
{%- endfor -%}
|
|
5
|
+
{% endmacro %}
|
|
6
|
+
|
|
7
|
+
{% macro slice_edges(
|
|
8
|
+
edge_types=None,
|
|
9
|
+
source_types=None,
|
|
10
|
+
target_types=None,
|
|
11
|
+
source_subtypes=None,
|
|
12
|
+
target_subtypes=None,
|
|
13
|
+
bot_policy='include_bots',
|
|
14
|
+
start_date=None,
|
|
15
|
+
end_date=None
|
|
16
|
+
) %}
|
|
17
|
+
SELECT e.*
|
|
18
|
+
FROM {{ ref('hin_edges') }} AS e
|
|
19
|
+
LEFT JOIN {{ ref('hin_nodes') }} AS source_node
|
|
20
|
+
ON source_node.node_id = e.source_node_id
|
|
21
|
+
LEFT JOIN {{ ref('hin_nodes') }} AS target_node
|
|
22
|
+
ON target_node.node_id = e.target_node_id
|
|
23
|
+
WHERE true
|
|
24
|
+
{% if edge_types is not none %}
|
|
25
|
+
AND e.edge_type IN ({{ _sql_string_list(edge_types) }})
|
|
26
|
+
{% endif %}
|
|
27
|
+
{% if source_types is not none %}
|
|
28
|
+
AND e.source_node_type IN ({{ _sql_string_list(source_types) }})
|
|
29
|
+
{% endif %}
|
|
30
|
+
{% if target_types is not none %}
|
|
31
|
+
AND e.target_node_type IN ({{ _sql_string_list(target_types) }})
|
|
32
|
+
{% endif %}
|
|
33
|
+
{% if source_subtypes is not none %}
|
|
34
|
+
AND e.source_node_subtype IN ({{ _sql_string_list(source_subtypes) }})
|
|
35
|
+
{% endif %}
|
|
36
|
+
{% if target_subtypes is not none %}
|
|
37
|
+
AND e.target_node_subtype IN ({{ _sql_string_list(target_subtypes) }})
|
|
38
|
+
{% endif %}
|
|
39
|
+
AND {{ filter_bots('source_node', 'target_node', bot_policy) }}
|
|
40
|
+
AND {{ apply_time_window('e.occurred_at', start_date, end_date) }}
|
|
41
|
+
{% endmacro %}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{{ config(materialized='table') }}
|
|
2
|
+
|
|
3
|
+
SELECT
|
|
4
|
+
account_key AS node_id,
|
|
5
|
+
github_id,
|
|
6
|
+
login,
|
|
7
|
+
coalesce(account_type, 'unknown') AS account_type,
|
|
8
|
+
coalesce(is_bot, false) AS is_bot,
|
|
9
|
+
bot_confidence,
|
|
10
|
+
bot_source,
|
|
11
|
+
created_at,
|
|
12
|
+
observed_at,
|
|
13
|
+
profile_json AS properties
|
|
14
|
+
FROM {{ ref('stg_accounts') }}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{{ config(materialized='table') }}
|
|
2
|
+
|
|
3
|
+
SELECT
|
|
4
|
+
artifact_key AS node_id,
|
|
5
|
+
artifact_type,
|
|
6
|
+
repo_key AS repo_node_id,
|
|
7
|
+
parent_artifact_key AS parent_artifact_node_id,
|
|
8
|
+
github_id,
|
|
9
|
+
number,
|
|
10
|
+
sha,
|
|
11
|
+
url,
|
|
12
|
+
title,
|
|
13
|
+
state,
|
|
14
|
+
file_path,
|
|
15
|
+
start_line,
|
|
16
|
+
end_line,
|
|
17
|
+
created_at,
|
|
18
|
+
updated_at,
|
|
19
|
+
closed_at,
|
|
20
|
+
merged_at,
|
|
21
|
+
observed_at,
|
|
22
|
+
artifact_json AS properties
|
|
23
|
+
FROM {{ ref('stg_artifacts') }}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{{ config(materialized='table') }}
|
|
2
|
+
|
|
3
|
+
WITH manifest AS (
|
|
4
|
+
SELECT *
|
|
5
|
+
FROM {{ ref('stg_adapter_manifest') }}
|
|
6
|
+
),
|
|
7
|
+
|
|
8
|
+
capabilities AS (
|
|
9
|
+
SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
10
|
+
'has_accounts' AS capability, has_accounts AS is_available FROM manifest
|
|
11
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
12
|
+
'has_repositories', has_repositories FROM manifest
|
|
13
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
14
|
+
'has_commits', has_commits FROM manifest
|
|
15
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
16
|
+
'has_file_touches', has_file_touches FROM manifest
|
|
17
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
18
|
+
'has_line_touches', has_line_touches FROM manifest
|
|
19
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
20
|
+
'has_pull_requests', has_pull_requests FROM manifest
|
|
21
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
22
|
+
'has_pr_reviews', has_pr_reviews FROM manifest
|
|
23
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
24
|
+
'has_issues', has_issues FROM manifest
|
|
25
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
26
|
+
'has_comments', has_comments FROM manifest
|
|
27
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
28
|
+
'has_stars', has_stars FROM manifest
|
|
29
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
30
|
+
'has_watches', has_watches FROM manifest
|
|
31
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
32
|
+
'has_forks', has_forks FROM manifest
|
|
33
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
34
|
+
'has_follows', has_follows FROM manifest
|
|
35
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
36
|
+
'has_mentions', has_mentions FROM manifest
|
|
37
|
+
UNION ALL SELECT adapter_run_id, adapter_name, adapter_version, source_name, coverage_start, coverage_end,
|
|
38
|
+
'has_artifact_refs', has_artifact_refs FROM manifest
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
SELECT *
|
|
42
|
+
FROM capabilities
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{{ config(materialized='table') }}
|
|
2
|
+
|
|
3
|
+
WITH relations AS (
|
|
4
|
+
SELECT
|
|
5
|
+
relation_key AS edge_id,
|
|
6
|
+
source_node_key AS source_node_id,
|
|
7
|
+
CASE WHEN source_node_type = 'account' THEN 'user' ELSE source_node_type END AS source_node_type,
|
|
8
|
+
target_node_key AS target_node_id,
|
|
9
|
+
CASE WHEN target_node_type = 'account' THEN 'user' ELSE target_node_type END AS target_node_type,
|
|
10
|
+
relation_type AS edge_type,
|
|
11
|
+
relation_subtype AS edge_subtype,
|
|
12
|
+
directed,
|
|
13
|
+
occurred_at,
|
|
14
|
+
observed_at,
|
|
15
|
+
valid_from,
|
|
16
|
+
valid_to,
|
|
17
|
+
weight,
|
|
18
|
+
event_count,
|
|
19
|
+
adapter_run_id,
|
|
20
|
+
source_record_id,
|
|
21
|
+
properties
|
|
22
|
+
FROM {{ ref('stg_relations') }}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
SELECT
|
|
26
|
+
r.edge_id,
|
|
27
|
+
r.source_node_id,
|
|
28
|
+
r.source_node_type,
|
|
29
|
+
sn.node_subtype AS source_node_subtype,
|
|
30
|
+
r.target_node_id,
|
|
31
|
+
r.target_node_type,
|
|
32
|
+
tn.node_subtype AS target_node_subtype,
|
|
33
|
+
r.edge_type,
|
|
34
|
+
r.edge_subtype,
|
|
35
|
+
r.directed,
|
|
36
|
+
r.occurred_at,
|
|
37
|
+
r.observed_at,
|
|
38
|
+
r.valid_from,
|
|
39
|
+
r.valid_to,
|
|
40
|
+
r.weight,
|
|
41
|
+
r.event_count,
|
|
42
|
+
CASE
|
|
43
|
+
WHEN r.source_node_type = 'repo' THEN r.source_node_id
|
|
44
|
+
WHEN r.target_node_type = 'repo' THEN r.target_node_id
|
|
45
|
+
END AS context_repo_node_id,
|
|
46
|
+
CASE
|
|
47
|
+
WHEN r.source_node_type = 'artifact' THEN r.source_node_id
|
|
48
|
+
WHEN r.target_node_type = 'artifact' THEN r.target_node_id
|
|
49
|
+
END AS context_artifact_node_id,
|
|
50
|
+
r.adapter_run_id,
|
|
51
|
+
r.source_record_id,
|
|
52
|
+
r.properties
|
|
53
|
+
FROM relations AS r
|
|
54
|
+
LEFT JOIN {{ ref('hin_nodes') }} AS sn
|
|
55
|
+
ON sn.node_id = r.source_node_id
|
|
56
|
+
LEFT JOIN {{ ref('hin_nodes') }} AS tn
|
|
57
|
+
ON tn.node_id = r.target_node_id
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{{ config(materialized='table') }}
|
|
2
|
+
|
|
3
|
+
SELECT
|
|
4
|
+
node_id,
|
|
5
|
+
'user' AS node_type,
|
|
6
|
+
account_type AS node_subtype,
|
|
7
|
+
node_id AS natural_key,
|
|
8
|
+
login AS display_name,
|
|
9
|
+
created_at,
|
|
10
|
+
NULL::TIMESTAMP AS updated_at,
|
|
11
|
+
observed_at,
|
|
12
|
+
false AS is_stub,
|
|
13
|
+
properties
|
|
14
|
+
FROM {{ ref('hin_accounts') }}
|
|
15
|
+
|
|
16
|
+
UNION ALL
|
|
17
|
+
|
|
18
|
+
SELECT
|
|
19
|
+
node_id,
|
|
20
|
+
'repo' AS node_type,
|
|
21
|
+
CASE WHEN is_fork THEN 'fork' ELSE 'repository' END AS node_subtype,
|
|
22
|
+
full_name AS natural_key,
|
|
23
|
+
full_name AS display_name,
|
|
24
|
+
created_at,
|
|
25
|
+
NULL::TIMESTAMP AS updated_at,
|
|
26
|
+
observed_at,
|
|
27
|
+
false AS is_stub,
|
|
28
|
+
properties
|
|
29
|
+
FROM {{ ref('hin_repositories') }}
|
|
30
|
+
|
|
31
|
+
UNION ALL
|
|
32
|
+
|
|
33
|
+
SELECT
|
|
34
|
+
node_id,
|
|
35
|
+
'artifact' AS node_type,
|
|
36
|
+
artifact_type AS node_subtype,
|
|
37
|
+
node_id AS natural_key,
|
|
38
|
+
coalesce(title, node_id) AS display_name,
|
|
39
|
+
created_at,
|
|
40
|
+
updated_at,
|
|
41
|
+
observed_at,
|
|
42
|
+
false AS is_stub,
|
|
43
|
+
properties
|
|
44
|
+
FROM {{ ref('hin_artifacts') }}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{{ config(materialized='table') }}
|
|
2
|
+
|
|
3
|
+
SELECT
|
|
4
|
+
repo_key AS node_id,
|
|
5
|
+
github_id,
|
|
6
|
+
full_name,
|
|
7
|
+
owner_account_key AS owner_account_node_id,
|
|
8
|
+
primary_language,
|
|
9
|
+
coalesce(is_fork, false) AS is_fork,
|
|
10
|
+
forked_from_repo_key AS forked_from_repo_node_id,
|
|
11
|
+
created_at,
|
|
12
|
+
archived_at,
|
|
13
|
+
observed_at,
|
|
14
|
+
repo_json AS properties
|
|
15
|
+
FROM {{ ref('stg_repositories') }}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{{ config(materialized='table') }}
|
|
2
|
+
|
|
3
|
+
WITH contains_edges AS (
|
|
4
|
+
{{ slice_edges(
|
|
5
|
+
edge_types=['contains'],
|
|
6
|
+
source_types=['repo'],
|
|
7
|
+
target_types=['artifact']
|
|
8
|
+
) }}
|
|
9
|
+
),
|
|
10
|
+
|
|
11
|
+
affiliations AS (
|
|
12
|
+
SELECT
|
|
13
|
+
incidence.left_node_id AS user_node_id,
|
|
14
|
+
contains_edges.source_node_id AS repo_node_id,
|
|
15
|
+
min(incidence.occurred_at) AS first_seen_at,
|
|
16
|
+
max(incidence.occurred_at) AS last_seen_at,
|
|
17
|
+
sum(incidence.weight)::DOUBLE AS weight,
|
|
18
|
+
count(DISTINCT incidence.evidence_edge_id) AS n_events,
|
|
19
|
+
list_distinct(list(incidence.role)) AS roles
|
|
20
|
+
FROM {{ ref('int_user_artifact_incidence') }} AS incidence
|
|
21
|
+
JOIN contains_edges
|
|
22
|
+
ON contains_edges.target_node_id = incidence.right_node_id
|
|
23
|
+
GROUP BY incidence.left_node_id, contains_edges.source_node_id
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
SELECT *
|
|
27
|
+
FROM affiliations
|