hinge 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hinge/__init__.py +3 -0
- hinge/config/__init__.py +0 -0
- hinge/config/logging_setup.py +63 -0
- hinge/config/settings.py +23 -0
- hinge/config/types.yaml +42 -0
- hinge/dbt/.user.yml +1 -0
- hinge/dbt/dbt_project.yml +18 -0
- hinge/dbt/macros/apply_time_window.sql +11 -0
- hinge/dbt/macros/assert_recipe_supported.sql +59 -0
- hinge/dbt/macros/build_incidence.sql +46 -0
- hinge/dbt/macros/collapse_path.sql +26 -0
- hinge/dbt/macros/filter_bots.sql +23 -0
- hinge/dbt/macros/network_edges.sql +39 -0
- hinge/dbt/macros/project_bipartite.sql +33 -0
- hinge/dbt/macros/slice_edges.sql +41 -0
- hinge/dbt/models/hin/hin_accounts.sql +14 -0
- hinge/dbt/models/hin/hin_artifacts.sql +23 -0
- hinge/dbt/models/hin/hin_capabilities.sql +42 -0
- hinge/dbt/models/hin/hin_edges.sql +57 -0
- hinge/dbt/models/hin/hin_nodes.sql +44 -0
- hinge/dbt/models/hin/hin_repositories.sql +15 -0
- hinge/dbt/models/intermediate/int_developer_repo_affiliation.sql +27 -0
- hinge/dbt/models/intermediate/int_user_artifact_incidence.sql +21 -0
- hinge/dbt/models/networks/artifact_reference.sql +27 -0
- hinge/dbt/models/networks/co_commit_user_user.sql +48 -0
- hinge/dbt/models/networks/co_edit_file_user_user.sql +48 -0
- hinge/dbt/models/networks/co_edit_line_user_user.sql +48 -0
- hinge/dbt/models/networks/dev_interaction.sql +98 -0
- hinge/dbt/models/networks/developer_repo_affiliation.sql +20 -0
- hinge/dbt/models/networks/follow_user_user.sql +27 -0
- hinge/dbt/models/networks/fork_repo_repo.sql +27 -0
- hinge/dbt/models/networks/issue_co_participation.sql +50 -0
- hinge/dbt/models/networks/issue_participation.sql +40 -0
- hinge/dbt/models/networks/pr_author_reviewer.sql +66 -0
- hinge/dbt/models/networks/pr_participation.sql +40 -0
- hinge/dbt/models/networks/pr_reviewer_coreview.sql +48 -0
- hinge/dbt/models/networks/repo_shared_contributors.sql +41 -0
- hinge/dbt/models/networks/star_user_repo.sql +27 -0
- hinge/dbt/models/networks/user_mention_user.sql +60 -0
- hinge/dbt/models/networks/watch_user_repo.sql +27 -0
- hinge/dbt/models/sources/sources.yml +63 -0
- hinge/dbt/models/staging/stg_accounts.sql +4 -0
- hinge/dbt/models/staging/stg_adapter_manifest.sql +4 -0
- hinge/dbt/models/staging/stg_artifacts.sql +4 -0
- hinge/dbt/models/staging/stg_relations.sql +4 -0
- hinge/dbt/models/staging/stg_repositories.sql +4 -0
- hinge/dbt/profiles.yml +9 -0
- hinge/dbt/seeds/ref_recipe_requirements.csv +35 -0
- hinge/frontends/__init__.py +0 -0
- hinge/frontends/cli/__init__.py +3 -0
- hinge/frontends/cli/app.py +189 -0
- hinge/frontends/lib.py +148 -0
- hinge/frontends/tui/__init__.py +7 -0
- hinge/frontends/web/__init__.py +6 -0
- hinge/kernel/__init__.py +0 -0
- hinge/kernel/projection/__init__.py +9 -0
- hinge/kernel/projection/projected_graph.py +32 -0
- hinge/kernel/projection/projection_spec.py +35 -0
- hinge/kernel/protocols/__init__.py +24 -0
- hinge/kernel/protocols/dataset_view.py +30 -0
- hinge/kernel/protocols/exporter_stage.py +24 -0
- hinge/kernel/protocols/projection_stage.py +36 -0
- hinge/kernel/protocols/reader_stage.py +52 -0
- hinge/kernel/protocols/store_stage.py +71 -0
- hinge/kernel/registry.py +71 -0
- hinge/kernel/runner.py +236 -0
- hinge/kernel/schema/__init__.py +12 -0
- hinge/kernel/schema/hin_schema.py +38 -0
- hinge/kernel/schema/schema_violation.py +21 -0
- hinge/kernel/schema/typed_edge.py +20 -0
- hinge/kernel/schema/typed_node.py +17 -0
- hinge/stages/__init__.py +0 -0
- hinge/stages/exporters/__init__.py +0 -0
- hinge/stages/exporters/csv_exporter.py +77 -0
- hinge/stages/exporters/dot_exporter.py +13 -0
- hinge/stages/exporters/gexf_exporter.py +13 -0
- hinge/stages/exporters/gml_exporter.py +79 -0
- hinge/stages/exporters/graphml_exporter.py +13 -0
- hinge/stages/exporters/jsonld_exporter.py +13 -0
- hinge/stages/exporters/networkx_exporter.py +57 -0
- hinge/stages/exporters/parquet_exporter.py +13 -0
- hinge/stages/projection/__init__.py +0 -0
- hinge/stages/projection/dbt_projection.py +342 -0
- hinge/stages/projection/specs/__init__.py +0 -0
- hinge/stages/projection/specs/artifact_reference.py +14 -0
- hinge/stages/projection/specs/co_commit_user_user.py +14 -0
- hinge/stages/projection/specs/co_edit_file_user_user.py +14 -0
- hinge/stages/projection/specs/co_edit_line_user_user.py +14 -0
- hinge/stages/projection/specs/dev_interaction.py +26 -0
- hinge/stages/projection/specs/developer_repo_affiliation.py +15 -0
- hinge/stages/projection/specs/follow_user_user.py +14 -0
- hinge/stages/projection/specs/fork_repo_repo.py +14 -0
- hinge/stages/projection/specs/issue_co_participation.py +14 -0
- hinge/stages/projection/specs/issue_participation.py +14 -0
- hinge/stages/projection/specs/pr_author_reviewer.py +14 -0
- hinge/stages/projection/specs/pr_participation.py +14 -0
- hinge/stages/projection/specs/pr_reviewer_coreview.py +14 -0
- hinge/stages/projection/specs/repo_shared_contributors.py +14 -0
- hinge/stages/projection/specs/star_user_repo.py +15 -0
- hinge/stages/projection/specs/user_mention_user.py +14 -0
- hinge/stages/projection/specs/watch_user_repo.py +14 -0
- hinge/stages/readers/__init__.py +0 -0
- hinge/stages/readers/_numfocus_contract_adapter.py +314 -0
- hinge/stages/readers/numfocus_reader.py +69 -0
- hinge/stages/store/__init__.py +0 -0
- hinge/stages/store/duckdb_store.py +466 -0
- hinge-0.1.0.dist-info/METADATA +309 -0
- hinge-0.1.0.dist-info/RECORD +111 -0
- hinge-0.1.0.dist-info/WHEEL +4 -0
- hinge-0.1.0.dist-info/entry_points.txt +35 -0
- hinge-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-co-commit",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-user co-commit network. Connects users who authored, "
|
|
9
|
+
"committed, or coauthored the same commit when commit-level data exists."
|
|
10
|
+
),
|
|
11
|
+
model_name="co_commit_user_user",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["co_committed"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-co-edit-file",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-user co-edit network. Connects users who touched the "
|
|
9
|
+
"same file when commit/file-touch data exists."
|
|
10
|
+
),
|
|
11
|
+
model_name="co_edit_file_user_user",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["co_edited_file"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-co-edit-line",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-user line-level co-edit network. Connects users who "
|
|
9
|
+
"touched the same line span when line-touch data exists."
|
|
10
|
+
),
|
|
11
|
+
model_name="co_edit_line_user_user",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["co_edited_line"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Projection spec: user-user-repo-collaboration.
|
|
2
|
+
|
|
3
|
+
The SQL lives at ``hinge/dbt/models/networks/dev_interaction.sql``.
|
|
4
|
+
This module exposes the spec under the ``hinge.projection_specs`` entry-point
|
|
5
|
+
group so the registry can discover it like any other plugin.
|
|
6
|
+
|
|
7
|
+
To add a new projection: copy this file, point ``model_name`` at your SQL
|
|
8
|
+
file's stem, register the entry-point in pyproject.toml.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
14
|
+
|
|
15
|
+
SPEC = ProjectionSpec(
|
|
16
|
+
name="user-user-repo-collaboration",
|
|
17
|
+
description=(
|
|
18
|
+
"Undirected user-to-user collaboration graph. Two developers are "
|
|
19
|
+
"connected by a 'collaborates_with' edge if they both made code "
|
|
20
|
+
"contributions (opened/reviewed/merged PRs, commented, pushed) to "
|
|
21
|
+
"the same repository. Edge attrs carry the count and list of shared repos."
|
|
22
|
+
),
|
|
23
|
+
model_name="dev_interaction",
|
|
24
|
+
output_node_types=["user"],
|
|
25
|
+
output_edge_types=["collaborates_with"],
|
|
26
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-repo-affiliation",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-repository affiliation network. Connects active "
|
|
9
|
+
"contributors to repositories through issue, PR, review, comment, "
|
|
10
|
+
"push, and lifecycle participation, excluding passive star/watch ties."
|
|
11
|
+
),
|
|
12
|
+
model_name="developer_repo_affiliation",
|
|
13
|
+
output_node_types=["user", "repo"],
|
|
14
|
+
output_edge_types=["affiliated_with"],
|
|
15
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-follow",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user follow network. Emits directed user -> user 'follows' "
|
|
9
|
+
"edges when the active adapter collected follower relations."
|
|
10
|
+
),
|
|
11
|
+
model_name="follow_user_user",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["follows"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="repo-repo-fork",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook repository fork network. Emits directed fork repo -> "
|
|
9
|
+
"upstream repo 'fork_of' edges when fork metadata is available."
|
|
10
|
+
),
|
|
11
|
+
model_name="fork_repo_repo",
|
|
12
|
+
output_node_types=["repo"],
|
|
13
|
+
output_edge_types=["fork_of"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-issue-co-participation",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-user issue participation network. Connects users who "
|
|
9
|
+
"opened, commented on, or closed the same issue."
|
|
10
|
+
),
|
|
11
|
+
model_name="issue_co_participation",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["co_participates_issue"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-issue-participation",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-issue participation network. Emits user -> issue "
|
|
9
|
+
"edges carrying roles such as opener, commenter, and closer."
|
|
10
|
+
),
|
|
11
|
+
model_name="issue_participation",
|
|
12
|
+
output_node_types=["user", "artifact"],
|
|
13
|
+
output_edge_types=["participated_in_issue"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-pr-author-reviewer",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook directed pull-request review network. Connects a PR opener "
|
|
9
|
+
"to users who reviewed that pull request."
|
|
10
|
+
),
|
|
11
|
+
model_name="pr_author_reviewer",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["reviewed_pr_from"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-pr-participation",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-pull-request participation network. Emits user -> PR "
|
|
9
|
+
"edges carrying roles such as opener, reviewer, commenter, and merger."
|
|
10
|
+
),
|
|
11
|
+
model_name="pr_participation",
|
|
12
|
+
output_node_types=["user", "artifact"],
|
|
13
|
+
output_edge_types=["participated_in_pr"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-pr-co-review",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook reviewer-reviewer PR network. Connects users who reviewed "
|
|
9
|
+
"the same pull request, weighted by shared PR count."
|
|
10
|
+
),
|
|
11
|
+
model_name="pr_reviewer_coreview",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["co_reviewed_pr"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="repo-repo-shared-contributors",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook repo-repo projection. Connects repositories that share at "
|
|
9
|
+
"least one active contributor, weighted by number of shared users."
|
|
10
|
+
),
|
|
11
|
+
model_name="repo_shared_contributors",
|
|
12
|
+
output_node_types=["repo"],
|
|
13
|
+
output_edge_types=["shared_contributors"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-repo-star",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-repository attention network. Emits one directed "
|
|
9
|
+
"user -> repo 'starred' edge for each GitHub star event available "
|
|
10
|
+
"in the active adapter run."
|
|
11
|
+
),
|
|
12
|
+
model_name="star_user_repo",
|
|
13
|
+
output_node_types=["user", "repo"],
|
|
14
|
+
output_edge_types=["starred"],
|
|
15
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-user-mention",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user mention network. Collapses user -> comment artifact -> "
|
|
9
|
+
"mentioned user paths into directed user-user mention edges."
|
|
10
|
+
),
|
|
11
|
+
model_name="user_mention_user",
|
|
12
|
+
output_node_types=["user"],
|
|
13
|
+
output_edge_types=["mentions_user"],
|
|
14
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from hinge.kernel.projection.projection_spec import ProjectionSpec
|
|
4
|
+
|
|
5
|
+
SPEC = ProjectionSpec(
|
|
6
|
+
name="user-repo-watch",
|
|
7
|
+
description=(
|
|
8
|
+
"Cookbook user-repository watch/subscription network. Emits directed "
|
|
9
|
+
"user -> repo 'watches' edges when the active adapter collected watch data."
|
|
10
|
+
),
|
|
11
|
+
model_name="watch_user_repo",
|
|
12
|
+
output_node_types=["user", "repo"],
|
|
13
|
+
output_edge_types=["watches"],
|
|
14
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""DuckDB contract-table adapter for the NumFocus Actions JSONL format.
|
|
2
|
+
|
|
3
|
+
This module is deliberately source-specific: it knows the JSON paths in the
|
|
4
|
+
NumFocus scrape (``actor.login``, ``details.pull_request.id``, etc.) and maps
|
|
5
|
+
those fields into the source-agnostic ``contract_*`` tables owned by
|
|
6
|
+
``DuckDBStore``. It is the bulk-ingest engine behind ``NumFocusReader``.
|
|
7
|
+
|
|
8
|
+
The reusable/core layer starts after this adapter has populated contract tables:
|
|
9
|
+
``contract_*`` -> canonical HIN views/models -> dbt projections. New data
|
|
10
|
+
sources should copy this shape, not this source mapping.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
import duckdb
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def ingest(
|
|
22
|
+
conn: duckdb.DuckDBPyConnection,
|
|
23
|
+
dataset_id: str,
|
|
24
|
+
path: str | Path,
|
|
25
|
+
*,
|
|
26
|
+
limit: int | None = None,
|
|
27
|
+
) -> tuple[int, int, int]:
|
|
28
|
+
source_path = str(Path(path))
|
|
29
|
+
row_limit_sql = "" if limit is None else f" LIMIT {int(limit)}"
|
|
30
|
+
conn.execute("DROP TABLE IF EXISTS _numfocus_raw")
|
|
31
|
+
conn.execute("DROP TABLE IF EXISTS _numfocus_flat")
|
|
32
|
+
conn.execute(
|
|
33
|
+
"CREATE TEMP TABLE _numfocus_raw AS "
|
|
34
|
+
"SELECT row_number() OVER () AS source_line, json "
|
|
35
|
+
"FROM read_json_objects(?)" + row_limit_sql,
|
|
36
|
+
[source_path],
|
|
37
|
+
)
|
|
38
|
+
conn.execute(
|
|
39
|
+
"CREATE TEMP TABLE _numfocus_flat AS "
|
|
40
|
+
"SELECT "
|
|
41
|
+
" json, source_line, "
|
|
42
|
+
" json_extract_string(json, '$.action') AS action, "
|
|
43
|
+
" json_extract_string(json, '$.event_id') AS event_id, "
|
|
44
|
+
" try_cast(replace(json_extract_string(json, '$.date'), 'Z', '') AS TIMESTAMP) AS occurred_at, "
|
|
45
|
+
" try_cast(json_extract_string(json, '$.actor.id') AS BIGINT) AS actor_id, "
|
|
46
|
+
" json_extract_string(json, '$.actor.login') AS actor_login, "
|
|
47
|
+
" try_cast(json_extract_string(json, '$.repository.id') AS BIGINT) AS repo_id, "
|
|
48
|
+
" json_extract_string(json, '$.repository.name') AS repo_full_name, "
|
|
49
|
+
" json_extract_string(json, '$.repository.organisation') AS org_login, "
|
|
50
|
+
" try_cast(json_extract_string(json, '$.repository.organisation_id') AS BIGINT) AS org_id, "
|
|
51
|
+
" try_cast(json_extract_string(json, '$.details.pull_request.id') AS BIGINT) AS pr_id, "
|
|
52
|
+
" try_cast(json_extract_string(json, '$.details.pull_request.number') AS INTEGER) AS pr_number, "
|
|
53
|
+
" json_extract_string(json, '$.details.pull_request.title') AS pr_title, "
|
|
54
|
+
" json_extract_string(json, '$.details.pull_request.state') AS pr_state, "
|
|
55
|
+
" try_cast(json_extract_string(json, '$.details.pull_request.author.id') AS BIGINT) AS pr_author_id, "
|
|
56
|
+
" json_extract_string(json, '$.details.pull_request.author.login') AS pr_author_login, "
|
|
57
|
+
" try_cast(replace(json_extract_string(json, '$.details.pull_request.created_date'), 'Z', '') AS TIMESTAMP) AS pr_created_at, "
|
|
58
|
+
" try_cast(replace(json_extract_string(json, '$.details.pull_request.updated_date'), 'Z', '') AS TIMESTAMP) AS pr_updated_at, "
|
|
59
|
+
" try_cast(replace(json_extract_string(json, '$.details.pull_request.closed_date'), 'Z', '') AS TIMESTAMP) AS pr_closed_at, "
|
|
60
|
+
" try_cast(replace(json_extract_string(json, '$.details.pull_request.merged_date'), 'Z', '') AS TIMESTAMP) AS pr_merged_at, "
|
|
61
|
+
" try_cast(json_extract_string(json, '$.details.review.id') AS BIGINT) AS review_id, "
|
|
62
|
+
" try_cast(replace(json_extract_string(json, '$.details.review.submitted_date'), 'Z', '') AS TIMESTAMP) AS review_submitted_at, "
|
|
63
|
+
" try_cast(replace(json_extract_string(json, '$.details.review.updated_date'), 'Z', '') AS TIMESTAMP) AS review_updated_at, "
|
|
64
|
+
" try_cast(json_extract_string(json, '$.details.comment.id') AS BIGINT) AS comment_id, "
|
|
65
|
+
" try_cast(json_extract_string(json, '$.details.comment.position') AS INTEGER) AS comment_position, "
|
|
66
|
+
" try_cast(json_extract_string(json, '$.details.comment.parent_comment_id') AS BIGINT) AS parent_comment_id, "
|
|
67
|
+
" try_cast(json_extract_string(json, '$.details.issue.id') AS BIGINT) AS issue_id, "
|
|
68
|
+
" try_cast(json_extract_string(json, '$.details.issue.number') AS INTEGER) AS issue_number, "
|
|
69
|
+
" json_extract_string(json, '$.details.issue.title') AS issue_title, "
|
|
70
|
+
" json_extract_string(json, '$.details.issue.state') AS issue_state, "
|
|
71
|
+
" try_cast(json_extract_string(json, '$.details.issue.author.id') AS BIGINT) AS issue_author_id, "
|
|
72
|
+
" json_extract_string(json, '$.details.issue.author.login') AS issue_author_login, "
|
|
73
|
+
" try_cast(replace(json_extract_string(json, '$.details.issue.created_date'), 'Z', '') AS TIMESTAMP) AS issue_created_at, "
|
|
74
|
+
" try_cast(replace(json_extract_string(json, '$.details.issue.updated_date'), 'Z', '') AS TIMESTAMP) AS issue_updated_at, "
|
|
75
|
+
" try_cast(replace(json_extract_string(json, '$.details.issue.closed_date'), 'Z', '') AS TIMESTAMP) AS issue_closed_at, "
|
|
76
|
+
" json_extract_string(json, '$.details.push.id') AS push_id, "
|
|
77
|
+
" json_extract_string(json, '$.details.push.ref') AS push_ref, "
|
|
78
|
+
" try_cast(json_extract_string(json, '$.details.push.commits') AS INTEGER) AS push_commits, "
|
|
79
|
+
" try_cast(json_extract_string(json, '$.details.release.id') AS BIGINT) AS release_id, "
|
|
80
|
+
" json_extract_string(json, '$.details.release.name') AS release_name, "
|
|
81
|
+
" json_extract_string(json, '$.details.release.tag') AS release_tag, "
|
|
82
|
+
" try_cast(json_extract_string(json, '$.details.release.author.id') AS BIGINT) AS release_author_id, "
|
|
83
|
+
" json_extract_string(json, '$.details.release.author.login') AS release_author_login, "
|
|
84
|
+
" try_cast(replace(json_extract_string(json, '$.details.release.created_date'), 'Z', '') AS TIMESTAMP) AS release_created_at, "
|
|
85
|
+
" try_cast(json_extract_string(json, '$.details.fork.id') AS BIGINT) AS fork_repo_id, "
|
|
86
|
+
" json_extract_string(json, '$.details.fork.name') AS fork_full_name "
|
|
87
|
+
"FROM _numfocus_raw"
|
|
88
|
+
)
|
|
89
|
+
_insert_contract_accounts(conn, dataset_id)
|
|
90
|
+
_insert_contract_repositories(conn, dataset_id)
|
|
91
|
+
_insert_contract_artifacts(conn, dataset_id)
|
|
92
|
+
_insert_contract_relations(conn, dataset_id)
|
|
93
|
+
_insert_contract_manifest(conn, dataset_id, source_path)
|
|
94
|
+
_backfill_graph_from_hin(conn, dataset_id)
|
|
95
|
+
record_count = _scalar(conn.execute("SELECT COUNT(*) FROM _numfocus_flat").fetchone())
|
|
96
|
+
node_count = _scalar(
|
|
97
|
+
conn.execute("SELECT COUNT(*) FROM nodes WHERE dataset_id = ?", [dataset_id]).fetchone()
|
|
98
|
+
)
|
|
99
|
+
edge_count = _scalar(
|
|
100
|
+
conn.execute("SELECT COUNT(*) FROM edges WHERE dataset_id = ?", [dataset_id]).fetchone()
|
|
101
|
+
)
|
|
102
|
+
return int(record_count), int(node_count), int(edge_count)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _scalar(row: tuple[Any, ...] | None) -> Any:
|
|
106
|
+
if row is None:
|
|
107
|
+
raise RuntimeError("expected one row from scalar query, got none")
|
|
108
|
+
return row[0]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _insert_contract_accounts(conn: duckdb.DuckDBPyConnection, dataset_id: str) -> None:
|
|
112
|
+
conn.execute(
|
|
113
|
+
"INSERT OR REPLACE INTO contract_accounts "
|
|
114
|
+
"SELECT ?, account_key, 'github', github_id, login, account_type, is_bot, "
|
|
115
|
+
" CASE WHEN is_bot THEN 0.8 ELSE 0.0 END, 'heuristic', NULL, NULL, "
|
|
116
|
+
" to_json({'login': login, 'github_id': github_id}), observed_at, ? "
|
|
117
|
+
"FROM ("
|
|
118
|
+
" SELECT DISTINCT "
|
|
119
|
+
" 'gh:user:' || coalesce(cast(actor_id AS TEXT), actor_login) AS account_key,"
|
|
120
|
+
" actor_id AS github_id, actor_login AS login,"
|
|
121
|
+
" CASE WHEN regexp_matches(lower(actor_login), '(\\[bot\\]|_bot)$') THEN 'bot' ELSE 'human' END AS account_type,"
|
|
122
|
+
" regexp_matches(lower(actor_login), '(\\[bot\\]|_bot)$') AS is_bot, occurred_at AS observed_at "
|
|
123
|
+
" FROM _numfocus_flat WHERE actor_login IS NOT NULL "
|
|
124
|
+
" UNION "
|
|
125
|
+
" SELECT DISTINCT 'gh:org:' || coalesce(cast(org_id AS TEXT), org_login), org_id, org_login,"
|
|
126
|
+
" 'organization', FALSE, occurred_at "
|
|
127
|
+
" FROM _numfocus_flat WHERE org_login IS NOT NULL "
|
|
128
|
+
" UNION "
|
|
129
|
+
" SELECT DISTINCT 'gh:user:' || coalesce(cast(pr_author_id AS TEXT), pr_author_login),"
|
|
130
|
+
" pr_author_id, pr_author_login,"
|
|
131
|
+
" CASE WHEN regexp_matches(lower(pr_author_login), '(\\[bot\\]|_bot)$') THEN 'bot' ELSE 'human' END,"
|
|
132
|
+
" regexp_matches(lower(pr_author_login), '(\\[bot\\]|_bot)$'), occurred_at "
|
|
133
|
+
" FROM _numfocus_flat WHERE pr_author_login IS NOT NULL "
|
|
134
|
+
" UNION "
|
|
135
|
+
" SELECT DISTINCT 'gh:user:' || coalesce(cast(issue_author_id AS TEXT), issue_author_login),"
|
|
136
|
+
" issue_author_id, issue_author_login,"
|
|
137
|
+
" CASE WHEN regexp_matches(lower(issue_author_login), '(\\[bot\\]|_bot)$') THEN 'bot' ELSE 'human' END,"
|
|
138
|
+
" regexp_matches(lower(issue_author_login), '(\\[bot\\]|_bot)$'), occurred_at "
|
|
139
|
+
" FROM _numfocus_flat WHERE issue_author_login IS NOT NULL "
|
|
140
|
+
" UNION "
|
|
141
|
+
" SELECT DISTINCT 'gh:user:' || coalesce(cast(release_author_id AS TEXT), release_author_login),"
|
|
142
|
+
" release_author_id, release_author_login,"
|
|
143
|
+
" CASE WHEN regexp_matches(lower(release_author_login), '(\\[bot\\]|_bot)$') THEN 'bot' ELSE 'human' END,"
|
|
144
|
+
" regexp_matches(lower(release_author_login), '(\\[bot\\]|_bot)$'), occurred_at "
|
|
145
|
+
" FROM _numfocus_flat WHERE release_author_login IS NOT NULL "
|
|
146
|
+
") a(account_key, github_id, login, account_type, is_bot, observed_at)",
|
|
147
|
+
[dataset_id, dataset_id],
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _insert_contract_repositories(conn: duckdb.DuckDBPyConnection, dataset_id: str) -> None:
|
|
152
|
+
conn.execute(
|
|
153
|
+
"INSERT OR REPLACE INTO contract_repositories "
|
|
154
|
+
"SELECT ?, repo_key, 'github', github_id, full_name, owner_account_key, name, NULL, NULL,"
|
|
155
|
+
" is_fork, forked_from_repo_key, NULL, NULL, NULL, NULL, NULL, NULL,"
|
|
156
|
+
" to_json({'full_name': full_name, 'github_id': github_id}), observed_at, ? "
|
|
157
|
+
"FROM ("
|
|
158
|
+
" SELECT DISTINCT "
|
|
159
|
+
" 'gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name) AS repo_key, repo_id AS github_id,"
|
|
160
|
+
" repo_full_name AS full_name, 'gh:org:' || coalesce(cast(org_id AS TEXT), org_login) AS owner_account_key,"
|
|
161
|
+
" regexp_replace(repo_full_name, '^.*/', '') AS name, FALSE AS is_fork, NULL AS forked_from_repo_key, occurred_at AS observed_at "
|
|
162
|
+
" FROM _numfocus_flat WHERE repo_full_name IS NOT NULL "
|
|
163
|
+
" UNION "
|
|
164
|
+
" SELECT DISTINCT "
|
|
165
|
+
" 'gh:repo:' || coalesce(cast(fork_repo_id AS TEXT), fork_full_name), fork_repo_id, fork_full_name, NULL,"
|
|
166
|
+
" regexp_replace(fork_full_name, '^.*/', ''), TRUE,"
|
|
167
|
+
" 'gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name), occurred_at "
|
|
168
|
+
" FROM _numfocus_flat WHERE fork_full_name IS NOT NULL OR fork_repo_id IS NOT NULL "
|
|
169
|
+
") r(repo_key, github_id, full_name, owner_account_key, name, is_fork, forked_from_repo_key, observed_at)",
|
|
170
|
+
[dataset_id, dataset_id],
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _insert_contract_artifacts(conn: duckdb.DuckDBPyConnection, dataset_id: str) -> None:
|
|
175
|
+
conn.execute(
|
|
176
|
+
"INSERT OR REPLACE INTO contract_artifacts "
|
|
177
|
+
"WITH base AS ("
|
|
178
|
+
" SELECT *, 'gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name) AS repo_key "
|
|
179
|
+
" FROM _numfocus_flat"
|
|
180
|
+
"), artifacts AS ("
|
|
181
|
+
" SELECT 'gh:artifact:pull_request:' || coalesce(cast(pr_id AS TEXT), repo_key || ':' || cast(pr_number AS TEXT)) AS artifact_key,"
|
|
182
|
+
" 'pull_request' AS artifact_type, repo_key, NULL AS parent_artifact_key, pr_id AS github_id, pr_number AS number, NULL AS sha,"
|
|
183
|
+
" pr_title AS title, pr_state AS state, pr_created_at AS created_at, pr_updated_at AS updated_at, pr_closed_at AS closed_at, pr_merged_at AS merged_at,"
|
|
184
|
+
" json_extract(json, '$.details.pull_request') AS artifact_json, occurred_at AS observed_at "
|
|
185
|
+
" FROM base WHERE pr_id IS NOT NULL OR pr_number IS NOT NULL "
|
|
186
|
+
" UNION ALL "
|
|
187
|
+
" SELECT 'gh:artifact:issue:' || coalesce(cast(issue_id AS TEXT), repo_key || ':' || cast(issue_number AS TEXT)),"
|
|
188
|
+
" 'issue', repo_key, NULL, issue_id, issue_number, NULL, issue_title, issue_state, issue_created_at, issue_updated_at, issue_closed_at, NULL,"
|
|
189
|
+
" json_extract(json, '$.details.issue'), occurred_at "
|
|
190
|
+
" FROM base WHERE issue_id IS NOT NULL OR issue_number IS NOT NULL "
|
|
191
|
+
" UNION ALL "
|
|
192
|
+
" SELECT 'gh:artifact:push:' || coalesce(cast(push_id AS TEXT), event_id),"
|
|
193
|
+
" 'push', repo_key, NULL, try_cast(push_id AS BIGINT), NULL, NULL, push_ref, NULL, occurred_at, occurred_at, NULL, NULL,"
|
|
194
|
+
" json_extract(json, '$.details.push'), occurred_at "
|
|
195
|
+
" FROM base WHERE action = 'PushCommits' AND (push_id IS NOT NULL OR event_id IS NOT NULL) "
|
|
196
|
+
" UNION ALL "
|
|
197
|
+
" SELECT 'gh:artifact:release:' || coalesce(cast(release_id AS TEXT), repo_key || ':' || release_tag),"
|
|
198
|
+
" 'release', repo_key, NULL, release_id, NULL, release_tag, release_name, NULL, release_created_at, NULL, NULL, NULL,"
|
|
199
|
+
" json_extract(json, '$.details.release'), occurred_at "
|
|
200
|
+
" FROM base WHERE release_id IS NOT NULL OR release_tag IS NOT NULL "
|
|
201
|
+
" UNION ALL "
|
|
202
|
+
" SELECT 'gh:artifact:comment:' || cast(comment_id AS TEXT),"
|
|
203
|
+
" CASE WHEN action = 'CreateIssueComment' THEN 'issue_comment' WHEN action = 'CreatePullRequestReviewComment' THEN 'review_comment' ELSE 'comment' END,"
|
|
204
|
+
" repo_key, coalesce("
|
|
205
|
+
" CASE WHEN pr_id IS NOT NULL OR pr_number IS NOT NULL THEN 'gh:artifact:pull_request:' || coalesce(cast(pr_id AS TEXT), repo_key || ':' || cast(pr_number AS TEXT)) END,"
|
|
206
|
+
" CASE WHEN issue_id IS NOT NULL OR issue_number IS NOT NULL THEN 'gh:artifact:issue:' || coalesce(cast(issue_id AS TEXT), repo_key || ':' || cast(issue_number AS TEXT)) END"
|
|
207
|
+
" ), comment_id, NULL, NULL, NULL, NULL, occurred_at, NULL, NULL, NULL,"
|
|
208
|
+
" json_extract(json, '$.details.comment'), occurred_at "
|
|
209
|
+
" FROM base WHERE comment_id IS NOT NULL "
|
|
210
|
+
" UNION ALL "
|
|
211
|
+
" SELECT 'gh:artifact:review:' || cast(review_id AS TEXT),"
|
|
212
|
+
" 'review', repo_key, 'gh:artifact:pull_request:' || coalesce(cast(pr_id AS TEXT), repo_key || ':' || cast(pr_number AS TEXT)),"
|
|
213
|
+
" review_id, NULL, NULL, NULL, NULL, review_submitted_at, review_updated_at, NULL, NULL,"
|
|
214
|
+
" json_extract(json, '$.details.review'), occurred_at "
|
|
215
|
+
" FROM base WHERE review_id IS NOT NULL "
|
|
216
|
+
" UNION ALL "
|
|
217
|
+
" SELECT 'gh:artifact:wiki_page:' || event_id, 'wiki_page', repo_key, NULL, NULL, NULL, NULL, 'wiki edit', NULL, occurred_at, NULL, NULL, NULL,"
|
|
218
|
+
" json_extract(json, '$.details.pages'), occurred_at "
|
|
219
|
+
" FROM base WHERE action = 'ManageWikiPage' AND event_id IS NOT NULL "
|
|
220
|
+
") "
|
|
221
|
+
"SELECT ?, artifact_key, 'github', artifact_type, repo_key, parent_artifact_key, github_id, NULL, number, sha, NULL,"
|
|
222
|
+
" title, state, NULL, NULL, NULL, NULL, NULL, NULL, created_at, updated_at, closed_at, merged_at, artifact_json, observed_at, ? "
|
|
223
|
+
"FROM artifacts WHERE artifact_key IS NOT NULL",
|
|
224
|
+
[dataset_id, dataset_id],
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _insert_contract_relations(conn: duckdb.DuckDBPyConnection, dataset_id: str) -> None:
|
|
229
|
+
conn.execute(
|
|
230
|
+
"INSERT OR REPLACE INTO contract_relations "
|
|
231
|
+
"WITH base AS ("
|
|
232
|
+
" SELECT *, 'gh:user:' || coalesce(cast(actor_id AS TEXT), actor_login) AS actor_key,"
|
|
233
|
+
" 'gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name) AS repo_key,"
|
|
234
|
+
" 'gh:artifact:pull_request:' || coalesce(cast(pr_id AS TEXT), 'gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name) || ':' || cast(pr_number AS TEXT)) AS pr_key,"
|
|
235
|
+
" 'gh:artifact:issue:' || coalesce(cast(issue_id AS TEXT), 'gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name) || ':' || cast(issue_number AS TEXT)) AS issue_key,"
|
|
236
|
+
" 'gh:artifact:push:' || coalesce(cast(push_id AS TEXT), event_id) AS push_key,"
|
|
237
|
+
" 'gh:artifact:release:' || coalesce(cast(release_id AS TEXT), 'gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name) || ':' || release_tag) AS release_key,"
|
|
238
|
+
" 'gh:repo:' || coalesce(cast(fork_repo_id AS TEXT), fork_full_name) AS fork_key "
|
|
239
|
+
" FROM _numfocus_flat WHERE actor_login IS NOT NULL"
|
|
240
|
+
"), event_relations AS ("
|
|
241
|
+
" SELECT 'gh:event:' || event_id || ':actor' AS relation_key, actor_key AS source_node_key, 'account' AS source_node_type,"
|
|
242
|
+
" CASE WHEN action IN ('StarRepository', 'AddMember') THEN repo_key ELSE "
|
|
243
|
+
" CASE "
|
|
244
|
+
" WHEN action IN ('OpenPullRequest','CreatePullRequestReview','CreatePullRequestComment','CreatePullRequestReviewComment','MergePullRequest','ClosePullRequest','ReopenPullRequest') THEN pr_key "
|
|
245
|
+
" WHEN action IN ('OpenIssue','CreateIssueComment','CloseIssue','ReopenIssue') THEN issue_key "
|
|
246
|
+
" WHEN action = 'PushCommits' THEN push_key "
|
|
247
|
+
" WHEN action = 'PublishRelease' THEN release_key "
|
|
248
|
+
" WHEN action = 'ForkRepository' THEN fork_key "
|
|
249
|
+
" WHEN action = 'ManageWikiPage' THEN 'gh:artifact:wiki_page:' || event_id "
|
|
250
|
+
" END END AS target_node_key,"
|
|
251
|
+
" CASE WHEN action IN ('StarRepository','AddMember','ForkRepository') THEN 'repo' ELSE 'artifact' END AS target_node_type,"
|
|
252
|
+
" CASE action "
|
|
253
|
+
" WHEN 'OpenPullRequest' THEN 'opened' WHEN 'OpenIssue' THEN 'opened'"
|
|
254
|
+
" WHEN 'CreatePullRequestReview' THEN 'reviewed'"
|
|
255
|
+
" WHEN 'CreatePullRequestComment' THEN 'commented_on' WHEN 'CreateIssueComment' THEN 'commented_on'"
|
|
256
|
+
" WHEN 'CreatePullRequestReviewComment' THEN 'review_commented_on'"
|
|
257
|
+
" WHEN 'MergePullRequest' THEN 'merged' WHEN 'ClosePullRequest' THEN 'closed' WHEN 'CloseIssue' THEN 'closed'"
|
|
258
|
+
" WHEN 'ReopenPullRequest' THEN 'reopened' WHEN 'ReopenIssue' THEN 'reopened'"
|
|
259
|
+
" WHEN 'PushCommits' THEN 'pushed' WHEN 'StarRepository' THEN 'starred' WHEN 'ForkRepository' THEN 'created_fork'"
|
|
260
|
+
" WHEN 'PublishRelease' THEN 'released' WHEN 'ManageWikiPage' THEN 'wiki_edited' WHEN 'AddMember' THEN 'member_of'"
|
|
261
|
+
" END AS relation_type, action AS relation_subtype, occurred_at, event_id, json AS properties "
|
|
262
|
+
" FROM base WHERE action NOT IN ('CreateBranch','DeleteBranch','CreateTag','DeleteTag','CreateRepository','MakeRepositoryPublic') "
|
|
263
|
+
"), contains_relations AS ("
|
|
264
|
+
" SELECT DISTINCT 'gh:contains:' || artifact_key AS relation_key, repo_key AS source_node_key, 'repo' AS source_node_type,"
|
|
265
|
+
" artifact_key AS target_node_key, 'artifact' AS target_node_type, 'contains' AS relation_type, artifact_type AS relation_subtype,"
|
|
266
|
+
" observed_at AS occurred_at, NULL AS event_id, artifact_json AS properties "
|
|
267
|
+
" FROM contract_artifacts WHERE dataset_id = ? AND repo_key IS NOT NULL "
|
|
268
|
+
"), parent_relations AS ("
|
|
269
|
+
" SELECT DISTINCT 'gh:parent:' || parent_artifact_key || ':' || artifact_key AS relation_key, parent_artifact_key, 'artifact',"
|
|
270
|
+
" artifact_key, 'artifact', 'contains', artifact_type, observed_at, NULL, artifact_json "
|
|
271
|
+
" FROM contract_artifacts WHERE dataset_id = ? AND parent_artifact_key IS NOT NULL "
|
|
272
|
+
"), fork_relations AS ("
|
|
273
|
+
" SELECT DISTINCT 'gh:fork_of:' || fork_key || ':' || repo_key, fork_key, 'repo', repo_key, 'repo', 'fork_of', NULL, occurred_at, event_id, json "
|
|
274
|
+
" FROM base WHERE action = 'ForkRepository' AND fork_key IS NOT NULL AND repo_key IS NOT NULL "
|
|
275
|
+
") "
|
|
276
|
+
"SELECT ?, relation_key, source_node_key, source_node_type, target_node_key, target_node_type, relation_type, relation_subtype,"
|
|
277
|
+
" TRUE, occurred_at, occurred_at, NULL, NULL, 1, 1.0, event_id, 'numfocus_actions', ?, properties "
|
|
278
|
+
"FROM (SELECT * FROM event_relations UNION ALL SELECT * FROM contains_relations UNION ALL SELECT * FROM parent_relations UNION ALL SELECT * FROM fork_relations) "
|
|
279
|
+
"WHERE relation_type IS NOT NULL AND target_node_key IS NOT NULL",
|
|
280
|
+
[dataset_id, dataset_id, dataset_id, dataset_id],
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _insert_contract_manifest(
|
|
285
|
+
conn: duckdb.DuckDBPyConnection, dataset_id: str, source_path: str
|
|
286
|
+
) -> None:
|
|
287
|
+
conn.execute(
|
|
288
|
+
"INSERT OR REPLACE INTO contract_adapter_manifest "
|
|
289
|
+
"SELECT ?, 'numfocus-actions-duckdb', '0.1.0', ?, now(), TRUE, TRUE, FALSE, FALSE, FALSE,"
|
|
290
|
+
" TRUE, TRUE, TRUE, TRUE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE,"
|
|
291
|
+
" min(occurred_at), max(occurred_at), list_distinct(list('gh:repo:' || coalesce(cast(repo_id AS TEXT), repo_full_name))),"
|
|
292
|
+
" 'Example fast DuckDB adapter for NumFocus GH_Actions.jsonl; source-specific, not core HIN logic.' "
|
|
293
|
+
"FROM _numfocus_flat",
|
|
294
|
+
[dataset_id, source_path],
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _backfill_graph_from_hin(conn: duckdb.DuckDBPyConnection, dataset_id: str) -> None:
|
|
299
|
+
conn.execute("DELETE FROM nodes WHERE dataset_id = ?", [dataset_id])
|
|
300
|
+
conn.execute("DELETE FROM edges WHERE dataset_id = ?", [dataset_id])
|
|
301
|
+
conn.execute(
|
|
302
|
+
"INSERT OR REPLACE INTO nodes "
|
|
303
|
+
"SELECT dataset_id, node_type AS type, node_id AS id, coalesce(updated_at, created_at, observed_at) AS ts,"
|
|
304
|
+
" to_json({'subtype': node_subtype, 'natural_key': natural_key, 'display_name': display_name}) AS attrs "
|
|
305
|
+
"FROM _store_hin_nodes WHERE dataset_id = ?",
|
|
306
|
+
[dataset_id],
|
|
307
|
+
)
|
|
308
|
+
conn.execute(
|
|
309
|
+
"INSERT INTO edges "
|
|
310
|
+
"SELECT dataset_id, edge_type AS type, source_node_id AS src_id, target_node_id AS dst_id, occurred_at AS ts,"
|
|
311
|
+
" to_json({'subtype': relation_subtype, 'source_record_id': source_record_id, 'weight': weight}) AS attrs "
|
|
312
|
+
"FROM _store_hin_edges WHERE dataset_id = ?",
|
|
313
|
+
[dataset_id],
|
|
314
|
+
)
|