pinky-snowpark 0.1.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pinky_snowpark-0.1.0.dev1/.github/workflows/ci.yml +39 -0
- pinky_snowpark-0.1.0.dev1/.github/workflows/publish.yml +32 -0
- pinky_snowpark-0.1.0.dev1/.github/workflows/release.yml +47 -0
- pinky_snowpark-0.1.0.dev1/.gitignore +27 -0
- pinky_snowpark-0.1.0.dev1/.pre-commit-config.yaml +107 -0
- pinky_snowpark-0.1.0.dev1/.secrets.baseline +127 -0
- pinky_snowpark-0.1.0.dev1/CONTEXT.md +16 -0
- pinky_snowpark-0.1.0.dev1/LICENSE +21 -0
- pinky_snowpark-0.1.0.dev1/PKG-INFO +74 -0
- pinky_snowpark-0.1.0.dev1/README.md +21 -0
- pinky_snowpark-0.1.0.dev1/docs/adr/0001-sp-telemetry-flat-attrs.md +31 -0
- pinky_snowpark-0.1.0.dev1/docs/explanation/design.md +39 -0
- pinky_snowpark-0.1.0.dev1/docs/explanation/features.md +58 -0
- pinky_snowpark-0.1.0.dev1/docs/index.md +17 -0
- pinky_snowpark-0.1.0.dev1/docs/reference/generated/api/excel.md +5 -0
- pinky_snowpark-0.1.0.dev1/docs/reference/generated/api/snowpark.md +5 -0
- pinky_snowpark-0.1.0.dev1/mkdocs.dev.yml +3 -0
- pinky_snowpark-0.1.0.dev1/mkdocs.yml +30 -0
- pinky_snowpark-0.1.0.dev1/pyproject.toml +96 -0
- pinky_snowpark-0.1.0.dev1/scripts/hooks/check_forbidden_keywords.py +76 -0
- pinky_snowpark-0.1.0.dev1/scripts/hooks/check_snowpark_api.py +160 -0
- pinky_snowpark-0.1.0.dev1/scripts/hooks/check_typed_api.py +140 -0
- pinky_snowpark-0.1.0.dev1/src/pinky_snowpark/__init__.py +90 -0
- pinky_snowpark-0.1.0.dev1/src/pinky_snowpark/excel.py +377 -0
- pinky_snowpark-0.1.0.dev1/src/pinky_snowpark/snowpark.py +1348 -0
- pinky_snowpark-0.1.0.dev1/tests/__init__.py +0 -0
- pinky_snowpark-0.1.0.dev1/tests/test_excel.py +61 -0
- pinky_snowpark-0.1.0.dev1/tests/test_formats.py +122 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, develop]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main, develop]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
ci:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
env:
|
|
13
|
+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
|
14
|
+
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Set up Python 3.11
|
|
19
|
+
uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.11"
|
|
22
|
+
|
|
23
|
+
- name: Install dependencies
|
|
24
|
+
run: pip install -e ".[dev]"
|
|
25
|
+
|
|
26
|
+
- name: Lint
|
|
27
|
+
run: ruff check src/ tests/
|
|
28
|
+
|
|
29
|
+
- name: Format check
|
|
30
|
+
run: ruff format --check src/ tests/
|
|
31
|
+
|
|
32
|
+
- name: Type check
|
|
33
|
+
run: mypy src/pinky_snowpark/
|
|
34
|
+
|
|
35
|
+
- name: Security scan
|
|
36
|
+
run: bandit -r src/pinky_snowpark/ -ll --skip B108
|
|
37
|
+
|
|
38
|
+
- name: Tests
|
|
39
|
+
run: pytest
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*.dev*"
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
publish:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
env:
|
|
12
|
+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
id-token: write # required for trusted publishing (OIDC)
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- name: Set up Python 3.11
|
|
21
|
+
uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.11"
|
|
24
|
+
|
|
25
|
+
- name: Install build
|
|
26
|
+
run: pip install build
|
|
27
|
+
|
|
28
|
+
- name: Build wheel and sdist
|
|
29
|
+
run: python -m build
|
|
30
|
+
|
|
31
|
+
- name: Publish to PyPI
|
|
32
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
release:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
env:
|
|
11
|
+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
|
12
|
+
permissions:
|
|
13
|
+
contents: write
|
|
14
|
+
id-token: write
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- name: Read version
|
|
20
|
+
id: version
|
|
21
|
+
run: echo "version=$(grep '^version' pyproject.toml | cut -d'"' -f2)" >> $GITHUB_OUTPUT
|
|
22
|
+
|
|
23
|
+
- name: Guard — no .dev version on main
|
|
24
|
+
run: |
|
|
25
|
+
if [[ "${{ steps.version.outputs.version }}" == *".dev"* ]]; then
|
|
26
|
+
echo "❌ version .dev sur main — bumper la version avant de merger"
|
|
27
|
+
exit 1
|
|
28
|
+
fi
|
|
29
|
+
|
|
30
|
+
- name: Create tag
|
|
31
|
+
run: |
|
|
32
|
+
git tag "v${{ steps.version.outputs.version }}"
|
|
33
|
+
git push origin "v${{ steps.version.outputs.version }}"
|
|
34
|
+
|
|
35
|
+
- name: Set up Python 3.11
|
|
36
|
+
uses: actions/setup-python@v5
|
|
37
|
+
with:
|
|
38
|
+
python-version: "3.11"
|
|
39
|
+
|
|
40
|
+
- name: Install build
|
|
41
|
+
run: pip install build
|
|
42
|
+
|
|
43
|
+
- name: Build
|
|
44
|
+
run: python -m build
|
|
45
|
+
|
|
46
|
+
- name: Publish to PyPI
|
|
47
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# MkDocs
|
|
2
|
+
docs/assets/
|
|
3
|
+
site/
|
|
4
|
+
|
|
5
|
+
# Python
|
|
6
|
+
__pycache__/
|
|
7
|
+
*.py[cod]
|
|
8
|
+
*.egg-info/
|
|
9
|
+
dist/
|
|
10
|
+
build/
|
|
11
|
+
.eggs/
|
|
12
|
+
|
|
13
|
+
# Environments
|
|
14
|
+
.env
|
|
15
|
+
.venv/
|
|
16
|
+
env/
|
|
17
|
+
|
|
18
|
+
# Pytest / coverage
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.coverage
|
|
21
|
+
htmlcov/
|
|
22
|
+
|
|
23
|
+
# Mypy
|
|
24
|
+
.mypy_cache/
|
|
25
|
+
|
|
26
|
+
# Ruff
|
|
27
|
+
.ruff_cache/
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
---
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# .pre-commit-config.yaml — pinky-snowpark
|
|
4
|
+
# =============================================================================
|
|
5
|
+
# Install:
|
|
6
|
+
# pip install pre-commit
|
|
7
|
+
# pre-commit install
|
|
8
|
+
# pre-commit install --hook-type commit-msg
|
|
9
|
+
#
|
|
10
|
+
# Update standard hooks:
|
|
11
|
+
# pre-commit autoupdate
|
|
12
|
+
# =============================================================================
|
|
13
|
+
default_install_hook_types: [pre-commit, commit-msg]
|
|
14
|
+
|
|
15
|
+
repos:
|
|
16
|
+
# ── Formatting: trailing spaces + newline ─────────────────────────────────
|
|
17
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
18
|
+
rev: v6.0.0
|
|
19
|
+
hooks:
|
|
20
|
+
- id: trailing-whitespace
|
|
21
|
+
- id: end-of-file-fixer
|
|
22
|
+
- id: check-yaml
|
|
23
|
+
- id: check-toml
|
|
24
|
+
- id: check-merge-conflict
|
|
25
|
+
- id: debug-statements
|
|
26
|
+
|
|
27
|
+
# ── Formatting: Markdown ──────────────────────────────────────────────────
|
|
28
|
+
- repo: local
|
|
29
|
+
hooks:
|
|
30
|
+
- id: mdformat
|
|
31
|
+
name: mdformat
|
|
32
|
+
entry: mdformat
|
|
33
|
+
language: system
|
|
34
|
+
files: ^README\.md$
|
|
35
|
+
|
|
36
|
+
# ── Security: secrets & forbidden files ──────────────────────────────────
|
|
37
|
+
- repo: https://github.com/gitleaks/gitleaks
|
|
38
|
+
rev: v8.30.0
|
|
39
|
+
hooks:
|
|
40
|
+
- id: gitleaks
|
|
41
|
+
|
|
42
|
+
# ── Security: alternative secret detection ────────────────────────────────
|
|
43
|
+
- repo: https://github.com/Yelp/detect-secrets
|
|
44
|
+
rev: v1.5.0
|
|
45
|
+
hooks:
|
|
46
|
+
- id: detect-secrets
|
|
47
|
+
args: ["--baseline", ".secrets.baseline"]
|
|
48
|
+
exclude: ^docs/
|
|
49
|
+
|
|
50
|
+
# ── Python: ruff lint + format ────────────────────────────────────────────
|
|
51
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
52
|
+
rev: v0.15.9
|
|
53
|
+
hooks:
|
|
54
|
+
- id: ruff-check
|
|
55
|
+
args: [--no-fix]
|
|
56
|
+
- id: ruff-format
|
|
57
|
+
args: [--check]
|
|
58
|
+
|
|
59
|
+
# ── Python: type checking ─────────────────────────────────────────────────
|
|
60
|
+
- repo: local
|
|
61
|
+
hooks:
|
|
62
|
+
- id: mypy
|
|
63
|
+
name: mypy
|
|
64
|
+
entry: mypy
|
|
65
|
+
language: system
|
|
66
|
+
types: [python]
|
|
67
|
+
args: [--ignore-missing-imports]
|
|
68
|
+
files: ^src/
|
|
69
|
+
|
|
70
|
+
# ── Python: security ─────────────────────────────────────────────────────
|
|
71
|
+
- repo: local
|
|
72
|
+
hooks:
|
|
73
|
+
- id: bandit
|
|
74
|
+
name: bandit
|
|
75
|
+
entry: bandit
|
|
76
|
+
language: system
|
|
77
|
+
pass_filenames: false
|
|
78
|
+
args: [-r, src/, -ll, --skip, B108]
|
|
79
|
+
|
|
80
|
+
# ── Typed API guard ──────────────────────────────────────────────────────
|
|
81
|
+
- repo: local
|
|
82
|
+
hooks:
|
|
83
|
+
- id: check-typed-api
|
|
84
|
+
name: "[TYPED-API] No raw dict/tuple/set in public annotations"
|
|
85
|
+
entry: python scripts/hooks/check_typed_api.py
|
|
86
|
+
language: system
|
|
87
|
+
types: [python]
|
|
88
|
+
files: ^src/
|
|
89
|
+
|
|
90
|
+
# ── Employer reference guard ──────────────────────────────────────────────
|
|
91
|
+
- repo: local
|
|
92
|
+
hooks:
|
|
93
|
+
- id: check-forbidden-keywords
|
|
94
|
+
name: "[FORBIDDEN] Employer references"
|
|
95
|
+
entry: python scripts/hooks/check_forbidden_keywords.py
|
|
96
|
+
language: system
|
|
97
|
+
pass_filenames: true
|
|
98
|
+
|
|
99
|
+
# ── Snowpark / Core API conventions ──────────────────────────────────────
|
|
100
|
+
- repo: local
|
|
101
|
+
hooks:
|
|
102
|
+
- id: check-snowpark-api
|
|
103
|
+
name: "[SNOWPARK-API] No session.sql(), canonical method names"
|
|
104
|
+
entry: python scripts/hooks/check_snowpark_api.py
|
|
105
|
+
language: system
|
|
106
|
+
types: [python]
|
|
107
|
+
files: ^src/
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "1.5.0",
|
|
3
|
+
"plugins_used": [
|
|
4
|
+
{
|
|
5
|
+
"name": "ArtifactoryDetector"
|
|
6
|
+
},
|
|
7
|
+
{
|
|
8
|
+
"name": "AWSKeyDetector"
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"name": "AzureStorageKeyDetector"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"name": "Base64HighEntropyString",
|
|
15
|
+
"limit": 4.5
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"name": "BasicAuthDetector"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"name": "CloudantDetector"
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"name": "DiscordBotTokenDetector"
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"name": "GitHubTokenDetector"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"name": "GitLabTokenDetector"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"name": "HexHighEntropyString",
|
|
34
|
+
"limit": 3.0
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"name": "IbmCloudIamDetector"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"name": "IbmCosHmacDetector"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"name": "IPPublicDetector"
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"name": "JwtTokenDetector"
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"name": "KeywordDetector",
|
|
50
|
+
"keyword_exclude": ""
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"name": "MailchimpDetector"
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"name": "NpmDetector"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"name": "OpenAIDetector"
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"name": "PrivateKeyDetector"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"name": "PypiTokenDetector"
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"name": "SendGridDetector"
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"name": "SlackDetector"
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"name": "SoftlayerDetector"
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"name": "SquareOAuthDetector"
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"name": "StripeDetector"
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
"name": "TelegramBotTokenDetector"
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"name": "TwilioKeyDetector"
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
"filters_used": [
|
|
90
|
+
{
|
|
91
|
+
"path": "detect_secrets.filters.allowlist.is_line_allowlisted"
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
|
|
95
|
+
"min_level": 2
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"path": "detect_secrets.filters.heuristic.is_indirect_reference"
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"path": "detect_secrets.filters.heuristic.is_likely_id_string"
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"path": "detect_secrets.filters.heuristic.is_lock_file"
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"path": "detect_secrets.filters.heuristic.is_potential_uuid"
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"path": "detect_secrets.filters.heuristic.is_sequential_string"
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"path": "detect_secrets.filters.heuristic.is_swagger_file"
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
"path": "detect_secrets.filters.heuristic.is_templated_secret"
|
|
123
|
+
}
|
|
124
|
+
],
|
|
125
|
+
"results": {},
|
|
126
|
+
"generated_at": "2026-06-04T04:24:36Z"
|
|
127
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# CONTEXT.md — pinky-snowpark
|
|
2
|
+
*Update date : 2026-05-31 20:22*
|
|
3
|
+
|
|
4
|
+
Single source of truth for naming in this repo. Variables, function names, module names, and documentation prose all follow this table.
|
|
5
|
+
|
|
6
|
+
| Term (preferred) | Avoid | Definition |
|
|
7
|
+
|---|---|---|
|
|
8
|
+
| `session` | `connection`, `client` | A live Snowpark `Session` — the entry point to Snowflake compute |
|
|
9
|
+
| `load_kwargs` | `load_args`, `load_params` | SP argument loader — reads CALL parameters from the Task context |
|
|
10
|
+
| `sp_trace` | `trace_sp`, `sp_telemetry` | SP lifecycle trace emitter — writes a span to the EVENT_TABLE |
|
|
11
|
+
| `stage_write` | `upload`, `export_to_stage` | Stage I/O helper — writes a DataFrame to a named stage with optional encryption |
|
|
12
|
+
| `with_role` | `set_role`, `use_role`, `switch_role` | Context manager for temporary role elevation — restores the original role on exit |
|
|
13
|
+
| `with_user` | `impersonate`, `set_user` | Context manager for temporary user impersonation — same pattern as `with_role` |
|
|
14
|
+
| `EnvPattern` | `EnvMode`, `EnvStrategy` | Enum of strategies to derive the current environment from the Snowflake context |
|
|
15
|
+
| `get_environment` | `detect_env`, `resolve_env` | Function that returns the current environment string using an `EnvPattern` |
|
|
16
|
+
| `CELL_FORMATS` | `FORMATS`, `EXCEL_STYLES` | Dict of named Excel cell format presets in `excel.py` |
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 pinky-and-co
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pinky-snowpark
|
|
3
|
+
Version: 0.1.0.dev1
|
|
4
|
+
Summary: Snowpark session utilities, stage I/O, and telemetry for Snowflake stored procedures
|
|
5
|
+
Project-URL: Homepage, https://github.com/pinky-and-co/pinky-snowpark
|
|
6
|
+
Project-URL: Repository, https://github.com/pinky-and-co/pinky-snowpark
|
|
7
|
+
Project-URL: Issues, https://github.com/pinky-and-co/pinky-snowpark/issues
|
|
8
|
+
Author: pinky-and-co
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2026 pinky-and-co
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: data-engineering,snowflake,snowpark
|
|
32
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
33
|
+
Classifier: Operating System :: OS Independent
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
35
|
+
Classifier: Topic :: Database
|
|
36
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
37
|
+
Requires-Python: >=3.11
|
|
38
|
+
Requires-Dist: openpyxl
|
|
39
|
+
Requires-Dist: pinky-core
|
|
40
|
+
Requires-Dist: snowflake-snowpark-python
|
|
41
|
+
Provides-Extra: dev
|
|
42
|
+
Requires-Dist: bandit; extra == 'dev'
|
|
43
|
+
Requires-Dist: mkdocs-material>=9.0; extra == 'dev'
|
|
44
|
+
Requires-Dist: mkdocstrings[python]>=0.25; extra == 'dev'
|
|
45
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
46
|
+
Requires-Dist: pre-commit; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
49
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
50
|
+
Provides-Extra: pgp
|
|
51
|
+
Requires-Dist: pgpy; extra == 'pgp'
|
|
52
|
+
Description-Content-Type: text/markdown
|
|
53
|
+
|
|
54
|
+
# pinky-snowpark
|
|
55
|
+
|
|
56
|
+
*Update date : 2026-06-04 00:00*
|
|
57
|
+
|
|
58
|
+
Snowpark session utilities, stage I/O, and telemetry for Snowflake stored procedures.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install pinky-snowpark
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Docs
|
|
67
|
+
|
|
68
|
+
- [Design](docs/explanation/design.md) — architecture and decisions
|
|
69
|
+
- [Features](docs/explanation/features.md) — feature list
|
|
70
|
+
- [ADR](docs/adr/) — architecture decision records
|
|
71
|
+
|
|
72
|
+
## License
|
|
73
|
+
|
|
74
|
+
MIT
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# pinky-snowpark
|
|
2
|
+
|
|
3
|
+
*Update date : 2026-06-04 00:00*
|
|
4
|
+
|
|
5
|
+
Snowpark session utilities, stage I/O, and telemetry for Snowflake stored procedures.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install pinky-snowpark
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Docs
|
|
14
|
+
|
|
15
|
+
- [Design](docs/explanation/design.md) — architecture and decisions
|
|
16
|
+
- [Features](docs/explanation/features.md) — feature list
|
|
17
|
+
- [ADR](docs/adr/) — architecture decision records
|
|
18
|
+
|
|
19
|
+
## License
|
|
20
|
+
|
|
21
|
+
MIT
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# ADR-0001 — SP telemetry via flat span attributes, not SYSTEM$SET_RETURN_VALUE
|
|
2
|
+
Status: accepted
|
|
3
|
+
Date: 2026-05-31
|
|
4
|
+
|
|
5
|
+
## Context
|
|
6
|
+
|
|
7
|
+
Snowflake Tasks support `SYSTEM$SET_RETURN_VALUE` to pass data between task runs.
|
|
8
|
+
Early SP implementations used it as the observability mechanism — the SP wrote a
|
|
9
|
+
JSON payload as the return value, and monitoring queries parsed it.
|
|
10
|
+
|
|
11
|
+
Two problems emerged:
|
|
12
|
+
1. `SYSTEM$SET_RETURN_VALUE` couples the SP to the Task runtime — a SP called directly
|
|
13
|
+
(in tests, on-demand) has no Task context and the call fails or silently does nothing.
|
|
14
|
+
2. JSON string payloads in QUERY_HISTORY / EVENT_TABLE require `PARSE_JSON()` at query
|
|
15
|
+
time — every monitoring query pays the parsing cost and becomes fragile against schema changes.
|
|
16
|
+
|
|
17
|
+
## Decision
|
|
18
|
+
|
|
19
|
+
SP observability uses `sp_trace()` with flat span attributes written to EVENT_TABLE.
|
|
20
|
+
`SYSTEM$SET_RETURN_VALUE` is never called from SP code.
|
|
21
|
+
|
|
22
|
+
DAG-to-SP data passing uses `load_kwargs()` on input and `v_return_values` dict on output —
|
|
23
|
+
the Task wrapper layer reads `v_return_values` and calls `SYSTEM$SET_RETURN_VALUE` on behalf
|
|
24
|
+
of the SP, keeping the SP decoupled.
|
|
25
|
+
|
|
26
|
+
## Consequences
|
|
27
|
+
|
|
28
|
+
- SPs are callable directly (tests, on-demand) without a Task context.
|
|
29
|
+
- EVENT_TABLE span attributes are flat key-value pairs — queryable with `WHERE resource_attributes['key'] = 'value'`, no JSON parsing.
|
|
30
|
+
- The Task wrapper is the only code that knows about `SYSTEM$SET_RETURN_VALUE` — one place to change if Snowflake evolves this API.
|
|
31
|
+
- Monitoring dashboards are simpler and faster.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Design — pinky-snowpark
|
|
2
|
+
*Update date : 2026-05-31 20:23*
|
|
3
|
+
|
|
4
|
+
Snowpark session utilities, telemetry, and stage I/O for Snowpark stored procedures.
|
|
5
|
+
The layer that makes SPs observable, testable, and production-grade — lifecycle tracing,
|
|
6
|
+
encrypted stage writes, Excel exports — without coupling business logic to infra concerns.
|
|
7
|
+
|
|
8
|
+
See [ADR-0001](../adr/0001-sp-telemetry-flat-attrs.md) for the SP telemetry design decision.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Module inventory
|
|
13
|
+
|
|
14
|
+
| Module | Description | Status |
|
|
15
|
+
|--------|-------------|--------|
|
|
16
|
+
| `snowpark.py` | Session helpers, SP lifecycle telemetry (`load_kwargs`, `sp_trace`), stage I/O, PGP, rotating filenames, `EnvPattern` | planned |
|
|
17
|
+
| `excel.py` | `create_workbook_from_df()`, `parse_worksheet()`, 30+ `CELL_FORMATS` presets | planned |
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## SP pattern — load_kwargs + sp_trace
|
|
22
|
+
|
|
23
|
+
SPs are fully decoupled from Snowflake Tasks. Two functions form the contract:
|
|
24
|
+
|
|
25
|
+
- **`load_kwargs(raw_args, required_keys, optional_keys)`** — normalises args whether called directly or via a Task DAG (`_TASK_NAME$key` → clean key). Emits `sp_start` telemetry automatically.
|
|
26
|
+
- **`sp_trace(v_result, v_return_values, status)`** — called in the `finally` block. Emits `sp_end` with flat span attributes queryable in EVENT_TABLE without JSON parsing.
|
|
27
|
+
|
|
28
|
+
The SP does not know it is inside a DAG. The DAG does not know the business logic.
|
|
29
|
+
Full DAG wiring design: `snowflake-provider/provider_design.md`.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## Future directions
|
|
34
|
+
|
|
35
|
+
- **SP test harness** — lightweight local Snowpark session mock for unit testing SPs without a live connection
|
|
36
|
+
- **Registry UI** — Streamlit component reading the `stage_write()` registry table (stage inventory with metadata)
|
|
37
|
+
- **Async stage write** — background thread upload for large DataFrames, returns a future
|
|
38
|
+
- **Conditional cell formatting** — alternating row colors, threshold-based highlights in Excel workbooks
|
|
39
|
+
- **Snowsight deep links** — extend `get_current_snowsight_url()` to direct links to query results / dashboards
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Features — pinky-snowpark
|
|
2
|
+
*Update date : 2026-05-31 20:23*
|
|
3
|
+
|
|
4
|
+
Feature list ordered by priority. `stable` = in production. `planned` = not yet built, order = priority.
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Planned
|
|
9
|
+
|
|
10
|
+
Ordered by priority — closest to implementation first.
|
|
11
|
+
|
|
12
|
+
### MVP — migration from backup
|
|
13
|
+
|
|
14
|
+
Both modules are production-ready in `_BACKUP/snowflake-kit-snowpark-bak`.
|
|
15
|
+
MVP = clean migration into the pinky-snowpark repo structure + repo scaffold.
|
|
16
|
+
|
|
17
|
+
**`snowpark.py`**
|
|
18
|
+
|
|
19
|
+
| Feature | Description |
|
|
20
|
+
|---------|-------------|
|
|
21
|
+
| Session helpers | `collect_dicts()`, `execute_statements()`, `run_generated_sql()`, `validate_sql()` |
|
|
22
|
+
| Role context manager | `with_role(session, role)` — temporary elevation, restores on exit |
|
|
23
|
+
| SP lifecycle telemetry | `load_kwargs()`, `sp_trace()`, `to_telemetry_attrs()` (flat span attributes for EVENT_TABLE) |
|
|
24
|
+
| Stage I/O | `stage_write()` — CSV / JSON / Parquet / Avro, optional PGP encryption, optional registry table append |
|
|
25
|
+
| PGP helpers | `load_pgp_key()`, `decrypt_file()` with auto format detection |
|
|
26
|
+
| Rotating filenames | `get_rotating_filename()` with 5 retention slots |
|
|
27
|
+
| Environment derivation | `get_environment(session, EnvPattern)` + `EnvPattern` enum (ACCOUNT / DATABASE_SUFFIX / DATABASE_PREFIX / SCHEMA_SUFFIX / SCHEMA_PREFIX) |
|
|
28
|
+
|
|
29
|
+
**`excel.py`**
|
|
30
|
+
|
|
31
|
+
| Feature | Description |
|
|
32
|
+
|---------|-------------|
|
|
33
|
+
| Workbook generation | `create_workbook_from_df()` — DataFrame → formatted `.xlsx` |
|
|
34
|
+
| Workbook parsing | `parse_worksheet()` — Excel → DataFrame |
|
|
35
|
+
| Format presets | `CELL_FORMATS` — 30+ presets (FR/EN dates, currencies, numeric, styles) |
|
|
36
|
+
|
|
37
|
+
### with_user — temporary user impersonation
|
|
38
|
+
|
|
39
|
+
`with_user(session, username)` context manager — same pattern as `with_role()`.
|
|
40
|
+
Use case: audit trails in multi-tenant SPs.
|
|
41
|
+
|
|
42
|
+
### decrypt_file — in-memory mode
|
|
43
|
+
|
|
44
|
+
Currently writes to `/tmp/`. Small files (<50 MB) should decrypt in-memory to avoid disk I/O inside Snowflake.
|
|
45
|
+
|
|
46
|
+
### excel.py — conditional cell formatting
|
|
47
|
+
|
|
48
|
+
Alternating row colors and threshold-based highlights. Extends existing `CELL_FORMATS` presets.
|
|
49
|
+
|
|
50
|
+
### SP test harness
|
|
51
|
+
|
|
52
|
+
Lightweight local Snowpark session mock for unit testing SPs without a live Snowflake connection.
|
|
53
|
+
|
|
54
|
+
### Backlog
|
|
55
|
+
|
|
56
|
+
- Async stage write — background thread upload, returns future
|
|
57
|
+
- Registry UI — Streamlit component reading the `stage_write()` registry table
|
|
58
|
+
- Snowsight deep links — direct links to query results / dashboards / worksheets
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# pinky-snowpark
|
|
2
|
+
*Update date : 2026-06-01*
|
|
3
|
+
|
|
4
|
+
Snowpark session utilities, stage I/O, and telemetry for Snowflake stored procedures.
|
|
5
|
+
|
|
6
|
+
The layer that makes SPs observable, testable, and production-grade — lifecycle tracing,
|
|
7
|
+
encrypted stage writes, Excel exports — without coupling business logic to infra concerns.
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
<!-- agent index — one line per file, what it covers and what NOT to put there -->
|
|
12
|
+
|
|
13
|
+
- [explanation/design.md](explanation/design.md) — architecture, SP pattern (load_kwargs + sp_trace), module inventory, future directions. Not for: decisions (→ adr/), feature list (→ features.md)
|
|
14
|
+
- [explanation/features.md](explanation/features.md) — feature list by priority: MVP migration + post-MVP backlog. Not for: design rationale (→ design.md)
|
|
15
|
+
- [reference/generated/api/snowpark.md](reference/generated/api/snowpark.md) — API reference for snowpark module (session helpers, telemetry, stage I/O, PGP, rotating filenames). Not for: hand-written content — edit docstrings in src/
|
|
16
|
+
- [reference/generated/api/excel.md](reference/generated/api/excel.md) — API reference for excel module (CellFormat, CELL_FORMATS, create_workbook_from_df, parse_worksheet). Not for: hand-written content — edit docstrings in src/
|
|
17
|
+
- [adr/0001-sp-telemetry-flat-attrs.md](adr/0001-sp-telemetry-flat-attrs.md) — sp_trace flat attrs over SYSTEM$SET_RETURN_VALUE; SPs decoupled from Task runtime
|