lambda-watcher 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/PKG-INFO +36 -21
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/README.md +35 -20
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/pyproject.toml +1 -1
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/__init__.py +1 -1
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/__init__.py +42 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/deps.py +77 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/envvars.py +25 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/handler.py +14 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/inventory.py +30 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/runtime.py +28 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/secrets.py +59 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/analysis/services.py +20 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/cli.py +200 -28
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/config.py +81 -11
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/db.py +152 -2
- lambda_watcher-0.2.0/src/lambda_watcher/diffing/compare.py +1054 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/diffing/highlight.py +24 -2
- lambda_watcher-0.2.0/src/lambda_watcher/diffing/intraline.py +339 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/diffing/render_html.py +237 -20
- lambda_watcher-0.2.0/src/lambda_watcher/diffing/render_text.py +378 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/extract.py +19 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/gitmirror.py +105 -6
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/identify.py +16 -2
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/ingest.py +51 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/notify.py +11 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/reindex.py +17 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/service.py +327 -36
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/store.py +51 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/templates.py +46 -1
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/utils.py +73 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/watcher.py +93 -6
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher.egg-info/PKG-INFO +36 -21
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_cli.py +140 -0
- lambda_watcher-0.2.0/tests/test_diff.py +708 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_docs.py +16 -6
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_render_html.py +32 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_service.py +96 -6
- lambda_watcher-0.1.0/src/lambda_watcher/diffing/compare.py +0 -525
- lambda_watcher-0.1.0/src/lambda_watcher/diffing/intraline.py +0 -162
- lambda_watcher-0.1.0/src/lambda_watcher/diffing/render_text.py +0 -198
- lambda_watcher-0.1.0/tests/test_diff.py +0 -159
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/LICENSE +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/setup.cfg +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/__main__.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/diffing/__init__.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/diffing/build.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher/diffing/icons.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher.egg-info/SOURCES.txt +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher.egg-info/dependency_links.txt +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher.egg-info/entry_points.txt +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher.egg-info/requires.txt +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/src/lambda_watcher.egg-info/top_level.txt +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_analysis.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_extract.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_identify.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_ingest.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_utils.py +0 -0
- {lambda_watcher-0.1.0 → lambda_watcher-0.2.0}/tests/test_watcher.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lambda-watcher
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Watch your Downloads folder for AWS Lambda deployment zips, archive them as versions, analyse them, and diff any two versions.
|
|
5
5
|
Author: Lambda Watcher contributors
|
|
6
6
|
License: Apache-2.0
|
|
@@ -45,13 +45,14 @@ to how you work. You keep downloading zips; it does the rest.
|
|
|
45
45
|
|
|
46
46
|
```
|
|
47
47
|
$ lambda-watcher watch
|
|
48
|
-
lambda-watcher 0.
|
|
48
|
+
lambda-watcher 0.2.0 — archiving into ~/.lambda-watcher
|
|
49
49
|
watching ~/Downloads. Press Ctrl-C to stop.
|
|
50
50
|
new order-processor v0001 order-processor.zip — archived a new version
|
|
51
|
-
new order-processor v0002 order-processor (1).zip — 2 added, 2 modified,
|
|
51
|
+
new order-processor v0002 order-processor (1).zip — 2 added, 2 modified, 9 renamed, 55 vendored
|
|
52
52
|
+24/-5 lines · new: 1 env var, 1 AWS service, 3 secrets
|
|
53
53
|
report: ~/.lambda-watcher/reports/order-processor/v0001-v0002.html
|
|
54
54
|
unchanged order-processor v0002 order-processor (2).zip — identical to version 2
|
|
55
|
+
done
|
|
55
56
|
```
|
|
56
57
|
|
|
57
58
|
That third line is the one that matters: the same code, downloaded again, is
|
|
@@ -89,6 +90,14 @@ lw open order-processor # the whole archive, in your editor
|
|
|
89
90
|
lw git order-processor log -p # or just use git
|
|
90
91
|
```
|
|
91
92
|
|
|
93
|
+
`lw diff` hides vendored dependency files and the git mirror keeps them, so the
|
|
94
|
+
two commands report different totals for the same pair of versions — the demo
|
|
95
|
+
below is *2 added, 2 modified, 9 renamed* from `lw diff` and *68 files changed*
|
|
96
|
+
from `lw git order-processor diff --stat v0001 v0002`. Neither is wrong, so each
|
|
97
|
+
one says the other exists: `lw diff --vendor` shows the hidden files, and
|
|
98
|
+
`lw diff --mirror` prints the mirror's own patch for whichever two versions you
|
|
99
|
+
asked for.
|
|
100
|
+
|
|
92
101
|
## Why the diffs are actually readable
|
|
93
102
|
|
|
94
103
|
A raw `diff -r` between two Lambda zips is unusable: thousands of vendored
|
|
@@ -100,6 +109,8 @@ answering the questions you actually have, in order:
|
|
|
100
109
|
| **Your code, separated from theirs** | `node_modules/`, `site-packages/` and friends are classified as vendored and hidden by default. Three changed files, not 3,000. |
|
|
101
110
|
| **Dependencies as versions, not files** | A `boto3` upgrade shows as `boto3 1.34.0 → 1.35.20`, parsed from the `dist-info` actually shipped in the zip — not 400 changed files. |
|
|
102
111
|
| **Config impact, called out** | A new `os.environ["QUEUE_URL"]` is flagged as *"this must exist in the function's environment before you deploy"*. A new `boto3.client("sqs")` is flagged as *"the execution role may need new IAM permissions"*. These are the changes that break a deploy and never show up in a file diff. |
|
|
112
|
+
| **A reindent is not a rewrite** | A file whose only change is indentation, trailing spaces, line endings or blank lines is labelled *whitespace only* instead of being reprinted line by line. One `black` run over the package would otherwise read as a total rewrite of every file it touched. `lw diff --whitespace` shows the hunk anyway. |
|
|
113
|
+
| **Minified bundles are diffed by word** | An 8 KB bundle on one line has no lines to diff, so a changed digit costs you the whole file quoted twice. Instead you get the changed run and the text either side of it: `@ 19 …var t=1 → 2;a=1;a=1…`. Lock files are skipped for the same reason — the dependency row above already says what moved. |
|
|
103
114
|
| **Renames survive edits** | A file that moved *and* changed is shown as one rename with a diff, not an unrelated add plus delete. |
|
|
104
115
|
| **Secrets are diffed too** | An AWS key or Stripe token that appears between v7 and v8 gets its own section. Values are stored redacted; the secret itself never enters the index. |
|
|
105
116
|
|
|
@@ -110,9 +121,10 @@ files and 56 of them are `site-packages/`:
|
|
|
110
121
|
$ lambda-watcher diff order-processor
|
|
111
122
|
╭──────────────────────────────────────────────────────────────────────╮
|
|
112
123
|
│ order-processor v0001 → v0002 │
|
|
113
|
-
│ 2 added 2 modified
|
|
124
|
+
│ 2 added 2 modified 9 renamed 55 vendored (hidden) +24 / -5 lines │
|
|
125
|
+
│ to see the 55 vendored files: lw diff order-processor --vendor │
|
|
114
126
|
╰──────────────────────────────────────────────────────────────────────╯
|
|
115
|
-
size 8.
|
|
127
|
+
size 8.3 KB → 9.2 KB (+886 B)
|
|
116
128
|
|
|
117
129
|
Dependencies
|
|
118
130
|
manager package from to origin
|
|
@@ -129,21 +141,24 @@ high aws-access-key-id config.py:6 AKIA…LE (20 chars)
|
|
|
129
141
|
high stripe-key config.py:7 sk_l…dc (32 chars)
|
|
130
142
|
low debug-flag config.py:8 DEBUG = True
|
|
131
143
|
|
|
132
|
-
Files
|
|
133
|
-
path
|
|
134
|
-
~ lambda_function.py
|
|
135
|
-
~ requirements.txt
|
|
136
|
-
+ config.py
|
|
137
|
-
+ helpers/__init__.py
|
|
138
|
-
→ {db → helpers/db}.py
|
|
139
|
-
→ site-packages/boto3-1.{34.0 → 35.20}.dist-info/
|
|
140
|
-
|
|
144
|
+
Files
|
|
145
|
+
path + − size
|
|
146
|
+
~ lambda_function.py 9 2 +322
|
|
147
|
+
~ requirements.txt 2 1 +17
|
|
148
|
+
+ config.py 10 +235
|
|
149
|
+
+ helpers/__init__.py
|
|
150
|
+
→ {db → helpers/db}.py 1 +33
|
|
151
|
+
→ site-packages/boto3-1.{34.0 → 35.20}.dist-info/ · 4 files, 1 1 1 +1
|
|
152
|
+
edited
|
|
153
|
+
→ site-packages/botocore-1.{34.0 → 35.20}.dist-info/ · 4 files, 1 1 1 +1
|
|
154
|
+
edited
|
|
141
155
|
```
|
|
142
156
|
|
|
143
|
-
The
|
|
144
|
-
package is one rename rather than a delete plus an add,
|
|
145
|
-
|
|
146
|
-
|
|
157
|
+
The 55 vendored files became three version numbers, `db.py` moving into a
|
|
158
|
+
package is one rename rather than a delete plus an add, each bumped package's
|
|
159
|
+
`dist-info` is one line rather than four, and the new environment variable, the
|
|
160
|
+
new AWS service and the three secret findings are changes that a file diff
|
|
161
|
+
cannot express at all.
|
|
147
162
|
|
|
148
163
|
That capture is not illustrative — it is the output of
|
|
149
164
|
[`docs/examples/build_demo.py`](docs/examples/build_demo.py), which builds the
|
|
@@ -244,7 +259,7 @@ the manual recipes.
|
|
|
244
259
|
| `ls` | Every function archived so far. |
|
|
245
260
|
| `versions FN` | Every archived version of one function. |
|
|
246
261
|
| `show FN [V]` | Runtime, handler, dependencies, env vars, services and findings for one version. `--files`, `--json`. |
|
|
247
|
-
| `diff FN` | Compare two versions. Defaults to the last two. `--from`/`--to`, `--html`, `--open`, `--vendor`, `--no-patch`, `--json`. |
|
|
262
|
+
| `diff FN` | Compare two versions. Defaults to the last two. `--from`/`--to`, `--html`, `--open`, `--vendor`, `--whitespace`, `--no-patch`, `--json`. |
|
|
248
263
|
| `report FN` | Build a browsable HTML history: an index plus a diff for every step. |
|
|
249
264
|
| `export FN [V]` | Get a version back out as a deployable zip (`--zip`) or a plain folder (`--tree`). |
|
|
250
265
|
| `open FN [V]` | Open the function's mirror in your editor — every version in one folder, with history. Name a version to open just its files. |
|
|
@@ -276,11 +291,11 @@ back from the newest.
|
|
|
276
291
|
└── functions/
|
|
277
292
|
└── order-processor/
|
|
278
293
|
└── versions/
|
|
279
|
-
├── 0001-
|
|
294
|
+
├── 0001-7fc98e0e/
|
|
280
295
|
│ ├── code/ # the extracted tree
|
|
281
296
|
│ ├── manifest.json # the full analysis
|
|
282
297
|
│ └── package.zip # the original download
|
|
283
|
-
└── 0002-
|
|
298
|
+
└── 0002-7f887035/
|
|
284
299
|
```
|
|
285
300
|
|
|
286
301
|
The directories are the source of truth. `index.db` is a cache you can delete
|
|
@@ -14,13 +14,14 @@ to how you work. You keep downloading zips; it does the rest.
|
|
|
14
14
|
|
|
15
15
|
```
|
|
16
16
|
$ lambda-watcher watch
|
|
17
|
-
lambda-watcher 0.
|
|
17
|
+
lambda-watcher 0.2.0 — archiving into ~/.lambda-watcher
|
|
18
18
|
watching ~/Downloads. Press Ctrl-C to stop.
|
|
19
19
|
new order-processor v0001 order-processor.zip — archived a new version
|
|
20
|
-
new order-processor v0002 order-processor (1).zip — 2 added, 2 modified,
|
|
20
|
+
new order-processor v0002 order-processor (1).zip — 2 added, 2 modified, 9 renamed, 55 vendored
|
|
21
21
|
+24/-5 lines · new: 1 env var, 1 AWS service, 3 secrets
|
|
22
22
|
report: ~/.lambda-watcher/reports/order-processor/v0001-v0002.html
|
|
23
23
|
unchanged order-processor v0002 order-processor (2).zip — identical to version 2
|
|
24
|
+
done
|
|
24
25
|
```
|
|
25
26
|
|
|
26
27
|
That third line is the one that matters: the same code, downloaded again, is
|
|
@@ -58,6 +59,14 @@ lw open order-processor # the whole archive, in your editor
|
|
|
58
59
|
lw git order-processor log -p # or just use git
|
|
59
60
|
```
|
|
60
61
|
|
|
62
|
+
`lw diff` hides vendored dependency files and the git mirror keeps them, so the
|
|
63
|
+
two commands report different totals for the same pair of versions — the demo
|
|
64
|
+
below is *2 added, 2 modified, 9 renamed* from `lw diff` and *68 files changed*
|
|
65
|
+
from `lw git order-processor diff --stat v0001 v0002`. Neither is wrong, so each
|
|
66
|
+
one says the other exists: `lw diff --vendor` shows the hidden files, and
|
|
67
|
+
`lw diff --mirror` prints the mirror's own patch for whichever two versions you
|
|
68
|
+
asked for.
|
|
69
|
+
|
|
61
70
|
## Why the diffs are actually readable
|
|
62
71
|
|
|
63
72
|
A raw `diff -r` between two Lambda zips is unusable: thousands of vendored
|
|
@@ -69,6 +78,8 @@ answering the questions you actually have, in order:
|
|
|
69
78
|
| **Your code, separated from theirs** | `node_modules/`, `site-packages/` and friends are classified as vendored and hidden by default. Three changed files, not 3,000. |
|
|
70
79
|
| **Dependencies as versions, not files** | A `boto3` upgrade shows as `boto3 1.34.0 → 1.35.20`, parsed from the `dist-info` actually shipped in the zip — not 400 changed files. |
|
|
71
80
|
| **Config impact, called out** | A new `os.environ["QUEUE_URL"]` is flagged as *"this must exist in the function's environment before you deploy"*. A new `boto3.client("sqs")` is flagged as *"the execution role may need new IAM permissions"*. These are the changes that break a deploy and never show up in a file diff. |
|
|
81
|
+
| **A reindent is not a rewrite** | A file whose only change is indentation, trailing spaces, line endings or blank lines is labelled *whitespace only* instead of being reprinted line by line. One `black` run over the package would otherwise read as a total rewrite of every file it touched. `lw diff --whitespace` shows the hunk anyway. |
|
|
82
|
+
| **Minified bundles are diffed by word** | An 8 KB bundle on one line has no lines to diff, so a changed digit costs you the whole file quoted twice. Instead you get the changed run and the text either side of it: `@ 19 …var t=1 → 2;a=1;a=1…`. Lock files are skipped for the same reason — the dependency row above already says what moved. |
|
|
72
83
|
| **Renames survive edits** | A file that moved *and* changed is shown as one rename with a diff, not an unrelated add plus delete. |
|
|
73
84
|
| **Secrets are diffed too** | An AWS key or Stripe token that appears between v7 and v8 gets its own section. Values are stored redacted; the secret itself never enters the index. |
|
|
74
85
|
|
|
@@ -79,9 +90,10 @@ files and 56 of them are `site-packages/`:
|
|
|
79
90
|
$ lambda-watcher diff order-processor
|
|
80
91
|
╭──────────────────────────────────────────────────────────────────────╮
|
|
81
92
|
│ order-processor v0001 → v0002 │
|
|
82
|
-
│ 2 added 2 modified
|
|
93
|
+
│ 2 added 2 modified 9 renamed 55 vendored (hidden) +24 / -5 lines │
|
|
94
|
+
│ to see the 55 vendored files: lw diff order-processor --vendor │
|
|
83
95
|
╰──────────────────────────────────────────────────────────────────────╯
|
|
84
|
-
size 8.
|
|
96
|
+
size 8.3 KB → 9.2 KB (+886 B)
|
|
85
97
|
|
|
86
98
|
Dependencies
|
|
87
99
|
manager package from to origin
|
|
@@ -98,21 +110,24 @@ high aws-access-key-id config.py:6 AKIA…LE (20 chars)
|
|
|
98
110
|
high stripe-key config.py:7 sk_l…dc (32 chars)
|
|
99
111
|
low debug-flag config.py:8 DEBUG = True
|
|
100
112
|
|
|
101
|
-
Files
|
|
102
|
-
path
|
|
103
|
-
~ lambda_function.py
|
|
104
|
-
~ requirements.txt
|
|
105
|
-
+ config.py
|
|
106
|
-
+ helpers/__init__.py
|
|
107
|
-
→ {db → helpers/db}.py
|
|
108
|
-
→ site-packages/boto3-1.{34.0 → 35.20}.dist-info/
|
|
109
|
-
|
|
113
|
+
Files
|
|
114
|
+
path + − size
|
|
115
|
+
~ lambda_function.py 9 2 +322
|
|
116
|
+
~ requirements.txt 2 1 +17
|
|
117
|
+
+ config.py 10 +235
|
|
118
|
+
+ helpers/__init__.py
|
|
119
|
+
→ {db → helpers/db}.py 1 +33
|
|
120
|
+
→ site-packages/boto3-1.{34.0 → 35.20}.dist-info/ · 4 files, 1 1 1 +1
|
|
121
|
+
edited
|
|
122
|
+
→ site-packages/botocore-1.{34.0 → 35.20}.dist-info/ · 4 files, 1 1 1 +1
|
|
123
|
+
edited
|
|
110
124
|
```
|
|
111
125
|
|
|
112
|
-
The
|
|
113
|
-
package is one rename rather than a delete plus an add,
|
|
114
|
-
|
|
115
|
-
|
|
126
|
+
The 55 vendored files became three version numbers, `db.py` moving into a
|
|
127
|
+
package is one rename rather than a delete plus an add, each bumped package's
|
|
128
|
+
`dist-info` is one line rather than four, and the new environment variable, the
|
|
129
|
+
new AWS service and the three secret findings are changes that a file diff
|
|
130
|
+
cannot express at all.
|
|
116
131
|
|
|
117
132
|
That capture is not illustrative — it is the output of
|
|
118
133
|
[`docs/examples/build_demo.py`](docs/examples/build_demo.py), which builds the
|
|
@@ -213,7 +228,7 @@ the manual recipes.
|
|
|
213
228
|
| `ls` | Every function archived so far. |
|
|
214
229
|
| `versions FN` | Every archived version of one function. |
|
|
215
230
|
| `show FN [V]` | Runtime, handler, dependencies, env vars, services and findings for one version. `--files`, `--json`. |
|
|
216
|
-
| `diff FN` | Compare two versions. Defaults to the last two. `--from`/`--to`, `--html`, `--open`, `--vendor`, `--no-patch`, `--json`. |
|
|
231
|
+
| `diff FN` | Compare two versions. Defaults to the last two. `--from`/`--to`, `--html`, `--open`, `--vendor`, `--whitespace`, `--no-patch`, `--json`. |
|
|
217
232
|
| `report FN` | Build a browsable HTML history: an index plus a diff for every step. |
|
|
218
233
|
| `export FN [V]` | Get a version back out as a deployable zip (`--zip`) or a plain folder (`--tree`). |
|
|
219
234
|
| `open FN [V]` | Open the function's mirror in your editor — every version in one folder, with history. Name a version to open just its files. |
|
|
@@ -245,11 +260,11 @@ back from the newest.
|
|
|
245
260
|
└── functions/
|
|
246
261
|
└── order-processor/
|
|
247
262
|
└── versions/
|
|
248
|
-
├── 0001-
|
|
263
|
+
├── 0001-7fc98e0e/
|
|
249
264
|
│ ├── code/ # the extracted tree
|
|
250
265
|
│ ├── manifest.json # the full analysis
|
|
251
266
|
│ └── package.zip # the original download
|
|
252
|
-
└── 0002-
|
|
267
|
+
└── 0002-7f887035/
|
|
253
268
|
```
|
|
254
269
|
|
|
255
270
|
The directories are the source of truth. `index.db` is a cache you can delete
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "lambda-watcher"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "Watch your Downloads folder for AWS Lambda deployment zips, archive them as versions, analyse them, and diff any two versions."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -46,26 +46,57 @@ class Analysis:
|
|
|
46
46
|
|
|
47
47
|
@property
|
|
48
48
|
def primary_handler(self) -> str | None:
|
|
49
|
+
"""The handler AWS would most likely invoke, or None if none was found.
|
|
50
|
+
|
|
51
|
+
:func:`~.handler.detect_handlers` returns its candidates best-first, so this
|
|
52
|
+
is simply the top one — something like ``lambda_function.lambda_handler``.
|
|
53
|
+
"""
|
|
49
54
|
return self.handlers[0].handler if self.handlers else None
|
|
50
55
|
|
|
51
56
|
@property
|
|
52
57
|
def vendor_file_count(self) -> int:
|
|
58
|
+
"""How many files came from ``node_modules``, ``site-packages`` and friends.
|
|
59
|
+
|
|
60
|
+
Usually most of the package. Reported separately so a diff can say "1 file
|
|
61
|
+
you wrote changed, 4,812 vendored files came along with it" instead of
|
|
62
|
+
burying the first number in the second.
|
|
63
|
+
"""
|
|
53
64
|
return sum(1 for f in self.inventory.files if f.is_vendor)
|
|
54
65
|
|
|
55
66
|
@property
|
|
56
67
|
def vendor_size(self) -> int:
|
|
68
|
+
"""Total bytes of vendored files, the companion to :attr:`vendor_file_count`."""
|
|
57
69
|
return sum(f.size for f in self.inventory.files if f.is_vendor)
|
|
58
70
|
|
|
59
71
|
def unique_env_vars(self, include_reserved: bool = False) -> list[str]:
|
|
72
|
+
"""The distinct environment variable names the code reads, sorted.
|
|
73
|
+
|
|
74
|
+
Collapses the per-reference list, which can name the same variable from a
|
|
75
|
+
dozen lines. Runtime-provided names (``AWS_REGION``, ``PATH``) are left out
|
|
76
|
+
unless ``include_reserved`` is set, because they are the same in every
|
|
77
|
+
package and only ever add noise to a diff.
|
|
78
|
+
"""
|
|
60
79
|
names = {
|
|
61
80
|
ref.name for ref in self.env_vars if include_reserved or not ref.is_reserved
|
|
62
81
|
}
|
|
63
82
|
return sorted(names)
|
|
64
83
|
|
|
65
84
|
def unique_services(self) -> list[str]:
|
|
85
|
+
"""The distinct AWS service ids the code talks to, sorted.
|
|
86
|
+
|
|
87
|
+
``["dynamodb", "s3", "sqs"]`` — one entry per service no matter how many
|
|
88
|
+
call sites mention it.
|
|
89
|
+
"""
|
|
66
90
|
return sorted({ref.service for ref in self.services})
|
|
67
91
|
|
|
68
92
|
def totals(self) -> dict[str, int]:
|
|
93
|
+
"""The headline counts for this package, as a flat dict.
|
|
94
|
+
|
|
95
|
+
The numbers a summary line is built from: how many files, how many bytes,
|
|
96
|
+
and how much of each is first-party code rather than vendored dependency.
|
|
97
|
+
Kept as one dict because it goes straight into the manifest and into the
|
|
98
|
+
index as a row.
|
|
99
|
+
"""
|
|
69
100
|
return {
|
|
70
101
|
"file_count": self.inventory.file_count,
|
|
71
102
|
"total_size": self.inventory.total_size,
|
|
@@ -77,6 +108,17 @@ class Analysis:
|
|
|
77
108
|
}
|
|
78
109
|
|
|
79
110
|
def to_manifest(self, extra: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
111
|
+
"""Render this analysis as the ``manifest.json`` written beside the version.
|
|
112
|
+
|
|
113
|
+
The manifest is the source of truth on disk: the SQLite index is rebuilt
|
|
114
|
+
from these files by :mod:`~lambda_watcher.reindex`, so anything the index
|
|
115
|
+
needs has to be here. ``extra`` carries the fields only the ingest knows —
|
|
116
|
+
the function name, the sequence number, the originating zip — which are
|
|
117
|
+
merged in over the top.
|
|
118
|
+
|
|
119
|
+
Bumping the shape of what this returns means bumping ``MANIFEST_SCHEMA``,
|
|
120
|
+
and old manifests still have to reindex.
|
|
121
|
+
"""
|
|
80
122
|
manifest: dict[str, Any] = {
|
|
81
123
|
"schema": MANIFEST_SCHEMA,
|
|
82
124
|
"tree_hash": self.inventory.tree_hash,
|
|
@@ -32,6 +32,14 @@ except ModuleNotFoundError: # pragma: no cover
|
|
|
32
32
|
|
|
33
33
|
@dataclass(frozen=True)
|
|
34
34
|
class Dependency:
|
|
35
|
+
"""One dependency, either declared in a manifest or installed in the zip.
|
|
36
|
+
|
|
37
|
+
``is_declared`` is the important flag. A declared entry came from
|
|
38
|
+
``requirements.txt`` and may be a range (``boto3>=1.34``); an installed
|
|
39
|
+
entry came from ``site-packages`` and is an exact version that really
|
|
40
|
+
shipped (``boto3 1.34.0``). Frozen so it can go in a set.
|
|
41
|
+
"""
|
|
42
|
+
|
|
35
43
|
manager: str # pip | npm | go | maven | gem
|
|
36
44
|
name: str
|
|
37
45
|
version: str | None
|
|
@@ -39,9 +47,16 @@ class Dependency:
|
|
|
39
47
|
is_declared: bool # False => vendored/installed
|
|
40
48
|
|
|
41
49
|
def key(self) -> tuple[str, str]:
|
|
50
|
+
"""Identity across versions: ``(manager, lowercased name)``.
|
|
51
|
+
|
|
52
|
+
Deliberately excludes the version, because this is what a diff groups on to
|
|
53
|
+
notice that ``boto3`` went from 1.34.0 to 1.35.20 rather than reporting one
|
|
54
|
+
package removed and a different one added.
|
|
55
|
+
"""
|
|
42
56
|
return (self.manager, self.name.lower())
|
|
43
57
|
|
|
44
58
|
def as_dict(self) -> dict:
|
|
59
|
+
"""This dependency as plain JSON-ready data, for the manifest."""
|
|
45
60
|
return {
|
|
46
61
|
"manager": self.manager,
|
|
47
62
|
"name": self.name,
|
|
@@ -58,6 +73,16 @@ _REQ_LINE = re.compile(
|
|
|
58
73
|
|
|
59
74
|
|
|
60
75
|
def _parse_requirements(text: str, source: str) -> list[Dependency]:
|
|
76
|
+
"""Parse a ``requirements.txt`` into declared pip dependencies.
|
|
77
|
+
|
|
78
|
+
Handles the ordinary ``boto3==1.34.0`` form plus extras (``requests[security]``),
|
|
79
|
+
direct URLs and ``git+`` references (the trailing path segment becomes the
|
|
80
|
+
name), and PEP 508 ``name @ url`` entries. Comments and the flag lines that
|
|
81
|
+
start with ``-`` (``-r base.txt``, ``-e .``, ``--index-url``) are skipped.
|
|
82
|
+
|
|
83
|
+
A bare ``boto3`` with no comparison operator records a None version — the
|
|
84
|
+
file asked for the package but not for any particular release.
|
|
85
|
+
"""
|
|
61
86
|
deps: list[Dependency] = []
|
|
62
87
|
for raw in text.splitlines():
|
|
63
88
|
line = raw.split("#", 1)[0].strip()
|
|
@@ -80,6 +105,16 @@ def _parse_requirements(text: str, source: str) -> list[Dependency]:
|
|
|
80
105
|
|
|
81
106
|
|
|
82
107
|
def _parse_pyproject(text: str, source: str) -> list[Dependency]:
|
|
108
|
+
"""Parse a ``pyproject.toml`` into declared pip dependencies.
|
|
109
|
+
|
|
110
|
+
Reads both the standard ``[project] dependencies`` list and Poetry's
|
|
111
|
+
``[tool.poetry.dependencies]`` table, whose values may be a bare version
|
|
112
|
+
string or a table with a ``version`` key. Poetry's ``python`` entry is
|
|
113
|
+
dropped: it constrains the interpreter, not the package set.
|
|
114
|
+
|
|
115
|
+
Returns nothing if TOML cannot be parsed — on Python 3.10 ``tomli`` may be
|
|
116
|
+
absent, and a malformed file is not worth failing an ingest over.
|
|
117
|
+
"""
|
|
83
118
|
if tomllib is None:
|
|
84
119
|
return []
|
|
85
120
|
try:
|
|
@@ -108,6 +143,15 @@ def _parse_pyproject(text: str, source: str) -> list[Dependency]:
|
|
|
108
143
|
|
|
109
144
|
|
|
110
145
|
def _parse_package_json(text: str, source: str, declared: bool = True) -> list[Dependency]:
|
|
146
|
+
"""Parse a ``package.json``, either as a manifest or as an installed package.
|
|
147
|
+
|
|
148
|
+
The same filename means two different things depending on where it sits.
|
|
149
|
+
At the package root it is a manifest, and ``declared=True`` reads the
|
|
150
|
+
``dependencies``/``devDependencies``/``optionalDependencies`` tables, whose
|
|
151
|
+
values are ranges like ``^4.17.21``. Inside ``node_modules/<pkg>/`` it
|
|
152
|
+
describes one installed package, and ``declared=False`` takes the file's own
|
|
153
|
+
``name`` and ``version`` — the exact release that shipped.
|
|
154
|
+
"""
|
|
111
155
|
try:
|
|
112
156
|
data = json.loads(text)
|
|
113
157
|
except json.JSONDecodeError:
|
|
@@ -129,6 +173,13 @@ def _parse_package_json(text: str, source: str, declared: bool = True) -> list[D
|
|
|
129
173
|
|
|
130
174
|
|
|
131
175
|
def _parse_package_lock(text: str, source: str) -> list[Dependency]:
|
|
176
|
+
"""Parse a ``package-lock.json`` into installed npm dependencies.
|
|
177
|
+
|
|
178
|
+
Supports both lockfile layouts: v2/v3 keep a flat ``packages`` map keyed by
|
|
179
|
+
path, where the name has to be recovered from the key when the entry omits
|
|
180
|
+
it, while v1 keeps a ``dependencies`` map keyed by name. Both give resolved
|
|
181
|
+
versions, so entries are recorded as installed rather than declared.
|
|
182
|
+
"""
|
|
132
183
|
try:
|
|
133
184
|
data = json.loads(text)
|
|
134
185
|
except json.JSONDecodeError:
|
|
@@ -154,6 +205,12 @@ _YARN_ENTRY = re.compile(r'^"?([^@\s"][^@\s"]*)@[^\n:]*:\s*$\n(?:.*\n)*?\s+versi
|
|
|
154
205
|
|
|
155
206
|
|
|
156
207
|
def _parse_yarn_lock(text: str, source: str) -> list[Dependency]:
|
|
208
|
+
"""Parse a ``yarn.lock`` into installed npm dependencies.
|
|
209
|
+
|
|
210
|
+
Yarn's format is not JSON, so this matches each ``name@range:`` header
|
|
211
|
+
against the indented ``version "1.2.3"`` line that follows it and takes the
|
|
212
|
+
resolved version.
|
|
213
|
+
"""
|
|
157
214
|
deps: list[Dependency] = []
|
|
158
215
|
for match in _YARN_ENTRY.finditer(text):
|
|
159
216
|
deps.append(Dependency("npm", match.group(1), match.group(2), source, False))
|
|
@@ -165,6 +222,11 @@ _GO_REQUIRE_LINE = re.compile(r"^\s*([^\s/]+\S*)\s+(v\S+)", re.MULTILINE)
|
|
|
165
222
|
|
|
166
223
|
|
|
167
224
|
def _parse_go_mod(text: str, source: str) -> list[Dependency]:
|
|
225
|
+
"""Parse a ``go.mod`` into declared Go dependencies.
|
|
226
|
+
|
|
227
|
+
Covers both spellings: the grouped ``require ( ... )`` block and the
|
|
228
|
+
single-line ``require example.com/mod v1.2.3`` form.
|
|
229
|
+
"""
|
|
168
230
|
deps: list[Dependency] = []
|
|
169
231
|
for block in _GO_REQUIRE_BLOCK.findall(text):
|
|
170
232
|
for name, version in _GO_REQUIRE_LINE.findall(block):
|
|
@@ -179,6 +241,16 @@ def _parse_go_mod(text: str, source: str) -> list[Dependency]:
|
|
|
179
241
|
|
|
180
242
|
|
|
181
243
|
def _parse_pom(text: str, source: str) -> list[Dependency]:
|
|
244
|
+
"""Parse a Maven ``pom.xml`` into declared Java dependencies.
|
|
245
|
+
|
|
246
|
+
Each ``<dependency>`` becomes one entry named ``groupId:artifactId``, the
|
|
247
|
+
coordinate Maven itself uses. A ``<version>`` that is a property reference
|
|
248
|
+
(``${aws.sdk.version}``) is recorded verbatim, since resolving it would mean
|
|
249
|
+
evaluating the build.
|
|
250
|
+
|
|
251
|
+
Regex rather than an XML parser because this only needs the common shape and
|
|
252
|
+
must not fail an ingest over an unusual document.
|
|
253
|
+
"""
|
|
182
254
|
deps: list[Dependency] = []
|
|
183
255
|
for block in re.findall(r"<dependency>(.*?)</dependency>", text, re.DOTALL):
|
|
184
256
|
group = re.search(r"<groupId>(.*?)</groupId>", block, re.DOTALL)
|
|
@@ -196,6 +268,11 @@ _GEMFILE_LOCK = re.compile(r"^\s{4}([a-zA-Z0-9_-]+)\s+\(([^)]+)\)", re.MULTILINE
|
|
|
196
268
|
|
|
197
269
|
|
|
198
270
|
def _parse_gemfile_lock(text: str, source: str) -> list[Dependency]:
|
|
271
|
+
"""Parse a ``Gemfile.lock`` into installed Ruby gems.
|
|
272
|
+
|
|
273
|
+
The four-space-indented ``name (1.2.3)`` lines under ``specs:`` are the
|
|
274
|
+
resolved versions, which is why these are recorded as installed.
|
|
275
|
+
"""
|
|
199
276
|
return [
|
|
200
277
|
Dependency("gem", name, version, source, False)
|
|
201
278
|
for name, version in _GEMFILE_LOCK.findall(text)
|
|
@@ -39,18 +39,43 @@ _SCANNABLE = {"python", "javascript", "typescript", "java", "ruby", "csharp", "g
|
|
|
39
39
|
|
|
40
40
|
@dataclass
|
|
41
41
|
class EnvVarRef:
|
|
42
|
+
"""One place in the code where an environment variable is read.
|
|
43
|
+
|
|
44
|
+
The same variable read from three files is three refs; collapse them with
|
|
45
|
+
:meth:`~lambda_watcher.analysis.Analysis.unique_env_vars` when you want the
|
|
46
|
+
set of names rather than the call sites.
|
|
47
|
+
"""
|
|
48
|
+
|
|
42
49
|
name: str
|
|
43
50
|
path: str
|
|
44
51
|
line: int
|
|
45
52
|
is_reserved: bool = False
|
|
46
53
|
|
|
47
54
|
def as_dict(self) -> dict:
|
|
55
|
+
"""This reference as plain JSON-ready data, for the manifest."""
|
|
48
56
|
return {"name": self.name, "path": self.path, "line": self.line, "is_reserved": self.is_reserved}
|
|
49
57
|
|
|
50
58
|
|
|
51
59
|
def detect_env_vars(
|
|
52
60
|
root: Path, inventory: Inventory, include_vendor: bool = False, max_files: int = 2000
|
|
53
61
|
) -> list[EnvVarRef]:
|
|
62
|
+
"""Find every environment variable the code reads, across languages.
|
|
63
|
+
|
|
64
|
+
Scans each text file line by line for the idioms that read configuration —
|
|
65
|
+
``os.environ["X"]`` and ``os.getenv("X")`` in Python, ``process.env.X`` in
|
|
66
|
+
JavaScript, ``System.getenv("X")`` in Java, and the Ruby and C# equivalents
|
|
67
|
+
— and records the name, file and line of each hit.
|
|
68
|
+
|
|
69
|
+
Vendored dependencies are skipped by default: they read hundreds of
|
|
70
|
+
variables that have nothing to do with this function. ``max_files`` caps how
|
|
71
|
+
many files are opened so a package with an enormous tree cannot make an
|
|
72
|
+
ingest crawl. Results are deduplicated by ``(name, path, line)`` and sorted,
|
|
73
|
+
so re-analysing an unchanged tree produces an identical list.
|
|
74
|
+
|
|
75
|
+
This is textual pattern matching, not parsing: a name built at runtime
|
|
76
|
+
(``os.environ[prefix + "_URL"]``) is invisible to it, and one inside a
|
|
77
|
+
comment still counts.
|
|
78
|
+
"""
|
|
54
79
|
refs: list[EnvVarRef] = []
|
|
55
80
|
seen: set[tuple[str, str, int]] = set()
|
|
56
81
|
entries = inventory.files if include_vendor else inventory.code_files
|
|
@@ -31,16 +31,30 @@ _PREFERRED = (
|
|
|
31
31
|
|
|
32
32
|
@dataclass
|
|
33
33
|
class HandlerCandidate:
|
|
34
|
+
"""One possible Lambda entry point, with a score saying how likely it is.
|
|
35
|
+
|
|
36
|
+
``handler`` is the string you would actually paste into the AWS console —
|
|
37
|
+
``lambda_function.lambda_handler`` — assembled from the file's module path
|
|
38
|
+
and the function's name.
|
|
39
|
+
"""
|
|
40
|
+
|
|
34
41
|
path: str
|
|
35
42
|
symbol: str
|
|
36
43
|
handler: str # "module.function", the value you paste into the console
|
|
37
44
|
score: int
|
|
38
45
|
|
|
39
46
|
def as_dict(self) -> dict:
|
|
47
|
+
"""This candidate as plain JSON-ready data, for the manifest."""
|
|
40
48
|
return {"path": self.path, "symbol": self.symbol, "handler": self.handler, "score": self.score}
|
|
41
49
|
|
|
42
50
|
|
|
43
51
|
def _module_name(path: str) -> str:
|
|
52
|
+
"""Turn a file path into the dotted module path AWS expects.
|
|
53
|
+
|
|
54
|
+
``src/app/handler.py`` -> ``src.app.handler``. The extension is dropped and
|
|
55
|
+
every directory separator becomes a dot, which is the form the handler
|
|
56
|
+
setting takes.
|
|
57
|
+
"""
|
|
44
58
|
pure = PurePosixPath(path)
|
|
45
59
|
stem = pure.stem
|
|
46
60
|
parts = list(pure.parts[:-1]) + [stem]
|
|
@@ -10,6 +10,14 @@ from ..utils import count_lines, is_probably_text, language_for, matches_any, sh
|
|
|
10
10
|
|
|
11
11
|
@dataclass
|
|
12
12
|
class FileEntry:
|
|
13
|
+
"""One file inside an extracted package, hashed and classified.
|
|
14
|
+
|
|
15
|
+
``path`` is always posix-style and relative to the package root, so the same
|
|
16
|
+
tree hashes identically on Windows and Linux. ``is_vendor`` is the flag most
|
|
17
|
+
of the tool keys off: it separates the handful of files somebody wrote from
|
|
18
|
+
the thousands that came out of ``pip install``.
|
|
19
|
+
"""
|
|
20
|
+
|
|
13
21
|
path: str # posix relative path
|
|
14
22
|
size: int
|
|
15
23
|
sha256: str
|
|
@@ -20,6 +28,7 @@ class FileEntry:
|
|
|
20
28
|
lines: int
|
|
21
29
|
|
|
22
30
|
def as_dict(self) -> dict:
|
|
31
|
+
"""This entry as plain JSON-ready data, for the manifest."""
|
|
23
32
|
return {
|
|
24
33
|
"path": self.path,
|
|
25
34
|
"size": self.size,
|
|
@@ -34,6 +43,14 @@ class FileEntry:
|
|
|
34
43
|
|
|
35
44
|
@dataclass
|
|
36
45
|
class Inventory:
|
|
46
|
+
"""Every file in one extracted package, plus the totals worth caching.
|
|
47
|
+
|
|
48
|
+
``tree_hash`` is the identity of the whole tree and what decides whether an
|
|
49
|
+
ingest has found a new version. The size and line counts are accumulated
|
|
50
|
+
during the walk rather than recomputed, since they are wanted on every
|
|
51
|
+
summary screen.
|
|
52
|
+
"""
|
|
53
|
+
|
|
37
54
|
files: list[FileEntry] = field(default_factory=list)
|
|
38
55
|
tree_hash: str = ""
|
|
39
56
|
total_size: int = 0
|
|
@@ -42,6 +59,7 @@ class Inventory:
|
|
|
42
59
|
|
|
43
60
|
@property
|
|
44
61
|
def file_count(self) -> int:
|
|
62
|
+
"""How many files the package contains, vendored ones included."""
|
|
45
63
|
return len(self.files)
|
|
46
64
|
|
|
47
65
|
@property
|
|
@@ -51,12 +69,24 @@ class Inventory:
|
|
|
51
69
|
|
|
52
70
|
@property
|
|
53
71
|
def code_file_count(self) -> int:
|
|
72
|
+
"""How many first-party files there are — the length of :attr:`code_files`."""
|
|
54
73
|
return len(self.code_files)
|
|
55
74
|
|
|
56
75
|
def by_path(self) -> dict[str, FileEntry]:
|
|
76
|
+
"""The files as a ``{path: entry}`` lookup.
|
|
77
|
+
|
|
78
|
+
Diffing two versions means asking "was this path in the other one too?"
|
|
79
|
+
thousands of times, which wants a dict rather than a scan of the list.
|
|
80
|
+
"""
|
|
57
81
|
return {f.path: f for f in self.files}
|
|
58
82
|
|
|
59
83
|
def language_breakdown(self) -> dict[str, int]:
|
|
84
|
+
"""Count first-party files per language, most common first.
|
|
85
|
+
|
|
86
|
+
``{"python": 12, "json": 3, "markdown": 1}``. Vendored files are excluded
|
|
87
|
+
deliberately — counting them would report the language of the dependencies
|
|
88
|
+
rather than of the function.
|
|
89
|
+
"""
|
|
60
90
|
counts: dict[str, int] = {}
|
|
61
91
|
for f in self.code_files:
|
|
62
92
|
counts[f.lang] = counts.get(f.lang, 0) + 1
|