ghwm 1.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ghwm-1.10.0/.agents/skills/code-review/SKILL.md +87 -0
- ghwm-1.10.0/.agents/skills/code-review/agents/openai.yaml +3 -0
- ghwm-1.10.0/.agents/skills/python-coding-standards/SKILL.md +78 -0
- ghwm-1.10.0/.agents/skills/python-performance-optimization/SKILL.md +100 -0
- ghwm-1.10.0/.agents/skills/python-performance-optimization/references/advanced-patterns.md +419 -0
- ghwm-1.10.0/.agents/skills/python-performance-optimization/references/details.md +342 -0
- ghwm-1.10.0/.agents/skills/tdd/SKILL.md +38 -0
- ghwm-1.10.0/.agents/skills/tdd/agents/openai.yaml +3 -0
- ghwm-1.10.0/.agents/skills/tdd/mocking.md +59 -0
- ghwm-1.10.0/.agents/skills/tdd/tests.md +77 -0
- ghwm-1.10.0/.agents/skills/testing-standards/SKILL.md +74 -0
- ghwm-1.10.0/.coveragerc +22 -0
- ghwm-1.10.0/.editorconfig +36 -0
- ghwm-1.10.0/.github/ISSUE_TEMPLATE/bug_report.yml +74 -0
- ghwm-1.10.0/.github/ISSUE_TEMPLATE/config.yml +9 -0
- ghwm-1.10.0/.github/ISSUE_TEMPLATE/feature_request.yml +54 -0
- ghwm-1.10.0/.github/ISSUE_TEMPLATE/security_report.yml +67 -0
- ghwm-1.10.0/.github/auto_assign.yaml +15 -0
- ghwm-1.10.0/.github/dependabot.yaml +40 -0
- ghwm-1.10.0/.github/linters/.gitleaks.toml +2 -0
- ghwm-1.10.0/.github/linters/.markdown-lint.yml +7 -0
- ghwm-1.10.0/.github/linters/.ruff.toml +9 -0
- ghwm-1.10.0/.github/linters/.textlintignore +5 -0
- ghwm-1.10.0/.github/linters/.textlintrc +16 -0
- ghwm-1.10.0/.github/linters/.yaml-lint.yml +14 -0
- ghwm-1.10.0/.github/linters/actionlint.yml +23 -0
- ghwm-1.10.0/.github/pr-title-checker-config.json +15 -0
- ghwm-1.10.0/.github/pull_request_template.md +13 -0
- ghwm-1.10.0/.github/release.yml +21 -0
- ghwm-1.10.0/.github/static/readme_header.jpg +0 -0
- ghwm-1.10.0/.github/super-linter-fix.env +9 -0
- ghwm-1.10.0/.github/super-linter.env +18 -0
- ghwm-1.10.0/.github/workflows/auto-assign-pr.yaml +37 -0
- ghwm-1.10.0/.github/workflows/pr-title-check.yaml +29 -0
- ghwm-1.10.0/.github/workflows/pre-commit.yaml +42 -0
- ghwm-1.10.0/.github/workflows/publish.yaml +60 -0
- ghwm-1.10.0/.github/workflows/release.yaml +69 -0
- ghwm-1.10.0/.github/workflows/super-linter.yaml +88 -0
- ghwm-1.10.0/.github/workflows/tests.yaml +127 -0
- ghwm-1.10.0/.gitignore +22 -0
- ghwm-1.10.0/.markdownlint.yaml +7 -0
- ghwm-1.10.0/.markdownlintignore +0 -0
- ghwm-1.10.0/.pre-commit-config.yaml +14 -0
- ghwm-1.10.0/AGENTS.md +189 -0
- ghwm-1.10.0/CONTRIBUTING.md +259 -0
- ghwm-1.10.0/LICENSE +21 -0
- ghwm-1.10.0/Makefile +134 -0
- ghwm-1.10.0/PKG-INFO +375 -0
- ghwm-1.10.0/README.md +347 -0
- ghwm-1.10.0/SECURITY.md +15 -0
- ghwm-1.10.0/codecov.yml +74 -0
- ghwm-1.10.0/docs/ARCHITECTURE.md +333 -0
- ghwm-1.10.0/docs/reference/frontmatter.md +266 -0
- ghwm-1.10.0/docs/reference/manifest.md +216 -0
- ghwm-1.10.0/ghwm.lock +27 -0
- ghwm-1.10.0/ghwm.yml +7 -0
- ghwm-1.10.0/pyproject.toml +107 -0
- ghwm-1.10.0/skills-lock.json +23 -0
- ghwm-1.10.0/src/ghwm/__init__.py +10 -0
- ghwm-1.10.0/src/ghwm/__main__.py +5 -0
- ghwm-1.10.0/src/ghwm/audit.py +187 -0
- ghwm-1.10.0/src/ghwm/cli.py +291 -0
- ghwm-1.10.0/src/ghwm/download.py +190 -0
- ghwm-1.10.0/src/ghwm/download_npm.py +287 -0
- ghwm-1.10.0/src/ghwm/install.py +238 -0
- ghwm-1.10.0/src/ghwm/lock.py +159 -0
- ghwm-1.10.0/src/ghwm/managed_files.py +270 -0
- ghwm-1.10.0/src/ghwm/manifest.py +213 -0
- ghwm-1.10.0/src/ghwm/metadata.py +187 -0
- ghwm-1.10.0/src/ghwm/package_names.py +13 -0
- ghwm-1.10.0/src/ghwm/paths.py +22 -0
- ghwm-1.10.0/src/ghwm/telemetry.py +146 -0
- ghwm-1.10.0/tests/__init__.py +1 -0
- ghwm-1.10.0/tests/conftest.py +32 -0
- ghwm-1.10.0/tests/shared.py +18 -0
- ghwm-1.10.0/tests/test_audit.py +231 -0
- ghwm-1.10.0/tests/test_cli.py +1359 -0
- ghwm-1.10.0/tests/test_download.py +287 -0
- ghwm-1.10.0/tests/test_download_npm.py +703 -0
- ghwm-1.10.0/tests/test_e2e.py +594 -0
- ghwm-1.10.0/tests/test_install.py +1001 -0
- ghwm-1.10.0/tests/test_lock.py +345 -0
- ghwm-1.10.0/tests/test_managed_files.py +196 -0
- ghwm-1.10.0/tests/test_manifest.py +213 -0
- ghwm-1.10.0/tests/test_metadata.py +414 -0
- ghwm-1.10.0/tests/test_security.py +115 -0
- ghwm-1.10.0/tests/test_telemetry.py +511 -0
- ghwm-1.10.0/uv.lock +808 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: code-review
|
|
3
|
+
description: "Review the changes since a fixed point (commit, branch, tag, or merge-base) along two axes: Standards (does the code follow this repo's documented coding standards?) and Spec (does the code match what the originating issue/spec asked for?). Runs both reviews in parallel sub-agents and reports them side by side. Use when the user wants to review a branch, a PR, work-in-progress changes, or asks to \"review since X\"."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Two-axis review of the diff between `HEAD` and a fixed point the user supplies:
|
|
7
|
+
|
|
8
|
+
- **Standards**: does the code conform to this repo's documented coding standards?
|
|
9
|
+
- **Spec**: does the code faithfully implement the originating issue / spec?
|
|
10
|
+
|
|
11
|
+
Both axes run as **parallel sub-agents** so they don't pollute each other's context, then this skill aggregates their findings.
|
|
12
|
+
|
|
13
|
+
The issue tracker should have been provided to you. If `docs/agents/issue-tracker.md` is missing, tell the user to run `/setup-matt-pocock-skills`.
|
|
14
|
+
|
|
15
|
+
## Process
|
|
16
|
+
|
|
17
|
+
### 1. Pin the fixed point
|
|
18
|
+
|
|
19
|
+
Whatever the user said is the fixed point (a commit SHA, branch name, tag, `main`, `HEAD~5`, etc.). If they didn't specify one, ask for it.
|
|
20
|
+
|
|
21
|
+
Capture the diff command once: `git diff <fixed-point>...HEAD` (three-dot, so the comparison is against the merge-base). Also note the list of commits via `git log <fixed-point>..HEAD --oneline`.
|
|
22
|
+
|
|
23
|
+
Before going further, confirm the fixed point resolves (`git rev-parse <fixed-point>`) and the diff is non-empty. A bad ref or empty diff should fail here, not inside two parallel sub-agents.
|
|
24
|
+
|
|
25
|
+
### 2. Identify the spec source
|
|
26
|
+
|
|
27
|
+
Look for the originating spec, in this order:
|
|
28
|
+
|
|
29
|
+
1. Issue references in the commit messages (`#123`, `Closes #45`, GitLab `!67`, etc.), fetched via the workflow in `docs/agents/issue-tracker.md`.
|
|
30
|
+
2. A path the user passed as an argument.
|
|
31
|
+
3. A spec file under `docs/`, `specs/`, or `.scratch/` matching the branch name or feature.
|
|
32
|
+
4. If nothing is found, ask the user where the spec is. If they say there isn't one, the **Spec** sub-agent will skip and report "no spec available".
|
|
33
|
+
|
|
34
|
+
### 3. Identify the standards sources
|
|
35
|
+
|
|
36
|
+
Anything in the repo that documents how code should be written, such as `CODING_STANDARDS.md` or `CONTRIBUTING.md`.
|
|
37
|
+
|
|
38
|
+
On top of whatever the repo documents, the Standards axis always carries the **smell baseline** below: a fixed set of Fowler code smells (_Refactoring_, ch.3) that applies even when a repo documents nothing. Two rules bind it:
|
|
39
|
+
|
|
40
|
+
- **The repo overrides.** A documented repo standard always wins; where it endorses something the baseline would flag, suppress the smell.
|
|
41
|
+
- **Always a judgement call.** Each smell is a labelled heuristic ("possible Feature Envy"), never a hard violation. Like any standard here, skip anything tooling already enforces.
|
|
42
|
+
|
|
43
|
+
Each smell reads *what it is* → *how to fix*; match it against the diff:
|
|
44
|
+
|
|
45
|
+
- **Mysterious Name**: a function, variable, or type whose name doesn't reveal what it does or holds. → rename it; if no honest name comes, the design's murky.
|
|
46
|
+
- **Duplicated Code**: the same logic shape appears in more than one hunk or file in the change. → extract the shared shape, call it from both.
|
|
47
|
+
- **Feature Envy**: a method that reaches into another object's data more than its own. → move the method onto the data it envies.
|
|
48
|
+
- **Data Clumps**: the same few fields or params keep travelling together (a type wanting to be born). → bundle them into one type, pass that.
|
|
49
|
+
- **Primitive Obsession**: a primitive or string standing in for a domain concept that deserves its own type. → give the concept its own small type.
|
|
50
|
+
- **Repeated Switches**: the same `switch`/`if`-cascade on the same type recurs across the change. → replace with polymorphism, or one map both sites share.
|
|
51
|
+
- **Shotgun Surgery**: one logical change forces scattered edits across many files in the diff. → gather what changes together into one module.
|
|
52
|
+
- **Divergent Change**: one file or module is edited for several unrelated reasons. → split so each module changes for one reason.
|
|
53
|
+
- **Speculative Generality**: abstraction, parameters, or hooks added for needs the spec doesn't have. → delete it; inline back until a real need shows.
|
|
54
|
+
- **Message Chains**: long `a.b().c().d()` navigation the caller shouldn't depend on. → hide the walk behind one method on the first object.
|
|
55
|
+
- **Middle Man**: a class or function that mostly just delegates onward. → cut it, call the real target direct.
|
|
56
|
+
- **Refused Bequest**: a subclass or implementer that ignores or overrides most of what it inherits. → drop the inheritance, use composition.
|
|
57
|
+
|
|
58
|
+
### 4. Spawn both sub-agents in parallel
|
|
59
|
+
|
|
60
|
+
**Standards sub-agent prompt** should include:
|
|
61
|
+
|
|
62
|
+
- The full diff command and commit list.
|
|
63
|
+
- The list of standards-source files you found in step 3, **plus the smell baseline from step 3** pasted in full (the sub-agent has no other access to it).
|
|
64
|
+
- The brief: "Report, per file/hunk where relevant, (a) every place the diff violates a documented standard: cite the standard (file + the rule); and (b) any baseline smell you spot: name it and quote the hunk. Distinguish hard violations from judgement calls: documented-standard breaches can be hard, but baseline smells are always judgement calls, and a documented repo standard overrides the baseline. Skip anything tooling enforces. Under 400 words."
|
|
65
|
+
|
|
66
|
+
**Spec sub-agent prompt** should include:
|
|
67
|
+
|
|
68
|
+
- The diff command and commit list.
|
|
69
|
+
- The path or fetched contents of the spec.
|
|
70
|
+
- The brief: "Report: (a) requirements the spec asked for that are missing or partial; (b) behaviour in the diff that wasn't asked for (scope creep); (c) requirements that look implemented but where the implementation looks wrong. Quote the spec line for each finding. Under 400 words."
|
|
71
|
+
|
|
72
|
+
If the spec is missing, skip the Spec sub-agent and note this in the final report.
|
|
73
|
+
|
|
74
|
+
### 5. Aggregate
|
|
75
|
+
|
|
76
|
+
Present the two reports under `## Standards` and `## Spec` headings, verbatim or lightly cleaned. Do **not** merge or rerank findings, because the two axes are deliberately separate (see _Why two axes_).
|
|
77
|
+
|
|
78
|
+
End with a one-line summary: total findings per axis, and the worst issue _within each axis_ (if any). Don't pick a single winner across axes: that's the reranking the separation exists to prevent.
|
|
79
|
+
|
|
80
|
+
## Why two axes
|
|
81
|
+
|
|
82
|
+
A change can pass one axis and fail the other:
|
|
83
|
+
|
|
84
|
+
- Code that follows every standard but implements the wrong thing → **Standards pass, Spec fail.**
|
|
85
|
+
- Code that does exactly what the issue asked but breaks the project's conventions → **Spec pass, Standards fail.**
|
|
86
|
+
|
|
87
|
+
Reporting them separately stops one axis from masking the other.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: python-coding-standards
|
|
3
|
+
description: "Reusable Python coding standards for implementation, refactoring, and review work. Use whenever writing or reviewing Python code, especially when you need guidance on typing, error handling, subprocess safety, module boundaries, or matching an existing codebase. Follow repository-specific instructions and existing project tooling when they are available."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# python-coding-standards
|
|
7
|
+
|
|
8
|
+
Use this skill as the generic baseline for Python work. Repository-specific rules may live in project instructions, `README.md`, `Makefile`, `pyproject.toml`, or CI config, so check those sources first when they are available.
|
|
9
|
+
|
|
10
|
+
## Compatibility
|
|
11
|
+
|
|
12
|
+
- Best for Python repositories with an existing formatter, linter, type checker, and test workflow.
|
|
13
|
+
- Works whether repository instructions live in dedicated instruction files, other docs, or only in project tooling.
|
|
14
|
+
- Safe to use as a standalone baseline when no repository-specific instructions are present.
|
|
15
|
+
|
|
16
|
+
## Start with repository context
|
|
17
|
+
|
|
18
|
+
1. Read repository-specific instructions if present.
|
|
19
|
+
2. Inspect `pyproject.toml`, `Makefile`, CI config, and the nearest existing implementation before changing code.
|
|
20
|
+
3. Match the repository's existing structure and naming unless the task explicitly requires a broader refactor.
|
|
21
|
+
|
|
22
|
+
## Core principles
|
|
23
|
+
|
|
24
|
+
- Prefer small, readable functions over clever or heavily abstracted code.
|
|
25
|
+
- Keep changes surgical; do not refactor unrelated areas.
|
|
26
|
+
- Reuse existing helpers before introducing new ones.
|
|
27
|
+
- Keep types explicit enough for the repository's configured type checker.
|
|
28
|
+
- Prefer descriptive local names for collections and loop items; avoid one-letter iteration variables unless a conventional index like `i` is clearly the most readable choice.
|
|
29
|
+
- Write error messages for operators and maintainers, not just for the current task.
|
|
30
|
+
- Add comments only when a block would otherwise be hard to understand.
|
|
31
|
+
|
|
32
|
+
## Error handling
|
|
33
|
+
|
|
34
|
+
- Do not add bare `except:` blocks.
|
|
35
|
+
- Catch only exceptions you can handle meaningfully.
|
|
36
|
+
- Prefer `except SomeError as exc` over `as e`.
|
|
37
|
+
- Do not silently swallow failures or return success-shaped defaults for unexpected errors.
|
|
38
|
+
- When falling back between values, be careful not to discard valid falsy values like `0`, `False`, `""`, or `[]`.
|
|
39
|
+
|
|
40
|
+
## Tooling
|
|
41
|
+
|
|
42
|
+
- Use the repository's existing formatter, linter, type checker, and test runner.
|
|
43
|
+
- Do not introduce new tooling when the repository already has established commands.
|
|
44
|
+
- When changing dependencies, follow the repository's existing dependency-management workflow.
|
|
45
|
+
|
|
46
|
+
## Module boundaries
|
|
47
|
+
|
|
48
|
+
- Do not import another module's private names.
|
|
49
|
+
- Keep entrypoints thin; move durable behavior into the owning module.
|
|
50
|
+
- Extract shared helpers only when reuse is real, not speculative.
|
|
51
|
+
- Remove duplicate helpers if a clear canonical implementation already exists.
|
|
52
|
+
- Keep constants in one place when they are shared across multiple functions or modules.
|
|
53
|
+
|
|
54
|
+
## Security and robustness
|
|
55
|
+
|
|
56
|
+
- Do not use `subprocess` with `shell=True` for untrusted input.
|
|
57
|
+
- Prefer argument lists to constructed shell strings.
|
|
58
|
+
- Use safe `tempfile` helpers instead of manual temp path creation.
|
|
59
|
+
- Do not log secrets, credentials, or full sensitive payloads.
|
|
60
|
+
- Treat archives, XML, and external data as untrusted input.
|
|
61
|
+
- Avoid code paths that depend on wall-clock timing, global mutable state, or ambient environment when a dependency can be injected instead.
|
|
62
|
+
|
|
63
|
+
## Code review guidance
|
|
64
|
+
|
|
65
|
+
When reviewing Python changes, focus on:
|
|
66
|
+
|
|
67
|
+
1. Correctness and unintended behavior changes
|
|
68
|
+
2. Type safety and data-shape clarity
|
|
69
|
+
3. Error handling and observability
|
|
70
|
+
4. Duplicate logic or unnecessary abstraction
|
|
71
|
+
5. Unsafe subprocess, file, tempdir, or parsing behavior
|
|
72
|
+
6. Missing or weak tests for changed behavior
|
|
73
|
+
|
|
74
|
+
## File-specific guidance
|
|
75
|
+
|
|
76
|
+
- `.py`: follow nearby module patterns, keep imports tidy, and keep public interfaces explicit.
|
|
77
|
+
- Tests: follow the repository's existing test style and helpers.
|
|
78
|
+
- Documentation: update the readme or architecture docs when behavior changes, following repository-specific documentation rules when they exist.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: python-performance-optimization
|
|
3
|
+
description: Profile and optimize Python code using cProfile, memory profilers, and performance best practices. Use when debugging slow Python code, optimizing bottlenecks, or improving application performance.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Python Performance Optimization
|
|
7
|
+
|
|
8
|
+
Comprehensive guide to profiling, analyzing, and optimizing Python code for better performance, including CPU profiling, memory optimization, and implementation best practices.
|
|
9
|
+
|
|
10
|
+
## When to Use This Skill
|
|
11
|
+
|
|
12
|
+
- Identifying performance bottlenecks in Python applications
|
|
13
|
+
- Reducing application latency and response times
|
|
14
|
+
- Optimizing CPU-intensive operations
|
|
15
|
+
- Reducing memory consumption and memory leaks
|
|
16
|
+
- Improving database query performance
|
|
17
|
+
- Optimizing I/O operations
|
|
18
|
+
- Speeding up data processing pipelines
|
|
19
|
+
- Implementing high-performance algorithms
|
|
20
|
+
- Profiling production applications
|
|
21
|
+
|
|
22
|
+
## Core Concepts
|
|
23
|
+
|
|
24
|
+
### 1. Profiling Types
|
|
25
|
+
|
|
26
|
+
- **CPU Profiling**: Identify time-consuming functions
|
|
27
|
+
- **Memory Profiling**: Track memory allocation and leaks
|
|
28
|
+
- **Line Profiling**: Profile at line-by-line granularity
|
|
29
|
+
- **Call Graph**: Visualize function call relationships
|
|
30
|
+
|
|
31
|
+
### 2. Performance Metrics
|
|
32
|
+
|
|
33
|
+
- **Execution Time**: How long operations take
|
|
34
|
+
- **Memory Usage**: Peak and average memory consumption
|
|
35
|
+
- **CPU Utilization**: Processor usage patterns
|
|
36
|
+
- **I/O Wait**: Time spent on I/O operations
|
|
37
|
+
|
|
38
|
+
### 3. Optimization Strategies
|
|
39
|
+
|
|
40
|
+
- **Algorithmic**: Better algorithms and data structures
|
|
41
|
+
- **Implementation**: More efficient code patterns
|
|
42
|
+
- **Parallelization**: Multi-threading/processing
|
|
43
|
+
- **Caching**: Avoid redundant computation
|
|
44
|
+
- **Native Extensions**: C/Rust for critical paths
|
|
45
|
+
|
|
46
|
+
## Quick Start
|
|
47
|
+
|
|
48
|
+
### Basic Timing
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
import time
|
|
52
|
+
|
|
53
|
+
def measure_time():
|
|
54
|
+
"""Simple timing measurement."""
|
|
55
|
+
start = time.time()
|
|
56
|
+
|
|
57
|
+
# Your code here
|
|
58
|
+
result = sum(range(1000000))
|
|
59
|
+
|
|
60
|
+
elapsed = time.time() - start
|
|
61
|
+
print(f"Execution time: {elapsed:.4f} seconds")
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
# Better: use timeit for accurate measurements
|
|
65
|
+
import timeit
|
|
66
|
+
|
|
67
|
+
execution_time = timeit.timeit(
|
|
68
|
+
"sum(range(1000000))",
|
|
69
|
+
number=100
|
|
70
|
+
)
|
|
71
|
+
print(f"Average time: {execution_time/100:.6f} seconds")
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Detailed patterns and worked examples
|
|
75
|
+
|
|
76
|
+
Detailed pattern documentation lives in `references/details.md`. Read that file when the navigation tier above is insufficient.
|
|
77
|
+
|
|
78
|
+
## Best Practices
|
|
79
|
+
|
|
80
|
+
1. **Profile before optimizing** - Measure to find real bottlenecks
|
|
81
|
+
2. **Focus on hot paths** - Optimize code that runs most frequently
|
|
82
|
+
3. **Use appropriate data structures** - Dict for lookups, set for membership
|
|
83
|
+
4. **Avoid premature optimization** - Clarity first, then optimize
|
|
84
|
+
5. **Use built-in functions** - They're implemented in C
|
|
85
|
+
6. **Cache expensive computations** - Use lru_cache
|
|
86
|
+
7. **Batch I/O operations** - Reduce system calls
|
|
87
|
+
8. **Use generators** for large datasets
|
|
88
|
+
9. **Consider NumPy** for numerical operations
|
|
89
|
+
10. **Profile production code** - Use py-spy for live systems
|
|
90
|
+
|
|
91
|
+
## Common Pitfalls
|
|
92
|
+
|
|
93
|
+
- Optimizing without profiling
|
|
94
|
+
- Using global variables unnecessarily
|
|
95
|
+
- Not using appropriate data structures
|
|
96
|
+
- Creating unnecessary copies of data
|
|
97
|
+
- Not using connection pooling for databases
|
|
98
|
+
- Ignoring algorithmic complexity
|
|
99
|
+
- Over-optimizing rare code paths
|
|
100
|
+
- Not considering memory usage
|
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
# Python Performance Optimization — Advanced Reference
|
|
2
|
+
|
|
3
|
+
Advanced optimization techniques including NumPy vectorization, caching, memory management, parallelization, async I/O, database optimization, and benchmarking tools.
|
|
4
|
+
|
|
5
|
+
## Advanced Optimization
|
|
6
|
+
|
|
7
|
+
### Pattern 11: NumPy for Numerical Operations
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
import timeit
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
def python_sum(n):
|
|
14
|
+
"""Sum using pure Python."""
|
|
15
|
+
return sum(range(n))
|
|
16
|
+
|
|
17
|
+
def numpy_sum(n):
|
|
18
|
+
"""Sum using NumPy."""
|
|
19
|
+
return np.arange(n).sum()
|
|
20
|
+
|
|
21
|
+
n = 1000000
|
|
22
|
+
|
|
23
|
+
python_time = timeit.timeit(lambda: python_sum(n), number=100)
|
|
24
|
+
numpy_time = timeit.timeit(lambda: numpy_sum(n), number=100)
|
|
25
|
+
|
|
26
|
+
print(f"Python: {python_time:.4f}s")
|
|
27
|
+
print(f"NumPy: {numpy_time:.4f}s")
|
|
28
|
+
print(f"Speedup: {python_time/numpy_time:.2f}x")
|
|
29
|
+
|
|
30
|
+
# Vectorized operations
|
|
31
|
+
def python_multiply():
|
|
32
|
+
"""Element-wise multiplication in Python."""
|
|
33
|
+
a = list(range(100000))
|
|
34
|
+
b = list(range(100000))
|
|
35
|
+
return [x * y for x, y in zip(a, b)]
|
|
36
|
+
|
|
37
|
+
def numpy_multiply():
|
|
38
|
+
"""Vectorized multiplication in NumPy."""
|
|
39
|
+
a = np.arange(100000)
|
|
40
|
+
b = np.arange(100000)
|
|
41
|
+
return a * b
|
|
42
|
+
|
|
43
|
+
py_time = timeit.timeit(python_multiply, number=100)
|
|
44
|
+
np_time = timeit.timeit(numpy_multiply, number=100)
|
|
45
|
+
|
|
46
|
+
print(f"\nPython multiply: {py_time:.4f}s")
|
|
47
|
+
print(f"NumPy multiply: {np_time:.4f}s")
|
|
48
|
+
print(f"Speedup: {py_time/np_time:.2f}x")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Pattern 12: Caching with functools.lru_cache
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from functools import lru_cache
|
|
55
|
+
import timeit
|
|
56
|
+
|
|
57
|
+
def fibonacci_slow(n):
|
|
58
|
+
"""Recursive fibonacci without caching."""
|
|
59
|
+
if n < 2:
|
|
60
|
+
return n
|
|
61
|
+
return fibonacci_slow(n-1) + fibonacci_slow(n-2)
|
|
62
|
+
|
|
63
|
+
@lru_cache(maxsize=None)
|
|
64
|
+
def fibonacci_fast(n):
|
|
65
|
+
"""Recursive fibonacci with caching."""
|
|
66
|
+
if n < 2:
|
|
67
|
+
return n
|
|
68
|
+
return fibonacci_fast(n-1) + fibonacci_fast(n-2)
|
|
69
|
+
|
|
70
|
+
# Massive speedup for recursive algorithms
|
|
71
|
+
n = 30
|
|
72
|
+
|
|
73
|
+
slow_time = timeit.timeit(lambda: fibonacci_slow(n), number=1)
|
|
74
|
+
fast_time = timeit.timeit(lambda: fibonacci_fast(n), number=1000)
|
|
75
|
+
|
|
76
|
+
print(f"Without cache (1 run): {slow_time:.4f}s")
|
|
77
|
+
print(f"With cache (1000 runs): {fast_time:.4f}s")
|
|
78
|
+
|
|
79
|
+
# Cache info
|
|
80
|
+
print(f"Cache info: {fibonacci_fast.cache_info()}")
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### Pattern 13: Using __slots__ for Memory
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
import sys
|
|
87
|
+
|
|
88
|
+
class RegularClass:
|
|
89
|
+
"""Regular class with __dict__."""
|
|
90
|
+
def __init__(self, x, y, z):
|
|
91
|
+
self.x = x
|
|
92
|
+
self.y = y
|
|
93
|
+
self.z = z
|
|
94
|
+
|
|
95
|
+
class SlottedClass:
|
|
96
|
+
"""Class with __slots__ for memory efficiency."""
|
|
97
|
+
__slots__ = ['x', 'y', 'z']
|
|
98
|
+
|
|
99
|
+
def __init__(self, x, y, z):
|
|
100
|
+
self.x = x
|
|
101
|
+
self.y = y
|
|
102
|
+
self.z = z
|
|
103
|
+
|
|
104
|
+
# Memory comparison
|
|
105
|
+
regular = RegularClass(1, 2, 3)
|
|
106
|
+
slotted = SlottedClass(1, 2, 3)
|
|
107
|
+
|
|
108
|
+
print(f"Regular class size: {sys.getsizeof(regular)} bytes")
|
|
109
|
+
print(f"Slotted class size: {sys.getsizeof(slotted)} bytes")
|
|
110
|
+
|
|
111
|
+
# Significant savings with many instances
|
|
112
|
+
regular_objects = [RegularClass(i, i+1, i+2) for i in range(10000)]
|
|
113
|
+
slotted_objects = [SlottedClass(i, i+1, i+2) for i in range(10000)]
|
|
114
|
+
|
|
115
|
+
print(f"\nMemory for 10000 regular objects: ~{sys.getsizeof(regular) * 10000} bytes")
|
|
116
|
+
print(f"Memory for 10000 slotted objects: ~{sys.getsizeof(slotted) * 10000} bytes")
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
### Pattern 14: Multiprocessing for CPU-Bound Tasks
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
import multiprocessing as mp
|
|
123
|
+
import time
|
|
124
|
+
|
|
125
|
+
def cpu_intensive_task(n):
|
|
126
|
+
"""CPU-intensive calculation."""
|
|
127
|
+
return sum(i**2 for i in range(n))
|
|
128
|
+
|
|
129
|
+
def sequential_processing():
|
|
130
|
+
"""Process tasks sequentially."""
|
|
131
|
+
start = time.time()
|
|
132
|
+
results = [cpu_intensive_task(1000000) for _ in range(4)]
|
|
133
|
+
elapsed = time.time() - start
|
|
134
|
+
return elapsed, results
|
|
135
|
+
|
|
136
|
+
def parallel_processing():
|
|
137
|
+
"""Process tasks in parallel."""
|
|
138
|
+
start = time.time()
|
|
139
|
+
with mp.Pool(processes=4) as pool:
|
|
140
|
+
results = pool.map(cpu_intensive_task, [1000000] * 4)
|
|
141
|
+
elapsed = time.time() - start
|
|
142
|
+
return elapsed, results
|
|
143
|
+
|
|
144
|
+
if __name__ == "__main__":
|
|
145
|
+
seq_time, seq_results = sequential_processing()
|
|
146
|
+
par_time, par_results = parallel_processing()
|
|
147
|
+
|
|
148
|
+
print(f"Sequential: {seq_time:.2f}s")
|
|
149
|
+
print(f"Parallel: {par_time:.2f}s")
|
|
150
|
+
print(f"Speedup: {seq_time/par_time:.2f}x")
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### Pattern 15: Async I/O for I/O-Bound Tasks
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
import asyncio
|
|
157
|
+
import aiohttp
|
|
158
|
+
import time
|
|
159
|
+
import requests
|
|
160
|
+
|
|
161
|
+
urls = [
|
|
162
|
+
"https://httpbin.org/delay/1",
|
|
163
|
+
"https://httpbin.org/delay/1",
|
|
164
|
+
"https://httpbin.org/delay/1",
|
|
165
|
+
"https://httpbin.org/delay/1",
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
def synchronous_requests():
|
|
169
|
+
"""Synchronous HTTP requests."""
|
|
170
|
+
start = time.time()
|
|
171
|
+
results = []
|
|
172
|
+
for url in urls:
|
|
173
|
+
response = requests.get(url)
|
|
174
|
+
results.append(response.status_code)
|
|
175
|
+
elapsed = time.time() - start
|
|
176
|
+
return elapsed, results
|
|
177
|
+
|
|
178
|
+
async def async_fetch(session, url):
|
|
179
|
+
"""Async HTTP request."""
|
|
180
|
+
async with session.get(url) as response:
|
|
181
|
+
return response.status
|
|
182
|
+
|
|
183
|
+
async def asynchronous_requests():
|
|
184
|
+
"""Asynchronous HTTP requests."""
|
|
185
|
+
start = time.time()
|
|
186
|
+
async with aiohttp.ClientSession() as session:
|
|
187
|
+
tasks = [async_fetch(session, url) for url in urls]
|
|
188
|
+
results = await asyncio.gather(*tasks)
|
|
189
|
+
elapsed = time.time() - start
|
|
190
|
+
return elapsed, results
|
|
191
|
+
|
|
192
|
+
# Async is much faster for I/O-bound work
|
|
193
|
+
sync_time, sync_results = synchronous_requests()
|
|
194
|
+
async_time, async_results = asyncio.run(asynchronous_requests())
|
|
195
|
+
|
|
196
|
+
print(f"Synchronous: {sync_time:.2f}s")
|
|
197
|
+
print(f"Asynchronous: {async_time:.2f}s")
|
|
198
|
+
print(f"Speedup: {sync_time/async_time:.2f}x")
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
## Database Optimization
|
|
202
|
+
|
|
203
|
+
### Pattern 16: Batch Database Operations
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
import sqlite3
|
|
207
|
+
import time
|
|
208
|
+
|
|
209
|
+
def create_db():
|
|
210
|
+
"""Create test database."""
|
|
211
|
+
conn = sqlite3.connect(":memory:")
|
|
212
|
+
conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)")
|
|
213
|
+
return conn
|
|
214
|
+
|
|
215
|
+
def slow_inserts(conn, count):
|
|
216
|
+
"""Insert records one at a time."""
|
|
217
|
+
start = time.time()
|
|
218
|
+
cursor = conn.cursor()
|
|
219
|
+
for i in range(count):
|
|
220
|
+
cursor.execute("INSERT INTO users (name) VALUES (?)", (f"User {i}",))
|
|
221
|
+
conn.commit() # Commit each insert
|
|
222
|
+
elapsed = time.time() - start
|
|
223
|
+
return elapsed
|
|
224
|
+
|
|
225
|
+
def fast_inserts(conn, count):
|
|
226
|
+
"""Batch insert with single commit."""
|
|
227
|
+
start = time.time()
|
|
228
|
+
cursor = conn.cursor()
|
|
229
|
+
data = [(f"User {i}",) for i in range(count)]
|
|
230
|
+
cursor.executemany("INSERT INTO users (name) VALUES (?)", data)
|
|
231
|
+
conn.commit() # Single commit
|
|
232
|
+
elapsed = time.time() - start
|
|
233
|
+
return elapsed
|
|
234
|
+
|
|
235
|
+
# Benchmark
|
|
236
|
+
conn1 = create_db()
|
|
237
|
+
slow_time = slow_inserts(conn1, 1000)
|
|
238
|
+
|
|
239
|
+
conn2 = create_db()
|
|
240
|
+
fast_time = fast_inserts(conn2, 1000)
|
|
241
|
+
|
|
242
|
+
print(f"Individual inserts: {slow_time:.4f}s")
|
|
243
|
+
print(f"Batch insert: {fast_time:.4f}s")
|
|
244
|
+
print(f"Speedup: {slow_time/fast_time:.2f}x")
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
### Pattern 17: Query Optimization
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
# Use indexes for frequently queried columns
|
|
251
|
+
"""
|
|
252
|
+
-- Slow: No index
|
|
253
|
+
SELECT * FROM users WHERE email = 'user@example.com';
|
|
254
|
+
|
|
255
|
+
-- Fast: With index
|
|
256
|
+
CREATE INDEX idx_users_email ON users(email);
|
|
257
|
+
SELECT * FROM users WHERE email = 'user@example.com';
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
# Use query planning
|
|
261
|
+
import sqlite3
|
|
262
|
+
|
|
263
|
+
conn = sqlite3.connect("example.db")
|
|
264
|
+
cursor = conn.cursor()
|
|
265
|
+
|
|
266
|
+
# Analyze query performance
|
|
267
|
+
cursor.execute("EXPLAIN QUERY PLAN SELECT * FROM users WHERE email = ?", ("test@example.com",))
|
|
268
|
+
print(cursor.fetchall())
|
|
269
|
+
|
|
270
|
+
# Use SELECT only needed columns
|
|
271
|
+
# Slow: SELECT *
|
|
272
|
+
# Fast: SELECT id, name
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
## Memory Optimization
|
|
276
|
+
|
|
277
|
+
### Pattern 18: Detecting Memory Leaks
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
import tracemalloc
|
|
281
|
+
import gc
|
|
282
|
+
|
|
283
|
+
def memory_leak_example():
|
|
284
|
+
"""Example that leaks memory."""
|
|
285
|
+
leaked_objects = []
|
|
286
|
+
|
|
287
|
+
for i in range(100000):
|
|
288
|
+
# Objects added but never removed
|
|
289
|
+
leaked_objects.append([i] * 100)
|
|
290
|
+
|
|
291
|
+
# In real code, this would be an unintended reference
|
|
292
|
+
|
|
293
|
+
def track_memory_usage():
|
|
294
|
+
"""Track memory allocations."""
|
|
295
|
+
tracemalloc.start()
|
|
296
|
+
|
|
297
|
+
# Take snapshot before
|
|
298
|
+
snapshot1 = tracemalloc.take_snapshot()
|
|
299
|
+
|
|
300
|
+
# Run code
|
|
301
|
+
memory_leak_example()
|
|
302
|
+
|
|
303
|
+
# Take snapshot after
|
|
304
|
+
snapshot2 = tracemalloc.take_snapshot()
|
|
305
|
+
|
|
306
|
+
# Compare
|
|
307
|
+
top_stats = snapshot2.compare_to(snapshot1, 'lineno')
|
|
308
|
+
|
|
309
|
+
print("Top 10 memory allocations:")
|
|
310
|
+
for stat in top_stats[:10]:
|
|
311
|
+
print(stat)
|
|
312
|
+
|
|
313
|
+
tracemalloc.stop()
|
|
314
|
+
|
|
315
|
+
# Monitor memory
|
|
316
|
+
track_memory_usage()
|
|
317
|
+
|
|
318
|
+
# Force garbage collection
|
|
319
|
+
gc.collect()
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
### Pattern 19: Iterators vs Lists
|
|
323
|
+
|
|
324
|
+
```python
|
|
325
|
+
import sys
|
|
326
|
+
|
|
327
|
+
def process_file_list(filename):
|
|
328
|
+
"""Load entire file into memory."""
|
|
329
|
+
with open(filename) as f:
|
|
330
|
+
lines = f.readlines() # Loads all lines
|
|
331
|
+
return sum(1 for line in lines if line.strip())
|
|
332
|
+
|
|
333
|
+
def process_file_iterator(filename):
|
|
334
|
+
"""Process file line by line."""
|
|
335
|
+
with open(filename) as f:
|
|
336
|
+
return sum(1 for line in f if line.strip())
|
|
337
|
+
|
|
338
|
+
# Iterator uses constant memory
|
|
339
|
+
# List loads entire file into memory
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
### Pattern 20: Weakref for Caches
|
|
343
|
+
|
|
344
|
+
```python
|
|
345
|
+
import weakref
|
|
346
|
+
|
|
347
|
+
class CachedResource:
|
|
348
|
+
"""Resource that can be garbage collected."""
|
|
349
|
+
def __init__(self, data):
|
|
350
|
+
self.data = data
|
|
351
|
+
|
|
352
|
+
# Regular cache prevents garbage collection
|
|
353
|
+
regular_cache = {}
|
|
354
|
+
|
|
355
|
+
def get_resource_regular(key):
|
|
356
|
+
"""Get resource from regular cache."""
|
|
357
|
+
if key not in regular_cache:
|
|
358
|
+
regular_cache[key] = CachedResource(f"Data for {key}")
|
|
359
|
+
return regular_cache[key]
|
|
360
|
+
|
|
361
|
+
# Weak reference cache allows garbage collection
|
|
362
|
+
weak_cache = weakref.WeakValueDictionary()
|
|
363
|
+
|
|
364
|
+
def get_resource_weak(key):
|
|
365
|
+
"""Get resource from weak cache."""
|
|
366
|
+
resource = weak_cache.get(key)
|
|
367
|
+
if resource is None:
|
|
368
|
+
resource = CachedResource(f"Data for {key}")
|
|
369
|
+
weak_cache[key] = resource
|
|
370
|
+
return resource
|
|
371
|
+
|
|
372
|
+
# When no strong references exist, objects can be GC'd
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
## Benchmarking Tools
|
|
376
|
+
|
|
377
|
+
### Custom Benchmark Decorator
|
|
378
|
+
|
|
379
|
+
```python
|
|
380
|
+
import time
|
|
381
|
+
from functools import wraps
|
|
382
|
+
|
|
383
|
+
def benchmark(func):
|
|
384
|
+
"""Decorator to benchmark function execution."""
|
|
385
|
+
@wraps(func)
|
|
386
|
+
def wrapper(*args, **kwargs):
|
|
387
|
+
start = time.perf_counter()
|
|
388
|
+
result = func(*args, **kwargs)
|
|
389
|
+
elapsed = time.perf_counter() - start
|
|
390
|
+
print(f"{func.__name__} took {elapsed:.6f} seconds")
|
|
391
|
+
return result
|
|
392
|
+
return wrapper
|
|
393
|
+
|
|
394
|
+
@benchmark
|
|
395
|
+
def slow_function():
|
|
396
|
+
"""Function to benchmark."""
|
|
397
|
+
time.sleep(0.5)
|
|
398
|
+
return sum(range(1000000))
|
|
399
|
+
|
|
400
|
+
result = slow_function()
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
### Performance Testing with pytest-benchmark
|
|
404
|
+
|
|
405
|
+
```python
|
|
406
|
+
# Install: pip install pytest-benchmark
|
|
407
|
+
|
|
408
|
+
def test_list_comprehension(benchmark):
|
|
409
|
+
"""Benchmark list comprehension."""
|
|
410
|
+
result = benchmark(lambda: [i**2 for i in range(10000)])
|
|
411
|
+
assert len(result) == 10000
|
|
412
|
+
|
|
413
|
+
def test_map_function(benchmark):
|
|
414
|
+
"""Benchmark map function."""
|
|
415
|
+
result = benchmark(lambda: list(map(lambda x: x**2, range(10000))))
|
|
416
|
+
assert len(result) == 10000
|
|
417
|
+
|
|
418
|
+
# Run with: pytest test_performance.py --benchmark-compare
|
|
419
|
+
```
|