pyforker 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyforker-2.0.0/LICENSE.md +20 -0
- pyforker-2.0.0/PKG-INFO +136 -0
- pyforker-2.0.0/README.md +112 -0
- pyforker-2.0.0/pyforker.egg-info/PKG-INFO +136 -0
- pyforker-2.0.0/pyforker.egg-info/SOURCES.txt +10 -0
- pyforker-2.0.0/pyforker.egg-info/dependency_links.txt +1 -0
- pyforker-2.0.0/pyforker.egg-info/entry_points.txt +2 -0
- pyforker-2.0.0/pyforker.egg-info/top_level.txt +1 -0
- pyforker-2.0.0/pyforker.py +477 -0
- pyforker-2.0.0/pyproject.toml +38 -0
- pyforker-2.0.0/setup.cfg +4 -0
- pyforker-2.0.0/setup.py +33 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sundaram Gupta
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY FROM OUTAISE OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
20
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
pyforker-2.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyforker
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Zero-dependency toolkit for selectively extracting sub-modules, inspecting ASTs, and operating remote library mirror servers.
|
|
5
|
+
Author: Sundaram Gupta
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: ast,extractor,git,cli,tree-sitter,refactoring
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Classifier: Topic :: Software Development :: Code Generators
|
|
19
|
+
Requires-Python: >=3.8
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE.md
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
Dynamic: requires-python
|
|
24
|
+
|
|
25
|
+
# pyforker
|
|
26
|
+
|
|
27
|
+
<p align="center">
|
|
28
|
+
<img src="logo.png" alt="pyforker logo" width="600"/>
|
|
29
|
+
</p>
|
|
30
|
+
|
|
31
|
+
**Production-Grade Single-File Python Library Extractor & Server Engine**
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
[](https://www.python.org/)
|
|
35
|
+
[](https://opensource.org/licenses/MIT)
|
|
36
|
+
[](#)
|
|
37
|
+
[](#)
|
|
38
|
+
|
|
39
|
+
`pyforker` is a lightweight, zero-dependency engine designed to selectively extract sub-modules from massive Python codebases or remote Git repositories. It automatically rewrites AST import graphs, resolves vendor dependencies, tracks upstream sync ledgers, and runs multi-threaded HTTP server daemons to mirror code snippets across networks.
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Key Features
|
|
46
|
+
|
|
47
|
+
* **Zero External Dependencies:** Built entirely on standard Python modules (`ast`, `urllib`, `subprocess`, `concurrent.futures`, `http.server`).
|
|
48
|
+
* **Sub-Module Slicing (`take`):** Extract specific folders from remote Git repositories without pulling unnecessary weight into your project.
|
|
49
|
+
* **AST Import Rewriting:** Dynamically rewrites import statements during extraction so isolated sub-modules map directly into your local package tree.
|
|
50
|
+
* **Dependency Discovery (`deps`):** AST-level scanning detects third-party PyPI requirements versus Python standard library imports.
|
|
51
|
+
* **Upstream Synchronization (`sync`):** Keeps track of commit SHAs in local `pyforker.json` manifests to pull updates from upstream sources seamlessly.
|
|
52
|
+
* **Multi-Threaded Server Engine (`serve`):** Built-in daemon allows remote extraction and repository mirroring over network endpoints.
|
|
53
|
+
* **Transactional File Safety:** Uses OS-level file locking (`O_CREAT | O_EXCL`) to ensure multi-threaded and concurrent CLI ops avoid race conditions.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Architecture Overview
|
|
58
|
+
|
|
59
|
+
```text
|
|
60
|
+
+-------------------------------------------------+
|
|
61
|
+
| CLI Entry |
|
|
62
|
+
| (pyforker take / deps / sync / serve) |
|
|
63
|
+
+-----------------------+-------------------------+
|
|
64
|
+
|
|
|
65
|
+
+-----------------+-----------------+
|
|
66
|
+
| |
|
|
67
|
+
+-----------v-----------+ +-----------v-----------+
|
|
68
|
+
| Git Engine | | Multi-Threaded |
|
|
69
|
+
| (Local Cache/Clones) | | HTTP Daemon Server |
|
|
70
|
+
+-----------+-----------+ +-----------+-----------+
|
|
71
|
+
| |
|
|
72
|
+
+-----------------+-----------------+
|
|
73
|
+
|
|
|
74
|
+
+-------------v-------------+
|
|
75
|
+
| AST Rewriter & Analyzer |
|
|
76
|
+
| (Dependency Scanning) |
|
|
77
|
+
+-------------+-------------+
|
|
78
|
+
|
|
|
79
|
+
+-------------v-------------+
|
|
80
|
+
| Storage & File Locks |
|
|
81
|
+
| (pyforker.json Manifest) |
|
|
82
|
+
+---------------------------+
|
|
83
|
+
InstallationFrom Source (Editable Mode)Clone your repository and install the binary link locally:Bashgit clone [https://github.com/your-username/pyforker.git](https://github.com/your-username/pyforker.git)
|
|
84
|
+
cd pyforker
|
|
85
|
+
pip install -e .
|
|
86
|
+
Direct Pip InstallationBashpip install .
|
|
87
|
+
CLI Reference & Usage1. Extracting a Sub-Module (take)Pull a single sub-folder out of a target repository, rewrite its namespace imports, and store it locally:Bashpyforker take [https://github.com/torvalds/linux.git](https://github.com/torvalds/linux.git) \
|
|
88
|
+
--sub-path tools/testing/kunit \
|
|
89
|
+
--out ./vendor/kunit \
|
|
90
|
+
--rewrite-from kunit \
|
|
91
|
+
--rewrite-to my_app.vendor.kunit
|
|
92
|
+
2. Discovering Third-Party Dependencies (deps)Scan an extracted module to discover external PyPI packages required to run it:Bashpyforker deps ./vendor/kunit
|
|
93
|
+
Output Example:Plaintext--- Discovered Third-Party Dependencies ---
|
|
94
|
+
- requests
|
|
95
|
+
- typing_extensions
|
|
96
|
+
-------------------------------------------
|
|
97
|
+
3. Syncing Extracted Code Upstream (sync)Check local pyforker.json manifests and update extracted modules against upstream Git repositories:Bashpyforker sync ./vendor/kunit
|
|
98
|
+
4. Running the Remote Mirror Server (serve)Start a multi-threaded HTTP server on a remote server or local machine:Bashpyforker serve --host 0.0.0.0 --port 8080
|
|
99
|
+
5. Managing Remote Server Aliases (server-add / server-list)Save and query remote server endpoints inside ~/.pyforker_config.json:Bash# Add server alias
|
|
100
|
+
pyforker server-add prod-mirror [http://192.168.1.100:8080](http://192.168.1.100:8080)
|
|
101
|
+
|
|
102
|
+
# List saved servers
|
|
103
|
+
pyforker server-list
|
|
104
|
+
6. Executing System Diagnostics (self-test)Validate local environment, AST transformation hooks, file locks, and configuration storage:Bashpyforker self-test
|
|
105
|
+
Manifest Specification (pyforker.json)When pyforker take completes an extraction, it generates or updates a local pyforker.json ledger:JSON{
|
|
106
|
+
"./vendor/kunit": {
|
|
107
|
+
"source_repo": "[https://github.com/torvalds/linux.git](https://github.com/torvalds/linux.git)",
|
|
108
|
+
"sub_path": "tools/testing/kunit",
|
|
109
|
+
"commit": "a1b2c3d4e5f67890",
|
|
110
|
+
"files_extracted": 14,
|
|
111
|
+
"extracted_at": 1773724800.0,
|
|
112
|
+
"detected_imports": [
|
|
113
|
+
"sys",
|
|
114
|
+
"os",
|
|
115
|
+
"requests"
|
|
116
|
+
]
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
Programmatic Python APIYou can import pyforker as an internal module inside your Python pipelines:Pythonfrom pyforker import ASTAnalyzer, GitEngine, FileLock
|
|
120
|
+
|
|
121
|
+
# Clone or update a repository into local cache
|
|
122
|
+
repo_path = GitEngine.clone_or_update(
|
|
123
|
+
"[https://github.com/example/repo.git](https://github.com/example/repo.git)",
|
|
124
|
+
"~/.pyforker_cache"
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
# Rewrite AST imports programmatically
|
|
128
|
+
detected_deps = ASTAnalyzer.process_file(
|
|
129
|
+
source_file="path/to/source.py",
|
|
130
|
+
target_file="path/to/output.py",
|
|
131
|
+
old_pkg="original_pkg",
|
|
132
|
+
new_pkg="my_app.vendor"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
print(f"Discovered dependencies: {detected_deps}")
|
|
136
|
+
HTTP REST API SpecificationWhen running pyforker serve, the server exposes the following endpoints:EndpointMethodParameters / BodyDescription/healthGETNoneReturns engine status and uptime./manifestGETNoneReturns history ledger stored in host configuration./takePOST{"repo_url": "...", "sub_path": "..."}Triggers background clone and path verification.Running Unit TestsRun the integrated self-test suite directly:Bashpython3 pyforker.py self-test
|
pyforker-2.0.0/README.md
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# pyforker
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<img src="logo.png" alt="pyforker logo" width="600"/>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
**Production-Grade Single-File Python Library Extractor & Server Engine**
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
[](https://www.python.org/)
|
|
11
|
+
[](https://opensource.org/licenses/MIT)
|
|
12
|
+
[](#)
|
|
13
|
+
[](#)
|
|
14
|
+
|
|
15
|
+
`pyforker` is a lightweight, zero-dependency engine designed to selectively extract sub-modules from massive Python codebases or remote Git repositories. It automatically rewrites AST import graphs, resolves vendor dependencies, tracks upstream sync ledgers, and runs multi-threaded HTTP server daemons to mirror code snippets across networks.
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## Key Features
|
|
22
|
+
|
|
23
|
+
* **Zero External Dependencies:** Built entirely on standard Python modules (`ast`, `urllib`, `subprocess`, `concurrent.futures`, `http.server`).
|
|
24
|
+
* **Sub-Module Slicing (`take`):** Extract specific folders from remote Git repositories without pulling unnecessary weight into your project.
|
|
25
|
+
* **AST Import Rewriting:** Dynamically rewrites import statements during extraction so isolated sub-modules map directly into your local package tree.
|
|
26
|
+
* **Dependency Discovery (`deps`):** AST-level scanning detects third-party PyPI requirements versus Python standard library imports.
|
|
27
|
+
* **Upstream Synchronization (`sync`):** Keeps track of commit SHAs in local `pyforker.json` manifests to pull updates from upstream sources seamlessly.
|
|
28
|
+
* **Multi-Threaded Server Engine (`serve`):** Built-in daemon allows remote extraction and repository mirroring over network endpoints.
|
|
29
|
+
* **Transactional File Safety:** Uses OS-level file locking (`O_CREAT | O_EXCL`) to ensure multi-threaded and concurrent CLI ops avoid race conditions.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## Architecture Overview
|
|
34
|
+
|
|
35
|
+
```text
|
|
36
|
+
+-------------------------------------------------+
|
|
37
|
+
| CLI Entry |
|
|
38
|
+
| (pyforker take / deps / sync / serve) |
|
|
39
|
+
+-----------------------+-------------------------+
|
|
40
|
+
|
|
|
41
|
+
+-----------------+-----------------+
|
|
42
|
+
| |
|
|
43
|
+
+-----------v-----------+ +-----------v-----------+
|
|
44
|
+
| Git Engine | | Multi-Threaded |
|
|
45
|
+
| (Local Cache/Clones) | | HTTP Daemon Server |
|
|
46
|
+
+-----------+-----------+ +-----------+-----------+
|
|
47
|
+
| |
|
|
48
|
+
+-----------------+-----------------+
|
|
49
|
+
|
|
|
50
|
+
+-------------v-------------+
|
|
51
|
+
| AST Rewriter & Analyzer |
|
|
52
|
+
| (Dependency Scanning) |
|
|
53
|
+
+-------------+-------------+
|
|
54
|
+
|
|
|
55
|
+
+-------------v-------------+
|
|
56
|
+
| Storage & File Locks |
|
|
57
|
+
| (pyforker.json Manifest) |
|
|
58
|
+
+---------------------------+
|
|
59
|
+
InstallationFrom Source (Editable Mode)Clone your repository and install the binary link locally:Bashgit clone [https://github.com/your-username/pyforker.git](https://github.com/your-username/pyforker.git)
|
|
60
|
+
cd pyforker
|
|
61
|
+
pip install -e .
|
|
62
|
+
Direct Pip InstallationBashpip install .
|
|
63
|
+
CLI Reference & Usage1. Extracting a Sub-Module (take)Pull a single sub-folder out of a target repository, rewrite its namespace imports, and store it locally:Bashpyforker take [https://github.com/torvalds/linux.git](https://github.com/torvalds/linux.git) \
|
|
64
|
+
--sub-path tools/testing/kunit \
|
|
65
|
+
--out ./vendor/kunit \
|
|
66
|
+
--rewrite-from kunit \
|
|
67
|
+
--rewrite-to my_app.vendor.kunit
|
|
68
|
+
2. Discovering Third-Party Dependencies (deps)Scan an extracted module to discover external PyPI packages required to run it:Bashpyforker deps ./vendor/kunit
|
|
69
|
+
Output Example:Plaintext--- Discovered Third-Party Dependencies ---
|
|
70
|
+
- requests
|
|
71
|
+
- typing_extensions
|
|
72
|
+
-------------------------------------------
|
|
73
|
+
3. Syncing Extracted Code Upstream (sync)Check local pyforker.json manifests and update extracted modules against upstream Git repositories:Bashpyforker sync ./vendor/kunit
|
|
74
|
+
4. Running the Remote Mirror Server (serve)Start a multi-threaded HTTP server on a remote server or local machine:Bashpyforker serve --host 0.0.0.0 --port 8080
|
|
75
|
+
5. Managing Remote Server Aliases (server-add / server-list)Save and query remote server endpoints inside ~/.pyforker_config.json:Bash# Add server alias
|
|
76
|
+
pyforker server-add prod-mirror [http://192.168.1.100:8080](http://192.168.1.100:8080)
|
|
77
|
+
|
|
78
|
+
# List saved servers
|
|
79
|
+
pyforker server-list
|
|
80
|
+
6. Executing System Diagnostics (self-test)Validate local environment, AST transformation hooks, file locks, and configuration storage:Bashpyforker self-test
|
|
81
|
+
Manifest Specification (pyforker.json)When pyforker take completes an extraction, it generates or updates a local pyforker.json ledger:JSON{
|
|
82
|
+
"./vendor/kunit": {
|
|
83
|
+
"source_repo": "[https://github.com/torvalds/linux.git](https://github.com/torvalds/linux.git)",
|
|
84
|
+
"sub_path": "tools/testing/kunit",
|
|
85
|
+
"commit": "a1b2c3d4e5f67890",
|
|
86
|
+
"files_extracted": 14,
|
|
87
|
+
"extracted_at": 1773724800.0,
|
|
88
|
+
"detected_imports": [
|
|
89
|
+
"sys",
|
|
90
|
+
"os",
|
|
91
|
+
"requests"
|
|
92
|
+
]
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
Programmatic Python APIYou can import pyforker as an internal module inside your Python pipelines:Pythonfrom pyforker import ASTAnalyzer, GitEngine, FileLock
|
|
96
|
+
|
|
97
|
+
# Clone or update a repository into local cache
|
|
98
|
+
repo_path = GitEngine.clone_or_update(
|
|
99
|
+
"[https://github.com/example/repo.git](https://github.com/example/repo.git)",
|
|
100
|
+
"~/.pyforker_cache"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# Rewrite AST imports programmatically
|
|
104
|
+
detected_deps = ASTAnalyzer.process_file(
|
|
105
|
+
source_file="path/to/source.py",
|
|
106
|
+
target_file="path/to/output.py",
|
|
107
|
+
old_pkg="original_pkg",
|
|
108
|
+
new_pkg="my_app.vendor"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
print(f"Discovered dependencies: {detected_deps}")
|
|
112
|
+
HTTP REST API SpecificationWhen running pyforker serve, the server exposes the following endpoints:EndpointMethodParameters / BodyDescription/healthGETNoneReturns engine status and uptime./manifestGETNoneReturns history ledger stored in host configuration./takePOST{"repo_url": "...", "sub_path": "..."}Triggers background clone and path verification.Running Unit TestsRun the integrated self-test suite directly:Bashpython3 pyforker.py self-test
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyforker
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Zero-dependency toolkit for selectively extracting sub-modules, inspecting ASTs, and operating remote library mirror servers.
|
|
5
|
+
Author: Sundaram Gupta
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: ast,extractor,git,cli,tree-sitter,refactoring
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Classifier: Topic :: Software Development :: Code Generators
|
|
19
|
+
Requires-Python: >=3.8
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE.md
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
Dynamic: requires-python
|
|
24
|
+
|
|
25
|
+
# pyforker
|
|
26
|
+
|
|
27
|
+
<p align="center">
|
|
28
|
+
<img src="logo.png" alt="pyforker logo" width="600"/>
|
|
29
|
+
</p>
|
|
30
|
+
|
|
31
|
+
**Production-Grade Single-File Python Library Extractor & Server Engine**
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
[](https://www.python.org/)
|
|
35
|
+
[](https://opensource.org/licenses/MIT)
|
|
36
|
+
[](#)
|
|
37
|
+
[](#)
|
|
38
|
+
|
|
39
|
+
`pyforker` is a lightweight, zero-dependency engine designed to selectively extract sub-modules from massive Python codebases or remote Git repositories. It automatically rewrites AST import graphs, resolves vendor dependencies, tracks upstream sync ledgers, and runs multi-threaded HTTP server daemons to mirror code snippets across networks.
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Key Features
|
|
46
|
+
|
|
47
|
+
* **Zero External Dependencies:** Built entirely on standard Python modules (`ast`, `urllib`, `subprocess`, `concurrent.futures`, `http.server`).
|
|
48
|
+
* **Sub-Module Slicing (`take`):** Extract specific folders from remote Git repositories without pulling unnecessary weight into your project.
|
|
49
|
+
* **AST Import Rewriting:** Dynamically rewrites import statements during extraction so isolated sub-modules map directly into your local package tree.
|
|
50
|
+
* **Dependency Discovery (`deps`):** AST-level scanning detects third-party PyPI requirements versus Python standard library imports.
|
|
51
|
+
* **Upstream Synchronization (`sync`):** Keeps track of commit SHAs in local `pyforker.json` manifests to pull updates from upstream sources seamlessly.
|
|
52
|
+
* **Multi-Threaded Server Engine (`serve`):** Built-in daemon allows remote extraction and repository mirroring over network endpoints.
|
|
53
|
+
* **Transactional File Safety:** Uses OS-level file locking (`O_CREAT | O_EXCL`) to ensure multi-threaded and concurrent CLI ops avoid race conditions.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Architecture Overview
|
|
58
|
+
|
|
59
|
+
```text
|
|
60
|
+
+-------------------------------------------------+
|
|
61
|
+
| CLI Entry |
|
|
62
|
+
| (pyforker take / deps / sync / serve) |
|
|
63
|
+
+-----------------------+-------------------------+
|
|
64
|
+
|
|
|
65
|
+
+-----------------+-----------------+
|
|
66
|
+
| |
|
|
67
|
+
+-----------v-----------+ +-----------v-----------+
|
|
68
|
+
| Git Engine | | Multi-Threaded |
|
|
69
|
+
| (Local Cache/Clones) | | HTTP Daemon Server |
|
|
70
|
+
+-----------+-----------+ +-----------+-----------+
|
|
71
|
+
| |
|
|
72
|
+
+-----------------+-----------------+
|
|
73
|
+
|
|
|
74
|
+
+-------------v-------------+
|
|
75
|
+
| AST Rewriter & Analyzer |
|
|
76
|
+
| (Dependency Scanning) |
|
|
77
|
+
+-------------+-------------+
|
|
78
|
+
|
|
|
79
|
+
+-------------v-------------+
|
|
80
|
+
| Storage & File Locks |
|
|
81
|
+
| (pyforker.json Manifest) |
|
|
82
|
+
+---------------------------+
|
|
83
|
+
InstallationFrom Source (Editable Mode)Clone your repository and install the binary link locally:Bashgit clone [https://github.com/your-username/pyforker.git](https://github.com/your-username/pyforker.git)
|
|
84
|
+
cd pyforker
|
|
85
|
+
pip install -e .
|
|
86
|
+
Direct Pip InstallationBashpip install .
|
|
87
|
+
CLI Reference & Usage1. Extracting a Sub-Module (take)Pull a single sub-folder out of a target repository, rewrite its namespace imports, and store it locally:Bashpyforker take [https://github.com/torvalds/linux.git](https://github.com/torvalds/linux.git) \
|
|
88
|
+
--sub-path tools/testing/kunit \
|
|
89
|
+
--out ./vendor/kunit \
|
|
90
|
+
--rewrite-from kunit \
|
|
91
|
+
--rewrite-to my_app.vendor.kunit
|
|
92
|
+
2. Discovering Third-Party Dependencies (deps)Scan an extracted module to discover external PyPI packages required to run it:Bashpyforker deps ./vendor/kunit
|
|
93
|
+
Output Example:Plaintext--- Discovered Third-Party Dependencies ---
|
|
94
|
+
- requests
|
|
95
|
+
- typing_extensions
|
|
96
|
+
-------------------------------------------
|
|
97
|
+
3. Syncing Extracted Code Upstream (sync)Check local pyforker.json manifests and update extracted modules against upstream Git repositories:Bashpyforker sync ./vendor/kunit
|
|
98
|
+
4. Running the Remote Mirror Server (serve)Start a multi-threaded HTTP server on a remote server or local machine:Bashpyforker serve --host 0.0.0.0 --port 8080
|
|
99
|
+
5. Managing Remote Server Aliases (server-add / server-list)Save and query remote server endpoints inside ~/.pyforker_config.json:Bash# Add server alias
|
|
100
|
+
pyforker server-add prod-mirror [http://192.168.1.100:8080](http://192.168.1.100:8080)
|
|
101
|
+
|
|
102
|
+
# List saved servers
|
|
103
|
+
pyforker server-list
|
|
104
|
+
6. Executing System Diagnostics (self-test)Validate local environment, AST transformation hooks, file locks, and configuration storage:Bashpyforker self-test
|
|
105
|
+
Manifest Specification (pyforker.json)When pyforker take completes an extraction, it generates or updates a local pyforker.json ledger:JSON{
|
|
106
|
+
"./vendor/kunit": {
|
|
107
|
+
"source_repo": "[https://github.com/torvalds/linux.git](https://github.com/torvalds/linux.git)",
|
|
108
|
+
"sub_path": "tools/testing/kunit",
|
|
109
|
+
"commit": "a1b2c3d4e5f67890",
|
|
110
|
+
"files_extracted": 14,
|
|
111
|
+
"extracted_at": 1773724800.0,
|
|
112
|
+
"detected_imports": [
|
|
113
|
+
"sys",
|
|
114
|
+
"os",
|
|
115
|
+
"requests"
|
|
116
|
+
]
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
Programmatic Python APIYou can import pyforker as an internal module inside your Python pipelines:Pythonfrom pyforker import ASTAnalyzer, GitEngine, FileLock
|
|
120
|
+
|
|
121
|
+
# Clone or update a repository into local cache
|
|
122
|
+
repo_path = GitEngine.clone_or_update(
|
|
123
|
+
"[https://github.com/example/repo.git](https://github.com/example/repo.git)",
|
|
124
|
+
"~/.pyforker_cache"
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
# Rewrite AST imports programmatically
|
|
128
|
+
detected_deps = ASTAnalyzer.process_file(
|
|
129
|
+
source_file="path/to/source.py",
|
|
130
|
+
target_file="path/to/output.py",
|
|
131
|
+
old_pkg="original_pkg",
|
|
132
|
+
new_pkg="my_app.vendor"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
print(f"Discovered dependencies: {detected_deps}")
|
|
136
|
+
HTTP REST API SpecificationWhen running pyforker serve, the server exposes the following endpoints:EndpointMethodParameters / BodyDescription/healthGETNoneReturns engine status and uptime./manifestGETNoneReturns history ledger stored in host configuration./takePOST{"repo_url": "...", "sub_path": "..."}Triggers background clone and path verification.Running Unit TestsRun the integrated self-test suite directly:Bashpython3 pyforker.py self-test
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pyforker
|
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
pyforker.py - Production-Grade Single-File Python Library Extractor & Server Engine
|
|
4
|
+
Zero-dependency toolkit for selectively extracting sub-modules, inspecting ASTs,
|
|
5
|
+
and operating remote library mirror servers.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
import os
|
|
10
|
+
import io
|
|
11
|
+
import re
|
|
12
|
+
import json
|
|
13
|
+
import time
|
|
14
|
+
import shutil
|
|
15
|
+
import hashlib
|
|
16
|
+
import logging
|
|
17
|
+
import argparse
|
|
18
|
+
import urllib.request
|
|
19
|
+
import urllib.parse
|
|
20
|
+
import urllib.error
|
|
21
|
+
import subprocess
|
|
22
|
+
import concurrent.futures
|
|
23
|
+
from http.server import HTTPServer, BaseHTTPRequestHandler
|
|
24
|
+
from socketserver import ThreadingMixIn
|
|
25
|
+
import ast
|
|
26
|
+
|
|
27
|
+
# Setup logging
|
|
28
|
+
logging.basicConfig(level=logging.INFO, format="[pyforker] %(levelname)s: %(message)s")
|
|
29
|
+
|
|
30
|
+
# Global Paths
|
|
31
|
+
CONFIG_FILE = os.path.expanduser("~/.pyforker_config.json")
|
|
32
|
+
MANIFEST_NAME = "pyforker.json"
|
|
33
|
+
|
|
34
|
+
# =====================================================================
|
|
35
|
+
# PERSISTENT STORAGE ENGINE
|
|
36
|
+
# =====================================================================
|
|
37
|
+
|
|
38
|
+
class StorageEngine:
|
|
39
|
+
@staticmethod
|
|
40
|
+
def load_config():
|
|
41
|
+
if not os.path.exists(CONFIG_FILE):
|
|
42
|
+
return {"servers": {}, "history": []}
|
|
43
|
+
try:
|
|
44
|
+
with open(CONFIG_FILE, "r", encoding="utf-8") as f:
|
|
45
|
+
return json.load(f)
|
|
46
|
+
except Exception:
|
|
47
|
+
return {"servers": {}, "history": []}
|
|
48
|
+
|
|
49
|
+
@staticmethod
|
|
50
|
+
def save_config(data):
|
|
51
|
+
with open(CONFIG_FILE, "w", encoding="utf-8") as f:
|
|
52
|
+
json.dump(data, f, indent=2)
|
|
53
|
+
|
|
54
|
+
@staticmethod
|
|
55
|
+
def save_manifest(target_dir, manifest_data):
|
|
56
|
+
path = os.path.join(target_dir, MANIFEST_NAME)
|
|
57
|
+
existing = {}
|
|
58
|
+
if os.path.exists(path):
|
|
59
|
+
try:
|
|
60
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
61
|
+
existing = json.load(f)
|
|
62
|
+
except Exception:
|
|
63
|
+
pass
|
|
64
|
+
existing.update(manifest_data)
|
|
65
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
66
|
+
json.dump(existing, f, indent=2)
|
|
67
|
+
|
|
68
|
+
# =====================================================================
|
|
69
|
+
# TRANSACTIONAL FILE LOCKING
|
|
70
|
+
# =====================================================================
|
|
71
|
+
|
|
72
|
+
class FileLock:
|
|
73
|
+
def __init__(self, lock_file, timeout=5):
|
|
74
|
+
self.lock_file = lock_file
|
|
75
|
+
self.timeout = timeout
|
|
76
|
+
self.fd = None
|
|
77
|
+
|
|
78
|
+
def __enter__(self):
|
|
79
|
+
start_time = time.time()
|
|
80
|
+
while True:
|
|
81
|
+
try:
|
|
82
|
+
self.fd = os.open(self.lock_file, os.O_CREAT | os.O_EXCL | os.O_RDWR)
|
|
83
|
+
return self
|
|
84
|
+
except OSError:
|
|
85
|
+
if time.time() - start_time > self.timeout:
|
|
86
|
+
raise TimeoutError(f"Could not acquire lock on {self.lock_file}")
|
|
87
|
+
time.sleep(0.05)
|
|
88
|
+
|
|
89
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
90
|
+
if self.fd is not None:
|
|
91
|
+
os.close(self.fd)
|
|
92
|
+
try:
|
|
93
|
+
os.remove(self.lock_file)
|
|
94
|
+
except OSError:
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
# =====================================================================
|
|
98
|
+
# REAL GIT & REPOSITORY CONNECTIONS
|
|
99
|
+
# =====================================================================
|
|
100
|
+
|
|
101
|
+
class GitEngine:
|
|
102
|
+
@staticmethod
|
|
103
|
+
def clone_or_update(repo_url, cache_dir):
|
|
104
|
+
repo_hash = hashlib.sha256(repo_url.encode()).hexdigest()[:12]
|
|
105
|
+
target_path = os.path.join(cache_dir, repo_hash)
|
|
106
|
+
|
|
107
|
+
if os.path.exists(target_path):
|
|
108
|
+
logging.info(f"Updating local cache for {repo_url}...")
|
|
109
|
+
try:
|
|
110
|
+
subprocess.run(["git", "-C", target_path, "pull"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
111
|
+
except Exception as e:
|
|
112
|
+
logging.warning(f"Failed to pull latest git changes: {e}")
|
|
113
|
+
else:
|
|
114
|
+
logging.info(f"Cloning real remote repository {repo_url}...")
|
|
115
|
+
os.makedirs(cache_dir, exist_ok=True)
|
|
116
|
+
subprocess.run(["git", "clone", "--depth", "1", repo_url, target_path], check=True)
|
|
117
|
+
|
|
118
|
+
return target_path
|
|
119
|
+
|
|
120
|
+
@staticmethod
|
|
121
|
+
def get_commit_hash(repo_path):
|
|
122
|
+
try:
|
|
123
|
+
res = subprocess.run(["git", "-C", repo_path, "rev-parse", "HEAD"], capture_output=True, text=True, check=True)
|
|
124
|
+
return res.stdout.strip()
|
|
125
|
+
except Exception:
|
|
126
|
+
return "unknown_commit"
|
|
127
|
+
|
|
128
|
+
# =====================================================================
|
|
129
|
+
# AST REWRITER & DEPENDENCY RESOLVER
|
|
130
|
+
# =====================================================================
|
|
131
|
+
|
|
132
|
+
class ASTDependencyRewriter(ast.NodeTransformer):
|
|
133
|
+
def __init__(self, root_package, target_package):
|
|
134
|
+
self.root_package = root_package
|
|
135
|
+
self.target_package = target_package
|
|
136
|
+
self.detected_imports = set()
|
|
137
|
+
|
|
138
|
+
def visit_Import(self, node):
|
|
139
|
+
for alias in node.names:
|
|
140
|
+
self.detected_imports.add(alias.name.split('.')[0])
|
|
141
|
+
return self.generic_visit(node)
|
|
142
|
+
|
|
143
|
+
def visit_ImportFrom(self, node):
|
|
144
|
+
if node.module:
|
|
145
|
+
self.detected_imports.add(node.module.split('.')[0])
|
|
146
|
+
if node.module == self.root_package or node.module.startswith(self.root_package + "."):
|
|
147
|
+
new_module = node.module.replace(self.root_package, self.target_package, 1)
|
|
148
|
+
return ast.copy_location(ast.ImportFrom(module=new_module, names=node.names, level=node.level), node)
|
|
149
|
+
return self.generic_visit(node)
|
|
150
|
+
|
|
151
|
+
class ASTAnalyzer:
|
|
152
|
+
@staticmethod
|
|
153
|
+
def process_file(source_file, target_file, old_pkg="", new_pkg=""):
|
|
154
|
+
with open(source_file, "r", encoding="utf-8") as f:
|
|
155
|
+
code = f.read()
|
|
156
|
+
|
|
157
|
+
try:
|
|
158
|
+
tree = ast.parse(code, filename=source_file)
|
|
159
|
+
rewriter = ASTDependencyRewriter(old_pkg, new_pkg) if old_pkg and new_pkg else ASTDependencyRewriter("", "")
|
|
160
|
+
new_tree = rewriter.visit(tree)
|
|
161
|
+
ast.fix_missing_locations(new_tree)
|
|
162
|
+
|
|
163
|
+
os.makedirs(os.path.dirname(target_file), exist_ok=True)
|
|
164
|
+
with open(target_file, "w", encoding="utf-8") as f:
|
|
165
|
+
f.write(ast.unparse(new_tree))
|
|
166
|
+
|
|
167
|
+
return rewriter.detected_imports
|
|
168
|
+
except Exception as e:
|
|
169
|
+
logging.warning(f"AST unparse skipped for {source_file}, falling back to plain copy. Reason: {e}")
|
|
170
|
+
os.makedirs(os.path.dirname(target_file), exist_ok=True)
|
|
171
|
+
shutil.copy2(source_file, target_file)
|
|
172
|
+
return set()
|
|
173
|
+
|
|
174
|
+
# =====================================================================
|
|
175
|
+
# REAL MULTI-THREADED HTTP SERVER ENGINE
|
|
176
|
+
# =====================================================================
|
|
177
|
+
|
|
178
|
+
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
|
|
179
|
+
"""Handles requests in separate threads for scalable remote service."""
|
|
180
|
+
daemon_threads = True
|
|
181
|
+
|
|
182
|
+
class PyForkerHTTPHandler(BaseHTTPRequestHandler):
|
|
183
|
+
def _send_json(self, status, payload):
|
|
184
|
+
self.send_response(status)
|
|
185
|
+
self.send_header("Content-Type", "application/json")
|
|
186
|
+
self.end_headers()
|
|
187
|
+
self.wfile.write(json.dumps(payload).encode("utf-8"))
|
|
188
|
+
|
|
189
|
+
def do_GET(self):
|
|
190
|
+
parsed = urllib.parse.urlparse(self.path)
|
|
191
|
+
if parsed.path == "/health":
|
|
192
|
+
self._send_json(200, {"status": "ok", "time": time.time(), "engine": "PyForker v2.0"})
|
|
193
|
+
elif parsed.path == "/manifest":
|
|
194
|
+
cfg = StorageEngine.load_config()
|
|
195
|
+
self._send_json(200, {"manifest": cfg.get("history", [])})
|
|
196
|
+
else:
|
|
197
|
+
self._send_json(404, {"error": "Endpoint not found"})
|
|
198
|
+
|
|
199
|
+
def do_POST(self):
|
|
200
|
+
parsed = urllib.parse.urlparse(self.path)
|
|
201
|
+
content_length = int(self.headers.get("Content-Length", 0))
|
|
202
|
+
body = self.rfile.read(content_length) if content_length > 0 else b"{}"
|
|
203
|
+
|
|
204
|
+
try:
|
|
205
|
+
payload = json.loads(body.decode("utf-8"))
|
|
206
|
+
except Exception:
|
|
207
|
+
self._send_json(400, {"error": "Invalid JSON"})
|
|
208
|
+
return
|
|
209
|
+
|
|
210
|
+
if parsed.path == "/take":
|
|
211
|
+
repo_url = payload.get("repo_url")
|
|
212
|
+
sub_path = payload.get("sub_path")
|
|
213
|
+
if not repo_url or not sub_path:
|
|
214
|
+
self._send_json(400, {"error": "Missing repo_url or sub_path"})
|
|
215
|
+
return
|
|
216
|
+
|
|
217
|
+
cache_dir = os.path.expanduser("~/.pyforker_cache")
|
|
218
|
+
try:
|
|
219
|
+
repo_path = GitEngine.clone_or_update(repo_url, cache_dir)
|
|
220
|
+
source = os.path.join(repo_path, sub_path)
|
|
221
|
+
if not os.path.exists(source):
|
|
222
|
+
self._send_json(404, {"error": f"Path {sub_path} not found in repo"})
|
|
223
|
+
return
|
|
224
|
+
|
|
225
|
+
commit_sha = GitEngine.get_commit_hash(repo_path)
|
|
226
|
+
self._send_json(200, {
|
|
227
|
+
"status": "success",
|
|
228
|
+
"commit": commit_sha,
|
|
229
|
+
"sub_path": sub_path,
|
|
230
|
+
"message": "Repository extraction target verified on remote server"
|
|
231
|
+
})
|
|
232
|
+
except Exception as e:
|
|
233
|
+
self._send_json(500, {"error": str(e)})
|
|
234
|
+
else:
|
|
235
|
+
self._send_json(404, {"error": "Endpoint not found"})
|
|
236
|
+
|
|
237
|
+
# =====================================================================
|
|
238
|
+
# CLI COMMAND PROCESSORS (SOLVING DEVELOPER PAIN POINTS)
|
|
239
|
+
# =====================================================================
|
|
240
|
+
|
|
241
|
+
def cmd_take(args):
|
|
242
|
+
"""Pulls directly from a local path or git URL and extracts a subfolder cleanly."""
|
|
243
|
+
target_out = os.path.abspath(args.out)
|
|
244
|
+
cache_dir = os.path.expanduser("~/.pyforker_cache")
|
|
245
|
+
|
|
246
|
+
if args.repo.startswith("http://") or args.repo.startswith("https://") or args.repo.startswith("git@"):
|
|
247
|
+
repo_path = GitEngine.clone_or_update(args.repo, cache_dir)
|
|
248
|
+
commit_hash = GitEngine.get_commit_hash(repo_path)
|
|
249
|
+
source_base = repo_path
|
|
250
|
+
else:
|
|
251
|
+
source_base = os.path.abspath(args.repo)
|
|
252
|
+
commit_hash = "local_filesystem"
|
|
253
|
+
|
|
254
|
+
source_dir = os.path.join(source_base, args.sub_path) if args.sub_path else source_base
|
|
255
|
+
|
|
256
|
+
if not os.path.exists(source_dir):
|
|
257
|
+
logging.error(f"Source sub-path does not exist: {source_dir}")
|
|
258
|
+
sys.exit(1)
|
|
259
|
+
|
|
260
|
+
logging.info(f"Extracting sub-module '{args.sub_path or '.'}' into '{target_out}'...")
|
|
261
|
+
|
|
262
|
+
detected_deps = set()
|
|
263
|
+
file_count = 0
|
|
264
|
+
|
|
265
|
+
with FileLock(os.path.join(os.getcwd(), ".pyforker.lock")):
|
|
266
|
+
for root, _, files in os.walk(source_dir):
|
|
267
|
+
for file in files:
|
|
268
|
+
if file.endswith(".py"):
|
|
269
|
+
src_file = os.path.join(root, file)
|
|
270
|
+
rel_file = os.path.relpath(src_file, source_dir)
|
|
271
|
+
dst_file = os.path.join(target_out, rel_file)
|
|
272
|
+
|
|
273
|
+
deps = ASTAnalyzer.process_file(src_file, dst_file, args.rewrite_from, args.rewrite_to)
|
|
274
|
+
detected_deps.update(deps)
|
|
275
|
+
file_count += 1
|
|
276
|
+
else:
|
|
277
|
+
src_file = os.path.join(root, file)
|
|
278
|
+
rel_file = os.path.relpath(src_file, source_dir)
|
|
279
|
+
dst_file = os.path.join(target_out, rel_file)
|
|
280
|
+
os.makedirs(os.path.dirname(dst_file), exist_ok=True)
|
|
281
|
+
shutil.copy2(src_file, dst_file)
|
|
282
|
+
|
|
283
|
+
manifest_entry = {
|
|
284
|
+
args.out: {
|
|
285
|
+
"source_repo": args.repo,
|
|
286
|
+
"sub_path": args.sub_path or "",
|
|
287
|
+
"commit": commit_hash,
|
|
288
|
+
"files_extracted": file_count,
|
|
289
|
+
"extracted_at": time.time(),
|
|
290
|
+
"detected_imports": list(detected_deps)
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
StorageEngine.save_manifest(target_out, manifest_entry)
|
|
294
|
+
logging.info(f"Successfully extracted {file_count} files to '{target_out}'. Ledger saved to {MANIFEST_NAME}.")
|
|
295
|
+
|
|
296
|
+
def cmd_deps(args):
|
|
297
|
+
"""Pain Point Solved: Scans an extracted module to discover external third-party dependencies."""
|
|
298
|
+
target_dir = os.path.abspath(args.path)
|
|
299
|
+
if not os.path.exists(target_dir):
|
|
300
|
+
logging.error(f"Path does not exist: {target_dir}")
|
|
301
|
+
sys.exit(1)
|
|
302
|
+
|
|
303
|
+
all_imports = set()
|
|
304
|
+
for root, _, files in os.walk(target_dir):
|
|
305
|
+
for file in files:
|
|
306
|
+
if file.endswith(".py"):
|
|
307
|
+
filepath = os.path.join(root, file)
|
|
308
|
+
try:
|
|
309
|
+
with open(filepath, "r", encoding="utf-8") as f:
|
|
310
|
+
tree = ast.parse(f.read())
|
|
311
|
+
for node in ast.walk(tree):
|
|
312
|
+
if isinstance(node, ast.Import):
|
|
313
|
+
for alias in node.names:
|
|
314
|
+
all_imports.add(alias.name.split(".")[0])
|
|
315
|
+
elif isinstance(node, ast.ImportFrom):
|
|
316
|
+
if node.module:
|
|
317
|
+
all_imports.add(node.module.split(".")[0])
|
|
318
|
+
except Exception:
|
|
319
|
+
pass
|
|
320
|
+
|
|
321
|
+
std_lib = sys.stdlib_module_names if hasattr(sys, "stdlib_module_names") else set()
|
|
322
|
+
third_party = sorted([imp for imp in all_imports if imp not in std_lib and not imp.startswith("_")])
|
|
323
|
+
|
|
324
|
+
print("\n--- Discovered Third-Party Dependencies ---")
|
|
325
|
+
if third_party:
|
|
326
|
+
for dep in third_party:
|
|
327
|
+
print(f" - {dep}")
|
|
328
|
+
else:
|
|
329
|
+
print(" No external third-party packages detected.")
|
|
330
|
+
print("-------------------------------------------\n")
|
|
331
|
+
|
|
332
|
+
def cmd_sync(args):
|
|
333
|
+
"""Pain Point Solved: Syncs local extracted sub-modules against updated upstream repos."""
|
|
334
|
+
manifest_path = os.path.join(args.path, MANIFEST_NAME)
|
|
335
|
+
if not os.path.exists(manifest_path):
|
|
336
|
+
logging.error(f"No {MANIFEST_NAME} found in {args.path}. Cannot sync.")
|
|
337
|
+
sys.exit(1)
|
|
338
|
+
|
|
339
|
+
with open(manifest_path, "r", encoding="utf-8") as f:
|
|
340
|
+
data = json.load(f)
|
|
341
|
+
|
|
342
|
+
for target, meta in data.items():
|
|
343
|
+
repo = meta.get("source_repo")
|
|
344
|
+
sub_path = meta.get("sub_path")
|
|
345
|
+
if repo and repo.startswith("http"):
|
|
346
|
+
logging.info(f"Syncing target '{target}' with {repo}...")
|
|
347
|
+
cache_dir = os.path.expanduser("~/.pyforker_cache")
|
|
348
|
+
repo_path = GitEngine.clone_or_update(repo, cache_dir)
|
|
349
|
+
|
|
350
|
+
src_dir = os.path.join(repo_path, sub_path) if sub_path else repo_path
|
|
351
|
+
dst_dir = os.path.abspath(args.path)
|
|
352
|
+
|
|
353
|
+
for root, _, files in os.walk(src_dir):
|
|
354
|
+
for file in files:
|
|
355
|
+
if file.endswith(".py"):
|
|
356
|
+
src_f = os.path.join(root, file)
|
|
357
|
+
rel_f = os.path.relpath(src_f, src_dir)
|
|
358
|
+
dst_f = os.path.join(dst_dir, rel_f)
|
|
359
|
+
ASTAnalyzer.process_file(src_f, dst_f)
|
|
360
|
+
|
|
361
|
+
meta["commit"] = GitEngine.get_commit_hash(repo_path)
|
|
362
|
+
meta["synced_at"] = time.time()
|
|
363
|
+
|
|
364
|
+
StorageEngine.save_manifest(args.path, data)
|
|
365
|
+
logging.info("Sync complete!")
|
|
366
|
+
|
|
367
|
+
def cmd_server_add(args):
|
|
368
|
+
"""Pain Point Solved: Save and manage multiple remote PyForker instances for team networks."""
|
|
369
|
+
cfg = StorageEngine.load_config()
|
|
370
|
+
cfg["servers"][args.name] = {"url": args.url, "added_at": time.time()}
|
|
371
|
+
StorageEngine.save_config(cfg)
|
|
372
|
+
logging.info(f"Saved remote server alias '{args.name}' -> {args.url}")
|
|
373
|
+
|
|
374
|
+
def cmd_server_list(args):
|
|
375
|
+
"""Lists saved server configurations."""
|
|
376
|
+
cfg = StorageEngine.load_config()
|
|
377
|
+
servers = cfg.get("servers", {})
|
|
378
|
+
print("\n--- Configured Remote PyForker Servers ---")
|
|
379
|
+
if not servers:
|
|
380
|
+
print(" No remote servers configured. Add one with 'server-add'.")
|
|
381
|
+
else:
|
|
382
|
+
for name, info in servers.items():
|
|
383
|
+
print(f" - [{name}] {info['url']}")
|
|
384
|
+
print("------------------------------------------\n")
|
|
385
|
+
|
|
386
|
+
def cmd_serve(args):
|
|
387
|
+
"""Starts a real multi-threaded PyForker HTTP server daemon."""
|
|
388
|
+
server_address = (args.host, args.port)
|
|
389
|
+
httpd = ThreadedHTTPServer(server_address, PyForkerHTTPHandler)
|
|
390
|
+
logging.info(f"Starting PyForker Server on http://{args.host}:{args.port}...")
|
|
391
|
+
try:
|
|
392
|
+
httpd.serve_forever()
|
|
393
|
+
except KeyboardInterrupt:
|
|
394
|
+
logging.info("Shutting down server...")
|
|
395
|
+
httpd.server_close()
|
|
396
|
+
|
|
397
|
+
def cmd_self_test(args):
|
|
398
|
+
"""Executes full diagnostic test suite over internal AST and storage subsystems."""
|
|
399
|
+
print("Running PyForker Diagnostic Suite...")
|
|
400
|
+
|
|
401
|
+
# 1. Test Storage
|
|
402
|
+
cfg = StorageEngine.load_config()
|
|
403
|
+
assert isinstance(cfg, dict), "Storage engine failed to return dict"
|
|
404
|
+
print(" [PASS] Storage Engine")
|
|
405
|
+
|
|
406
|
+
# 2. Test File Locking
|
|
407
|
+
lock_file = ".test.lock"
|
|
408
|
+
with FileLock(lock_file):
|
|
409
|
+
assert os.path.exists(lock_file), "Lock file creation failed"
|
|
410
|
+
assert not os.path.exists(lock_file), "Lock file cleanup failed"
|
|
411
|
+
print(" [PASS] Transactional File Locking")
|
|
412
|
+
|
|
413
|
+
# 3. Test AST Engine
|
|
414
|
+
test_code = "from original import module\nimport os"
|
|
415
|
+
tree = ast.parse(test_code)
|
|
416
|
+
rewriter = ASTDependencyRewriter("original", "refactored")
|
|
417
|
+
new_tree = rewriter.visit(tree)
|
|
418
|
+
output = ast.unparse(new_tree)
|
|
419
|
+
assert "from refactored import module" in output, "AST rewrite failed"
|
|
420
|
+
print(" [PASS] AST Dependency Rewriting Engine")
|
|
421
|
+
|
|
422
|
+
print("\nAll diagnostics passed successfully!")
|
|
423
|
+
|
|
424
|
+
# =====================================================================
|
|
425
|
+
# MAIN ENTRY POINT
|
|
426
|
+
# =====================================================================
|
|
427
|
+
|
|
428
|
+
def main():
|
|
429
|
+
parser = argparse.ArgumentParser(
|
|
430
|
+
description="pyforker - Production-Grade Single-File Python Library Extractor & Server Engine"
|
|
431
|
+
)
|
|
432
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
433
|
+
|
|
434
|
+
# Command: take
|
|
435
|
+
p_take = subparsers.add_parser("take", help="Extract sub-module from local path or remote Git URL")
|
|
436
|
+
p_take.add_argument("repo", help="Git repository URL or local directory path")
|
|
437
|
+
p_take.add_argument("--sub-path", help="Relative sub-path inside repo to extract", default="")
|
|
438
|
+
p_take.add_argument("--out", help="Target output directory", required=True)
|
|
439
|
+
p_take.add_argument("--rewrite-from", help="Original package name to rewrite in imports", default="")
|
|
440
|
+
p_take.add_argument("--rewrite-to", help="Target package name to rewrite in imports", default="")
|
|
441
|
+
p_take.set_defaults(func=cmd_take)
|
|
442
|
+
|
|
443
|
+
# Command: deps
|
|
444
|
+
p_deps = subparsers.add_parser("deps", help="Scan extracted module to list third-party dependencies")
|
|
445
|
+
p_deps.add_argument("path", help="Path to extracted python code directory")
|
|
446
|
+
p_deps.set_defaults(func=cmd_deps)
|
|
447
|
+
|
|
448
|
+
# Command: sync
|
|
449
|
+
p_sync = subparsers.add_parser("sync", help="Sync extracted module with updated upstream git repository")
|
|
450
|
+
p_sync.add_argument("path", help="Path to directory containing pyforker.json manifest")
|
|
451
|
+
p_sync.set_defaults(func=cmd_sync)
|
|
452
|
+
|
|
453
|
+
# Command: server-add
|
|
454
|
+
p_sadd = subparsers.add_parser("server-add", help="Save a remote PyForker server alias")
|
|
455
|
+
p_sadd.add_argument("name", help="Server alias name (e.g., prod-server)")
|
|
456
|
+
p_sadd.add_argument("url", help="Server URL (e.g., http://192.168.1.50:8080)")
|
|
457
|
+
p_sadd.set_defaults(func=cmd_server_add)
|
|
458
|
+
|
|
459
|
+
# Command: server-list
|
|
460
|
+
p_slist = subparsers.add_parser("server-list", help="List saved PyForker server aliases")
|
|
461
|
+
p_slist.set_defaults(func=cmd_server_list)
|
|
462
|
+
|
|
463
|
+
# Command: serve
|
|
464
|
+
p_serve = subparsers.add_parser("serve", help="Launch multi-threaded PyForker HTTP server daemon")
|
|
465
|
+
p_serve.add_argument("--host", default="0.0.0.0", help="Binding host address")
|
|
466
|
+
p_serve.add_argument("--port", type=int, default=8080, help="Binding port")
|
|
467
|
+
p_serve.set_defaults(func=cmd_serve)
|
|
468
|
+
|
|
469
|
+
# Command: self-test
|
|
470
|
+
p_test = subparsers.add_parser("self-test", help="Run local diagnostic test suite")
|
|
471
|
+
p_test.set_defaults(func=cmd_self_test)
|
|
472
|
+
|
|
473
|
+
args = parser.parse_args()
|
|
474
|
+
args.func(args)
|
|
475
|
+
|
|
476
|
+
if __name__ == "__main__":
|
|
477
|
+
main()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pyforker"
|
|
7
|
+
version = "2.0.0"
|
|
8
|
+
description = "Zero-dependency toolkit for selectively extracting sub-modules, inspecting ASTs, and operating remote library mirror servers."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Sundaram Gupta" }
|
|
14
|
+
]
|
|
15
|
+
keywords = ["ast", "extractor", "git", "cli", "tree-sitter", "refactoring"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.8",
|
|
22
|
+
"Programming Language :: Python :: 3.9",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
27
|
+
"Topic :: Software Development :: Code Generators",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# Standard library only - zero external dependencies required
|
|
31
|
+
dependencies = []
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
pyforker = "pyforker:main"
|
|
35
|
+
|
|
36
|
+
[tool.setuptools]
|
|
37
|
+
# Exposes pyforker.py directly as a single-module package
|
|
38
|
+
py-modules = ["pyforker"]
|
pyforker-2.0.0/setup.cfg
ADDED
pyforker-2.0.0/setup.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import os
|
|
3
|
+
from setuptools import setup
|
|
4
|
+
|
|
5
|
+
# Read long description from README.md if present
|
|
6
|
+
long_description = ""
|
|
7
|
+
if os.path.exists("README.md"):
|
|
8
|
+
with open("README.md", "r", encoding="utf-8") as f:
|
|
9
|
+
long_description = f.read()
|
|
10
|
+
|
|
11
|
+
setup(
|
|
12
|
+
name="pyforker",
|
|
13
|
+
version="2.0.0",
|
|
14
|
+
description="Zero-dependency toolkit for selectively extracting sub-modules and running mirror servers.",
|
|
15
|
+
long_description=long_description,
|
|
16
|
+
long_description_content_type="text/markdown",
|
|
17
|
+
author="Sundaram Gupta",
|
|
18
|
+
py_modules=["pyforker"],
|
|
19
|
+
python_requires=">=3.8",
|
|
20
|
+
install_requires=[],
|
|
21
|
+
entry_points={
|
|
22
|
+
"console_scripts": [
|
|
23
|
+
"pyforker = pyforker:main",
|
|
24
|
+
],
|
|
25
|
+
},
|
|
26
|
+
classifiers=[
|
|
27
|
+
"Development Status :: 4 - Beta",
|
|
28
|
+
"Intended Audience :: Developers",
|
|
29
|
+
"License :: OSI Approved :: MIT License",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Topic :: Software Development :: Code Generators",
|
|
32
|
+
],
|
|
33
|
+
)
|