pydorky 2.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devcontainer/devcontainer.json +17 -0
- package/.github/FUNDING.yml +15 -0
- package/.github/workflows/e2e-integration.yml +57 -0
- package/.github/workflows/publish.yml +24 -0
- package/.nvmrc +1 -0
- package/LICENSE +21 -0
- package/README.md +156 -0
- package/bin/index.js +19 -0
- package/bin/legacy.js +432 -0
- package/docs/doc#1 get-started/README.md +105 -0
- package/docs/doc#2 features-wishlist +33 -0
- package/docs/doc#2.5 python-port +31 -0
- package/docs/doc#3 the-correct-node-version +107 -0
- package/docs/doc#4 why-where-python +42 -0
- package/docs/doc#5 how-do-endpoints-cli-work +0 -0
- package/dorky-usage-aws.svg +1 -0
- package/dorky-usage-google-drive.svg +1 -0
- package/google-drive-credentials.json +16 -0
- package/openapi/openapi.yaml +257 -0
- package/package.json +46 -0
- package/python-client/README.md +19 -0
- package/python-client/dorky_client/__init__.py +3 -0
- package/python-client/dorky_client/client.py +32 -0
- package/python-client/pyproject.toml +13 -0
- package/python-client/tests/test_integration.py +20 -0
- package/rectdorky.png +0 -0
- package/server/index.js +193 -0
- package/server/package.json +12 -0
- package/todo/01-core-infrastructure.md +84 -0
- package/todo/02-storage-providers.md +104 -0
- package/todo/03-compression-formats.md +94 -0
- package/todo/04-python-client.md +126 -0
- package/todo/05-metadata-versioning.md +116 -0
- package/todo/06-performance-concurrency.md +130 -0
- package/todo/07-security-encryption.md +114 -0
- package/todo/08-developer-experience.md +175 -0
- package/todo/README.md +37 -0
- package/web-app/README.md +70 -0
- package/web-app/package-lock.json +17915 -0
- package/web-app/package.json +43 -0
- package/web-app/public/favicon.ico +0 -0
- package/web-app/public/index.html +43 -0
- package/web-app/public/logo192.png +0 -0
- package/web-app/public/logo512.png +0 -0
- package/web-app/public/manifest.json +25 -0
- package/web-app/public/robots.txt +3 -0
- package/web-app/src/App.css +23 -0
- package/web-app/src/App.js +84 -0
- package/web-app/src/App.test.js +8 -0
- package/web-app/src/PrivacyPolicy.js +26 -0
- package/web-app/src/TermsAndConditions.js +41 -0
- package/web-app/src/index.css +3 -0
- package/web-app/src/index.js +26 -0
- package/web-app/src/logo.svg +1 -0
- package/web-app/src/reportWebVitals.js +13 -0
- package/web-app/src/setupTests.js +5 -0
- package/web-app/tailwind.config.js +10 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Pydorky Dev Container",
|
|
3
|
+
"image": "mcr.microsoft.com/vscode/devcontainers/javascript-node:16",
|
|
4
|
+
"updateRemoteUserUID": false,
|
|
5
|
+
"customizations": {
|
|
6
|
+
"vscode": {
|
|
7
|
+
"extensions": [
|
|
8
|
+
"esbenp.prettier-vscode",
|
|
9
|
+
"dbaeumer.vscode-eslint",
|
|
10
|
+
"ms-python.python"
|
|
11
|
+
]
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"postCreateCommand": "npm install",
|
|
15
|
+
"forwardPorts": [3000, 9229],
|
|
16
|
+
"remoteUser": "node"
|
|
17
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# These are supported funding model platforms
|
|
2
|
+
|
|
3
|
+
github: Pratham-Jain-3903
|
|
4
|
+
patreon: # Replace with a single Patreon username
|
|
5
|
+
open_collective: # Replace with a single Open Collective username
|
|
6
|
+
ko_fi: # Replace with a single Ko-fi username
|
|
7
|
+
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
|
8
|
+
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
|
9
|
+
liberapay: # Replace with a single Liberapay username
|
|
10
|
+
issuehunt: # Replace with a single IssueHunt username
|
|
11
|
+
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
|
|
12
|
+
polar: # Replace with a single Polar username
|
|
13
|
+
buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
|
|
14
|
+
thanks_dev: # Replace with a single thanks.dev username
|
|
15
|
+
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
name: E2E integration (Node CLI + Python client)
|
|
2
|
+
on:
|
|
3
|
+
pull_request:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- '**'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
e2e:
|
|
10
|
+
name: Node16 + Python integration
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- name: Checkout
|
|
14
|
+
uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- name: Setup Node.js 16
|
|
17
|
+
uses: actions/setup-node@v4
|
|
18
|
+
with:
|
|
19
|
+
node-version: '16'
|
|
20
|
+
|
|
21
|
+
- name: Install root dependencies
|
|
22
|
+
run: npm ci
|
|
23
|
+
|
|
24
|
+
- name: Install server dependencies
|
|
25
|
+
run: |
|
|
26
|
+
cd server
|
|
27
|
+
npm ci
|
|
28
|
+
|
|
29
|
+
- name: Start server stub
|
|
30
|
+
run: |
|
|
31
|
+
cd server
|
|
32
|
+
nohup npm start > server.log 2>&1 &
|
|
33
|
+
# wait for server to be available
|
|
34
|
+
for i in {1..20}; do
|
|
35
|
+
if curl -sSf http://localhost:3000/ >/dev/null; then break; fi
|
|
36
|
+
sleep 1
|
|
37
|
+
done
|
|
38
|
+
|
|
39
|
+
- name: Node CLI smoke test
|
|
40
|
+
run: |
|
|
41
|
+
node --version
|
|
42
|
+
node bin/index.js --help || true
|
|
43
|
+
|
|
44
|
+
- name: Setup Python
|
|
45
|
+
uses: actions/setup-python@v4
|
|
46
|
+
with:
|
|
47
|
+
python-version: '3.10'
|
|
48
|
+
|
|
49
|
+
- name: Install Python test deps and client
|
|
50
|
+
run: |
|
|
51
|
+
python -m pip install --upgrade pip
|
|
52
|
+
pip install pytest
|
|
53
|
+
pip install -e python-client
|
|
54
|
+
|
|
55
|
+
- name: Run Python client integration tests
|
|
56
|
+
run: |
|
|
57
|
+
pytest -q python-client/tests/test_integration.py
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: "🚀 Publish on npm"
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
release:
|
|
10
|
+
name: 🚀 release
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- name: Checkout
|
|
14
|
+
uses: actions/checkout@v4
|
|
15
|
+
- name: Setup node
|
|
16
|
+
uses: actions/setup-node@v3
|
|
17
|
+
with:
|
|
18
|
+
node-version: "16.x"
|
|
19
|
+
registry-url: 'https://registry.npmjs.org'
|
|
20
|
+
- run: npm install
|
|
21
|
+
- name: 🚀 publish
|
|
22
|
+
run: npm publish --access public
|
|
23
|
+
env:
|
|
24
|
+
NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
|
package/.nvmrc
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
16
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Trishant Pahwa
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<!-- Logo Placeholder - Replace with actual logo -->
|
|
4
|
+
<img
|
|
5
|
+
src="https://raw.githubusercontent.com/Pratham-Jain-3903/pydorky/main/rectdorky.png"
|
|
6
|
+
alt="Pydorky Logo"
|
|
7
|
+
width="500"
|
|
8
|
+
style="object-fit: cover; object-position: center;"
|
|
9
|
+
/>
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# Pydorky
|
|
13
|
+
|
|
14
|
+
**Practical artifact storage for teams**
|
|
15
|
+
|
|
16
|
+
[](https://github.com/Pratham-Jain-3903/pydorky/actions/workflows/e2e-integration.yml)
|
|
17
|
+
[](https://www.npmjs.com/package/dorky)
|
|
18
|
+
[](https://opensource.org/licenses/ISC)
|
|
19
|
+
[](https://www.python.org/downloads/)
|
|
20
|
+
[](https://nodejs.org/)
|
|
21
|
+
|
|
22
|
+
[Features](#vision) • [Quick Start](#quick-start) • [Documentation](docs/) • [CLI Usage](#node-npm-users--cli-and-legacy-usage)
|
|
23
|
+
|
|
24
|
+
</div>
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## Technology Stack
|
|
29
|
+
|
|
30
|
+
<div align="center">
|
|
31
|
+
|
|
32
|
+
### Core Technologies
|
|
33
|
+

|
|
34
|
+

|
|
35
|
+

|
|
36
|
+
|
|
37
|
+
### Documentation Stack Migration Pending
|
|
38
|
+

|
|
39
|
+

|
|
40
|
+
|
|
41
|
+
### Cloud Storage Platforms
|
|
42
|
+

|
|
43
|
+

|
|
44
|
+

|
|
45
|
+
|
|
46
|
+
</div>
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## Introduction
|
|
51
|
+
|
|
52
|
+
This repository is Pydorky, a Python-first client and ecosystem built around a small language-agnostic artifact service. Pydorky was inspired by the original `dorky` tool — originally authored by Trishant Pahwa — and extends the idea with data-focused features and a Python client optimized for data teams. The original project provided the foundation; this fork focuses the design on reliability, data ergonomics, and reproducible sharing.
|
|
53
|
+
|
|
54
|
+
## Vision
|
|
55
|
+
|
|
56
|
+
Teams should never lose important artifacts in chat threads, ad-hoc paste tools, or personal drives. Pydorky offers a reproducible, privacy-aware, and developer-friendly way to store project artifacts (secrets, environment files, small datasets, and binaries) in user-controlled storage (S3, GCS, etc.).
|
|
57
|
+
|
|
58
|
+
## Why we built this
|
|
59
|
+
|
|
60
|
+
I grew tired of important artifacts being scattered across chat apps (Teams, Slack), quick paste services (sharetext.io), and personal drives. That pattern causes lost context, duplicate uploads, and potential leaks. Pydorky provides a minimal, auditable, and automated alternative that:
|
|
61
|
+
|
|
62
|
+
- keeps artifacts out of VCS while enabling reproducible sharing
|
|
63
|
+
- integrates with existing cloud storage and IAM controls
|
|
64
|
+
- provides lightweight metadata, idempotency, and streaming-friendly APIs
|
|
65
|
+
- offers a Python client for data teams (Parquet/pyarrow integration) and thin clients for other languages
|
|
66
|
+
|
|
67
|
+
## Approach
|
|
68
|
+
|
|
69
|
+
Keep the service small and language-agnostic. Implement a canonical HTTP service for storage semantics (streaming upload/download, idempotency keys, metadata, hierarchical sync). Language clients — Python and Node — remain thin translators that implement ergonomic workflows (`commit`, `stage`, `push`). Heavy data-format work (Parquet conversions, large-table transforms) is handled by optional Python tools or sidecar converters.
|
|
70
|
+
|
|
71
|
+
## Quick start
|
|
72
|
+
|
|
73
|
+
### Install Python Client
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
pip install -e python-client
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Run Local Service
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
cd server
|
|
83
|
+
npm install
|
|
84
|
+
npm start
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Upload with Python Client
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from dorky_client import DorkyClient
|
|
91
|
+
client = DorkyClient('http://localhost:3000')
|
|
92
|
+
client.upload('path/to/file', metadata={'note':'example'})
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Where to look next
|
|
96
|
+
|
|
97
|
+
Implementation notes and design rationale: see `docs/` (architecture, migration plan, Python client notes).
|
|
98
|
+
|
|
99
|
+
## Acknowledgements
|
|
100
|
+
|
|
101
|
+
This project builds on and refines the ideas in the original `dorky` project. See `docs/` for more on differences and the migration path.
|
|
102
|
+
|
|
103
|
+
## Node (npm) users — CLI and legacy usage
|
|
104
|
+
|
|
105
|
+
We continue to support the original Node/npm CLI workflows so JavaScript and Ops users can keep using the familiar commands.
|
|
106
|
+
|
|
107
|
+
### Install CLI Globally
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
npm install -g dorky
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### Quick CLI Commands
|
|
114
|
+
|
|
115
|
+
- Initialize for AWS S3: `dorky --init aws`
|
|
116
|
+
- Initialize for Google Drive: `dorky --init google-drive`
|
|
117
|
+
- List files: `dorky --list` (respects `.dorkyignore`)
|
|
118
|
+
- Add to stage: `dorky --add <file>`
|
|
119
|
+
- Remove from stage: `dorky --rm <file>`
|
|
120
|
+
- Push staged files: `dorky --push`
|
|
121
|
+
- Pull files from remote: `dorky --pull`
|
|
122
|
+
|
|
123
|
+
## AWS S3 Notes
|
|
124
|
+
|
|
125
|
+
1. Create an S3 bucket and configure `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in your environment.
|
|
126
|
+
2. Use `dorky --init aws` to set up local metadata and `.dorkyignore`.
|
|
127
|
+
3. Use `dorky --add`, `dorky --push`, `dorky --pull` as your normal workflow.
|
|
128
|
+
|
|
129
|
+
## Google Drive Notes
|
|
130
|
+
|
|
131
|
+
1. Use `dorky --init google-drive` and follow the OAuth setup instructions.
|
|
132
|
+
2. Use the same `--add`, `--push`, `--pull` flow as above.
|
|
133
|
+
|
|
134
|
+
## Developer & Compatibility Notes
|
|
135
|
+
|
|
136
|
+
- The repo contains both the Node CLI and a Python client. The canonical storage semantics are provided by a small HTTP service; clients are thin translators.
|
|
137
|
+
- Current Node compatibility: project targets Node 16 (`.nvmrc` + `engines.node`), with a planned migration to Node 20. See `docs/doc#3 the-correct-node-version`.
|
|
138
|
+
- Python users: see `python-client/` for the PyPI-ready client scaffold and `docs/doc#2.5 python-port` for design notes.
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
<div align="center">
|
|
143
|
+
|
|
144
|
+
## License
|
|
145
|
+
|
|
146
|
+
This project is licensed under the ISC License - see the [LICENSE](LICENSE) file for details.
|
|
147
|
+
|
|
148
|
+
## Contributing
|
|
149
|
+
|
|
150
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
**Made with care by the Pydorky contributors**
|
|
155
|
+
|
|
156
|
+
</div>
|
package/bin/index.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// Entry point that preserves legacy CLI behavior while we layer in a new
|
|
4
|
+
// command-based CLI. For now, everything routes to the legacy implementation
|
|
5
|
+
// to maintain backward compatibility.
|
|
6
|
+
|
|
7
|
+
const { run: runLegacy } = require('./legacy');
|
|
8
|
+
|
|
9
|
+
// Detect legacy-style invocations (flags like --init/--push/etc.)
|
|
10
|
+
const rawArgs = process.argv.slice(2);
|
|
11
|
+
const isLegacyInvocation = rawArgs.length === 0 || rawArgs.some(arg => arg.startsWith('--'));
|
|
12
|
+
|
|
13
|
+
if (isLegacyInvocation) {
|
|
14
|
+
runLegacy(rawArgs);
|
|
15
|
+
} else {
|
|
16
|
+
// Placeholder for future command-based CLI; keep legacy behavior for now.
|
|
17
|
+
runLegacy(rawArgs);
|
|
18
|
+
}
|
|
19
|
+
|