pydorky 2.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/.devcontainer/devcontainer.json +17 -0
  2. package/.github/FUNDING.yml +15 -0
  3. package/.github/workflows/e2e-integration.yml +57 -0
  4. package/.github/workflows/publish.yml +24 -0
  5. package/.nvmrc +1 -0
  6. package/LICENSE +21 -0
  7. package/README.md +156 -0
  8. package/bin/index.js +19 -0
  9. package/bin/legacy.js +432 -0
  10. package/docs/doc#1 get-started/README.md +105 -0
  11. package/docs/doc#2 features-wishlist +33 -0
  12. package/docs/doc#2.5 python-port +31 -0
  13. package/docs/doc#3 the-correct-node-version +107 -0
  14. package/docs/doc#4 why-where-python +42 -0
  15. package/docs/doc#5 how-do-endpoints-cli-work +0 -0
  16. package/dorky-usage-aws.svg +1 -0
  17. package/dorky-usage-google-drive.svg +1 -0
  18. package/google-drive-credentials.json +16 -0
  19. package/openapi/openapi.yaml +257 -0
  20. package/package.json +46 -0
  21. package/python-client/README.md +19 -0
  22. package/python-client/dorky_client/__init__.py +3 -0
  23. package/python-client/dorky_client/client.py +32 -0
  24. package/python-client/pyproject.toml +13 -0
  25. package/python-client/tests/test_integration.py +20 -0
  26. package/rectdorky.png +0 -0
  27. package/server/index.js +193 -0
  28. package/server/package.json +12 -0
  29. package/todo/01-core-infrastructure.md +84 -0
  30. package/todo/02-storage-providers.md +104 -0
  31. package/todo/03-compression-formats.md +94 -0
  32. package/todo/04-python-client.md +126 -0
  33. package/todo/05-metadata-versioning.md +116 -0
  34. package/todo/06-performance-concurrency.md +130 -0
  35. package/todo/07-security-encryption.md +114 -0
  36. package/todo/08-developer-experience.md +175 -0
  37. package/todo/README.md +37 -0
  38. package/web-app/README.md +70 -0
  39. package/web-app/package-lock.json +17915 -0
  40. package/web-app/package.json +43 -0
  41. package/web-app/public/favicon.ico +0 -0
  42. package/web-app/public/index.html +43 -0
  43. package/web-app/public/logo192.png +0 -0
  44. package/web-app/public/logo512.png +0 -0
  45. package/web-app/public/manifest.json +25 -0
  46. package/web-app/public/robots.txt +3 -0
  47. package/web-app/src/App.css +23 -0
  48. package/web-app/src/App.js +84 -0
  49. package/web-app/src/App.test.js +8 -0
  50. package/web-app/src/PrivacyPolicy.js +26 -0
  51. package/web-app/src/TermsAndConditions.js +41 -0
  52. package/web-app/src/index.css +3 -0
  53. package/web-app/src/index.js +26 -0
  54. package/web-app/src/logo.svg +1 -0
  55. package/web-app/src/reportWebVitals.js +13 -0
  56. package/web-app/src/setupTests.js +5 -0
  57. package/web-app/tailwind.config.js +10 -0
@@ -0,0 +1,17 @@
1
+ {
2
+ "name": "Pydorky Dev Container",
3
+ "image": "mcr.microsoft.com/vscode/devcontainers/javascript-node:16",
4
+ "updateRemoteUserUID": false,
5
+ "customizations": {
6
+ "vscode": {
7
+ "extensions": [
8
+ "esbenp.prettier-vscode",
9
+ "dbaeumer.vscode-eslint",
10
+ "ms-python.python"
11
+ ]
12
+ }
13
+ },
14
+ "postCreateCommand": "npm install",
15
+ "forwardPorts": [3000, 9229],
16
+ "remoteUser": "node"
17
+ }
@@ -0,0 +1,15 @@
1
+ # These are supported funding model platforms
2
+
3
+ github: Pratham-Jain-3903
4
+ patreon: # Replace with a single Patreon username
5
+ open_collective: # Replace with a single Open Collective username
6
+ ko_fi: # Replace with a single Ko-fi username
7
+ tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
8
+ community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
9
+ liberapay: # Replace with a single Liberapay username
10
+ issuehunt: # Replace with a single IssueHunt username
11
+ lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
12
+ polar: # Replace with a single Polar username
13
+ buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
14
+ thanks_dev: # Replace with a single thanks.dev username
15
+ custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
@@ -0,0 +1,57 @@
1
+ name: E2E integration (Node CLI + Python client)
2
+ on:
3
+ pull_request:
4
+ push:
5
+ branches:
6
+ - '**'
7
+
8
+ jobs:
9
+ e2e:
10
+ name: Node16 + Python integration
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - name: Checkout
14
+ uses: actions/checkout@v4
15
+
16
+ - name: Setup Node.js 16
17
+ uses: actions/setup-node@v4
18
+ with:
19
+ node-version: '16'
20
+
21
+ - name: Install root dependencies
22
+ run: npm ci
23
+
24
+ - name: Install server dependencies
25
+ run: |
26
+ cd server
27
+ npm ci
28
+
29
+ - name: Start server stub
30
+ run: |
31
+ cd server
32
+ nohup npm start > server.log 2>&1 &
33
+ # wait for server to be available
34
+ for i in {1..20}; do
35
+ if curl -sSf http://localhost:3000/ >/dev/null; then break; fi
36
+ sleep 1
37
+ done
38
+
39
+ - name: Node CLI smoke test
40
+ run: |
41
+ node --version
42
+ node bin/index.js --help || true
43
+
44
+ - name: Setup Python
45
+ uses: actions/setup-python@v4
46
+ with:
47
+ python-version: '3.10'
48
+
49
+ - name: Install Python test deps and client
50
+ run: |
51
+ python -m pip install --upgrade pip
52
+ pip install pytest
53
+ pip install -e python-client
54
+
55
+ - name: Run Python client integration tests
56
+ run: |
57
+ pytest -q python-client/tests/test_integration.py
@@ -0,0 +1,24 @@
1
+ name: "🚀 Publish on npm"
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+
8
+ jobs:
9
+ release:
10
+ name: 🚀 release
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - name: Checkout
14
+ uses: actions/checkout@v4
15
+ - name: Setup node
16
+ uses: actions/setup-node@v3
17
+ with:
18
+ node-version: "16.x"
19
+ registry-url: 'https://registry.npmjs.org'
20
+ - run: npm install
21
+ - name: 🚀 publish
22
+ run: npm publish --access public
23
+ env:
24
+ NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
package/.nvmrc ADDED
@@ -0,0 +1 @@
1
+ 16
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2022 Trishant Pahwa
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,156 @@
1
+ <div align="center">
2
+
3
+ <!-- Logo Placeholder - Replace with actual logo -->
4
+ <img
5
+ src="https://raw.githubusercontent.com/Pratham-Jain-3903/pydorky/main/rectdorky.png"
6
+ alt="Pydorky Logo"
7
+ width="500"
8
+ style="object-fit: cover; object-position: center;"
9
+ />
10
+
11
+
12
+ # Pydorky
13
+
14
+ **Practical artifact storage for teams**
15
+
16
+ [![Build Status](https://github.com/Pratham-Jain-3903/pydorky/actions/workflows/e2e-integration.yml/badge.svg)](https://github.com/Pratham-Jain-3903/pydorky/actions/workflows/e2e-integration.yml)
17
+ [![npm version](https://img.shields.io/npm/v/dorky.svg)](https://www.npmjs.com/package/dorky)
18
+ [![License: ISC](https://img.shields.io/badge/License-ISC-blue.svg)](https://opensource.org/licenses/ISC)
19
+ [![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
20
+ [![Node.js 16+](https://img.shields.io/badge/node.js-16+-green.svg)](https://nodejs.org/)
21
+
22
+ [Features](#vision) • [Quick Start](#quick-start) • [Documentation](docs/) • [CLI Usage](#node-npm-users--cli-and-legacy-usage)
23
+
24
+ </div>
25
+
26
+ ---
27
+
28
+ ## Technology Stack
29
+
30
+ <div align="center">
31
+
32
+ ### Core Technologies
33
+ ![Python](https://img.shields.io/badge/Python-3776AB?style=for-the-badge&logo=python&logoColor=white)
34
+ ![JavaScript](https://img.shields.io/badge/JavaScript-F7DF1E?style=for-the-badge&logo=javascript&logoColor=black)
35
+ ![Express](https://img.shields.io/badge/Express-000000?style=for-the-badge&logo=express&logoColor=white)
36
+
37
+ ### Documentation Stack Migration Pending
38
+ ![Node.js](https://img.shields.io/badge/Node.js-339933?style=for-the-badge&logo=nodedotjs&logoColor=white)
39
+ ![React](https://img.shields.io/badge/React-20232A?style=for-the-badge&logo=react&logoColor=61DAFB)
40
+
41
+ ### Cloud Storage Platforms
42
+ ![AWS](https://img.shields.io/badge/AWS_S3-232F3E?style=for-the-badge&logo=amazonaws&logoColor=white)
43
+ ![Google Cloud](https://img.shields.io/badge/Google_Cloud-4285F4?style=for-the-badge&logo=googlecloud&logoColor=white)
44
+ ![Azure](https://img.shields.io/badge/Azure-0078D4?style=for-the-badge&logo=microsoftazure&logoColor=white)
45
+
46
+ </div>
47
+
48
+ ---
49
+
50
+ ## Introduction
51
+
52
+ This repository is Pydorky, a Python-first client and ecosystem built around a small language-agnostic artifact service. Pydorky was inspired by the original `dorky` tool — originally authored by Trishant Pahwa — and extends the idea with data-focused features and a Python client optimized for data teams. The original project provided the foundation; this fork focuses the design on reliability, data ergonomics, and reproducible sharing.
53
+
54
+ ## Vision
55
+
56
+ Teams should never lose important artifacts in chat threads, ad-hoc paste tools, or personal drives. Pydorky offers a reproducible, privacy-aware, and developer-friendly way to store project artifacts (secrets, environment files, small datasets, and binaries) in user-controlled storage (S3, GCS, etc.).
57
+
58
+ ## Why we built this
59
+
60
+ I grew tired of important artifacts being scattered across chat apps (Teams, Slack), quick paste services (sharetext.io), and personal drives. That pattern causes lost context, duplicate uploads, and potential leaks. Pydorky provides a minimal, auditable, and automated alternative that:
61
+
62
+ - keeps artifacts out of VCS while enabling reproducible sharing
63
+ - integrates with existing cloud storage and IAM controls
64
+ - provides lightweight metadata, idempotency, and streaming-friendly APIs
65
+ - offers a Python client for data teams (Parquet/pyarrow integration) and thin clients for other languages
66
+
67
+ ## Approach
68
+
69
+ Keep the service small and language-agnostic. Implement a canonical HTTP service for storage semantics (streaming upload/download, idempotency keys, metadata, hierarchical sync). Language clients — Python and Node — remain thin translators that implement ergonomic workflows (`commit`, `stage`, `push`). Heavy data-format work (Parquet conversions, large-table transforms) is handled by optional Python tools or sidecar converters.
70
+
71
+ ## Quick start
72
+
73
+ ### Install Python Client
74
+
75
+ ```bash
76
+ pip install -e python-client
77
+ ```
78
+
79
+ ### Run Local Service
80
+
81
+ ```bash
82
+ cd server
83
+ npm install
84
+ npm start
85
+ ```
86
+
87
+ ### Upload with Python Client
88
+
89
+ ```python
90
+ from dorky_client import DorkyClient
91
+ client = DorkyClient('http://localhost:3000')
92
+ client.upload('path/to/file', metadata={'note':'example'})
93
+ ```
94
+
95
+ ## Where to look next
96
+
97
+ Implementation notes and design rationale: see `docs/` (architecture, migration plan, Python client notes).
98
+
99
+ ## Acknowledgements
100
+
101
+ This project builds on and refines the ideas in the original `dorky` project. See `docs/` for more on differences and the migration path.
102
+
103
+ ## Node (npm) users — CLI and legacy usage
104
+
105
+ We continue to support the original Node/npm CLI workflows so JavaScript and Ops users can keep using the familiar commands.
106
+
107
+ ### Install CLI Globally
108
+
109
+ ```bash
110
+ npm install -g dorky
111
+ ```
112
+
113
+ ### Quick CLI Commands
114
+
115
+ - Initialize for AWS S3: `dorky --init aws`
116
+ - Initialize for Google Drive: `dorky --init google-drive`
117
+ - List files: `dorky --list` (respects `.dorkyignore`)
118
+ - Add to stage: `dorky --add <file>`
119
+ - Remove from stage: `dorky --rm <file>`
120
+ - Push staged files: `dorky --push`
121
+ - Pull files from remote: `dorky --pull`
122
+
123
+ ## AWS S3 Notes
124
+
125
+ 1. Create an S3 bucket and configure `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in your environment.
126
+ 2. Use `dorky --init aws` to set up local metadata and `.dorkyignore`.
127
+ 3. Use `dorky --add`, `dorky --push`, `dorky --pull` as your normal workflow.
128
+
129
+ ## Google Drive Notes
130
+
131
+ 1. Use `dorky --init google-drive` and follow the OAuth setup instructions.
132
+ 2. Use the same `--add`, `--push`, `--pull` flow as above.
133
+
134
+ ## Developer & Compatibility Notes
135
+
136
+ - The repo contains both the Node CLI and a Python client. The canonical storage semantics are provided by a small HTTP service; clients are thin translators.
137
+ - Current Node compatibility: project targets Node 16 (`.nvmrc` + `engines.node`), with a planned migration to Node 20. See `docs/doc#3 the-correct-node-version`.
138
+ - Python users: see `python-client/` for the PyPI-ready client scaffold and `docs/doc#2.5 python-port` for design notes.
139
+
140
+ ---
141
+
142
+ <div align="center">
143
+
144
+ ## License
145
+
146
+ This project is licensed under the ISC License - see the [LICENSE](LICENSE) file for details.
147
+
148
+ ## Contributing
149
+
150
+ Contributions are welcome! Please feel free to submit a Pull Request.
151
+
152
+ ---
153
+
154
+ **Made with care by the Pydorky contributors**
155
+
156
+ </div>
package/bin/index.js ADDED
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env node
2
+
3
+ // Entry point that preserves legacy CLI behavior while we layer in a new
4
+ // command-based CLI. For now, everything routes to the legacy implementation
5
+ // to maintain backward compatibility.
6
+
7
+ const { run: runLegacy } = require('./legacy');
8
+
9
+ // Detect legacy-style invocations (flags like --init/--push/etc.)
10
+ const rawArgs = process.argv.slice(2);
11
+ const isLegacyInvocation = rawArgs.length === 0 || rawArgs.some(arg => arg.startsWith('--'));
12
+
13
+ if (isLegacyInvocation) {
14
+ runLegacy(rawArgs);
15
+ } else {
16
+ // Placeholder for future command-based CLI; keep legacy behavior for now.
17
+ runLegacy(rawArgs);
18
+ }
19
+