cloudcat 0.1.4__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudcat-0.2.1/PKG-INFO +702 -0
- cloudcat-0.2.1/README.md +625 -0
- {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat/__init__.py +1 -1
- cloudcat-0.2.1/cloudcat/cli.py +770 -0
- cloudcat-0.2.1/cloudcat/compression.py +114 -0
- cloudcat-0.2.1/cloudcat/config.py +54 -0
- cloudcat-0.2.1/cloudcat/filtering.py +120 -0
- cloudcat-0.2.1/cloudcat/formatters.py +93 -0
- cloudcat-0.2.1/cloudcat/readers/__init__.py +20 -0
- cloudcat-0.2.1/cloudcat/readers/avro.py +77 -0
- cloudcat-0.2.1/cloudcat/readers/csv.py +53 -0
- cloudcat-0.2.1/cloudcat/readers/json.py +102 -0
- cloudcat-0.2.1/cloudcat/readers/orc.py +86 -0
- cloudcat-0.2.1/cloudcat/readers/parquet.py +115 -0
- cloudcat-0.2.1/cloudcat/readers/text.py +57 -0
- cloudcat-0.2.1/cloudcat/storage/__init__.py +24 -0
- cloudcat-0.2.1/cloudcat/storage/azure.py +109 -0
- cloudcat-0.2.1/cloudcat/storage/base.py +91 -0
- cloudcat-0.2.1/cloudcat/storage/gcs.py +92 -0
- cloudcat-0.2.1/cloudcat/storage/s3.py +88 -0
- cloudcat-0.2.1/cloudcat.egg-info/PKG-INFO +702 -0
- cloudcat-0.2.1/cloudcat.egg-info/SOURCES.txt +40 -0
- cloudcat-0.2.1/cloudcat.egg-info/requires.txt +38 -0
- {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat.egg-info/top_level.txt +1 -0
- cloudcat-0.2.1/setup.py +95 -0
- cloudcat-0.2.1/tests/__init__.py +0 -0
- cloudcat-0.2.1/tests/conftest.py +45 -0
- cloudcat-0.2.1/tests/test_cli.py +255 -0
- cloudcat-0.2.1/tests/test_compression.py +163 -0
- cloudcat-0.2.1/tests/test_config.py +111 -0
- cloudcat-0.2.1/tests/test_data_reading.py +133 -0
- cloudcat-0.2.1/tests/test_file_filtering.py +145 -0
- cloudcat-0.2.1/tests/test_filtering.py +240 -0
- cloudcat-0.2.1/tests/test_format_detection.py +65 -0
- cloudcat-0.2.1/tests/test_integration.py +158 -0
- cloudcat-0.2.1/tests/test_output_formatting.py +104 -0
- cloudcat-0.2.1/tests/test_path_parsing.py +48 -0
- cloudcat-0.1.4/PKG-INFO +0 -60
- cloudcat-0.1.4/README.md +0 -161
- cloudcat-0.1.4/cloudcat/cli.py +0 -787
- cloudcat-0.1.4/cloudcat.egg-info/PKG-INFO +0 -60
- cloudcat-0.1.4/cloudcat.egg-info/SOURCES.txt +0 -12
- cloudcat-0.1.4/cloudcat.egg-info/requires.txt +0 -18
- cloudcat-0.1.4/setup.py +0 -60
- {cloudcat-0.1.4 → cloudcat-0.2.1}/LICENSE +0 -0
- {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat.egg-info/dependency_links.txt +0 -0
- {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat.egg-info/entry_points.txt +0 -0
- {cloudcat-0.1.4 → cloudcat-0.2.1}/pyproject.toml +0 -0
- {cloudcat-0.1.4 → cloudcat-0.2.1}/setup.cfg +0 -0
cloudcat-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,702 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cloudcat
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Blob Storage from your terminal
|
|
5
|
+
Home-page: https://github.com/jonathansudhakar1/cloudcat
|
|
6
|
+
Author: Jonathan Sudhakar
|
|
7
|
+
Author-email: jonathan@example.com
|
|
8
|
+
Project-URL: Bug Reports, https://github.com/jonathansudhakar1/cloudcat/issues
|
|
9
|
+
Project-URL: Source, https://github.com/jonathansudhakar1/cloudcat
|
|
10
|
+
Project-URL: Documentation, https://github.com/jonathansudhakar1/cloudcat#readme
|
|
11
|
+
Keywords: cloud,gcs,s3,azure,cli,storage,data,parquet,csv,json,avro,orc,google-cloud,aws,azure-blob,data-engineering,etl,spark,bigquery,databricks,kafka,hive
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: System Administrators
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Topic :: Utilities
|
|
17
|
+
Classifier: Topic :: Database
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
19
|
+
Classifier: Topic :: System :: Systems Administration
|
|
20
|
+
Classifier: Environment :: Console
|
|
21
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
22
|
+
Classifier: Operating System :: OS Independent
|
|
23
|
+
Classifier: Programming Language :: Python :: 3
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
27
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
28
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
29
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
30
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
31
|
+
Requires-Python: >=3.7
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
License-File: LICENSE
|
|
34
|
+
Requires-Dist: click>=8.0.0
|
|
35
|
+
Requires-Dist: pandas>=1.3.0
|
|
36
|
+
Requires-Dist: tabulate>=0.8.9
|
|
37
|
+
Requires-Dist: colorama>=0.4.4
|
|
38
|
+
Requires-Dist: google-cloud-storage>=2.0.0
|
|
39
|
+
Requires-Dist: boto3>=1.18.0
|
|
40
|
+
Requires-Dist: azure-storage-blob>=12.0.0
|
|
41
|
+
Requires-Dist: azure-identity>=1.0.0
|
|
42
|
+
Provides-Extra: parquet
|
|
43
|
+
Requires-Dist: pyarrow>=5.0.0; extra == "parquet"
|
|
44
|
+
Provides-Extra: avro
|
|
45
|
+
Requires-Dist: fastavro>=1.4.0; extra == "avro"
|
|
46
|
+
Provides-Extra: orc
|
|
47
|
+
Requires-Dist: pyarrow>=5.0.0; extra == "orc"
|
|
48
|
+
Provides-Extra: zstd
|
|
49
|
+
Requires-Dist: zstandard>=0.15.0; extra == "zstd"
|
|
50
|
+
Provides-Extra: lz4
|
|
51
|
+
Requires-Dist: lz4>=3.0.0; extra == "lz4"
|
|
52
|
+
Provides-Extra: snappy
|
|
53
|
+
Requires-Dist: python-snappy>=0.6.0; extra == "snappy"
|
|
54
|
+
Provides-Extra: compression
|
|
55
|
+
Requires-Dist: zstandard>=0.15.0; extra == "compression"
|
|
56
|
+
Requires-Dist: lz4>=3.0.0; extra == "compression"
|
|
57
|
+
Requires-Dist: python-snappy>=0.6.0; extra == "compression"
|
|
58
|
+
Provides-Extra: all
|
|
59
|
+
Requires-Dist: pyarrow>=5.0.0; extra == "all"
|
|
60
|
+
Requires-Dist: fastavro>=1.4.0; extra == "all"
|
|
61
|
+
Requires-Dist: zstandard>=0.15.0; extra == "all"
|
|
62
|
+
Requires-Dist: lz4>=3.0.0; extra == "all"
|
|
63
|
+
Requires-Dist: python-snappy>=0.6.0; extra == "all"
|
|
64
|
+
Dynamic: author
|
|
65
|
+
Dynamic: author-email
|
|
66
|
+
Dynamic: classifier
|
|
67
|
+
Dynamic: description
|
|
68
|
+
Dynamic: description-content-type
|
|
69
|
+
Dynamic: home-page
|
|
70
|
+
Dynamic: keywords
|
|
71
|
+
Dynamic: license-file
|
|
72
|
+
Dynamic: project-url
|
|
73
|
+
Dynamic: provides-extra
|
|
74
|
+
Dynamic: requires-dist
|
|
75
|
+
Dynamic: requires-python
|
|
76
|
+
Dynamic: summary
|
|
77
|
+
|
|
78
|
+
<p align="center">
|
|
79
|
+
<img src="https://raw.githubusercontent.com/jonathansudhakar1/cloudcat/main/assets/logo.png" alt="CloudCat Logo" width="200">
|
|
80
|
+
</p>
|
|
81
|
+
|
|
82
|
+
<h1 align="center">CloudCat</h1>
|
|
83
|
+
|
|
84
|
+
<p align="center">
|
|
85
|
+
<strong>The Swiss Army knife for viewing cloud storage data from your terminal</strong>
|
|
86
|
+
</p>
|
|
87
|
+
|
|
88
|
+
<p align="center">
|
|
89
|
+
<a href="https://pypi.org/project/cloudcat/"><img src="https://img.shields.io/pypi/v/cloudcat.svg?style=flat-square&logo=pypi&logoColor=white" alt="PyPI version"></a>
|
|
90
|
+
<a href="https://pypi.org/project/cloudcat/"><img src="https://img.shields.io/pypi/pyversions/cloudcat.svg?style=flat-square&logo=python&logoColor=white" alt="Python versions"></a>
|
|
91
|
+
<a href="https://pypi.org/project/cloudcat/"><img src="https://img.shields.io/pypi/dm/cloudcat.svg?style=flat-square&logo=pypi&logoColor=white" alt="PyPI Downloads"></a>
|
|
92
|
+
<a href="https://github.com/jonathansudhakar1/cloudcat/releases"><img src="https://img.shields.io/github/downloads/jonathansudhakar1/cloudcat/total.svg?style=flat-square&logo=homebrew&logoColor=white&label=homebrew" alt="Homebrew Downloads"></a>
|
|
93
|
+
<a href="https://github.com/jonathansudhakar1/cloudcat/blob/main/LICENSE"><img src="https://img.shields.io/github/license/jonathansudhakar1/cloudcat.svg?style=flat-square" alt="License"></a>
|
|
94
|
+
</p>
|
|
95
|
+
|
|
96
|
+
<p align="center">
|
|
97
|
+
<a href="#installation">Installation</a> •
|
|
98
|
+
<a href="#quick-start">Quick Start</a> •
|
|
99
|
+
<a href="#features">Features</a> •
|
|
100
|
+
<a href="#examples">Examples</a> •
|
|
101
|
+
<a href="#documentation">Documentation</a>
|
|
102
|
+
</p>
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
**CloudCat** is a powerful command-line tool that lets you instantly preview and analyze data files stored in **Google Cloud Storage (GCS)**, **Amazon S3**, and **Azure Blob Storage** — without downloading entire files. Think of it as `cat`, `head`, and `less` combined, but for cloud storage with built-in support for CSV, JSON, Parquet, Avro, ORC, and plain text formats.
|
|
107
|
+
|
|
108
|
+
## Why CloudCat?
|
|
109
|
+
|
|
110
|
+
- **No Downloads Required** — Stream and preview data directly from cloud storage
|
|
111
|
+
- **Format-Aware** — Intelligently handles CSV, JSON, Parquet, Avro, ORC, and plain text files
|
|
112
|
+
- **Directory Smart** — Automatically discovers data files in Spark/Hive/Kafka output directories
|
|
113
|
+
- **Beautiful Output** — Colorized tables, pretty-printed JSON, and schema visualization
|
|
114
|
+
- **Developer Friendly** — Simple CLI with sensible defaults and powerful options
|
|
115
|
+
- **Compression Support** — Automatic decompression of gzip, zstd, lz4, snappy, and bz2 files
|
|
116
|
+
- **SQL-like Filtering** — Filter rows with WHERE clauses (e.g., `--where "status=active"`)
|
|
117
|
+
|
|
118
|
+
## Installation
|
|
119
|
+
|
|
120
|
+
### Homebrew (macOS)
|
|
121
|
+
|
|
122
|
+
The easiest way to install on macOS — no Python required:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
brew install jonathansudhakar1/cloudcat/cloudcat
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
This installs a self-contained binary that includes Python and all dependencies.
|
|
129
|
+
|
|
130
|
+
To upgrade:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
brew upgrade cloudcat
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
> **Note:** On first run, macOS may block the app. Go to System Settings > Privacy & Security and click "Allow", or run:
|
|
137
|
+
> ```bash
|
|
138
|
+
> xattr -d com.apple.quarantine $(which cloudcat)
|
|
139
|
+
> ```
|
|
140
|
+
|
|
141
|
+
### pip (Python)
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
# Full installation with all formats and compression
|
|
145
|
+
pip install 'cloudcat[all]'
|
|
146
|
+
|
|
147
|
+
# Standard installation (includes GCS, S3, and Azure support)
|
|
148
|
+
pip install cloudcat
|
|
149
|
+
|
|
150
|
+
# With Parquet file support
|
|
151
|
+
pip install 'cloudcat[parquet]'
|
|
152
|
+
|
|
153
|
+
# With Avro file support
|
|
154
|
+
pip install 'cloudcat[avro]'
|
|
155
|
+
|
|
156
|
+
# With ORC file support (uses pyarrow)
|
|
157
|
+
pip install 'cloudcat[orc]'
|
|
158
|
+
|
|
159
|
+
# With compression support (zstd, lz4, snappy)
|
|
160
|
+
pip install 'cloudcat[compression]'
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
> **Note:** If using zsh (default on macOS), quotes around extras are required to prevent shell interpretation of brackets.
|
|
164
|
+
|
|
165
|
+
To upgrade:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
pip install --upgrade 'cloudcat[all]'
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Requirements
|
|
172
|
+
|
|
173
|
+
- **Homebrew**: macOS (Intel or Apple Silicon)
|
|
174
|
+
- **pip**: Python 3.7+
|
|
175
|
+
- Cloud provider credentials configured (see [Authentication](#authentication))
|
|
176
|
+
|
|
177
|
+
## Quick Start
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
# Preview a CSV file from GCS
|
|
181
|
+
cloudcat -p gcs://my-bucket/data.csv
|
|
182
|
+
|
|
183
|
+
# Preview a Parquet file from S3
|
|
184
|
+
cloudcat -p s3://my-bucket/analytics/events.parquet
|
|
185
|
+
|
|
186
|
+
# Preview JSON data from Azure with pretty formatting
|
|
187
|
+
cloudcat -p az://my-container/logs.json -o jsonp
|
|
188
|
+
|
|
189
|
+
# Read Avro files from Kafka
|
|
190
|
+
cloudcat -p s3://my-bucket/kafka-export.avro
|
|
191
|
+
|
|
192
|
+
# Read ORC files from Hive
|
|
193
|
+
cloudcat -p gcs://my-bucket/hive-table.orc
|
|
194
|
+
|
|
195
|
+
# Read log files as plain text
|
|
196
|
+
cloudcat -p az://logs/app.log -i text
|
|
197
|
+
|
|
198
|
+
# Read from a Spark output directory
|
|
199
|
+
cloudcat -p s3://my-bucket/spark-output/ -i parquet
|
|
200
|
+
|
|
201
|
+
# Read compressed files (auto-detected)
|
|
202
|
+
cloudcat -p gcs://my-bucket/data.csv.gz
|
|
203
|
+
|
|
204
|
+
# Filter rows with WHERE clause
|
|
205
|
+
cloudcat -p s3://bucket/users.parquet --where "status=active"
|
|
206
|
+
|
|
207
|
+
# Skip first 100 rows (pagination)
|
|
208
|
+
cloudcat -p gcs://bucket/data.csv --offset 100 -n 10
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
## Features
|
|
212
|
+
|
|
213
|
+
### Cloud Storage Support
|
|
214
|
+
|
|
215
|
+
| Provider | URL Scheme | Status |
|
|
216
|
+
|----------|------------|--------|
|
|
217
|
+
| Google Cloud Storage | `gcs://` or `gs://` | ✅ Supported |
|
|
218
|
+
| Amazon S3 | `s3://` | ✅ Supported |
|
|
219
|
+
| Azure Blob Storage | `az://` or `azure://` | ✅ Supported |
|
|
220
|
+
|
|
221
|
+
### File Format Support
|
|
222
|
+
|
|
223
|
+
| Format | Read | Auto-Detect | Streaming | Use Case |
|
|
224
|
+
|--------|------|-------------|-----------|----------|
|
|
225
|
+
| CSV | ✅ | ✅ | ✅ | General data files |
|
|
226
|
+
| JSON | ✅ | ✅ | ✅ | API responses, configs |
|
|
227
|
+
| JSON Lines | ✅ | ✅ | ✅ | Log files, streaming data |
|
|
228
|
+
| Parquet | ✅ | ✅ | ✅ | Spark/analytics data |
|
|
229
|
+
| Avro | ✅ | ✅ | ✅ | Kafka, data pipelines |
|
|
230
|
+
| ORC | ✅ | ✅ | ✅ | Hive, Hadoop ecosystem |
|
|
231
|
+
| Text | ✅ | ✅ | ✅ | Log files, plain text |
|
|
232
|
+
| TSV | ✅ | Via `--delimiter` | ✅ | Tab-separated data |
|
|
233
|
+
|
|
234
|
+
### Compression Support
|
|
235
|
+
|
|
236
|
+
| Format | Extension | Built-in | Use Case |
|
|
237
|
+
|--------|-----------|----------|----------|
|
|
238
|
+
| Gzip | `.gz`, `.gzip` | ✅ | Most common, universal |
|
|
239
|
+
| Bzip2 | `.bz2` | ✅ | High compression ratio |
|
|
240
|
+
| Zstandard | `.zst`, `.zstd` | Optional | Fast, modern compression |
|
|
241
|
+
| LZ4 | `.lz4` | Optional | Very fast decompression |
|
|
242
|
+
| Snappy | `.snappy` | Optional | Hadoop ecosystem |
|
|
243
|
+
|
|
244
|
+
CloudCat automatically detects and decompresses files based on extension (e.g., `data.csv.gz`, `logs.json.zst`).
|
|
245
|
+
|
|
246
|
+
### Output Formats
|
|
247
|
+
|
|
248
|
+
| Format | Flag | Description |
|
|
249
|
+
|--------|------|-------------|
|
|
250
|
+
| Table | `-o table` | Beautiful ASCII table with colored headers (default) |
|
|
251
|
+
| JSON | `-o json` | Standard JSON Lines output |
|
|
252
|
+
| Pretty JSON | `-o jsonp` | Syntax-highlighted, indented JSON |
|
|
253
|
+
| CSV | `-o csv` | Comma-separated values |
|
|
254
|
+
|
|
255
|
+
### Key Capabilities
|
|
256
|
+
|
|
257
|
+
- **Schema Inspection** — View column names and data types
|
|
258
|
+
- **Column Selection** — Display only the columns you need
|
|
259
|
+
- **Row Limiting** — Control how many rows to preview
|
|
260
|
+
- **Row Offset** — Skip first N rows for pagination/sampling
|
|
261
|
+
- **WHERE Filtering** — Filter rows with SQL-like conditions
|
|
262
|
+
- **Record Counting** — Get total record counts (with Parquet metadata optimization)
|
|
263
|
+
- **Multi-File Reading** — Combine data from multiple files in a directory
|
|
264
|
+
- **Custom Delimiters** — Support for tab, pipe, semicolon, and other delimiters
|
|
265
|
+
- **Auto Decompression** — Transparent handling of compressed files
|
|
266
|
+
|
|
267
|
+
## Examples
|
|
268
|
+
|
|
269
|
+
### Basic Usage
|
|
270
|
+
|
|
271
|
+
```bash
|
|
272
|
+
# Preview first 10 rows (default)
|
|
273
|
+
cloudcat -p gcs://bucket/data.csv
|
|
274
|
+
|
|
275
|
+
# Preview 50 rows
|
|
276
|
+
cloudcat -p s3://bucket/data.parquet -n 50
|
|
277
|
+
|
|
278
|
+
# Show only specific columns
|
|
279
|
+
cloudcat -p gcs://bucket/users.json -c id,name,email
|
|
280
|
+
|
|
281
|
+
# View schema only (no data)
|
|
282
|
+
cloudcat -p s3://bucket/events.parquet -s schema_only
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
### Working with Different Formats
|
|
286
|
+
|
|
287
|
+
```bash
|
|
288
|
+
# CSV with custom delimiter (tab-separated)
|
|
289
|
+
cloudcat -p gcs://bucket/data.tsv -d "\t"
|
|
290
|
+
|
|
291
|
+
# Pipe-delimited file
|
|
292
|
+
cloudcat -p s3://bucket/export.txt -d "|"
|
|
293
|
+
|
|
294
|
+
# Semicolon-delimited (common in European data)
|
|
295
|
+
cloudcat -p gcs://bucket/report.csv -d ";"
|
|
296
|
+
|
|
297
|
+
# JSON array file
|
|
298
|
+
cloudcat -p s3://bucket/config.json
|
|
299
|
+
|
|
300
|
+
# JSON Lines file (auto-detected)
|
|
301
|
+
cloudcat -p gcs://bucket/events.jsonl
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
### Filtering and Pagination
|
|
305
|
+
|
|
306
|
+
```bash
|
|
307
|
+
# Filter rows with WHERE clause
|
|
308
|
+
cloudcat -p s3://bucket/users.parquet --where "status=active"
|
|
309
|
+
cloudcat -p gcs://bucket/events.json --where "age>30"
|
|
310
|
+
cloudcat -p s3://bucket/logs.csv --where "level=ERROR"
|
|
311
|
+
|
|
312
|
+
# String matching filters
|
|
313
|
+
cloudcat -p gcs://bucket/data.csv --where "name contains john"
|
|
314
|
+
cloudcat -p s3://bucket/emails.json --where "email endswith @gmail.com"
|
|
315
|
+
cloudcat -p az://logs/app.log --where "message startswith ERROR"
|
|
316
|
+
|
|
317
|
+
# Skip first N rows (pagination)
|
|
318
|
+
cloudcat -p gcs://bucket/data.csv --offset 100 -n 10
|
|
319
|
+
|
|
320
|
+
# Combine offset with filters
|
|
321
|
+
cloudcat -p s3://bucket/users.parquet --where "active=true" --offset 50 -n 20
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
### Compressed Files
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
# Gzip compressed (built-in)
|
|
328
|
+
cloudcat -p gcs://bucket/data.csv.gz
|
|
329
|
+
cloudcat -p s3://bucket/logs.json.gz
|
|
330
|
+
|
|
331
|
+
# Zstandard compressed (requires: pip install cloudcat[zstd])
|
|
332
|
+
cloudcat -p gcs://bucket/events.parquet.zst
|
|
333
|
+
|
|
334
|
+
# LZ4 compressed (requires: pip install cloudcat[lz4])
|
|
335
|
+
cloudcat -p s3://bucket/data.csv.lz4
|
|
336
|
+
|
|
337
|
+
# Bzip2 compressed (built-in)
|
|
338
|
+
cloudcat -p az://container/archive.json.bz2
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
### Directory Operations
|
|
342
|
+
|
|
343
|
+
CloudCat intelligently handles directories containing multiple data files (common with Spark, Hive, and distributed processing outputs):
|
|
344
|
+
|
|
345
|
+
```bash
|
|
346
|
+
# Auto-detect and read first data file in directory
|
|
347
|
+
cloudcat -p gcs://bucket/spark-output/
|
|
348
|
+
|
|
349
|
+
# Read and combine multiple files (up to 25MB by default)
|
|
350
|
+
cloudcat -p s3://bucket/daily-logs/ -m all
|
|
351
|
+
|
|
352
|
+
# Read up to 100MB of data from multiple files
|
|
353
|
+
cloudcat -p gcs://bucket/events/ -m all --max-size-mb 100
|
|
354
|
+
|
|
355
|
+
# Force reading only the first file
|
|
356
|
+
cloudcat -p s3://bucket/output/ -m first
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
**CloudCat automatically:**
|
|
360
|
+
- Skips empty files
|
|
361
|
+
- Ignores metadata files (`_SUCCESS`, `_metadata`, `.crc`, etc.)
|
|
362
|
+
- Prioritizes files matching the specified format
|
|
363
|
+
- Reports which files were selected
|
|
364
|
+
|
|
365
|
+
### Output Format Examples
|
|
366
|
+
|
|
367
|
+
```bash
|
|
368
|
+
# Default table output (great for terminals)
|
|
369
|
+
cloudcat -p gcs://bucket/data.csv
|
|
370
|
+
# ┌────┬────────────┬─────────┐
|
|
371
|
+
# │ id │ name │ value │
|
|
372
|
+
# ├────┼────────────┼─────────┤
|
|
373
|
+
# │ 1 │ Alice │ 100 │
|
|
374
|
+
# │ 2 │ Bob │ 200 │
|
|
375
|
+
# └────┴────────────┴─────────┘
|
|
376
|
+
|
|
377
|
+
# Pretty JSON (great for nested data)
|
|
378
|
+
cloudcat -p s3://bucket/events.json -o jsonp
|
|
379
|
+
# {
|
|
380
|
+
# "id": 1,
|
|
381
|
+
# "name": "Alice",
|
|
382
|
+
# "metadata": {
|
|
383
|
+
# "created": "2024-01-15"
|
|
384
|
+
# }
|
|
385
|
+
# }
|
|
386
|
+
|
|
387
|
+
# JSON Lines (great for piping to jq)
|
|
388
|
+
cloudcat -p gcs://bucket/data.parquet -o json | jq '.name'
|
|
389
|
+
|
|
390
|
+
# CSV (great for further processing)
|
|
391
|
+
cloudcat -p s3://bucket/data.json -o csv > output.csv
|
|
392
|
+
```
|
|
393
|
+
|
|
394
|
+
### Data Pipeline Examples
|
|
395
|
+
|
|
396
|
+
```bash
|
|
397
|
+
# Convert Parquet to CSV
|
|
398
|
+
cloudcat -p gcs://bucket/data.parquet -o csv -n 0 > data.csv
|
|
399
|
+
|
|
400
|
+
# Preview and filter with jq
|
|
401
|
+
cloudcat -p s3://bucket/events.json -o json | jq 'select(.status == "error")'
|
|
402
|
+
|
|
403
|
+
# Quick data validation
|
|
404
|
+
cloudcat -p gcs://bucket/import.csv -s schema_only
|
|
405
|
+
|
|
406
|
+
# Sample data from large dataset
|
|
407
|
+
cloudcat -p s3://bucket/big-table.parquet -n 100 -c user_id,event_type
|
|
408
|
+
|
|
409
|
+
# Export specific columns to CSV
|
|
410
|
+
cloudcat -p gcs://bucket/users.parquet -c email,created_at -o csv -n 0 > emails.csv
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
### Real-World Use Cases
|
|
414
|
+
|
|
415
|
+
#### Debugging Spark Jobs
|
|
416
|
+
```bash
|
|
417
|
+
# Check output of a Spark job
|
|
418
|
+
cloudcat -p gcs://data-lake/jobs/daily-etl/output/ -i parquet -n 20
|
|
419
|
+
|
|
420
|
+
# Verify schema matches expectations
|
|
421
|
+
cloudcat -p s3://analytics/spark-output/ -s schema_only
|
|
422
|
+
```
|
|
423
|
+
|
|
424
|
+
#### Log Analysis
|
|
425
|
+
```bash
|
|
426
|
+
# Preview recent logs
|
|
427
|
+
cloudcat -p gcs://logs/app/2024-01-15/ -m all -n 50
|
|
428
|
+
|
|
429
|
+
# Check error logs (combine with grep)
|
|
430
|
+
cloudcat -p s3://logs/errors/ -o json | grep "ERROR"
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
#### Data Validation
|
|
434
|
+
```bash
|
|
435
|
+
# Quick sanity check on data export
|
|
436
|
+
cloudcat -p gcs://exports/daily/users.csv -s show
|
|
437
|
+
|
|
438
|
+
# Verify record count
|
|
439
|
+
cloudcat -p s3://warehouse/transactions.parquet --no-count
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
#### Format Conversion
|
|
443
|
+
```bash
|
|
444
|
+
# Convert tab-separated to comma-separated
|
|
445
|
+
cloudcat -p gcs://imports/data.tsv -d "\t" -o csv > converted.csv
|
|
446
|
+
|
|
447
|
+
# Convert JSON to CSV for spreadsheet import
|
|
448
|
+
cloudcat -p s3://api-dumps/response.json -o csv > data.csv
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
## Command Reference
|
|
452
|
+
|
|
453
|
+
```
|
|
454
|
+
Usage: cloudcat [OPTIONS]
|
|
455
|
+
|
|
456
|
+
Options:
|
|
457
|
+
-p, --path TEXT Cloud storage path (required)
|
|
458
|
+
Format: gcs://bucket/path, s3://bucket/path,
|
|
459
|
+
or az://container/path
|
|
460
|
+
|
|
461
|
+
-o, --output-format TEXT Output format: table, json, jsonp, csv
|
|
462
|
+
[default: table]
|
|
463
|
+
|
|
464
|
+
-i, --input-format TEXT Input format: csv, json, parquet, avro, orc, text
|
|
465
|
+
[default: auto-detect from extension]
|
|
466
|
+
|
|
467
|
+
-c, --columns TEXT Comma-separated list of columns to display
|
|
468
|
+
[default: all columns]
|
|
469
|
+
|
|
470
|
+
-n, --num-rows INTEGER Number of rows to display (0 for all)
|
|
471
|
+
[default: 10]
|
|
472
|
+
|
|
473
|
+
--offset INTEGER Skip first N rows
|
|
474
|
+
[default: 0]
|
|
475
|
+
|
|
476
|
+
-w, --where TEXT Filter rows with SQL-like conditions
|
|
477
|
+
Examples: "status=active", "age>30",
|
|
478
|
+
"name contains john", "email endswith @gmail.com"
|
|
479
|
+
|
|
480
|
+
-s, --schema TEXT Schema display: show, dont_show, schema_only
|
|
481
|
+
[default: show]
|
|
482
|
+
|
|
483
|
+
--no-count Disable automatic record counting
|
|
484
|
+
|
|
485
|
+
-m, --multi-file-mode TEXT Directory handling: auto, first, all
|
|
486
|
+
[default: auto]
|
|
487
|
+
|
|
488
|
+
--max-size-mb INTEGER Max data size for multi-file mode in MB
|
|
489
|
+
[default: 25]
|
|
490
|
+
|
|
491
|
+
-d, --delimiter TEXT CSV delimiter (use \t for tab)
|
|
492
|
+
[default: comma]
|
|
493
|
+
|
|
494
|
+
--profile TEXT AWS profile name (for S3 access)
|
|
495
|
+
|
|
496
|
+
--project TEXT GCP project ID (for GCS access)
|
|
497
|
+
|
|
498
|
+
--credentials TEXT Path to GCP service account JSON file
|
|
499
|
+
|
|
500
|
+
--account TEXT Azure storage account name
|
|
501
|
+
|
|
502
|
+
--help Show this message and exit
|
|
503
|
+
```
|
|
504
|
+
|
|
505
|
+
### WHERE Clause Operators
|
|
506
|
+
|
|
507
|
+
| Operator | Example | Description |
|
|
508
|
+
|----------|---------|-------------|
|
|
509
|
+
| `=` | `status=active` | Exact match |
|
|
510
|
+
| `!=` | `type!=deleted` | Not equal |
|
|
511
|
+
| `>` | `age>30` | Greater than |
|
|
512
|
+
| `<` | `price<100` | Less than |
|
|
513
|
+
| `>=` | `count>=10` | Greater than or equal |
|
|
514
|
+
| `<=` | `score<=50` | Less than or equal |
|
|
515
|
+
| `contains` | `name contains john` | Case-insensitive substring match |
|
|
516
|
+
| `startswith` | `email startswith admin` | String prefix match |
|
|
517
|
+
| `endswith` | `file endswith .csv` | String suffix match |
|
|
518
|
+
|
|
519
|
+
## Authentication
|
|
520
|
+
|
|
521
|
+
### Google Cloud Storage
|
|
522
|
+
|
|
523
|
+
CloudCat uses [Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials). Set up authentication using one of these methods:
|
|
524
|
+
|
|
525
|
+
```bash
|
|
526
|
+
# Option 1: User credentials (for development)
|
|
527
|
+
gcloud auth application-default login
|
|
528
|
+
|
|
529
|
+
# Option 2: Service account via environment variable
|
|
530
|
+
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
|
|
531
|
+
|
|
532
|
+
# Option 3: Service account via CLI option
|
|
533
|
+
cloudcat -p gcs://bucket/data.csv --credentials /path/to/service-account.json
|
|
534
|
+
|
|
535
|
+
# Option 4: Specify GCP project
|
|
536
|
+
cloudcat -p gcs://bucket/data.csv --project my-gcp-project
|
|
537
|
+
```
|
|
538
|
+
|
|
539
|
+
### Amazon S3
|
|
540
|
+
|
|
541
|
+
CloudCat uses the standard [AWS credential chain](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html):
|
|
542
|
+
|
|
543
|
+
```bash
|
|
544
|
+
# Option 1: Environment variables
|
|
545
|
+
export AWS_ACCESS_KEY_ID="your-access-key"
|
|
546
|
+
export AWS_SECRET_ACCESS_KEY="your-secret-key"
|
|
547
|
+
export AWS_DEFAULT_REGION="us-east-1"
|
|
548
|
+
|
|
549
|
+
# Option 2: AWS credentials file (~/.aws/credentials)
|
|
550
|
+
aws configure
|
|
551
|
+
|
|
552
|
+
# Option 3: AWS named profile
|
|
553
|
+
cloudcat -p s3://bucket/data.csv --profile production
|
|
554
|
+
|
|
555
|
+
# Option 4: IAM role (for EC2/ECS/Lambda)
|
|
556
|
+
# Automatically detected
|
|
557
|
+
```
|
|
558
|
+
|
|
559
|
+
### Azure Blob Storage
|
|
560
|
+
|
|
561
|
+
CloudCat supports multiple authentication methods for Azure:
|
|
562
|
+
|
|
563
|
+
```bash
|
|
564
|
+
# Option 1: Connection string (simplest)
|
|
565
|
+
export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net"
|
|
566
|
+
|
|
567
|
+
# Option 2: Account URL with DefaultAzureCredential (for Azure AD auth)
|
|
568
|
+
export AZURE_STORAGE_ACCOUNT_URL="https://youraccount.blob.core.windows.net"
|
|
569
|
+
az login
|
|
570
|
+
|
|
571
|
+
# Option 3: Specify storage account via CLI option
|
|
572
|
+
cloudcat -p az://container/data.csv --account mystorageaccount
|
|
573
|
+
```
|
|
574
|
+
|
|
575
|
+
**Path format:** `az://container-name/path/to/blob`
|
|
576
|
+
|
|
577
|
+
## Performance Tips
|
|
578
|
+
|
|
579
|
+
1. **Use `--no-count`** for large files when you don't need the total record count
|
|
580
|
+
2. **Prefer Parquet** format when possible — record counts are instant from metadata
|
|
581
|
+
3. **Use `--num-rows`** to limit data transfer for large files
|
|
582
|
+
4. **Use `--columns`** to select only needed columns (especially effective with Parquet)
|
|
583
|
+
5. **Use `-m first`** when you only need a sample from directories with many files
|
|
584
|
+
|
|
585
|
+
## Troubleshooting
|
|
586
|
+
|
|
587
|
+
### Common Issues
|
|
588
|
+
|
|
589
|
+
**"google-cloud-storage package is required"**
|
|
590
|
+
```bash
|
|
591
|
+
pip install cloudcat[gcs]
|
|
592
|
+
```
|
|
593
|
+
|
|
594
|
+
**"boto3 package is required"**
|
|
595
|
+
```bash
|
|
596
|
+
pip install cloudcat[s3]
|
|
597
|
+
```
|
|
598
|
+
|
|
599
|
+
**"pyarrow package is required"**
|
|
600
|
+
```bash
|
|
601
|
+
pip install cloudcat[parquet]
|
|
602
|
+
```
|
|
603
|
+
|
|
604
|
+
**"azure-storage-blob package is required"**
|
|
605
|
+
```bash
|
|
606
|
+
pip install cloudcat[azure]
|
|
607
|
+
```
|
|
608
|
+
|
|
609
|
+
**"fastavro package is required"**
|
|
610
|
+
```bash
|
|
611
|
+
pip install cloudcat[avro]
|
|
612
|
+
```
|
|
613
|
+
|
|
614
|
+
**"pyarrow with ORC support is required"**
|
|
615
|
+
```bash
|
|
616
|
+
pip install cloudcat[orc]
|
|
617
|
+
```
|
|
618
|
+
|
|
619
|
+
**"zstandard package is required for .zst files"**
|
|
620
|
+
```bash
|
|
621
|
+
pip install cloudcat[zstd]
|
|
622
|
+
# or for all compression formats:
|
|
623
|
+
pip install cloudcat[compression]
|
|
624
|
+
```
|
|
625
|
+
|
|
626
|
+
**"lz4 package is required for .lz4 files"**
|
|
627
|
+
```bash
|
|
628
|
+
pip install cloudcat[lz4]
|
|
629
|
+
```
|
|
630
|
+
|
|
631
|
+
**"python-snappy package is required for .snappy files"**
|
|
632
|
+
```bash
|
|
633
|
+
pip install cloudcat[snappy]
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
**Authentication errors**
|
|
637
|
+
- GCS: Run `gcloud auth application-default login`
|
|
638
|
+
- S3: Run `aws configure` or check your credentials
|
|
639
|
+
- Azure: Set `AZURE_STORAGE_CONNECTION_STRING` or `AZURE_STORAGE_ACCOUNT_URL` and run `az login`
|
|
640
|
+
|
|
641
|
+
**"Could not infer format from path"**
|
|
642
|
+
```bash
|
|
643
|
+
# Specify the format explicitly
|
|
644
|
+
cloudcat -p gcs://bucket/data -i parquet
|
|
645
|
+
```
|
|
646
|
+
|
|
647
|
+
## Contributing
|
|
648
|
+
|
|
649
|
+
Contributions are welcome! Here's how you can help:
|
|
650
|
+
|
|
651
|
+
1. **Report bugs** — Open an issue with reproduction steps
|
|
652
|
+
2. **Suggest features** — Open an issue describing the use case
|
|
653
|
+
3. **Submit PRs** — Fork, create a branch, and submit a pull request
|
|
654
|
+
|
|
655
|
+
### Development Setup
|
|
656
|
+
|
|
657
|
+
```bash
|
|
658
|
+
# Clone the repository
|
|
659
|
+
git clone https://github.com/jonathansudhakar1/cloudcat.git
|
|
660
|
+
cd cloudcat
|
|
661
|
+
|
|
662
|
+
# Create virtual environment
|
|
663
|
+
python -m venv venv
|
|
664
|
+
source venv/bin/activate # or `venv\Scripts\activate` on Windows
|
|
665
|
+
|
|
666
|
+
# Install in development mode with all dependencies
|
|
667
|
+
pip install -e ".[all]"
|
|
668
|
+
|
|
669
|
+
# Run tests
|
|
670
|
+
pytest
|
|
671
|
+
```
|
|
672
|
+
|
|
673
|
+
## Roadmap
|
|
674
|
+
|
|
675
|
+
- [x] Azure Blob Storage support
|
|
676
|
+
- [x] Avro format support
|
|
677
|
+
- [x] ORC format support
|
|
678
|
+
- [x] Plain text format support
|
|
679
|
+
- [x] SQL-like filtering (`--where` clause)
|
|
680
|
+
- [x] Compression support (gzip, zstd, lz4, snappy, bz2)
|
|
681
|
+
- [x] Row offset/pagination (`--offset`)
|
|
682
|
+
- [ ] Interactive mode with pagination
|
|
683
|
+
- [ ] Output to file with `--output-file`
|
|
684
|
+
- [ ] Configuration file support
|
|
685
|
+
|
|
686
|
+
## Related Projects
|
|
687
|
+
|
|
688
|
+
- [s3cmd](https://s3tools.org/s3cmd) — S3 command-line tool
|
|
689
|
+
- [gsutil](https://cloud.google.com/storage/docs/gsutil) — Google Cloud Storage CLI
|
|
690
|
+
- [aws-cli](https://aws.amazon.com/cli/) — AWS command-line interface
|
|
691
|
+
- [azcopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10) — Azure Storage data transfer tool
|
|
692
|
+
- [duckdb](https://duckdb.org/) — In-process SQL OLAP database
|
|
693
|
+
|
|
694
|
+
## License
|
|
695
|
+
|
|
696
|
+
MIT License — see [LICENSE](LICENSE) for details.
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
<p align="center">
|
|
700
|
+
<a href="https://github.com/jonathansudhakar1/cloudcat/issues">Report Bug</a> •
|
|
701
|
+
<a href="https://github.com/jonathansudhakar1/cloudcat/issues">Request Feature</a>
|
|
702
|
+
</p>
|