cloudcat 0.1.4__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. cloudcat-0.2.1/PKG-INFO +702 -0
  2. cloudcat-0.2.1/README.md +625 -0
  3. {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat/__init__.py +1 -1
  4. cloudcat-0.2.1/cloudcat/cli.py +770 -0
  5. cloudcat-0.2.1/cloudcat/compression.py +114 -0
  6. cloudcat-0.2.1/cloudcat/config.py +54 -0
  7. cloudcat-0.2.1/cloudcat/filtering.py +120 -0
  8. cloudcat-0.2.1/cloudcat/formatters.py +93 -0
  9. cloudcat-0.2.1/cloudcat/readers/__init__.py +20 -0
  10. cloudcat-0.2.1/cloudcat/readers/avro.py +77 -0
  11. cloudcat-0.2.1/cloudcat/readers/csv.py +53 -0
  12. cloudcat-0.2.1/cloudcat/readers/json.py +102 -0
  13. cloudcat-0.2.1/cloudcat/readers/orc.py +86 -0
  14. cloudcat-0.2.1/cloudcat/readers/parquet.py +115 -0
  15. cloudcat-0.2.1/cloudcat/readers/text.py +57 -0
  16. cloudcat-0.2.1/cloudcat/storage/__init__.py +24 -0
  17. cloudcat-0.2.1/cloudcat/storage/azure.py +109 -0
  18. cloudcat-0.2.1/cloudcat/storage/base.py +91 -0
  19. cloudcat-0.2.1/cloudcat/storage/gcs.py +92 -0
  20. cloudcat-0.2.1/cloudcat/storage/s3.py +88 -0
  21. cloudcat-0.2.1/cloudcat.egg-info/PKG-INFO +702 -0
  22. cloudcat-0.2.1/cloudcat.egg-info/SOURCES.txt +40 -0
  23. cloudcat-0.2.1/cloudcat.egg-info/requires.txt +38 -0
  24. {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat.egg-info/top_level.txt +1 -0
  25. cloudcat-0.2.1/setup.py +95 -0
  26. cloudcat-0.2.1/tests/__init__.py +0 -0
  27. cloudcat-0.2.1/tests/conftest.py +45 -0
  28. cloudcat-0.2.1/tests/test_cli.py +255 -0
  29. cloudcat-0.2.1/tests/test_compression.py +163 -0
  30. cloudcat-0.2.1/tests/test_config.py +111 -0
  31. cloudcat-0.2.1/tests/test_data_reading.py +133 -0
  32. cloudcat-0.2.1/tests/test_file_filtering.py +145 -0
  33. cloudcat-0.2.1/tests/test_filtering.py +240 -0
  34. cloudcat-0.2.1/tests/test_format_detection.py +65 -0
  35. cloudcat-0.2.1/tests/test_integration.py +158 -0
  36. cloudcat-0.2.1/tests/test_output_formatting.py +104 -0
  37. cloudcat-0.2.1/tests/test_path_parsing.py +48 -0
  38. cloudcat-0.1.4/PKG-INFO +0 -60
  39. cloudcat-0.1.4/README.md +0 -161
  40. cloudcat-0.1.4/cloudcat/cli.py +0 -787
  41. cloudcat-0.1.4/cloudcat.egg-info/PKG-INFO +0 -60
  42. cloudcat-0.1.4/cloudcat.egg-info/SOURCES.txt +0 -12
  43. cloudcat-0.1.4/cloudcat.egg-info/requires.txt +0 -18
  44. cloudcat-0.1.4/setup.py +0 -60
  45. {cloudcat-0.1.4 → cloudcat-0.2.1}/LICENSE +0 -0
  46. {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat.egg-info/dependency_links.txt +0 -0
  47. {cloudcat-0.1.4 → cloudcat-0.2.1}/cloudcat.egg-info/entry_points.txt +0 -0
  48. {cloudcat-0.1.4 → cloudcat-0.2.1}/pyproject.toml +0 -0
  49. {cloudcat-0.1.4 → cloudcat-0.2.1}/setup.cfg +0 -0
@@ -0,0 +1,702 @@
1
+ Metadata-Version: 2.4
2
+ Name: cloudcat
3
+ Version: 0.2.1
4
+ Summary: Preview and analyze data files in Google Cloud Storage, AWS S3, and Azure Blob Storage from your terminal
5
+ Home-page: https://github.com/jonathansudhakar1/cloudcat
6
+ Author: Jonathan Sudhakar
7
+ Author-email: jonathan@example.com
8
+ Project-URL: Bug Reports, https://github.com/jonathansudhakar1/cloudcat/issues
9
+ Project-URL: Source, https://github.com/jonathansudhakar1/cloudcat
10
+ Project-URL: Documentation, https://github.com/jonathansudhakar1/cloudcat#readme
11
+ Keywords: cloud,gcs,s3,azure,cli,storage,data,parquet,csv,json,avro,orc,google-cloud,aws,azure-blob,data-engineering,etl,spark,bigquery,databricks,kafka,hive
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: System Administrators
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: Topic :: Utilities
17
+ Classifier: Topic :: Database
18
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
19
+ Classifier: Topic :: System :: Systems Administration
20
+ Classifier: Environment :: Console
21
+ Classifier: License :: OSI Approved :: MIT License
22
+ Classifier: Operating System :: OS Independent
23
+ Classifier: Programming Language :: Python :: 3
24
+ Classifier: Programming Language :: Python :: 3.7
25
+ Classifier: Programming Language :: Python :: 3.8
26
+ Classifier: Programming Language :: Python :: 3.9
27
+ Classifier: Programming Language :: Python :: 3.10
28
+ Classifier: Programming Language :: Python :: 3.11
29
+ Classifier: Programming Language :: Python :: 3.12
30
+ Classifier: Programming Language :: Python :: 3.13
31
+ Requires-Python: >=3.7
32
+ Description-Content-Type: text/markdown
33
+ License-File: LICENSE
34
+ Requires-Dist: click>=8.0.0
35
+ Requires-Dist: pandas>=1.3.0
36
+ Requires-Dist: tabulate>=0.8.9
37
+ Requires-Dist: colorama>=0.4.4
38
+ Requires-Dist: google-cloud-storage>=2.0.0
39
+ Requires-Dist: boto3>=1.18.0
40
+ Requires-Dist: azure-storage-blob>=12.0.0
41
+ Requires-Dist: azure-identity>=1.0.0
42
+ Provides-Extra: parquet
43
+ Requires-Dist: pyarrow>=5.0.0; extra == "parquet"
44
+ Provides-Extra: avro
45
+ Requires-Dist: fastavro>=1.4.0; extra == "avro"
46
+ Provides-Extra: orc
47
+ Requires-Dist: pyarrow>=5.0.0; extra == "orc"
48
+ Provides-Extra: zstd
49
+ Requires-Dist: zstandard>=0.15.0; extra == "zstd"
50
+ Provides-Extra: lz4
51
+ Requires-Dist: lz4>=3.0.0; extra == "lz4"
52
+ Provides-Extra: snappy
53
+ Requires-Dist: python-snappy>=0.6.0; extra == "snappy"
54
+ Provides-Extra: compression
55
+ Requires-Dist: zstandard>=0.15.0; extra == "compression"
56
+ Requires-Dist: lz4>=3.0.0; extra == "compression"
57
+ Requires-Dist: python-snappy>=0.6.0; extra == "compression"
58
+ Provides-Extra: all
59
+ Requires-Dist: pyarrow>=5.0.0; extra == "all"
60
+ Requires-Dist: fastavro>=1.4.0; extra == "all"
61
+ Requires-Dist: zstandard>=0.15.0; extra == "all"
62
+ Requires-Dist: lz4>=3.0.0; extra == "all"
63
+ Requires-Dist: python-snappy>=0.6.0; extra == "all"
64
+ Dynamic: author
65
+ Dynamic: author-email
66
+ Dynamic: classifier
67
+ Dynamic: description
68
+ Dynamic: description-content-type
69
+ Dynamic: home-page
70
+ Dynamic: keywords
71
+ Dynamic: license-file
72
+ Dynamic: project-url
73
+ Dynamic: provides-extra
74
+ Dynamic: requires-dist
75
+ Dynamic: requires-python
76
+ Dynamic: summary
77
+
78
+ <p align="center">
79
+ <img src="https://raw.githubusercontent.com/jonathansudhakar1/cloudcat/main/assets/logo.png" alt="CloudCat Logo" width="200">
80
+ </p>
81
+
82
+ <h1 align="center">CloudCat</h1>
83
+
84
+ <p align="center">
85
+ <strong>The Swiss Army knife for viewing cloud storage data from your terminal</strong>
86
+ </p>
87
+
88
+ <p align="center">
89
+ <a href="https://pypi.org/project/cloudcat/"><img src="https://img.shields.io/pypi/v/cloudcat.svg?style=flat-square&logo=pypi&logoColor=white" alt="PyPI version"></a>
90
+ <a href="https://pypi.org/project/cloudcat/"><img src="https://img.shields.io/pypi/pyversions/cloudcat.svg?style=flat-square&logo=python&logoColor=white" alt="Python versions"></a>
91
+ <a href="https://pypi.org/project/cloudcat/"><img src="https://img.shields.io/pypi/dm/cloudcat.svg?style=flat-square&logo=pypi&logoColor=white" alt="PyPI Downloads"></a>
92
+ <a href="https://github.com/jonathansudhakar1/cloudcat/releases"><img src="https://img.shields.io/github/downloads/jonathansudhakar1/cloudcat/total.svg?style=flat-square&logo=homebrew&logoColor=white&label=homebrew" alt="Homebrew Downloads"></a>
93
+ <a href="https://github.com/jonathansudhakar1/cloudcat/blob/main/LICENSE"><img src="https://img.shields.io/github/license/jonathansudhakar1/cloudcat.svg?style=flat-square" alt="License"></a>
94
+ </p>
95
+
96
+ <p align="center">
97
+ <a href="#installation">Installation</a> •
98
+ <a href="#quick-start">Quick Start</a> •
99
+ <a href="#features">Features</a> •
100
+ <a href="#examples">Examples</a> •
101
+ <a href="#documentation">Documentation</a>
102
+ </p>
103
+
104
+ ---
105
+
106
+ **CloudCat** is a powerful command-line tool that lets you instantly preview and analyze data files stored in **Google Cloud Storage (GCS)**, **Amazon S3**, and **Azure Blob Storage** — without downloading entire files. Think of it as `cat`, `head`, and `less` combined, but for cloud storage with built-in support for CSV, JSON, Parquet, Avro, ORC, and plain text formats.
107
+
108
+ ## Why CloudCat?
109
+
110
+ - **No Downloads Required** — Stream and preview data directly from cloud storage
111
+ - **Format-Aware** — Intelligently handles CSV, JSON, Parquet, Avro, ORC, and plain text files
112
+ - **Directory Smart** — Automatically discovers data files in Spark/Hive/Kafka output directories
113
+ - **Beautiful Output** — Colorized tables, pretty-printed JSON, and schema visualization
114
+ - **Developer Friendly** — Simple CLI with sensible defaults and powerful options
115
+ - **Compression Support** — Automatic decompression of gzip, zstd, lz4, snappy, and bz2 files
116
+ - **SQL-like Filtering** — Filter rows with WHERE clauses (e.g., `--where "status=active"`)
117
+
118
+ ## Installation
119
+
120
+ ### Homebrew (macOS)
121
+
122
+ The easiest way to install on macOS — no Python required:
123
+
124
+ ```bash
125
+ brew install jonathansudhakar1/cloudcat/cloudcat
126
+ ```
127
+
128
+ This installs a self-contained binary that includes Python and all dependencies.
129
+
130
+ To upgrade:
131
+
132
+ ```bash
133
+ brew upgrade cloudcat
134
+ ```
135
+
136
+ > **Note:** On first run, macOS may block the app. Go to System Settings > Privacy & Security and click "Allow", or run:
137
+ > ```bash
138
+ > xattr -d com.apple.quarantine $(which cloudcat)
139
+ > ```
140
+
141
+ ### pip (Python)
142
+
143
+ ```bash
144
+ # Full installation with all formats and compression
145
+ pip install 'cloudcat[all]'
146
+
147
+ # Standard installation (includes GCS, S3, and Azure support)
148
+ pip install cloudcat
149
+
150
+ # With Parquet file support
151
+ pip install 'cloudcat[parquet]'
152
+
153
+ # With Avro file support
154
+ pip install 'cloudcat[avro]'
155
+
156
+ # With ORC file support (uses pyarrow)
157
+ pip install 'cloudcat[orc]'
158
+
159
+ # With compression support (zstd, lz4, snappy)
160
+ pip install 'cloudcat[compression]'
161
+ ```
162
+
163
+ > **Note:** If using zsh (default on macOS), quotes around extras are required to prevent shell interpretation of brackets.
164
+
165
+ To upgrade:
166
+
167
+ ```bash
168
+ pip install --upgrade 'cloudcat[all]'
169
+ ```
170
+
171
+ ### Requirements
172
+
173
+ - **Homebrew**: macOS (Intel or Apple Silicon)
174
+ - **pip**: Python 3.7+
175
+ - Cloud provider credentials configured (see [Authentication](#authentication))
176
+
177
+ ## Quick Start
178
+
179
+ ```bash
180
+ # Preview a CSV file from GCS
181
+ cloudcat -p gcs://my-bucket/data.csv
182
+
183
+ # Preview a Parquet file from S3
184
+ cloudcat -p s3://my-bucket/analytics/events.parquet
185
+
186
+ # Preview JSON data from Azure with pretty formatting
187
+ cloudcat -p az://my-container/logs.json -o jsonp
188
+
189
+ # Read Avro files from Kafka
190
+ cloudcat -p s3://my-bucket/kafka-export.avro
191
+
192
+ # Read ORC files from Hive
193
+ cloudcat -p gcs://my-bucket/hive-table.orc
194
+
195
+ # Read log files as plain text
196
+ cloudcat -p az://logs/app.log -i text
197
+
198
+ # Read from a Spark output directory
199
+ cloudcat -p s3://my-bucket/spark-output/ -i parquet
200
+
201
+ # Read compressed files (auto-detected)
202
+ cloudcat -p gcs://my-bucket/data.csv.gz
203
+
204
+ # Filter rows with WHERE clause
205
+ cloudcat -p s3://bucket/users.parquet --where "status=active"
206
+
207
+ # Skip first 100 rows (pagination)
208
+ cloudcat -p gcs://bucket/data.csv --offset 100 -n 10
209
+ ```
210
+
211
+ ## Features
212
+
213
+ ### Cloud Storage Support
214
+
215
+ | Provider | URL Scheme | Status |
216
+ |----------|------------|--------|
217
+ | Google Cloud Storage | `gcs://` or `gs://` | ✅ Supported |
218
+ | Amazon S3 | `s3://` | ✅ Supported |
219
+ | Azure Blob Storage | `az://` or `azure://` | ✅ Supported |
220
+
221
+ ### File Format Support
222
+
223
+ | Format | Read | Auto-Detect | Streaming | Use Case |
224
+ |--------|------|-------------|-----------|----------|
225
+ | CSV | ✅ | ✅ | ✅ | General data files |
226
+ | JSON | ✅ | ✅ | ✅ | API responses, configs |
227
+ | JSON Lines | ✅ | ✅ | ✅ | Log files, streaming data |
228
+ | Parquet | ✅ | ✅ | ✅ | Spark/analytics data |
229
+ | Avro | ✅ | ✅ | ✅ | Kafka, data pipelines |
230
+ | ORC | ✅ | ✅ | ✅ | Hive, Hadoop ecosystem |
231
+ | Text | ✅ | ✅ | ✅ | Log files, plain text |
232
+ | TSV | ✅ | Via `--delimiter` | ✅ | Tab-separated data |
233
+
234
+ ### Compression Support
235
+
236
+ | Format | Extension | Built-in | Use Case |
237
+ |--------|-----------|----------|----------|
238
+ | Gzip | `.gz`, `.gzip` | ✅ | Most common, universal |
239
+ | Bzip2 | `.bz2` | ✅ | High compression ratio |
240
+ | Zstandard | `.zst`, `.zstd` | Optional | Fast, modern compression |
241
+ | LZ4 | `.lz4` | Optional | Very fast decompression |
242
+ | Snappy | `.snappy` | Optional | Hadoop ecosystem |
243
+
244
+ CloudCat automatically detects and decompresses files based on extension (e.g., `data.csv.gz`, `logs.json.zst`).
245
+
246
+ ### Output Formats
247
+
248
+ | Format | Flag | Description |
249
+ |--------|------|-------------|
250
+ | Table | `-o table` | Beautiful ASCII table with colored headers (default) |
251
+ | JSON | `-o json` | Standard JSON Lines output |
252
+ | Pretty JSON | `-o jsonp` | Syntax-highlighted, indented JSON |
253
+ | CSV | `-o csv` | Comma-separated values |
254
+
255
+ ### Key Capabilities
256
+
257
+ - **Schema Inspection** — View column names and data types
258
+ - **Column Selection** — Display only the columns you need
259
+ - **Row Limiting** — Control how many rows to preview
260
+ - **Row Offset** — Skip first N rows for pagination/sampling
261
+ - **WHERE Filtering** — Filter rows with SQL-like conditions
262
+ - **Record Counting** — Get total record counts (with Parquet metadata optimization)
263
+ - **Multi-File Reading** — Combine data from multiple files in a directory
264
+ - **Custom Delimiters** — Support for tab, pipe, semicolon, and other delimiters
265
+ - **Auto Decompression** — Transparent handling of compressed files
266
+
267
+ ## Examples
268
+
269
+ ### Basic Usage
270
+
271
+ ```bash
272
+ # Preview first 10 rows (default)
273
+ cloudcat -p gcs://bucket/data.csv
274
+
275
+ # Preview 50 rows
276
+ cloudcat -p s3://bucket/data.parquet -n 50
277
+
278
+ # Show only specific columns
279
+ cloudcat -p gcs://bucket/users.json -c id,name,email
280
+
281
+ # View schema only (no data)
282
+ cloudcat -p s3://bucket/events.parquet -s schema_only
283
+ ```
284
+
285
+ ### Working with Different Formats
286
+
287
+ ```bash
288
+ # CSV with custom delimiter (tab-separated)
289
+ cloudcat -p gcs://bucket/data.tsv -d "\t"
290
+
291
+ # Pipe-delimited file
292
+ cloudcat -p s3://bucket/export.txt -d "|"
293
+
294
+ # Semicolon-delimited (common in European data)
295
+ cloudcat -p gcs://bucket/report.csv -d ";"
296
+
297
+ # JSON array file
298
+ cloudcat -p s3://bucket/config.json
299
+
300
+ # JSON Lines file (auto-detected)
301
+ cloudcat -p gcs://bucket/events.jsonl
302
+ ```
303
+
304
+ ### Filtering and Pagination
305
+
306
+ ```bash
307
+ # Filter rows with WHERE clause
308
+ cloudcat -p s3://bucket/users.parquet --where "status=active"
309
+ cloudcat -p gcs://bucket/events.json --where "age>30"
310
+ cloudcat -p s3://bucket/logs.csv --where "level=ERROR"
311
+
312
+ # String matching filters
313
+ cloudcat -p gcs://bucket/data.csv --where "name contains john"
314
+ cloudcat -p s3://bucket/emails.json --where "email endswith @gmail.com"
315
+ cloudcat -p az://logs/app.log --where "message startswith ERROR"
316
+
317
+ # Skip first N rows (pagination)
318
+ cloudcat -p gcs://bucket/data.csv --offset 100 -n 10
319
+
320
+ # Combine offset with filters
321
+ cloudcat -p s3://bucket/users.parquet --where "active=true" --offset 50 -n 20
322
+ ```
323
+
324
+ ### Compressed Files
325
+
326
+ ```bash
327
+ # Gzip compressed (built-in)
328
+ cloudcat -p gcs://bucket/data.csv.gz
329
+ cloudcat -p s3://bucket/logs.json.gz
330
+
331
+ # Zstandard compressed (requires: pip install cloudcat[zstd])
332
+ cloudcat -p gcs://bucket/events.parquet.zst
333
+
334
+ # LZ4 compressed (requires: pip install cloudcat[lz4])
335
+ cloudcat -p s3://bucket/data.csv.lz4
336
+
337
+ # Bzip2 compressed (built-in)
338
+ cloudcat -p az://container/archive.json.bz2
339
+ ```
340
+
341
+ ### Directory Operations
342
+
343
+ CloudCat intelligently handles directories containing multiple data files (common with Spark, Hive, and distributed processing outputs):
344
+
345
+ ```bash
346
+ # Auto-detect and read first data file in directory
347
+ cloudcat -p gcs://bucket/spark-output/
348
+
349
+ # Read and combine multiple files (up to 25MB by default)
350
+ cloudcat -p s3://bucket/daily-logs/ -m all
351
+
352
+ # Read up to 100MB of data from multiple files
353
+ cloudcat -p gcs://bucket/events/ -m all --max-size-mb 100
354
+
355
+ # Force reading only the first file
356
+ cloudcat -p s3://bucket/output/ -m first
357
+ ```
358
+
359
+ **CloudCat automatically:**
360
+ - Skips empty files
361
+ - Ignores metadata files (`_SUCCESS`, `_metadata`, `.crc`, etc.)
362
+ - Prioritizes files matching the specified format
363
+ - Reports which files were selected
364
+
365
+ ### Output Format Examples
366
+
367
+ ```bash
368
+ # Default table output (great for terminals)
369
+ cloudcat -p gcs://bucket/data.csv
370
+ # ┌────┬────────────┬─────────┐
371
+ # │ id │ name │ value │
372
+ # ├────┼────────────┼─────────┤
373
+ # │ 1 │ Alice │ 100 │
374
+ # │ 2 │ Bob │ 200 │
375
+ # └────┴────────────┴─────────┘
376
+
377
+ # Pretty JSON (great for nested data)
378
+ cloudcat -p s3://bucket/events.json -o jsonp
379
+ # {
380
+ # "id": 1,
381
+ # "name": "Alice",
382
+ # "metadata": {
383
+ # "created": "2024-01-15"
384
+ # }
385
+ # }
386
+
387
+ # JSON Lines (great for piping to jq)
388
+ cloudcat -p gcs://bucket/data.parquet -o json | jq '.name'
389
+
390
+ # CSV (great for further processing)
391
+ cloudcat -p s3://bucket/data.json -o csv > output.csv
392
+ ```
393
+
394
+ ### Data Pipeline Examples
395
+
396
+ ```bash
397
+ # Convert Parquet to CSV
398
+ cloudcat -p gcs://bucket/data.parquet -o csv -n 0 > data.csv
399
+
400
+ # Preview and filter with jq
401
+ cloudcat -p s3://bucket/events.json -o json | jq 'select(.status == "error")'
402
+
403
+ # Quick data validation
404
+ cloudcat -p gcs://bucket/import.csv -s schema_only
405
+
406
+ # Sample data from large dataset
407
+ cloudcat -p s3://bucket/big-table.parquet -n 100 -c user_id,event_type
408
+
409
+ # Export specific columns to CSV
410
+ cloudcat -p gcs://bucket/users.parquet -c email,created_at -o csv -n 0 > emails.csv
411
+ ```
412
+
413
+ ### Real-World Use Cases
414
+
415
+ #### Debugging Spark Jobs
416
+ ```bash
417
+ # Check output of a Spark job
418
+ cloudcat -p gcs://data-lake/jobs/daily-etl/output/ -i parquet -n 20
419
+
420
+ # Verify schema matches expectations
421
+ cloudcat -p s3://analytics/spark-output/ -s schema_only
422
+ ```
423
+
424
+ #### Log Analysis
425
+ ```bash
426
+ # Preview recent logs
427
+ cloudcat -p gcs://logs/app/2024-01-15/ -m all -n 50
428
+
429
+ # Check error logs (combine with grep)
430
+ cloudcat -p s3://logs/errors/ -o json | grep "ERROR"
431
+ ```
432
+
433
+ #### Data Validation
434
+ ```bash
435
+ # Quick sanity check on data export
436
+ cloudcat -p gcs://exports/daily/users.csv -s show
437
+
438
+ # Verify record count
439
+ cloudcat -p s3://warehouse/transactions.parquet --no-count
440
+ ```
441
+
442
+ #### Format Conversion
443
+ ```bash
444
+ # Convert tab-separated to comma-separated
445
+ cloudcat -p gcs://imports/data.tsv -d "\t" -o csv > converted.csv
446
+
447
+ # Convert JSON to CSV for spreadsheet import
448
+ cloudcat -p s3://api-dumps/response.json -o csv > data.csv
449
+ ```
450
+
451
+ ## Command Reference
452
+
453
+ ```
454
+ Usage: cloudcat [OPTIONS]
455
+
456
+ Options:
457
+ -p, --path TEXT Cloud storage path (required)
458
+ Format: gcs://bucket/path, s3://bucket/path,
459
+ or az://container/path
460
+
461
+ -o, --output-format TEXT Output format: table, json, jsonp, csv
462
+ [default: table]
463
+
464
+ -i, --input-format TEXT Input format: csv, json, parquet, avro, orc, text
465
+ [default: auto-detect from extension]
466
+
467
+ -c, --columns TEXT Comma-separated list of columns to display
468
+ [default: all columns]
469
+
470
+ -n, --num-rows INTEGER Number of rows to display (0 for all)
471
+ [default: 10]
472
+
473
+ --offset INTEGER Skip first N rows
474
+ [default: 0]
475
+
476
+ -w, --where TEXT Filter rows with SQL-like conditions
477
+ Examples: "status=active", "age>30",
478
+ "name contains john", "email endswith @gmail.com"
479
+
480
+ -s, --schema TEXT Schema display: show, dont_show, schema_only
481
+ [default: show]
482
+
483
+ --no-count Disable automatic record counting
484
+
485
+ -m, --multi-file-mode TEXT Directory handling: auto, first, all
486
+ [default: auto]
487
+
488
+ --max-size-mb INTEGER Max data size for multi-file mode in MB
489
+ [default: 25]
490
+
491
+ -d, --delimiter TEXT CSV delimiter (use \t for tab)
492
+ [default: comma]
493
+
494
+ --profile TEXT AWS profile name (for S3 access)
495
+
496
+ --project TEXT GCP project ID (for GCS access)
497
+
498
+ --credentials TEXT Path to GCP service account JSON file
499
+
500
+ --account TEXT Azure storage account name
501
+
502
+ --help Show this message and exit
503
+ ```
504
+
505
+ ### WHERE Clause Operators
506
+
507
+ | Operator | Example | Description |
508
+ |----------|---------|-------------|
509
+ | `=` | `status=active` | Exact match |
510
+ | `!=` | `type!=deleted` | Not equal |
511
+ | `>` | `age>30` | Greater than |
512
+ | `<` | `price<100` | Less than |
513
+ | `>=` | `count>=10` | Greater than or equal |
514
+ | `<=` | `score<=50` | Less than or equal |
515
+ | `contains` | `name contains john` | Case-insensitive substring match |
516
+ | `startswith` | `email startswith admin` | String prefix match |
517
+ | `endswith` | `file endswith .csv` | String suffix match |
518
+
519
+ ## Authentication
520
+
521
+ ### Google Cloud Storage
522
+
523
+ CloudCat uses [Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials). Set up authentication using one of these methods:
524
+
525
+ ```bash
526
+ # Option 1: User credentials (for development)
527
+ gcloud auth application-default login
528
+
529
+ # Option 2: Service account via environment variable
530
+ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
531
+
532
+ # Option 3: Service account via CLI option
533
+ cloudcat -p gcs://bucket/data.csv --credentials /path/to/service-account.json
534
+
535
+ # Option 4: Specify GCP project
536
+ cloudcat -p gcs://bucket/data.csv --project my-gcp-project
537
+ ```
538
+
539
+ ### Amazon S3
540
+
541
+ CloudCat uses the standard [AWS credential chain](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html):
542
+
543
+ ```bash
544
+ # Option 1: Environment variables
545
+ export AWS_ACCESS_KEY_ID="your-access-key"
546
+ export AWS_SECRET_ACCESS_KEY="your-secret-key"
547
+ export AWS_DEFAULT_REGION="us-east-1"
548
+
549
+ # Option 2: AWS credentials file (~/.aws/credentials)
550
+ aws configure
551
+
552
+ # Option 3: AWS named profile
553
+ cloudcat -p s3://bucket/data.csv --profile production
554
+
555
+ # Option 4: IAM role (for EC2/ECS/Lambda)
556
+ # Automatically detected
557
+ ```
558
+
559
+ ### Azure Blob Storage
560
+
561
+ CloudCat supports multiple authentication methods for Azure:
562
+
563
+ ```bash
564
+ # Option 1: Connection string (simplest)
565
+ export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net"
566
+
567
+ # Option 2: Account URL with DefaultAzureCredential (for Azure AD auth)
568
+ export AZURE_STORAGE_ACCOUNT_URL="https://youraccount.blob.core.windows.net"
569
+ az login
570
+
571
+ # Option 3: Specify storage account via CLI option
572
+ cloudcat -p az://container/data.csv --account mystorageaccount
573
+ ```
574
+
575
+ **Path format:** `az://container-name/path/to/blob`
576
+
577
+ ## Performance Tips
578
+
579
+ 1. **Use `--no-count`** for large files when you don't need the total record count
580
+ 2. **Prefer Parquet** format when possible — record counts are instant from metadata
581
+ 3. **Use `--num-rows`** to limit data transfer for large files
582
+ 4. **Use `--columns`** to select only needed columns (especially effective with Parquet)
583
+ 5. **Use `-m first`** when you only need a sample from directories with many files
584
+
585
+ ## Troubleshooting
586
+
587
+ ### Common Issues
588
+
589
+ **"google-cloud-storage package is required"**
590
+ ```bash
591
+ pip install cloudcat[gcs]
592
+ ```
593
+
594
+ **"boto3 package is required"**
595
+ ```bash
596
+ pip install cloudcat[s3]
597
+ ```
598
+
599
+ **"pyarrow package is required"**
600
+ ```bash
601
+ pip install cloudcat[parquet]
602
+ ```
603
+
604
+ **"azure-storage-blob package is required"**
605
+ ```bash
606
+ pip install cloudcat[azure]
607
+ ```
608
+
609
+ **"fastavro package is required"**
610
+ ```bash
611
+ pip install cloudcat[avro]
612
+ ```
613
+
614
+ **"pyarrow with ORC support is required"**
615
+ ```bash
616
+ pip install cloudcat[orc]
617
+ ```
618
+
619
+ **"zstandard package is required for .zst files"**
620
+ ```bash
621
+ pip install cloudcat[zstd]
622
+ # or for all compression formats:
623
+ pip install cloudcat[compression]
624
+ ```
625
+
626
+ **"lz4 package is required for .lz4 files"**
627
+ ```bash
628
+ pip install cloudcat[lz4]
629
+ ```
630
+
631
+ **"python-snappy package is required for .snappy files"**
632
+ ```bash
633
+ pip install cloudcat[snappy]
634
+ ```
635
+
636
+ **Authentication errors**
637
+ - GCS: Run `gcloud auth application-default login`
638
+ - S3: Run `aws configure` or check your credentials
639
+ - Azure: Set `AZURE_STORAGE_CONNECTION_STRING` or `AZURE_STORAGE_ACCOUNT_URL` and run `az login`
640
+
641
+ **"Could not infer format from path"**
642
+ ```bash
643
+ # Specify the format explicitly
644
+ cloudcat -p gcs://bucket/data -i parquet
645
+ ```
646
+
647
+ ## Contributing
648
+
649
+ Contributions are welcome! Here's how you can help:
650
+
651
+ 1. **Report bugs** — Open an issue with reproduction steps
652
+ 2. **Suggest features** — Open an issue describing the use case
653
+ 3. **Submit PRs** — Fork, create a branch, and submit a pull request
654
+
655
+ ### Development Setup
656
+
657
+ ```bash
658
+ # Clone the repository
659
+ git clone https://github.com/jonathansudhakar1/cloudcat.git
660
+ cd cloudcat
661
+
662
+ # Create virtual environment
663
+ python -m venv venv
664
+ source venv/bin/activate # or `venv\Scripts\activate` on Windows
665
+
666
+ # Install in development mode with all dependencies
667
+ pip install -e ".[all]"
668
+
669
+ # Run tests
670
+ pytest
671
+ ```
672
+
673
+ ## Roadmap
674
+
675
+ - [x] Azure Blob Storage support
676
+ - [x] Avro format support
677
+ - [x] ORC format support
678
+ - [x] Plain text format support
679
+ - [x] SQL-like filtering (`--where` clause)
680
+ - [x] Compression support (gzip, zstd, lz4, snappy, bz2)
681
+ - [x] Row offset/pagination (`--offset`)
682
+ - [ ] Interactive mode with pagination
683
+ - [ ] Output to file with `--output-file`
684
+ - [ ] Configuration file support
685
+
686
+ ## Related Projects
687
+
688
+ - [s3cmd](https://s3tools.org/s3cmd) — S3 command-line tool
689
+ - [gsutil](https://cloud.google.com/storage/docs/gsutil) — Google Cloud Storage CLI
690
+ - [aws-cli](https://aws.amazon.com/cli/) — AWS command-line interface
691
+ - [azcopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10) — Azure Storage data transfer tool
692
+ - [duckdb](https://duckdb.org/) — In-process SQL OLAP database
693
+
694
+ ## License
695
+
696
+ MIT License — see [LICENSE](LICENSE) for details.
697
+
698
+
699
+ <p align="center">
700
+ <a href="https://github.com/jonathansudhakar1/cloudcat/issues">Report Bug</a> •
701
+ <a href="https://github.com/jonathansudhakar1/cloudcat/issues">Request Feature</a>
702
+ </p>