pyckup 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyckup-0.1.2/PKG-INFO +190 -0
- pyckup-0.1.2/pyckup.egg-info/PKG-INFO +190 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/setup.cfg +1 -2
- pyckup-0.1.1/PKG-INFO +0 -190
- pyckup-0.1.1/pyckup.egg-info/PKG-INFO +0 -190
- {pyckup-0.1.1 → pyckup-0.1.2}/LICENSE +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/README.md +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/pyckup/__init__.py +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/pyckup/base.py +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/pyckup.egg-info/SOURCES.txt +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/pyckup.egg-info/dependency_links.txt +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/pyckup.egg-info/not-zip-safe +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/pyckup.egg-info/requires.txt +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/pyckup.egg-info/top_level.txt +0 -0
- {pyckup-0.1.1 → pyckup-0.1.2}/setup.py +0 -0
pyckup-0.1.2/PKG-INFO
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: pyckup
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
|
|
5
|
+
Home-page: https://github.com/i2mint/pyckup
|
|
6
|
+
License: apache-2.0
|
|
7
|
+
Keywords: data,data access,data preparation,machine learning,artificial intelligence
|
|
8
|
+
Platform: any
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
|
|
12
|
+
# pyckup
|
|
13
|
+
|
|
14
|
+
Tools to provide easy access to prepared data to data scientists that can't
|
|
15
|
+
be asked.
|
|
16
|
+
|
|
17
|
+
They just want to get on with the fun -- **not** get stuck in data access and
|
|
18
|
+
data preparation concerns. And they should want that!
|
|
19
|
+
|
|
20
|
+
Of course, someone needed to do the work of getting the data from where and how it is,
|
|
21
|
+
to where and how it needs to be (for a particular problem and context).
|
|
22
|
+
|
|
23
|
+
What we believe is that this work should not only be less tedious and less
|
|
24
|
+
time-consuming (see py2store and related for that!), but also, once it's done,
|
|
25
|
+
it shouldn't have to be re-done every time someone wants to kick the data
|
|
26
|
+
around.
|
|
27
|
+
|
|
28
|
+
So we made ``pyckup``.
|
|
29
|
+
|
|
30
|
+
We hope it helps.
|
|
31
|
+
|
|
32
|
+
# install
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
pip install pyckup
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
# Usage
|
|
39
|
+
|
|
40
|
+
A protocol is what's to the left of ``://`` in a url.
|
|
41
|
+
You've seen'em. For example, ``http`` and ``https``.
|
|
42
|
+
Well, if you try grabbing a valid url, you'll get ``bytes`` out of it,
|
|
43
|
+
without having to make an actual http request yourself.
|
|
44
|
+
|
|
45
|
+
>>> b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
46
|
+
>>> isinstance(b, bytes)
|
|
47
|
+
True
|
|
48
|
+
|
|
49
|
+
Now let's put those bytes in a file:
|
|
50
|
+
|
|
51
|
+
>>> from tempfile import mktemp
|
|
52
|
+
>>> filepath = mktemp()
|
|
53
|
+
>>> with open(filepath, 'wb') as fp:
|
|
54
|
+
... n_bytes_written = fp.write(b);
|
|
55
|
+
>>> assert n_bytes_written == len(b)
|
|
56
|
+
|
|
57
|
+
A filepath is a url. So you should be able to grab it's contents too.
|
|
58
|
+
|
|
59
|
+
>>> file_bytes = grab(filepath)
|
|
60
|
+
>>> assert file_bytes == b
|
|
61
|
+
|
|
62
|
+
There wasn't really a protocol there, but when ``grab`` sees that the url
|
|
63
|
+
you're passing it starts with a slash, it automatically prepends a ``file://``
|
|
64
|
+
to it, like most browswers do.
|
|
65
|
+
You can also specify the ``file://`` protocol explicitly (again, like in
|
|
66
|
+
most browsers):
|
|
67
|
+
|
|
68
|
+
>>> file_bytes_2 = grab('file://' + filepath)
|
|
69
|
+
>>> assert file_bytes_2 == file_bytes == b
|
|
70
|
+
|
|
71
|
+
If your filepath points to a folder, you'll get back a ``dol.Files`` object,
|
|
72
|
+
which is a ``Mapping`` whose keys are file paths (relative to the folder) and
|
|
73
|
+
the values are the bytes (acquired lazily) of the corresponding file's contents.
|
|
74
|
+
|
|
75
|
+
>>> import os, typing
|
|
76
|
+
>>> folder, filename = os.path.dirname(filepath), os.path.basename(filepath)
|
|
77
|
+
>>> f = grab(folder)
|
|
78
|
+
>>> isinstance(f, typing.Mapping)
|
|
79
|
+
True
|
|
80
|
+
>>> filename in f
|
|
81
|
+
True
|
|
82
|
+
>>> file_bytes_3 = f[filename]
|
|
83
|
+
>>> assert file_bytes_3 == file_bytes_2 == file_bytes == b
|
|
84
|
+
|
|
85
|
+
``grab`` can handle various protocols, according to what packages it finds
|
|
86
|
+
on your system. For example, if you have ``haggle``
|
|
87
|
+
(https://pypi.org/project/haggle/) installed, you'll find that ``kaggle``
|
|
88
|
+
is also a valid protocol.
|
|
89
|
+
|
|
90
|
+
>>> sorted(grab.dflt_protocols) # doctest: +SKIP
|
|
91
|
+
['file', 'http', 'https', 'kaggle']
|
|
92
|
+
|
|
93
|
+
Notice that ``grab`` has an argument called ``protocols``.
|
|
94
|
+
Yes, this means you have control.
|
|
95
|
+
You just need to specify a mapping between protocol strings and the "url_to_content"
|
|
96
|
+
function that should be used to get the content from that url.
|
|
97
|
+
|
|
98
|
+
In the following we'll add a fake ``foo`` protocol that doesn't really fetch any
|
|
99
|
+
data, but applies ``str.upper`` to the url, but you get the point.
|
|
100
|
+
|
|
101
|
+
>>> from functools import partial
|
|
102
|
+
>>> from pyckup.base import dflt_protocols
|
|
103
|
+
>>> mygrab = partial(grab, protocols=dict(dflt_protocols, foo=str.upper))
|
|
104
|
+
>>> mygrab('foo://a_fake_url')
|
|
105
|
+
'FOO://A_FAKE_URL'
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# Examples
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from pyckup import grab
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
See what (default) protocols you have access to.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from pyckup import grab
|
|
118
|
+
|
|
119
|
+
grab.procotols
|
|
120
|
+
# ['file', 'kaggle', 'http', 'https']
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Grab file contents
|
|
124
|
+
|
|
125
|
+
Specifying a "file" protocol (i.e. prefixing your string with
|
|
126
|
+
"file://" -- followed by a full path) will give you the contents of the file
|
|
127
|
+
in bytes.
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from pyckup import grab
|
|
131
|
+
|
|
132
|
+
b = grab('file:///Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
133
|
+
assert isinstance(b, bytes)
|
|
134
|
+
print(b.decode())
|
|
135
|
+
# from pyckup.base import grab, protocols
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
But you can also use a full path, or other natural means of specifying files.
|
|
139
|
+
In that case though, `grab` will try to give you the contents in a convenient type
|
|
140
|
+
(e.g. a `dict` for `.json`, a python object of `.pickle`, string for `.txt`...).
|
|
141
|
+
This is convenient, but don't depend on the type to strongly
|
|
142
|
+
since it depends on what `py2store.misc` sets it to be.
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from pyckup import grab
|
|
146
|
+
|
|
147
|
+
grab('/Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
148
|
+
# b'from pyckup.base import grab, protocols\n\n\n'
|
|
149
|
+
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.json')
|
|
150
|
+
# {'hello': 'world', 'abc': [1, 2, 3]}
|
|
151
|
+
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.pickle')
|
|
152
|
+
# [1, 2, 3]
|
|
153
|
+
print(grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.txt'))
|
|
154
|
+
# This
|
|
155
|
+
# is
|
|
156
|
+
# text
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## Grab the contents of a url
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from pyckup import grab
|
|
163
|
+
|
|
164
|
+
b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
165
|
+
type(b), len(b)
|
|
166
|
+
# (bytes, 11357)
|
|
167
|
+
print(b[:100].decode())
|
|
168
|
+
# Apache License
|
|
169
|
+
# Version 2.0, January 2004
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Grab stuff from kaggle
|
|
173
|
+
|
|
174
|
+
Note: If you want to access kaggle datasets with ``pyckup``,
|
|
175
|
+
you'll need to get an account.
|
|
176
|
+
See [haggle](https://github.com/otosense/haggle#api-credentials)
|
|
177
|
+
for more information.
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from pyckup import grab
|
|
182
|
+
|
|
183
|
+
z = grab('kaggle://drgilermo/face-images-with-marked-landmark-points')
|
|
184
|
+
list(z)
|
|
185
|
+
# ['face_images.npz', 'facial_keypoints.csv']
|
|
186
|
+
print(z['facial_keypoints.csv'][:100].decode())
|
|
187
|
+
# left_eye_center_x,left_eye_center_y,right_eye_center_x,right_eye_center_y,left_eye_inner_corner_x,le
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: pyckup
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
|
|
5
|
+
Home-page: https://github.com/i2mint/pyckup
|
|
6
|
+
License: apache-2.0
|
|
7
|
+
Keywords: data,data access,data preparation,machine learning,artificial intelligence
|
|
8
|
+
Platform: any
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
|
|
12
|
+
# pyckup
|
|
13
|
+
|
|
14
|
+
Tools to provide easy access to prepared data to data scientists that can't
|
|
15
|
+
be asked.
|
|
16
|
+
|
|
17
|
+
They just want to get on with the fun -- **not** get stuck in data access and
|
|
18
|
+
data preparation concerns. And they should want that!
|
|
19
|
+
|
|
20
|
+
Of course, someone needed to do the work of getting the data from where and how it is,
|
|
21
|
+
to where and how it needs to be (for a particular problem and context).
|
|
22
|
+
|
|
23
|
+
What we believe is that this work should not only be less tedious and less
|
|
24
|
+
time-consuming (see py2store and related for that!), but also, once it's done,
|
|
25
|
+
it shouldn't have to be re-done every time someone wants to kick the data
|
|
26
|
+
around.
|
|
27
|
+
|
|
28
|
+
So we made ``pyckup``.
|
|
29
|
+
|
|
30
|
+
We hope it helps.
|
|
31
|
+
|
|
32
|
+
# install
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
pip install pyckup
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
# Usage
|
|
39
|
+
|
|
40
|
+
A protocol is what's to the left of ``://`` in a url.
|
|
41
|
+
You've seen'em. For example, ``http`` and ``https``.
|
|
42
|
+
Well, if you try grabbing a valid url, you'll get ``bytes`` out of it,
|
|
43
|
+
without having to make an actual http request yourself.
|
|
44
|
+
|
|
45
|
+
>>> b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
46
|
+
>>> isinstance(b, bytes)
|
|
47
|
+
True
|
|
48
|
+
|
|
49
|
+
Now let's put those bytes in a file:
|
|
50
|
+
|
|
51
|
+
>>> from tempfile import mktemp
|
|
52
|
+
>>> filepath = mktemp()
|
|
53
|
+
>>> with open(filepath, 'wb') as fp:
|
|
54
|
+
... n_bytes_written = fp.write(b);
|
|
55
|
+
>>> assert n_bytes_written == len(b)
|
|
56
|
+
|
|
57
|
+
A filepath is a url. So you should be able to grab it's contents too.
|
|
58
|
+
|
|
59
|
+
>>> file_bytes = grab(filepath)
|
|
60
|
+
>>> assert file_bytes == b
|
|
61
|
+
|
|
62
|
+
There wasn't really a protocol there, but when ``grab`` sees that the url
|
|
63
|
+
you're passing it starts with a slash, it automatically prepends a ``file://``
|
|
64
|
+
to it, like most browswers do.
|
|
65
|
+
You can also specify the ``file://`` protocol explicitly (again, like in
|
|
66
|
+
most browsers):
|
|
67
|
+
|
|
68
|
+
>>> file_bytes_2 = grab('file://' + filepath)
|
|
69
|
+
>>> assert file_bytes_2 == file_bytes == b
|
|
70
|
+
|
|
71
|
+
If your filepath points to a folder, you'll get back a ``dol.Files`` object,
|
|
72
|
+
which is a ``Mapping`` whose keys are file paths (relative to the folder) and
|
|
73
|
+
the values are the bytes (acquired lazily) of the corresponding file's contents.
|
|
74
|
+
|
|
75
|
+
>>> import os, typing
|
|
76
|
+
>>> folder, filename = os.path.dirname(filepath), os.path.basename(filepath)
|
|
77
|
+
>>> f = grab(folder)
|
|
78
|
+
>>> isinstance(f, typing.Mapping)
|
|
79
|
+
True
|
|
80
|
+
>>> filename in f
|
|
81
|
+
True
|
|
82
|
+
>>> file_bytes_3 = f[filename]
|
|
83
|
+
>>> assert file_bytes_3 == file_bytes_2 == file_bytes == b
|
|
84
|
+
|
|
85
|
+
``grab`` can handle various protocols, according to what packages it finds
|
|
86
|
+
on your system. For example, if you have ``haggle``
|
|
87
|
+
(https://pypi.org/project/haggle/) installed, you'll find that ``kaggle``
|
|
88
|
+
is also a valid protocol.
|
|
89
|
+
|
|
90
|
+
>>> sorted(grab.dflt_protocols) # doctest: +SKIP
|
|
91
|
+
['file', 'http', 'https', 'kaggle']
|
|
92
|
+
|
|
93
|
+
Notice that ``grab`` has an argument called ``protocols``.
|
|
94
|
+
Yes, this means you have control.
|
|
95
|
+
You just need to specify a mapping between protocol strings and the "url_to_content"
|
|
96
|
+
function that should be used to get the content from that url.
|
|
97
|
+
|
|
98
|
+
In the following we'll add a fake ``foo`` protocol that doesn't really fetch any
|
|
99
|
+
data, but applies ``str.upper`` to the url, but you get the point.
|
|
100
|
+
|
|
101
|
+
>>> from functools import partial
|
|
102
|
+
>>> from pyckup.base import dflt_protocols
|
|
103
|
+
>>> mygrab = partial(grab, protocols=dict(dflt_protocols, foo=str.upper))
|
|
104
|
+
>>> mygrab('foo://a_fake_url')
|
|
105
|
+
'FOO://A_FAKE_URL'
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# Examples
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from pyckup import grab
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
See what (default) protocols you have access to.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from pyckup import grab
|
|
118
|
+
|
|
119
|
+
grab.procotols
|
|
120
|
+
# ['file', 'kaggle', 'http', 'https']
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Grab file contents
|
|
124
|
+
|
|
125
|
+
Specifying a "file" protocol (i.e. prefixing your string with
|
|
126
|
+
"file://" -- followed by a full path) will give you the contents of the file
|
|
127
|
+
in bytes.
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from pyckup import grab
|
|
131
|
+
|
|
132
|
+
b = grab('file:///Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
133
|
+
assert isinstance(b, bytes)
|
|
134
|
+
print(b.decode())
|
|
135
|
+
# from pyckup.base import grab, protocols
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
But you can also use a full path, or other natural means of specifying files.
|
|
139
|
+
In that case though, `grab` will try to give you the contents in a convenient type
|
|
140
|
+
(e.g. a `dict` for `.json`, a python object of `.pickle`, string for `.txt`...).
|
|
141
|
+
This is convenient, but don't depend on the type to strongly
|
|
142
|
+
since it depends on what `py2store.misc` sets it to be.
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from pyckup import grab
|
|
146
|
+
|
|
147
|
+
grab('/Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
148
|
+
# b'from pyckup.base import grab, protocols\n\n\n'
|
|
149
|
+
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.json')
|
|
150
|
+
# {'hello': 'world', 'abc': [1, 2, 3]}
|
|
151
|
+
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.pickle')
|
|
152
|
+
# [1, 2, 3]
|
|
153
|
+
print(grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.txt'))
|
|
154
|
+
# This
|
|
155
|
+
# is
|
|
156
|
+
# text
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## Grab the contents of a url
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from pyckup import grab
|
|
163
|
+
|
|
164
|
+
b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
165
|
+
type(b), len(b)
|
|
166
|
+
# (bytes, 11357)
|
|
167
|
+
print(b[:100].decode())
|
|
168
|
+
# Apache License
|
|
169
|
+
# Version 2.0, January 2004
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Grab stuff from kaggle
|
|
173
|
+
|
|
174
|
+
Note: If you want to access kaggle datasets with ``pyckup``,
|
|
175
|
+
you'll need to get an account.
|
|
176
|
+
See [haggle](https://github.com/otosense/haggle#api-credentials)
|
|
177
|
+
for more information.
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from pyckup import grab
|
|
182
|
+
|
|
183
|
+
z = grab('kaggle://drgilermo/face-images-with-marked-landmark-points')
|
|
184
|
+
list(z)
|
|
185
|
+
# ['face_images.npz', 'facial_keypoints.csv']
|
|
186
|
+
print(z['facial_keypoints.csv'][:100].decode())
|
|
187
|
+
# left_eye_center_x,left_eye_center_y,right_eye_center_x,right_eye_center_y,left_eye_inner_corner_x,le
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[metadata]
|
|
2
2
|
name = pyckup
|
|
3
|
-
version = 0.1.
|
|
3
|
+
version = 0.1.2
|
|
4
4
|
url = https://github.com/i2mint/pyckup
|
|
5
5
|
platforms = any
|
|
6
6
|
description_file = README.md
|
|
@@ -15,7 +15,6 @@ keywords =
|
|
|
15
15
|
data preparation
|
|
16
16
|
machine learning
|
|
17
17
|
artificial intelligence
|
|
18
|
-
author = Otosense
|
|
19
18
|
display_name = pyckup
|
|
20
19
|
|
|
21
20
|
[options]
|
pyckup-0.1.1/PKG-INFO
DELETED
|
@@ -1,190 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: pyckup
|
|
3
|
-
Version: 0.1.1
|
|
4
|
-
Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
|
|
5
|
-
Home-page: https://github.com/i2mint/pyckup
|
|
6
|
-
Author: Otosense
|
|
7
|
-
License: apache-2.0
|
|
8
|
-
Description: # pyckup
|
|
9
|
-
|
|
10
|
-
Tools to provide easy access to prepared data to data scientists that can't
|
|
11
|
-
be asked.
|
|
12
|
-
|
|
13
|
-
They just want to get on with the fun -- **not** get stuck in data access and
|
|
14
|
-
data preparation concerns. And they should want that!
|
|
15
|
-
|
|
16
|
-
Of course, someone needed to do the work of getting the data from where and how it is,
|
|
17
|
-
to where and how it needs to be (for a particular problem and context).
|
|
18
|
-
|
|
19
|
-
What we believe is that this work should not only be less tedious and less
|
|
20
|
-
time-consuming (see py2store and related for that!), but also, once it's done,
|
|
21
|
-
it shouldn't have to be re-done every time someone wants to kick the data
|
|
22
|
-
around.
|
|
23
|
-
|
|
24
|
-
So we made ``pyckup``.
|
|
25
|
-
|
|
26
|
-
We hope it helps.
|
|
27
|
-
|
|
28
|
-
# install
|
|
29
|
-
|
|
30
|
-
```
|
|
31
|
-
pip install pyckup
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
# Usage
|
|
35
|
-
|
|
36
|
-
A protocol is what's to the left of ``://`` in a url.
|
|
37
|
-
You've seen'em. For example, ``http`` and ``https``.
|
|
38
|
-
Well, if you try grabbing a valid url, you'll get ``bytes`` out of it,
|
|
39
|
-
without having to make an actual http request yourself.
|
|
40
|
-
|
|
41
|
-
>>> b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
42
|
-
>>> isinstance(b, bytes)
|
|
43
|
-
True
|
|
44
|
-
|
|
45
|
-
Now let's put those bytes in a file:
|
|
46
|
-
|
|
47
|
-
>>> from tempfile import mktemp
|
|
48
|
-
>>> filepath = mktemp()
|
|
49
|
-
>>> with open(filepath, 'wb') as fp:
|
|
50
|
-
... n_bytes_written = fp.write(b);
|
|
51
|
-
>>> assert n_bytes_written == len(b)
|
|
52
|
-
|
|
53
|
-
A filepath is a url. So you should be able to grab it's contents too.
|
|
54
|
-
|
|
55
|
-
>>> file_bytes = grab(filepath)
|
|
56
|
-
>>> assert file_bytes == b
|
|
57
|
-
|
|
58
|
-
There wasn't really a protocol there, but when ``grab`` sees that the url
|
|
59
|
-
you're passing it starts with a slash, it automatically prepends a ``file://``
|
|
60
|
-
to it, like most browswers do.
|
|
61
|
-
You can also specify the ``file://`` protocol explicitly (again, like in
|
|
62
|
-
most browsers):
|
|
63
|
-
|
|
64
|
-
>>> file_bytes_2 = grab('file://' + filepath)
|
|
65
|
-
>>> assert file_bytes_2 == file_bytes == b
|
|
66
|
-
|
|
67
|
-
If your filepath points to a folder, you'll get back a ``dol.Files`` object,
|
|
68
|
-
which is a ``Mapping`` whose keys are file paths (relative to the folder) and
|
|
69
|
-
the values are the bytes (acquired lazily) of the corresponding file's contents.
|
|
70
|
-
|
|
71
|
-
>>> import os, typing
|
|
72
|
-
>>> folder, filename = os.path.dirname(filepath), os.path.basename(filepath)
|
|
73
|
-
>>> f = grab(folder)
|
|
74
|
-
>>> isinstance(f, typing.Mapping)
|
|
75
|
-
True
|
|
76
|
-
>>> filename in f
|
|
77
|
-
True
|
|
78
|
-
>>> file_bytes_3 = f[filename]
|
|
79
|
-
>>> assert file_bytes_3 == file_bytes_2 == file_bytes == b
|
|
80
|
-
|
|
81
|
-
``grab`` can handle various protocols, according to what packages it finds
|
|
82
|
-
on your system. For example, if you have ``haggle``
|
|
83
|
-
(https://pypi.org/project/haggle/) installed, you'll find that ``kaggle``
|
|
84
|
-
is also a valid protocol.
|
|
85
|
-
|
|
86
|
-
>>> sorted(grab.dflt_protocols) # doctest: +SKIP
|
|
87
|
-
['file', 'http', 'https', 'kaggle']
|
|
88
|
-
|
|
89
|
-
Notice that ``grab`` has an argument called ``protocols``.
|
|
90
|
-
Yes, this means you have control.
|
|
91
|
-
You just need to specify a mapping between protocol strings and the "url_to_content"
|
|
92
|
-
function that should be used to get the content from that url.
|
|
93
|
-
|
|
94
|
-
In the following we'll add a fake ``foo`` protocol that doesn't really fetch any
|
|
95
|
-
data, but applies ``str.upper`` to the url, but you get the point.
|
|
96
|
-
|
|
97
|
-
>>> from functools import partial
|
|
98
|
-
>>> from pyckup.base import dflt_protocols
|
|
99
|
-
>>> mygrab = partial(grab, protocols=dict(dflt_protocols, foo=str.upper))
|
|
100
|
-
>>> mygrab('foo://a_fake_url')
|
|
101
|
-
'FOO://A_FAKE_URL'
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
# Examples
|
|
105
|
-
|
|
106
|
-
```python
|
|
107
|
-
from pyckup import grab
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
See what (default) protocols you have access to.
|
|
111
|
-
|
|
112
|
-
```python
|
|
113
|
-
from pyckup import grab
|
|
114
|
-
|
|
115
|
-
grab.procotols
|
|
116
|
-
# ['file', 'kaggle', 'http', 'https']
|
|
117
|
-
```
|
|
118
|
-
|
|
119
|
-
## Grab file contents
|
|
120
|
-
|
|
121
|
-
Specifying a "file" protocol (i.e. prefixing your string with
|
|
122
|
-
"file://" -- followed by a full path) will give you the contents of the file
|
|
123
|
-
in bytes.
|
|
124
|
-
|
|
125
|
-
```python
|
|
126
|
-
from pyckup import grab
|
|
127
|
-
|
|
128
|
-
b = grab('file:///Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
129
|
-
assert isinstance(b, bytes)
|
|
130
|
-
print(b.decode())
|
|
131
|
-
# from pyckup.base import grab, protocols
|
|
132
|
-
```
|
|
133
|
-
|
|
134
|
-
But you can also use a full path, or other natural means of specifying files.
|
|
135
|
-
In that case though, `grab` will try to give you the contents in a convenient type
|
|
136
|
-
(e.g. a `dict` for `.json`, a python object of `.pickle`, string for `.txt`...).
|
|
137
|
-
This is convenient, but don't depend on the type to strongly
|
|
138
|
-
since it depends on what `py2store.misc` sets it to be.
|
|
139
|
-
|
|
140
|
-
```python
|
|
141
|
-
from pyckup import grab
|
|
142
|
-
|
|
143
|
-
grab('/Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
144
|
-
# b'from pyckup.base import grab, protocols\n\n\n'
|
|
145
|
-
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.json')
|
|
146
|
-
# {'hello': 'world', 'abc': [1, 2, 3]}
|
|
147
|
-
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.pickle')
|
|
148
|
-
# [1, 2, 3]
|
|
149
|
-
print(grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.txt'))
|
|
150
|
-
# This
|
|
151
|
-
# is
|
|
152
|
-
# text
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
## Grab the contents of a url
|
|
156
|
-
|
|
157
|
-
```python
|
|
158
|
-
from pyckup import grab
|
|
159
|
-
|
|
160
|
-
b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
161
|
-
type(b), len(b)
|
|
162
|
-
# (bytes, 11357)
|
|
163
|
-
print(b[:100].decode())
|
|
164
|
-
# Apache License
|
|
165
|
-
# Version 2.0, January 2004
|
|
166
|
-
```
|
|
167
|
-
|
|
168
|
-
## Grab stuff from kaggle
|
|
169
|
-
|
|
170
|
-
Note: If you want to access kaggle datasets with ``pyckup``,
|
|
171
|
-
you'll need to get an account.
|
|
172
|
-
See [haggle](https://github.com/otosense/haggle#api-credentials)
|
|
173
|
-
for more information.
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
```python
|
|
177
|
-
from pyckup import grab
|
|
178
|
-
|
|
179
|
-
z = grab('kaggle://drgilermo/face-images-with-marked-landmark-points')
|
|
180
|
-
list(z)
|
|
181
|
-
# ['face_images.npz', 'facial_keypoints.csv']
|
|
182
|
-
print(z['facial_keypoints.csv'][:100].decode())
|
|
183
|
-
# left_eye_center_x,left_eye_center_y,right_eye_center_x,right_eye_center_y,left_eye_inner_corner_x,le
|
|
184
|
-
```
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
Keywords: data,data access,data preparation,machine learning,artificial intelligence
|
|
189
|
-
Platform: any
|
|
190
|
-
Description-Content-Type: text/markdown
|
|
@@ -1,190 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: pyckup
|
|
3
|
-
Version: 0.1.1
|
|
4
|
-
Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
|
|
5
|
-
Home-page: https://github.com/i2mint/pyckup
|
|
6
|
-
Author: Otosense
|
|
7
|
-
License: apache-2.0
|
|
8
|
-
Description: # pyckup
|
|
9
|
-
|
|
10
|
-
Tools to provide easy access to prepared data to data scientists that can't
|
|
11
|
-
be asked.
|
|
12
|
-
|
|
13
|
-
They just want to get on with the fun -- **not** get stuck in data access and
|
|
14
|
-
data preparation concerns. And they should want that!
|
|
15
|
-
|
|
16
|
-
Of course, someone needed to do the work of getting the data from where and how it is,
|
|
17
|
-
to where and how it needs to be (for a particular problem and context).
|
|
18
|
-
|
|
19
|
-
What we believe is that this work should not only be less tedious and less
|
|
20
|
-
time-consuming (see py2store and related for that!), but also, once it's done,
|
|
21
|
-
it shouldn't have to be re-done every time someone wants to kick the data
|
|
22
|
-
around.
|
|
23
|
-
|
|
24
|
-
So we made ``pyckup``.
|
|
25
|
-
|
|
26
|
-
We hope it helps.
|
|
27
|
-
|
|
28
|
-
# install
|
|
29
|
-
|
|
30
|
-
```
|
|
31
|
-
pip install pyckup
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
# Usage
|
|
35
|
-
|
|
36
|
-
A protocol is what's to the left of ``://`` in a url.
|
|
37
|
-
You've seen'em. For example, ``http`` and ``https``.
|
|
38
|
-
Well, if you try grabbing a valid url, you'll get ``bytes`` out of it,
|
|
39
|
-
without having to make an actual http request yourself.
|
|
40
|
-
|
|
41
|
-
>>> b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
42
|
-
>>> isinstance(b, bytes)
|
|
43
|
-
True
|
|
44
|
-
|
|
45
|
-
Now let's put those bytes in a file:
|
|
46
|
-
|
|
47
|
-
>>> from tempfile import mktemp
|
|
48
|
-
>>> filepath = mktemp()
|
|
49
|
-
>>> with open(filepath, 'wb') as fp:
|
|
50
|
-
... n_bytes_written = fp.write(b);
|
|
51
|
-
>>> assert n_bytes_written == len(b)
|
|
52
|
-
|
|
53
|
-
A filepath is a url. So you should be able to grab it's contents too.
|
|
54
|
-
|
|
55
|
-
>>> file_bytes = grab(filepath)
|
|
56
|
-
>>> assert file_bytes == b
|
|
57
|
-
|
|
58
|
-
There wasn't really a protocol there, but when ``grab`` sees that the url
|
|
59
|
-
you're passing it starts with a slash, it automatically prepends a ``file://``
|
|
60
|
-
to it, like most browswers do.
|
|
61
|
-
You can also specify the ``file://`` protocol explicitly (again, like in
|
|
62
|
-
most browsers):
|
|
63
|
-
|
|
64
|
-
>>> file_bytes_2 = grab('file://' + filepath)
|
|
65
|
-
>>> assert file_bytes_2 == file_bytes == b
|
|
66
|
-
|
|
67
|
-
If your filepath points to a folder, you'll get back a ``dol.Files`` object,
|
|
68
|
-
which is a ``Mapping`` whose keys are file paths (relative to the folder) and
|
|
69
|
-
the values are the bytes (acquired lazily) of the corresponding file's contents.
|
|
70
|
-
|
|
71
|
-
>>> import os, typing
|
|
72
|
-
>>> folder, filename = os.path.dirname(filepath), os.path.basename(filepath)
|
|
73
|
-
>>> f = grab(folder)
|
|
74
|
-
>>> isinstance(f, typing.Mapping)
|
|
75
|
-
True
|
|
76
|
-
>>> filename in f
|
|
77
|
-
True
|
|
78
|
-
>>> file_bytes_3 = f[filename]
|
|
79
|
-
>>> assert file_bytes_3 == file_bytes_2 == file_bytes == b
|
|
80
|
-
|
|
81
|
-
``grab`` can handle various protocols, according to what packages it finds
|
|
82
|
-
on your system. For example, if you have ``haggle``
|
|
83
|
-
(https://pypi.org/project/haggle/) installed, you'll find that ``kaggle``
|
|
84
|
-
is also a valid protocol.
|
|
85
|
-
|
|
86
|
-
>>> sorted(grab.dflt_protocols) # doctest: +SKIP
|
|
87
|
-
['file', 'http', 'https', 'kaggle']
|
|
88
|
-
|
|
89
|
-
Notice that ``grab`` has an argument called ``protocols``.
|
|
90
|
-
Yes, this means you have control.
|
|
91
|
-
You just need to specify a mapping between protocol strings and the "url_to_content"
|
|
92
|
-
function that should be used to get the content from that url.
|
|
93
|
-
|
|
94
|
-
In the following we'll add a fake ``foo`` protocol that doesn't really fetch any
|
|
95
|
-
data, but applies ``str.upper`` to the url, but you get the point.
|
|
96
|
-
|
|
97
|
-
>>> from functools import partial
|
|
98
|
-
>>> from pyckup.base import dflt_protocols
|
|
99
|
-
>>> mygrab = partial(grab, protocols=dict(dflt_protocols, foo=str.upper))
|
|
100
|
-
>>> mygrab('foo://a_fake_url')
|
|
101
|
-
'FOO://A_FAKE_URL'
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
# Examples
|
|
105
|
-
|
|
106
|
-
```python
|
|
107
|
-
from pyckup import grab
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
See what (default) protocols you have access to.
|
|
111
|
-
|
|
112
|
-
```python
|
|
113
|
-
from pyckup import grab
|
|
114
|
-
|
|
115
|
-
grab.procotols
|
|
116
|
-
# ['file', 'kaggle', 'http', 'https']
|
|
117
|
-
```
|
|
118
|
-
|
|
119
|
-
## Grab file contents
|
|
120
|
-
|
|
121
|
-
Specifying a "file" protocol (i.e. prefixing your string with
|
|
122
|
-
"file://" -- followed by a full path) will give you the contents of the file
|
|
123
|
-
in bytes.
|
|
124
|
-
|
|
125
|
-
```python
|
|
126
|
-
from pyckup import grab
|
|
127
|
-
|
|
128
|
-
b = grab('file:///Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
129
|
-
assert isinstance(b, bytes)
|
|
130
|
-
print(b.decode())
|
|
131
|
-
# from pyckup.base import grab, protocols
|
|
132
|
-
```
|
|
133
|
-
|
|
134
|
-
But you can also use a full path, or other natural means of specifying files.
|
|
135
|
-
In that case though, `grab` will try to give you the contents in a convenient type
|
|
136
|
-
(e.g. a `dict` for `.json`, a python object of `.pickle`, string for `.txt`...).
|
|
137
|
-
This is convenient, but don't depend on the type to strongly
|
|
138
|
-
since it depends on what `py2store.misc` sets it to be.
|
|
139
|
-
|
|
140
|
-
```python
|
|
141
|
-
from pyckup import grab
|
|
142
|
-
|
|
143
|
-
grab('/Users/Thor.Whalen/Dropbox/dev/p3/proj/i/pyckup/pyckup/__init__.py')
|
|
144
|
-
# b'from pyckup.base import grab, protocols\n\n\n'
|
|
145
|
-
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.json')
|
|
146
|
-
# {'hello': 'world', 'abc': [1, 2, 3]}
|
|
147
|
-
grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.pickle')
|
|
148
|
-
# [1, 2, 3]
|
|
149
|
-
print(grab('~/Dropbox/dev/p3/proj/i/pyckup/data/example.txt'))
|
|
150
|
-
# This
|
|
151
|
-
# is
|
|
152
|
-
# text
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
## Grab the contents of a url
|
|
156
|
-
|
|
157
|
-
```python
|
|
158
|
-
from pyckup import grab
|
|
159
|
-
|
|
160
|
-
b = grab('https://raw.githubusercontent.com/i2mint/pyckup/master/LICENSE')
|
|
161
|
-
type(b), len(b)
|
|
162
|
-
# (bytes, 11357)
|
|
163
|
-
print(b[:100].decode())
|
|
164
|
-
# Apache License
|
|
165
|
-
# Version 2.0, January 2004
|
|
166
|
-
```
|
|
167
|
-
|
|
168
|
-
## Grab stuff from kaggle
|
|
169
|
-
|
|
170
|
-
Note: If you want to access kaggle datasets with ``pyckup``,
|
|
171
|
-
you'll need to get an account.
|
|
172
|
-
See [haggle](https://github.com/otosense/haggle#api-credentials)
|
|
173
|
-
for more information.
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
```python
|
|
177
|
-
from pyckup import grab
|
|
178
|
-
|
|
179
|
-
z = grab('kaggle://drgilermo/face-images-with-marked-landmark-points')
|
|
180
|
-
list(z)
|
|
181
|
-
# ['face_images.npz', 'facial_keypoints.csv']
|
|
182
|
-
print(z['facial_keypoints.csv'][:100].decode())
|
|
183
|
-
# left_eye_center_x,left_eye_center_y,right_eye_center_x,right_eye_center_y,left_eye_inner_corner_x,le
|
|
184
|
-
```
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
Keywords: data,data access,data preparation,machine learning,artificial intelligence
|
|
189
|
-
Platform: any
|
|
190
|
-
Description-Content-Type: text/markdown
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|