nimporter-plus 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nimporter_plus-0.0.2/.gitignore +126 -0
- nimporter_plus-0.0.2/LICENSE +20 -0
- nimporter_plus-0.0.2/PKG-INFO +535 -0
- nimporter_plus-0.0.2/README.md +501 -0
- nimporter_plus-0.0.2/nimporter_plus/__init__.py +21 -0
- nimporter_plus-0.0.2/nimporter_plus/cli.py +206 -0
- nimporter_plus-0.0.2/nimporter_plus/lib.py +345 -0
- nimporter_plus-0.0.2/nimporter_plus/nexporter.py +302 -0
- nimporter_plus-0.0.2/nimporter_plus/nimporter.py +288 -0
- nimporter_plus-0.0.2/pyproject.toml +90 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Nimporter
|
|
2
|
+
nim-extensions/
|
|
3
|
+
|
|
4
|
+
# VS Code
|
|
5
|
+
.vscode/
|
|
6
|
+
|
|
7
|
+
# MacOS
|
|
8
|
+
*.DS_STORE
|
|
9
|
+
|
|
10
|
+
# Nim
|
|
11
|
+
nimcache/
|
|
12
|
+
nimblecache/
|
|
13
|
+
|
|
14
|
+
# Byte-compiled / optimized / DLL files
|
|
15
|
+
__pycache__/
|
|
16
|
+
*.py[cod]
|
|
17
|
+
*$py.class
|
|
18
|
+
|
|
19
|
+
# C extensions
|
|
20
|
+
*.so
|
|
21
|
+
|
|
22
|
+
# Distribution / packaging
|
|
23
|
+
.Python
|
|
24
|
+
build/
|
|
25
|
+
develop-eggs/
|
|
26
|
+
dist/
|
|
27
|
+
downloads/
|
|
28
|
+
eggs/
|
|
29
|
+
.eggs/
|
|
30
|
+
lib/
|
|
31
|
+
lib64/
|
|
32
|
+
parts/
|
|
33
|
+
sdist/
|
|
34
|
+
var/
|
|
35
|
+
wheels/
|
|
36
|
+
*.egg-info/
|
|
37
|
+
.installed.cfg
|
|
38
|
+
*.egg
|
|
39
|
+
MANIFEST
|
|
40
|
+
|
|
41
|
+
# PyInstaller
|
|
42
|
+
# Usually these files are written by a python script from a template
|
|
43
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
44
|
+
*.manifest
|
|
45
|
+
*.spec
|
|
46
|
+
|
|
47
|
+
# Installer logs
|
|
48
|
+
pip-log.txt
|
|
49
|
+
pip-delete-this-directory.txt
|
|
50
|
+
|
|
51
|
+
# Unit test / coverage reports
|
|
52
|
+
htmlcov/
|
|
53
|
+
.tox/
|
|
54
|
+
.coverage
|
|
55
|
+
.coverage.*
|
|
56
|
+
.cache
|
|
57
|
+
nosetests.xml
|
|
58
|
+
coverage.xml
|
|
59
|
+
*.cover
|
|
60
|
+
.hypothesis/
|
|
61
|
+
.pytest_cache/
|
|
62
|
+
|
|
63
|
+
# Translations
|
|
64
|
+
*.mo
|
|
65
|
+
*.pot
|
|
66
|
+
|
|
67
|
+
# Django stuff:
|
|
68
|
+
*.log
|
|
69
|
+
local_settings.py
|
|
70
|
+
db.sqlite3
|
|
71
|
+
|
|
72
|
+
# Flask stuff:
|
|
73
|
+
instance/
|
|
74
|
+
.webassets-cache
|
|
75
|
+
|
|
76
|
+
# Scrapy stuff:
|
|
77
|
+
.scrapy
|
|
78
|
+
|
|
79
|
+
# Sphinx documentation
|
|
80
|
+
docs/_build/
|
|
81
|
+
|
|
82
|
+
# PyBuilder
|
|
83
|
+
target/
|
|
84
|
+
|
|
85
|
+
# Jupyter Notebook
|
|
86
|
+
.ipynb_checkpoints
|
|
87
|
+
|
|
88
|
+
# pyenv
|
|
89
|
+
.python-version
|
|
90
|
+
|
|
91
|
+
# celery beat schedule file
|
|
92
|
+
celerybeat-schedule
|
|
93
|
+
|
|
94
|
+
# SageMath parsed files
|
|
95
|
+
*.sage.py
|
|
96
|
+
|
|
97
|
+
# Environments
|
|
98
|
+
.env
|
|
99
|
+
.venv
|
|
100
|
+
env/
|
|
101
|
+
venv/
|
|
102
|
+
ENV/
|
|
103
|
+
env.bak/
|
|
104
|
+
venv.bak/
|
|
105
|
+
|
|
106
|
+
# Spyder project settings
|
|
107
|
+
.spyderproject
|
|
108
|
+
.spyproject
|
|
109
|
+
|
|
110
|
+
# Rope project settings
|
|
111
|
+
.ropeproject
|
|
112
|
+
|
|
113
|
+
# mkdocs documentation
|
|
114
|
+
/site
|
|
115
|
+
|
|
116
|
+
# mypy
|
|
117
|
+
.mypy_cache/
|
|
118
|
+
|
|
119
|
+
# Pycharm settings
|
|
120
|
+
.idea/uv_publish.sh
|
|
121
|
+
|
|
122
|
+
# Nimporter Test Cache & Generated Files
|
|
123
|
+
test_cache/
|
|
124
|
+
tests/data/**/.gitignore
|
|
125
|
+
tests/data/**/config.nims
|
|
126
|
+
test-projects/
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
The MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Samuel Wilder, https://github.com/Pebaz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
|
7
|
+
the Software without restriction, including without limitation the rights to
|
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
|
10
|
+
subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nimporter-plus
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Compile Nim extensions for Python when imported!
|
|
5
|
+
Project-URL: Homepage, https://github.com/lucidrains/nimporter
|
|
6
|
+
Project-URL: Repository, https://github.com/lucidrains/nimporter
|
|
7
|
+
Project-URL: Documentation, https://pebaz.github.io/nimporter/index.html
|
|
8
|
+
Author-email: Pebaz <pebazium@gmail.com>, SekouDiaoNlp <diao.sekou.nlp@gmail.com>
|
|
9
|
+
Maintainer-email: Phil Wang <lucidrains@gmail.com>
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: compiler,cython,cython-alternative,import,nim,nim-compiler,nim-source,nimporter-library,nimpy,performance,python,transpiler
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Education
|
|
16
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
|
+
Classifier: Natural Language :: English
|
|
20
|
+
Classifier: Operating System :: OS Independent
|
|
21
|
+
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
27
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
28
|
+
Classifier: Topic :: Utilities
|
|
29
|
+
Requires-Python: >=3.7
|
|
30
|
+
Requires-Dist: cookiecutter>=2.1.1
|
|
31
|
+
Requires-Dist: icecream>=2.1.3
|
|
32
|
+
Requires-Dist: py-cpuinfo>=9.0.0
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
<p align="center">
|
|
37
|
+
<img src=misc/nimporter-logo.svg>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
# Nimporter
|
|
41
|
+
|
|
42
|
+
> **Note:** Fork of the unmaintained original [Nimporter by Pebaz](https://github.com/Pebaz/nimporter) with threading and packaging fixes.
|
|
43
|
+
> *Directly import [Nim](<https://nim-lang.org/>) extensions for Python and
|
|
44
|
+
seamlessly package them for distribution in **1 line of code.***
|
|
45
|
+
|
|
46
|
+
<p align="center">
|
|
47
|
+
<img src=misc/Nimporter-Functionality.png>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
<p align="center">
|
|
51
|
+
<img src=misc/Nimporter-Setup.py.png>
|
|
52
|
+
</p>
|
|
53
|
+
|
|
54
|
+
## ๐ฑ Benefits
|
|
55
|
+
|
|
56
|
+
* **๐ Performance**: Nim compiles to C
|
|
57
|
+
|
|
58
|
+
* **๐ Distribution**: Packaging Nimporter libraries is the primary use case
|
|
59
|
+
|
|
60
|
+
* **๐ฆ Invisible**: End users do not need to install Nim for source or binary
|
|
61
|
+
distributions
|
|
62
|
+
|
|
63
|
+
* **โป๏ธ Ecosystem**: Leverage [Python](https://pypi.org/) libraries for breadth
|
|
64
|
+
and [Nim](https://nimble.directory/) libraries for performance.
|
|
65
|
+
|
|
66
|
+
* **๐งฃ Seamless**: Integration with existing Nim code uses the
|
|
67
|
+
[Nimpy](https://github.com/yglukhov/nimpy) library.
|
|
68
|
+
|
|
69
|
+
* **๐ Simple**: Nimporter barely has a user interface at all
|
|
70
|
+
|
|
71
|
+
## ๐ฃ Installation
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# ๐ From Pypi:
|
|
75
|
+
$ pip install nimporter-plus
|
|
76
|
+
|
|
77
|
+
# โ๏ธ From GitHub:
|
|
78
|
+
$ pip install git+https://github.com/lucidrains/nimporter
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**Library Author Dependencies:**
|
|
82
|
+
|
|
83
|
+
1. [Nim Compiler](<https://nim-lang.org/install.html>) (for compiling Nim
|
|
84
|
+
source files)
|
|
85
|
+
2. [Nimpy library](https://github.com/yglukhov/nimpy) (installed automatically
|
|
86
|
+
if `nimporter init lib` is used)
|
|
87
|
+
3. [Nimporter library](https://github.com/lucidrains/nimporter) (distributed
|
|
88
|
+
libraries will need access to Nimporter).
|
|
89
|
+
|
|
90
|
+
Nimporter can work seamlessly when Nim is installed via
|
|
91
|
+
[Choosenim](https://nim-lang.org/install_unix.html#installation-using-choosenim)
|
|
92
|
+
or manually. No additional configuration is necessary once installed since
|
|
93
|
+
Nimporter can find the Nim standard library and install
|
|
94
|
+
[Nimpy library](https://github.com/yglukhov/nimpy) if Nimble is on your path.
|
|
95
|
+
|
|
96
|
+
**End User Dependencies:**
|
|
97
|
+
|
|
98
|
+
Users of Nimporter libraries only need Nimporter! ๐
|
|
99
|
+
|
|
100
|
+
## ๐ Documentation
|
|
101
|
+
|
|
102
|
+
To get started, first look at the [Nimpy](https://github.com/yglukhov/nimpy)
|
|
103
|
+
project as that is how Nim libraries are created that can be imported into
|
|
104
|
+
Python. During development, Python can directly import the Nim file and build
|
|
105
|
+
the public user-facing Python API in tandem with the Nim library extension. For
|
|
106
|
+
assistance with the Nim language, look at
|
|
107
|
+
[Nim For Python Programmers](https://github.com/nim-lang/Nim/wiki/Nim-for-Python-Programmers#table-of-contents)
|
|
108
|
+
as it is a great resource for getting up to speed quickly with Nim. Finally,
|
|
109
|
+
Nimporter's unit tests all make use of a
|
|
110
|
+
[reference project](https://github.com/lucidrains/nimporter/tree/master/tests/data)
|
|
111
|
+
that was designed to use each of Nimporter's features.
|
|
112
|
+
|
|
113
|
+
* ๐ฆ [Nim For Python Programmers](https://github.com/nim-lang/Nim/wiki/Nim-for-Python-Programmers#table-of-contents)
|
|
114
|
+
* ๐ฆ [Nimpy](https://github.com/yglukhov/nimpy)
|
|
115
|
+
* ๐
[Reference Project](https://github.com/lucidrains/nimporter/tree/master/tests/data)
|
|
116
|
+
|
|
117
|
+
Additionally, the
|
|
118
|
+
[Nimpy tests folder](https://github.com/yglukhov/nimpy/tree/master/tests)
|
|
119
|
+
contains code examples on these topics:
|
|
120
|
+
|
|
121
|
+
* Passing/Returning None, booleans, integers, floats, strings, lists, tuples, dictionaries, JSON, and objects.
|
|
122
|
+
* Defining/Raising Python exceptions from Nim.
|
|
123
|
+
* Yielding values back to Python using iterators in Nim.
|
|
124
|
+
* Exposing Nim functions with custom names.
|
|
125
|
+
* Exposing Nim extension modules with customized names and docstrings.
|
|
126
|
+
* Using Python builtin functions in Nim.
|
|
127
|
+
* Using passed Python objects and accessing methods and fields.
|
|
128
|
+
* Passing keyword arguments to a Python function.
|
|
129
|
+
|
|
130
|
+
## ๐ Features
|
|
131
|
+
|
|
132
|
+
* Directly import Nim Extension Modules & Extension Libraries using Nimpy
|
|
133
|
+
* Cache build artifacts for quicker subsequent runs
|
|
134
|
+
* Invalidate cache using hash files
|
|
135
|
+
* Stores artifacts and hash files in `__pycache__` to not clutter project
|
|
136
|
+
* Build Source & Binary Distributions using Nimporer with 1 line of code
|
|
137
|
+
* Command Line Interface for introspecting, initializing, and compiling
|
|
138
|
+
projects
|
|
139
|
+
* Nimporter does not require that library end-users install a Nim compiler
|
|
140
|
+
|
|
141
|
+
## ๐ ๏ธ Usage
|
|
142
|
+
|
|
143
|
+

|
|
144
|
+
|
|
145
|
+
Nimporter is a library that allows the seamless import & packaging of Nim
|
|
146
|
+
extensions for Python built with [Nimpy](https://github.com/yglukhov/nimpy).
|
|
147
|
+
Nimpy is a library that is used on the Nim side for iteroperability with
|
|
148
|
+
Python. All Nimporter libraries rely on Nimpy in order to expose Nim functions
|
|
149
|
+
to Python. Nimporter's role in this is to formalize a method of distributing
|
|
150
|
+
Nimpy libraries to ease the burden on library maintainers and end users so that
|
|
151
|
+
they do not have to even have knowledge of Nim in order to use the library.
|
|
152
|
+
|
|
153
|
+
Nimpy is a complete library by itself. For information on how to integrate Nim
|
|
154
|
+
and Python, look at the
|
|
155
|
+
[Nimpy documentation](https://github.com/yglukhov/nimpy) as it will be the Nim
|
|
156
|
+
day-to-day development experience. Nimporter's role comes into play when a
|
|
157
|
+
library is ready to be distributed. **Nimporter handles the entire packaging
|
|
158
|
+
for source and binary distributions in 1 line of code.**
|
|
159
|
+
|
|
160
|
+
**Important Considerations**
|
|
161
|
+
|
|
162
|
+
Nimporter was designed to help bridge the ecosystem gap between Python and Nim
|
|
163
|
+
while utilizing Nimpy so that library authors could seamlessly develop and
|
|
164
|
+
distribute their libraries. Due to this fact, there are important limitations
|
|
165
|
+
to consider when using Nimporter. They are described below:
|
|
166
|
+
|
|
167
|
+
1. Importing: Nimporter uses the C compiler that was used to build Python when
|
|
168
|
+
importing a Nim module/library. This can be overridden in a
|
|
169
|
+
`<lib name>.nim.cfg` but doing so means that the library will most likely
|
|
170
|
+
not work on other platforms.
|
|
171
|
+
|
|
172
|
+
2. Distributing Sources: Nimporter sets the C compiler automatically by
|
|
173
|
+
iterating through MSVC and GCC for each platform and architecture combo.
|
|
174
|
+
This means that there will likely be several copies of the generated C
|
|
175
|
+
source code for each supported platform (given in `get_nim_extensions()`).
|
|
176
|
+
|
|
177
|
+
3. Distributing Binaries: Nimporter uses the same process described for direct
|
|
178
|
+
import of Nim code and will use the same C compiler that was used to build
|
|
179
|
+
Python itself.
|
|
180
|
+
|
|
181
|
+
### ๐ป Instrumentation
|
|
182
|
+
|
|
183
|
+
To enable Nimporter debug traces, define `NIMPORTER_INSTRUMENT` in the
|
|
184
|
+
environment and Nimporter will use
|
|
185
|
+
[IceCream](https://github.com/gruns/icecream) to show output from Nim and other
|
|
186
|
+
interesting bits necessary for debugging any issues that could arise.
|
|
187
|
+
|
|
188
|
+
### ๐ฆ Extension Modules & Extension Libraries
|
|
189
|
+
|
|
190
|
+
Extension Modules are distinct from Extension Libraries. Nimporter (not Nimpy)
|
|
191
|
+
makes a distinction here. However, it is of special note that distribution of
|
|
192
|
+
either extension type is the same (`nimporter.get_nim_extensions()`).
|
|
193
|
+
|
|
194
|
+
**๐ฆ Extension Libraries**
|
|
195
|
+
|
|
196
|
+
Extension Libraries are entire Nim projects exposed as a single module from the
|
|
197
|
+
perspective of Python. They are comprised of a single folder containing all
|
|
198
|
+
code and configuration for the extension. *It is important to note that they
|
|
199
|
+
are a concept formalized by the Nimporter project and must accept some
|
|
200
|
+
limitations.*
|
|
201
|
+
|
|
202
|
+
These limitations (and capabilities) are listed below:
|
|
203
|
+
|
|
204
|
+
* โ๏ธ Can have external Nim dependencies: inside the Extension Library folder,
|
|
205
|
+
use a `<library name>.nimble` in order to depend upon other Nim libraries.
|
|
206
|
+
|
|
207
|
+
* โ๏ธ Can be split up into any number of Nim modules: the Extension Library
|
|
208
|
+
folder can contain any desired inner folder structure.
|
|
209
|
+
|
|
210
|
+
* โ๏ธ CLI switches used by Nim & the C compiler can be customized: this can be
|
|
211
|
+
useful but be cognizant about cross-platform compatibility. Remember, if
|
|
212
|
+
the C compiler used by Python is different than the one used by Nim, there
|
|
213
|
+
*will definitely without a doubt* be strange issues arising from this. Note
|
|
214
|
+
that choosing a different C compiler may result in the `setup.py` not being
|
|
215
|
+
able to compile the extension. Use a `<library name>.nim.cfg` for this use
|
|
216
|
+
case.
|
|
217
|
+
|
|
218
|
+
* โ Must use folder structure known to Nimporter: the below folder structure
|
|
219
|
+
is generated when `nimporter init lib` is used:
|
|
220
|
+
|
|
221
|
+
```
|
|
222
|
+
the_library_name/
|
|
223
|
+
the_library_name.nim # Must be present
|
|
224
|
+
the_library_name.nim.cfg # Must be present even if empty
|
|
225
|
+
the_library_name.nimble # Must contain `requires "nimpy"`
|
|
226
|
+
```
|
|
227
|
+
**๐ด Extension Modules**
|
|
228
|
+
|
|
229
|
+
Extension Modules are the simplest form of using Nimpy libraries with existing
|
|
230
|
+
Python code. Once Nimporter is imported, Nimpy libraries can be directly
|
|
231
|
+
imported like normal Python modules. However, there are a few restrictions on
|
|
232
|
+
what is supported when importing a Nim module in this way. It is important to
|
|
233
|
+
remember that Nim compiles to C and therefore could theoretically integrate
|
|
234
|
+
with a build system that is extremely brittle. To completely solve this,
|
|
235
|
+
Nimporter disallows certain use cases that are technically possible but would
|
|
236
|
+
otherwise prevent widespread use of the resulting technology.
|
|
237
|
+
|
|
238
|
+
Below are the restrictions present when importing a Nim Extension Module:
|
|
239
|
+
|
|
240
|
+
* โ Cannot have any dependencies other than `Nimpy`: this is due to the fact
|
|
241
|
+
that Nimporter disallows multiple `*.nimble` files strewn about in a Python
|
|
242
|
+
project. Use an Extension Library for this use case.
|
|
243
|
+
|
|
244
|
+
* โ Cannot import other Nim modules in same directory: this is because there
|
|
245
|
+
is no way to tell which files pertain to each extension and knowing this is
|
|
246
|
+
a prerequisite to packaging the extension up for distribution.
|
|
247
|
+
Additionally, Nimporter moves extensions to temporary directories during
|
|
248
|
+
compilation in order to control where the Nim compiler places the resultant
|
|
249
|
+
C sources.
|
|
250
|
+
|
|
251
|
+
* โ Cannot customize Nim or C compiler switches: proliferating a Python
|
|
252
|
+
package with these extra files would be unsightly and it is possible to
|
|
253
|
+
have two different Nim modules with custom configurations collide in
|
|
254
|
+
awkward ways. If CLI configuration is required, use an Extension Library.
|
|
255
|
+
|
|
256
|
+
* โ Cannot override the C compiler used to build the extension: Although this
|
|
257
|
+
practice is certainly and technically possible, it is unequivocally a bad
|
|
258
|
+
decision when integrating software originating from a different compilers.
|
|
259
|
+
If an expert user is in need of this capability, use an Extension Library.
|
|
260
|
+
|
|
261
|
+
Although these restrictions limit the number of possible use cases for the
|
|
262
|
+
integration of Nim & Python, portability, compatibility, and stability were
|
|
263
|
+
chosen as the guiding principles for Nimporter.
|
|
264
|
+
|
|
265
|
+
## ๐ฆ Distribution
|
|
266
|
+
|
|
267
|
+
There are a few ways to use Nimporter to integrate Nim & Python code:
|
|
268
|
+
|
|
269
|
+
1. ๐ฅ Library uses Nim code internally but exposes a Python API: this is the
|
|
270
|
+
reason why Nimporter was built. It was built to allow Python library
|
|
271
|
+
authors to use Nim to speed up their library.
|
|
272
|
+
|
|
273
|
+
2. ๐ฅ Application uses Nim code: this is very possible but it is recommended to
|
|
274
|
+
pull out the Nim code into a Python library that imports that Nim code in
|
|
275
|
+
order to take advantage of the amazing distribution features of Nimporter.
|
|
276
|
+
Having a separately-updatable library that the application imports greatly
|
|
277
|
+
streamlines development and reduces packaging difficulty (the Python
|
|
278
|
+
library dependency that imports Nim code behaves exactly like a pure-Python
|
|
279
|
+
dependency).
|
|
280
|
+
|
|
281
|
+
3. ๐ฅ Docker: this is a possible application of Nimporter, but it requires the
|
|
282
|
+
use of `nimporter compile` in order to let the Docker container not have to
|
|
283
|
+
contain a Nim & C compiler and to ensure that the builds are cached.
|
|
284
|
+
|
|
285
|
+
Amazingly, Nimporter allows the end user installing a library built with
|
|
286
|
+
Nimporter to not have to install Nim! ๐ฅณ This is incredible and is accomplished
|
|
287
|
+
by recompiling the same Nim extension to every desired platform, architecture,
|
|
288
|
+
and C compiler that the library is supported on. Specifically, Nimporter tells
|
|
289
|
+
the Nim compiler to compile the extension to C once for Windows, MacOS, and
|
|
290
|
+
Linux and and then bundles all of the resulting C source files into the source
|
|
291
|
+
distribution. At the time of the installation on the end user's machine, the
|
|
292
|
+
appropriate set of C source files is selected that matches the user's
|
|
293
|
+
environment! ๐
|
|
294
|
+
|
|
295
|
+
For binary distributions, this process just skips to the one set of C source
|
|
296
|
+
files that matches the host's environment. One binary distribution per
|
|
297
|
+
supported platform must then be built.
|
|
298
|
+
|
|
299
|
+
This might sound complicated but Nimporter accomplishes this by requesting that
|
|
300
|
+
the `setup.py` contain 1 line of code to find, compile, and bundle all of the C
|
|
301
|
+
files necessary to be portable across platform, architecture, and C compilers.
|
|
302
|
+
|
|
303
|
+
### ๐ง Source Distributions
|
|
304
|
+
|
|
305
|
+
To create a source distribution, it is assumed that the `setup.py` contains a
|
|
306
|
+
dependency on Nimporter as well as a call to `get_nim_extensions()`.
|
|
307
|
+
|
|
308
|
+
```python
|
|
309
|
+
# Example setup.py
|
|
310
|
+
|
|
311
|
+
import setuptools
|
|
312
|
+
from nimporter import get_nim_extensions, WINDOWS, MACOS, LINUX
|
|
313
|
+
|
|
314
|
+
setuptools.setup(
|
|
315
|
+
name='calculatorlib',
|
|
316
|
+
install_requires=['nimporter'],
|
|
317
|
+
py_modules=['calculatorlib.py'],
|
|
318
|
+
ext_modules=get_nim_extensions(platforms=[WINDOWS, LINUX, MACOS])
|
|
319
|
+
)
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
The below command will create a source distribution in the `dist/` directory
|
|
323
|
+
and can be easily uploaded to PyPI.
|
|
324
|
+
|
|
325
|
+
```bash
|
|
326
|
+
$ python setup.py sdist # Contains entire matrix of supported platforms, etc.
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
> Note: when an end-user tries to install a Nimporter library from GitHub
|
|
330
|
+
directly, it is required that the Nim compiler and a compatible C compiler
|
|
331
|
+
is installed because `setup.py install` is invoked which is equivalent to a
|
|
332
|
+
binary distribution but does require the Nim & C compilers to be installed.
|
|
333
|
+
|
|
334
|
+
### ๐ฟ Binary Distributions
|
|
335
|
+
|
|
336
|
+
Binary distributions use the same `setup.py` structure mentioned above.
|
|
337
|
+
|
|
338
|
+
The below command will create a Python Wheel in the `dist/` directory that can
|
|
339
|
+
be easily uploaded to PyPI.
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
$ python setup.py bdist_wheel # Contains a single supported platform, etc.
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
> Note: A Nim compiler and C compiler is required when creating a binary
|
|
346
|
+
distribution.
|
|
347
|
+
|
|
348
|
+
> Special note for Linux users: Unfortunately, PyPi will not allow you to
|
|
349
|
+
upload just any Linux wheel. There is a special compilation process that
|
|
350
|
+
can be explained [here](https://github.com/pypa/manylinux). Interestingly
|
|
351
|
+
enough, I got around this by simply renaming the resulting Linux build
|
|
352
|
+
according to the **manylinux1** naming convention. You can see my solution
|
|
353
|
+
in the `examples/github_actions_template.yml` file for the `build-linux`
|
|
354
|
+
job. I expect that there could be many downsides of using this hack but it
|
|
355
|
+
worked for me on 2 different Linux platforms.
|
|
356
|
+
|
|
357
|
+
### โญ Publish Build Artifacts to PyPi Automatically
|
|
358
|
+
|
|
359
|
+
For a dead-simple way to publish Windows, MacOS, and Linux packages to PyPi
|
|
360
|
+
automatically, use the `github_actions_template.yml` template found in the
|
|
361
|
+
`examples/` directory. This template integrates with your repository's GitHub
|
|
362
|
+
Actions runner to build, package, and deploy your library on Windows, MacOS,
|
|
363
|
+
and Linux automatically when you create a new "Release" is created.
|
|
364
|
+
|
|
365
|
+
## ๐ฝ Computer Hardware Actually Exists
|
|
366
|
+
|
|
367
|
+
Dynamic, safe programming languages are great, but naturally, when integrating
|
|
368
|
+
with native code, there are limitations to what is possible to accomplish in
|
|
369
|
+
certain situations. On Windows, a DLL that has been loaded into a process
|
|
370
|
+
cannot be deleted while it is in use. Additionally, Windows has a path length
|
|
371
|
+
limit of 260 characters by default (and therefore relying on the user having
|
|
372
|
+
disabled this limit in the system registry is not possible). This severely
|
|
373
|
+
limits how deep a Nim extension can be placed into a Python package hierarchy.
|
|
374
|
+
Furthermore, generously-named Nim extensions may fail to compile with a message
|
|
375
|
+
that resembles:
|
|
376
|
+
|
|
377
|
+
```
|
|
378
|
+
failed to open compiler generated file: ''
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
> If this message occurs, it is due to the path length limit of 260 characters.
|
|
382
|
+
Shorten the name of the Nim extension and make the package hierarchy
|
|
383
|
+
shallower. More information about the 260 character path limit can be found
|
|
384
|
+
[here](https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd).
|
|
385
|
+
|
|
386
|
+
Nimporter comes with a defense against this behavior by automatically renaming
|
|
387
|
+
the generated C sources to not contain the `@m` and `@s` symbols that
|
|
388
|
+
proliferate the filename and soak up most of the 260 character budget. For
|
|
389
|
+
instance, the filename:
|
|
390
|
+
|
|
391
|
+
```
|
|
392
|
+
@m..@s..@s..@s..@s..@s..@sUsers@s<USERNAME>@s.nimble@spkgs@snimpy-0.2.0@snimpy@spy_utils.nim.c
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
Gets turned into:
|
|
396
|
+
|
|
397
|
+
```
|
|
398
|
+
NIMPORTER@nimpy-0.2.0@nimpy@py_utils.nim.c
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
Much shorter! ๐
|
|
402
|
+
|
|
403
|
+
## ๐งโ๐ป Nimporter Command Line Interface
|
|
404
|
+
|
|
405
|
+
Nimporter provides a CLI that you can use to easily clean all cached build and
|
|
406
|
+
hash files from your project recursively. This can be very useful for debugging
|
|
407
|
+
situations arising from stale builds.
|
|
408
|
+
|
|
409
|
+
Usage example:
|
|
410
|
+
|
|
411
|
+
```bash
|
|
412
|
+
# Removes all __pycache__ directories with .hash, .pyd/.so, and .pyc files
|
|
413
|
+
$ nimporter clean
|
|
414
|
+
```
|
|
415
|
+
|
|
416
|
+
The Nimporter CLI can also precompile all extensions within a project without
|
|
417
|
+
needing to run the project. This is useful in situations where you do not want
|
|
418
|
+
to package your application using a `setup.py` (such as a zip file) or for use
|
|
419
|
+
within Docker containers.
|
|
420
|
+
|
|
421
|
+
```bash
|
|
422
|
+
# Recursively compile all Nim extension modules and libraries:
|
|
423
|
+
$ nimporter compile
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
Finally, the CLI has provisions for listing out the extensions that it can
|
|
427
|
+
auto-detect. This is useful to identify if an extension folder structure is
|
|
428
|
+
properly setup.
|
|
429
|
+
|
|
430
|
+
```bash
|
|
431
|
+
# List all extensions that Nimporter will find when handling imports
|
|
432
|
+
$ nimporter list
|
|
433
|
+
```
|
|
434
|
+
|
|
435
|
+
## โ Usage with Docker
|
|
436
|
+
|
|
437
|
+
Nimporter can easily be used within a Docker container. To prevent the need for
|
|
438
|
+
a Nim compiler toolchain to be installed into the container to run Nim code,
|
|
439
|
+
the extensions can be precompiled and copied into the container. This process
|
|
440
|
+
is roughly as follows:
|
|
441
|
+
|
|
442
|
+
1. Create a project that uses Python and Nim
|
|
443
|
+
2. Run `nimporter compile` to recursively-compile all extensions in the project
|
|
444
|
+
3. Ensure that in your Dockerfile that the `__pycache__` directories are
|
|
445
|
+
included as they will contain the Nim shared objects as well as the
|
|
446
|
+
Nimporter hash files to prevent a recompilation (which would fail without a
|
|
447
|
+
Nim & C compiler installed in the container).
|
|
448
|
+
|
|
449
|
+
## ๐งช Running The Tests
|
|
450
|
+
|
|
451
|
+
To run Nimporter's test suite on your local machine, you will need to install a
|
|
452
|
+
Nim compiler. This example will assume you are cloning the GitHub repository.
|
|
453
|
+
|
|
454
|
+
```bash
|
|
455
|
+
$ git clone https://github.com/lucidrains/nimporter
|
|
456
|
+
$ cd nimporter
|
|
457
|
+
$ pip install -r requirements_dev.txt
|
|
458
|
+
$ pip install . # Nimporter is needed for the integration tests
|
|
459
|
+
$ pytest --cov=. --cov-report=html tests
|
|
460
|
+
```
|
|
461
|
+
|
|
462
|
+
## โ How Does Nimporter Work?
|
|
463
|
+
|
|
464
|
+
Nimporter provides essentially two capabilities:
|
|
465
|
+
|
|
466
|
+
* The ability to directly import Nim code
|
|
467
|
+
* The ability to bundle Python-compatible extensions for any supported platform
|
|
468
|
+
|
|
469
|
+
The way it accomplishes the ability to import Nim code is by adding two custom
|
|
470
|
+
importers to the Python import machinery. This is why it is required to import
|
|
471
|
+
Nimporter before importing any Nim code because the Python import machinery
|
|
472
|
+
must be amended with the custom importers.
|
|
473
|
+
|
|
474
|
+
The first one is for the ability to search and import Nim modules. When a Nim
|
|
475
|
+
module is found, Nimporter first looks in the `__pycache__` directory to see if
|
|
476
|
+
there is already a built version of the module. If there is not, it builds a
|
|
477
|
+
new one and stores it in the `__pycache__` directory.
|
|
478
|
+
|
|
479
|
+
If one is found, it could be stale, meaning the Nim file could have been
|
|
480
|
+
modified since it was built. To keep track of this, a hash of the source file
|
|
481
|
+
is also kept in the `__pycache__` directory and is consulted whenever there is
|
|
482
|
+
a possibility that a stale build could be imported.
|
|
483
|
+
|
|
484
|
+
When a Nim module and a Python module have the same name and reside in the same
|
|
485
|
+
folder, the Python module is given precedence. *Please don't do this.*
|
|
486
|
+
|
|
487
|
+
The second custom importer has the exact same purpose of the first one except
|
|
488
|
+
it is used to import Nim extension libraries. A library is any folder within a
|
|
489
|
+
Python project that contains a `<lib name>.nim`, a `<lib name>.nimble`, and a
|
|
490
|
+
`<lib name>.nim.cfg`.
|
|
491
|
+
|
|
492
|
+
These files mark that the folder should be treated as one unit. It also makes
|
|
493
|
+
it so that Nimble dependencies can be installed.
|
|
494
|
+
|
|
495
|
+
As for the second capability, Nimporter helps you bundle and distribute Nim
|
|
496
|
+
code as part of a source or binary distribution extremely easily.
|
|
497
|
+
|
|
498
|
+
The way it works is by iterating through your entire project and identifying
|
|
499
|
+
any Nim module and Nim library that it finds and compiling them to C using a
|
|
500
|
+
feature of Nim that specifically supports this.
|
|
501
|
+
|
|
502
|
+
Why compile to C? Because Python already has extensive infrastructure to
|
|
503
|
+
support the compilation and distribution of C extensions.
|
|
504
|
+
|
|
505
|
+
Once each Nim module and library is compiled to C, Python deals with them the
|
|
506
|
+
exact same way as a typical C extension. These extensions are then bundled into
|
|
507
|
+
the resulting binary distribution and can be uploaded to PyPi or similar.
|
|
508
|
+
|
|
509
|
+
For source distributions, Nimporter instructs the Nim compiler to output a copy
|
|
510
|
+
of the generated C code for each platform, architecture, and C compiler that is
|
|
511
|
+
supported by the library author (some libraries only make sense to work on
|
|
512
|
+
Windows for example like DirectX). It then bundles all of these as individual C
|
|
513
|
+
extensions into the source distribution. At installation time, Nimporter then
|
|
514
|
+
selects the C extension that matches the end-user's host machine target triple.
|
|
515
|
+
|
|
516
|
+
## ๐ท Contributing
|
|
517
|
+
|
|
518
|
+
[Pull requests](https://github.com/lucidrains/nimporter/pulls) are welcome,
|
|
519
|
+
especially for fixing bugs! ๐
|
|
520
|
+
|
|
521
|
+
Feel free to [open an issue](https://github.com/lucidrains/nimporter/issues) if
|
|
522
|
+
something seems to be broken but please look through the README first if time
|
|
523
|
+
allows.
|
|
524
|
+
|
|
525
|
+
## ๐ Special Thanks
|
|
526
|
+
|
|
527
|
+
Nimporter would not be possible without
|
|
528
|
+
[Nimpy](https://github.com/yglukhov/nimpy). Thank you
|
|
529
|
+
[Yuriy Glukhov](https://github.com/yglukhov) for making this project possible!
|
|
530
|
+
|
|
531
|
+
## ๐ Stargazers Over Time
|
|
532
|
+
|
|
533
|
+
[](https://starchart.cc/lucidrains/nimporter)
|
|
534
|
+
|
|
535
|
+
> Made using <https://starchart.cc/>
|