pymoku-sdk 0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. pymoku_sdk-0.3/.gitignore +12 -0
  2. pymoku_sdk-0.3/.gitlab-ci.yml +48 -0
  3. pymoku_sdk-0.3/LICENSE +22 -0
  4. pymoku_sdk-0.3/PKG-INFO +138 -0
  5. pymoku_sdk-0.3/README.md +120 -0
  6. pymoku_sdk-0.3/pyproject.toml +51 -0
  7. pymoku_sdk-0.3/setup.cfg +4 -0
  8. pymoku_sdk-0.3/src/pymoku/sdk/__init__.py +61 -0
  9. pymoku_sdk-0.3/src/pymoku/sdk/barfile.py +81 -0
  10. pymoku_sdk-0.3/src/pymoku/sdk/checkpoints.py +145 -0
  11. pymoku_sdk-0.3/src/pymoku/sdk/cocotb.py +39 -0
  12. pymoku_sdk-0.3/src/pymoku/sdk/keys.py +89 -0
  13. pymoku_sdk-0.3/src/pymoku/sdk/main.py +285 -0
  14. pymoku_sdk-0.3/src/pymoku/sdk/manifests.py +128 -0
  15. pymoku_sdk-0.3/src/pymoku/sdk/support/constraints/slot_default.xdc +2 -0
  16. pymoku_sdk-0.3/src/pymoku/sdk/support/parts.def +10 -0
  17. pymoku_sdk-0.3/src/pymoku/sdk/support/python/api_main.py +61 -0
  18. pymoku_sdk-0.3/src/pymoku/sdk/support/sim/amba.py +732 -0
  19. pymoku_sdk-0.3/src/pymoku/sdk/support/sim/blackbox_models.py +223 -0
  20. pymoku_sdk-0.3/src/pymoku/sdk/support/sim/cocotb_test.py +58 -0
  21. pymoku_sdk-0.3/src/pymoku/sdk/support/sim/spiregs.py +70 -0
  22. pymoku_sdk-0.3/src/pymoku/sdk/support/sim/startup.py +110 -0
  23. pymoku_sdk-0.3/src/pymoku/sdk/support/tcl/bitstream.tcl +11 -0
  24. pymoku_sdk-0.3/src/pymoku/sdk/support/tcl/reports.tcl +7 -0
  25. pymoku_sdk-0.3/src/pymoku/sdk/support/tcl/route.tcl +17 -0
  26. pymoku_sdk-0.3/src/pymoku/sdk/support/tcl/shell-zynq.tcl +9 -0
  27. pymoku_sdk-0.3/src/pymoku/sdk/support/tcl/shell-zynqmp.tcl +6 -0
  28. pymoku_sdk-0.3/src/pymoku/sdk/support/tcl/synth.tcl +48 -0
  29. pymoku_sdk-0.3/src/pymoku/sdk/support/vhdl/MemoryMap.vhd +222 -0
  30. pymoku_sdk-0.3/src/pymoku/sdk/support/vhdl/MokuSupport.vhd +1102 -0
  31. pymoku_sdk-0.3/src/pymoku/sdk/templates/TestTop.vhd +64 -0
  32. pymoku_sdk-0.3/src/pymoku/sdk/templates/Top.vhd +75 -0
  33. pymoku_sdk-0.3/src/pymoku/sdk/templates/api.py +11 -0
  34. pymoku_sdk-0.3/src/pymoku/sdk/templates/applet.py +7 -0
  35. pymoku_sdk-0.3/src/pymoku/sdk/templates/gitignore +6 -0
  36. pymoku_sdk-0.3/src/pymoku/sdk/templates/module.py +7 -0
  37. pymoku_sdk-0.3/src/pymoku/sdk/templates/parts.def +70 -0
  38. pymoku_sdk-0.3/src/pymoku/sdk/templates/parts_platform.def +19 -0
  39. pymoku_sdk-0.3/src/pymoku/sdk/templates/plugin.py +27 -0
  40. pymoku_sdk-0.3/src/pymoku/sdk/templates/test_module.py +30 -0
  41. pymoku_sdk-0.3/src/pymoku/sdk/vivado.py +667 -0
  42. pymoku_sdk-0.3/src/pymoku_sdk.egg-info/PKG-INFO +138 -0
  43. pymoku_sdk-0.3/src/pymoku_sdk.egg-info/SOURCES.txt +46 -0
  44. pymoku_sdk-0.3/src/pymoku_sdk.egg-info/dependency_links.txt +1 -0
  45. pymoku_sdk-0.3/src/pymoku_sdk.egg-info/entry_points.txt +2 -0
  46. pymoku_sdk-0.3/src/pymoku_sdk.egg-info/requires.txt +6 -0
  47. pymoku_sdk-0.3/src/pymoku_sdk.egg-info/top_level.txt +1 -0
  48. pymoku_sdk-0.3/tests/test_import.py +4 -0
@@ -0,0 +1,12 @@
1
+ pymoku_sdk.egg-info
2
+ *.pyc
3
+ .DS_Store
4
+ build/
5
+ .vscode
6
+ .pytest_cache
7
+ .coverage
8
+ htmlcov/
9
+ venv
10
+ venv2
11
+ venv3
12
+ .ipynb_checkpoints
@@ -0,0 +1,48 @@
1
+ default:
2
+ tags:
3
+ - docker
4
+ image: python:3.10
5
+ cache:
6
+ paths:
7
+ - .pip-cache/
8
+ before_script:
9
+ - python --version
10
+ - pip install --upgrade pip
11
+ - pip install build twine
12
+
13
+ stages:
14
+ - build
15
+ - style
16
+ - test
17
+ - publish
18
+
19
+ variables:
20
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip-cache"
21
+
22
+ style:
23
+ stage: style
24
+ script:
25
+ - pip install flake8 flake8-pyproject
26
+ - flake8
27
+
28
+ build:
29
+ stage: build
30
+ script:
31
+ - python -m build
32
+ artifacts:
33
+ paths:
34
+ - dist/
35
+
36
+ test:
37
+ stage: test
38
+ script:
39
+ - pip install pytest
40
+ - pip install dist/*.whl
41
+ - pytest
42
+
43
+ publish:
44
+ stage: publish
45
+ script:
46
+ - TWINE_PASSWORD=${PYPI_TOKEN} TWINE_USERNAME=__token__ python -m twine upload dist/*
47
+ rules:
48
+ - if: $CI_COMMIT_TAG
pymoku_sdk-0.3/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2026 Liquid Instruments Pty. Ltd.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
@@ -0,0 +1,138 @@
1
+ Metadata-Version: 2.4
2
+ Name: pymoku-sdk
3
+ Version: 0.3
4
+ Summary: CLI tool to create pymoku projects
5
+ Author-email: Ben Coughlan <ben@liquidinstruments.com>
6
+ Maintainer-email: Ben Coughlan <ben@liquidinstruments.com>
7
+ License-Expression: MIT
8
+ Requires-Python: >=3.9
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Requires-Dist: pymoku>=4.7.1
12
+ Requires-Dist: pymoku-parts>=0.9
13
+ Requires-Dist: click
14
+ Requires-Dist: rich
15
+ Requires-Dist: Jinja2
16
+ Requires-Dist: cryptography
17
+ Dynamic: license-file
18
+
19
+ # Getting Started
20
+
21
+ ## Prerequisites
22
+
23
+ ### Simulation
24
+
25
+ Simulations are run in a docker container with the image `li-cocotb-pymoku`.
26
+ You will need a docker environment with access to this image (currently unpublished).
27
+
28
+ The resulting `wave.ghw` files can be viewed with GtkWave or Surfer.
29
+
30
+ ### Building bitstreams
31
+
32
+ #### Vivado
33
+ Bitstreams require a local verision of Vivado (>= 2024.2 recommended) installed
34
+ with appropriate licences configured.
35
+
36
+ **TODO**
37
+
38
+ Parts can be exteneded to package build context for a remote worker, avoiding the
39
+ need for a local Vivado installation.
40
+
41
+ ### Keys
42
+
43
+ Creating barfiles that can be deployed on a Moku require them to be signed by
44
+ Liquid Instruments or with developer keys that have been enabled on the target
45
+ Moku.
46
+
47
+ See below for generating keys; these must be signed by Liquid Instruments to
48
+ enable specific devices.
49
+
50
+ ## (interim) Installation
51
+
52
+ These are the current instructions until these packages are publicly available.
53
+ It does require access to the LI internal GitLab.
54
+
55
+ Put these in `requirements.txt`
56
+
57
+ pymoku @ git+https://git.liquidinstruments.com/pymoku/pymoku.git
58
+ pymoku-parts @ git+https://git.liquidinstruments.com/pymoku/pymoku-parts.git
59
+ pymoku-sdk @ git+https://git.liquidinstruments.com/pymoku/pymoku-sdk.git
60
+ docker
61
+
62
+ create a virtual environment
63
+
64
+ python -m virtualenv venv
65
+ source venv/bin/activate
66
+
67
+ and then run
68
+
69
+ pip install -r requirements.txt
70
+
71
+ ## Generate a signing key
72
+
73
+ This only needs to be done once, and the key will be stored in the user home directory.
74
+
75
+ pymokusdk keys create my-key
76
+
77
+ The resulting public key will need to be signed by Liquid Instruments.
78
+
79
+ ## Create a new project
80
+
81
+ pymokusdk new MySlot --hw mokugo
82
+ cd myslot
83
+
84
+ ## Run the simulaition
85
+
86
+ parts tests
87
+
88
+ Results will available in the `build/test-slot` folder. Of interest will be
89
+ `wave.ghw` and `stdout.log`. Build failures in the VHDL will be printed to the
90
+ console, but errors during execution will be `stdout.log`.
91
+
92
+ ## Run a build
93
+
94
+ parts mokugo python --stats
95
+
96
+ The resulting barfiles will be in `output/`.
97
+
98
+ ## Deploying from Python
99
+
100
+ import pymoku
101
+ pymoku.plugins.DEV_CACHE = './output' # reference our local output
102
+ moku = pymoku.connect('10.0.0.1')
103
+ moku.deploy_platform('pymoku:platform_2') # deploy a platform (only needed once)
104
+ myslot = moku.deploy('custom:myslot') # this name should match the specific id used in parts.def
105
+
106
+ The `myslot` object will be created dynamically from your custom python plugin
107
+ and attached to the slot instance on the Moku.
108
+
109
+ ## Deploying from the GUI
110
+
111
+ First you will need to have installed `pymoku[extras]` to include GUI support.
112
+
113
+ Then run pymoku with reference to your local `output` folder
114
+
115
+ pymoku --dev output/
116
+
117
+ This will launch the GUI. After connecting to a Moku and deploying a platform,
118
+ your custom implementation will be available from the functions palette (assuming
119
+ it was built for the deployed platform).
120
+
121
+ Launching the slot instance will open the applet defined by your custom python
122
+ plugin. The default will appear as a python console.
123
+
124
+ # TODO
125
+
126
+ - [ ] 'clean' targets
127
+ - [x] Create barfiles for checkpoints to use as platforms
128
+ - [x] Don't use GPG for signatures
129
+ - [x] Use personal signing keys to sign manifests
130
+ - [x] Generate signing keys
131
+ - [ ] Auto generate "custom wrapper"
132
+ - [ ] Bitstream encryption
133
+ - [ ] Sanity check environment
134
+ - [x] Fetch checkpoints from cloud
135
+ - [x] Plugin versions (or lack of) break pymoku prioritisation
136
+ - [x] API deployment
137
+ - [x] If Git on PATH or CLI arg, add .gitignore and init repo
138
+ - [ ] Auto generate top/test bench with new generics and custom number of Ain/Aout
@@ -0,0 +1,120 @@
1
+ # Getting Started
2
+
3
+ ## Prerequisites
4
+
5
+ ### Simulation
6
+
7
+ Simulations are run in a docker container with the image `li-cocotb-pymoku`.
8
+ You will need a docker environment with access to this image (currently unpublished).
9
+
10
+ The resulting `wave.ghw` files can be viewed with GtkWave or Surfer.
11
+
12
+ ### Building bitstreams
13
+
14
+ #### Vivado
15
+ Bitstreams require a local verision of Vivado (>= 2024.2 recommended) installed
16
+ with appropriate licences configured.
17
+
18
+ **TODO**
19
+
20
+ Parts can be exteneded to package build context for a remote worker, avoiding the
21
+ need for a local Vivado installation.
22
+
23
+ ### Keys
24
+
25
+ Creating barfiles that can be deployed on a Moku require them to be signed by
26
+ Liquid Instruments or with developer keys that have been enabled on the target
27
+ Moku.
28
+
29
+ See below for generating keys; these must be signed by Liquid Instruments to
30
+ enable specific devices.
31
+
32
+ ## (interim) Installation
33
+
34
+ These are the current instructions until these packages are publicly available.
35
+ It does require access to the LI internal GitLab.
36
+
37
+ Put these in `requirements.txt`
38
+
39
+ pymoku @ git+https://git.liquidinstruments.com/pymoku/pymoku.git
40
+ pymoku-parts @ git+https://git.liquidinstruments.com/pymoku/pymoku-parts.git
41
+ pymoku-sdk @ git+https://git.liquidinstruments.com/pymoku/pymoku-sdk.git
42
+ docker
43
+
44
+ create a virtual environment
45
+
46
+ python -m virtualenv venv
47
+ source venv/bin/activate
48
+
49
+ and then run
50
+
51
+ pip install -r requirements.txt
52
+
53
+ ## Generate a signing key
54
+
55
+ This only needs to be done once, and the key will be stored in the user home directory.
56
+
57
+ pymokusdk keys create my-key
58
+
59
+ The resulting public key will need to be signed by Liquid Instruments.
60
+
61
+ ## Create a new project
62
+
63
+ pymokusdk new MySlot --hw mokugo
64
+ cd myslot
65
+
66
+ ## Run the simulaition
67
+
68
+ parts tests
69
+
70
+ Results will available in the `build/test-slot` folder. Of interest will be
71
+ `wave.ghw` and `stdout.log`. Build failures in the VHDL will be printed to the
72
+ console, but errors during execution will be `stdout.log`.
73
+
74
+ ## Run a build
75
+
76
+ parts mokugo python --stats
77
+
78
+ The resulting barfiles will be in `output/`.
79
+
80
+ ## Deploying from Python
81
+
82
+ import pymoku
83
+ pymoku.plugins.DEV_CACHE = './output' # reference our local output
84
+ moku = pymoku.connect('10.0.0.1')
85
+ moku.deploy_platform('pymoku:platform_2') # deploy a platform (only needed once)
86
+ myslot = moku.deploy('custom:myslot') # this name should match the specific id used in parts.def
87
+
88
+ The `myslot` object will be created dynamically from your custom python plugin
89
+ and attached to the slot instance on the Moku.
90
+
91
+ ## Deploying from the GUI
92
+
93
+ First you will need to have installed `pymoku[extras]` to include GUI support.
94
+
95
+ Then run pymoku with reference to your local `output` folder
96
+
97
+ pymoku --dev output/
98
+
99
+ This will launch the GUI. After connecting to a Moku and deploying a platform,
100
+ your custom implementation will be available from the functions palette (assuming
101
+ it was built for the deployed platform).
102
+
103
+ Launching the slot instance will open the applet defined by your custom python
104
+ plugin. The default will appear as a python console.
105
+
106
+ # TODO
107
+
108
+ - [ ] 'clean' targets
109
+ - [x] Create barfiles for checkpoints to use as platforms
110
+ - [x] Don't use GPG for signatures
111
+ - [x] Use personal signing keys to sign manifests
112
+ - [x] Generate signing keys
113
+ - [ ] Auto generate "custom wrapper"
114
+ - [ ] Bitstream encryption
115
+ - [ ] Sanity check environment
116
+ - [x] Fetch checkpoints from cloud
117
+ - [x] Plugin versions (or lack of) break pymoku prioritisation
118
+ - [x] API deployment
119
+ - [x] If Git on PATH or CLI arg, add .gitignore and init repo
120
+ - [ ] Auto generate top/test bench with new generics and custom number of Ain/Aout
@@ -0,0 +1,51 @@
1
+ [build-system]
2
+ requires = ["setuptools>=64", "setuptools_scm>=8", "Flake8-pyproject"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "pymoku-sdk"
7
+ dynamic = ["version"]
8
+ requires-python = ">=3.9"
9
+ description = "CLI tool to create pymoku projects"
10
+ readme = "README.md"
11
+ authors = [
12
+ {name = "Ben Coughlan", email = "ben@liquidinstruments.com"},
13
+ ]
14
+ maintainers = [
15
+ {name = "Ben Coughlan", email = "ben@liquidinstruments.com"},
16
+ ]
17
+ license = "MIT"
18
+ license-files = ["LICENSE"]
19
+ dependencies = [
20
+ 'pymoku >= 4.7.1',
21
+ 'pymoku-parts >= 0.9',
22
+ 'click',
23
+ 'rich',
24
+ 'Jinja2',
25
+ 'cryptography',
26
+ ]
27
+
28
+ [project.scripts]
29
+ pymokusdk = "pymoku.sdk.main:cli"
30
+
31
+ [tool.setuptools_scm]
32
+
33
+ [tool.flake8]
34
+ exclude = [
35
+ "src/pymoku/sdk/support",
36
+ "src/pymoku/sdk/templates",
37
+ ".git",
38
+ "build",
39
+ "dist",
40
+ "venv*",
41
+ ".ipynb*",
42
+ ]
43
+ ignore = [
44
+ "E501", # line too long
45
+ "E741", # ambiguous variable name
46
+ "W503", # line break before binary operator
47
+ "F405", # may be undefined, or defined from star imports
48
+ ]
49
+
50
+ [tool.pytest.ini_options]
51
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,61 @@
1
+ try:
2
+ import setuptools_scm
3
+ SCM_VERSION = setuptools_scm.get_version()
4
+ except ImportError:
5
+ SCM_VERSION = ''
6
+
7
+ FIRMWARE_VERSION = 616
8
+
9
+
10
+ from .vivado import * # noqa
11
+ from .manifests import * # noqa
12
+ from .barfile import * # noqa
13
+ from .cocotb import * # noqa
14
+ from .checkpoints import * # noqa
15
+
16
+
17
+ def generate_slot_bitstreams(platform, synth,
18
+ constraints='support-slot-constraints',
19
+ route_tcl='support-slot-route-tcl',
20
+ bitstream_tcl='support-slot-bitstream-tcl',
21
+ reports_tcl='support-slot-reports-tcl'):
22
+ """
23
+ There is a distinction between arbitrary RMs and Slot RMs, in that Slot RMs are
24
+ expected to be implemented within the group of Slots defined as "Instruments",
25
+ whereas any RM can be implemented within any Slot as long as it meets the
26
+ interface.
27
+
28
+ This API simply provides a shortcut to the common case of "Instruments".
29
+ """
30
+
31
+ routes = [RMRoute(synth, shell, route_tcl=route_tcl, constraints=constraints) for shell in platform.slot_shells()]
32
+ rms = [RMInstance(route, bitstream_tcl=bitstream_tcl, reports_tcl=reports_tcl) for route in routes]
33
+ return rms
34
+
35
+
36
+ def generate_slot_manifests(platform, synth, manifest_id,
37
+ constraints='support-slot-constraints',
38
+ route_tcl='support-slot-route-tcl',
39
+ bitstream_tcl='support-slot-bitstream-tcl',
40
+ reports_tcl='support-slot-reports-tcl'):
41
+ rms = generate_slot_bitstreams(platform, synth,
42
+ constraints=constraints,
43
+ route_tcl=route_tcl,
44
+ bitstream_tcl=bitstream_tcl,
45
+ reports_tcl=reports_tcl)
46
+ manifests = [Manifest(manifest_id, rm) for rm in rms]
47
+ return manifests
48
+
49
+
50
+ def include_support():
51
+ import pymoku.sdk # noqa
52
+ import shutil # noqa
53
+ from pymoku.parts import include, ConfigGlobals # noqa
54
+ from importlib import resources # noqa
55
+
56
+ support_path = ConfigGlobals.build_path / 'support'
57
+ if not support_path.exists():
58
+ with resources.path(pymoku.sdk, 'support') as support_res:
59
+ print(support_res, support_path)
60
+ shutil.copytree(support_res, support_path)
61
+ include(support_path, prefix='support')
@@ -0,0 +1,81 @@
1
+ import itertools
2
+ import tarfile
3
+ import json
4
+ import io
5
+ import hashlib
6
+ from pathlib import Path
7
+ from pymoku.parts import BuildStep, SourceFile
8
+ from pymoku.sdk import SCM_VERSION, log, keys
9
+
10
+
11
+ KEY_DIR = Path.home() / '.config/pymoku/keys'
12
+
13
+
14
+ class BarFile(BuildStep):
15
+ VERSION = 0
16
+
17
+ def __init__(self, barfile, *manifests):
18
+ super().__init__(name=barfile)
19
+
20
+ # list of manifests or list of lists of manifests
21
+ try:
22
+ self.manifests = list(itertools.chain.from_iterable(manifests))
23
+ except TypeError:
24
+ self.manifests = manifests
25
+
26
+ self.barfile = self.build_path / barfile
27
+ for m in self.manifests:
28
+ self.depends_on(m.bitstreams())
29
+ self.creates(SourceFile(self.barfile))
30
+ self._bs_arcnames = {}
31
+
32
+ def arcname_for(self, bitstream):
33
+ # generate a unique filename for bitstreams in the archive
34
+ if bitstream not in self._bs_arcnames:
35
+ self._bs_arcnames[bitstream] = f'{len(self._bs_arcnames):03d}'
36
+ return self._bs_arcnames[bitstream]
37
+
38
+ def config_state(self):
39
+ # Manifests aren't build steps, and we don't know when they change
40
+ # until their own dependencies are built.
41
+ sha = hashlib.sha256(SCM_VERSION.encode())
42
+ sha.update(json.dumps(dict(keys.list_public_keys())).encode())
43
+ return sha.digest()
44
+
45
+ async def do_step(self):
46
+ manifests = []
47
+ for m in self.manifests:
48
+ manifests.extend(m(self.arcname_for))
49
+
50
+ self.update_progress(total=2 + len(self._bs_arcnames))
51
+
52
+ manifest_bytes = self.create_manifest([m for m, bs in manifests])
53
+ self.update_progress(advance=1)
54
+
55
+ with tarfile.open(self.barfile, mode='w') as tf:
56
+ m_tinfo = tarfile.TarInfo(name='MANIFEST')
57
+ m_tinfo.size = len(manifest_bytes)
58
+ tf.addfile(m_tinfo, io.BytesIO(manifest_bytes))
59
+
60
+ # sign with all the keys TODO maybe specify which ones?
61
+ for keyname, private_key in keys.list_private_keys():
62
+ sig_bytes = private_key.sign(manifest_bytes)
63
+ sig_tinfo = tarfile.TarInfo(name=f'MANIFEST.{keyname}.sig')
64
+ sig_tinfo.size = len(sig_bytes)
65
+ tf.addfile(sig_tinfo, io.BytesIO(sig_bytes))
66
+
67
+ for bs, fname in self._bs_arcnames.items():
68
+ bs_info = tf.gettarinfo(bs.path, arcname=fname)
69
+ with open(bs.path, 'rb') as f:
70
+ tf.addfile(bs_info, fileobj=f)
71
+ self.update_progress(advance=1)
72
+
73
+ # 32 MB is the upload limit through the web interface, but also probably
74
+ # a reasonable limit in general
75
+ size = self.barfile.stat().st_size
76
+ if size > 1024 * 2**15:
77
+ log.warning(f'Barfile {self.barfile} exceeds 32MB ({size:d})')
78
+
79
+ def create_manifest(self, items):
80
+ manifest = dict(version=self.VERSION, items=items)
81
+ return json.dumps(manifest, indent=2).encode()
@@ -0,0 +1,145 @@
1
+ import io
2
+ import re
3
+ import hashlib
4
+ import tarfile
5
+ import json
6
+ import logging
7
+ import asyncio
8
+ import concurrent.futures
9
+ from functools import cache
10
+ from pymoku.sdk import BuildStep
11
+ from pymoku.plugins import PluginCache
12
+ from pymoku.parts import SourceFile
13
+
14
+
15
+ _cache = PluginCache()
16
+ log = logging.getLogger("pymokusdk")
17
+
18
+
19
+ @cache
20
+ def get_platform(platform_name):
21
+ """ Find a Platform checkpoint
22
+ """
23
+ manifests = _cache.checkpoint_manifests(_id=f'pymoku:{platform_name}')
24
+
25
+ if not manifests:
26
+ _cache.store_manifests(_cache.cloud_manifests(_type='checkpoint'))
27
+ manifests = _cache.checkpoint_manifests(_id=f'pymoku:{platform_name}')
28
+
29
+ if not manifests:
30
+ log.error(f"No checkpoints found for {platform_name}")
31
+ return None
32
+
33
+ log.debug(f"Using platform {platform_name} from {manifests[0]['location']}")
34
+
35
+ return CheckpointPlatform(platform_name, manifests)
36
+
37
+
38
+ class CheckpointPlatform(BuildStep):
39
+ """ This mimics the vivado.Platform interface but loads the checkpoint from
40
+ a barfile rather than trying to build it.
41
+
42
+ This step downloads the barfile to the local pymoku cache; the
43
+ CheckpointShell unpacks each DCP into the build directory.
44
+ """
45
+ def __init__(self, name, manifests):
46
+ super().__init__(name=name)
47
+ # there is no "platform" checkpoint, only the individual cells
48
+ self.manifests = manifests
49
+
50
+ self.hwver = self.slot_shells()[0].hwver
51
+ self.part = self.slot_shells()[0].part
52
+ self.arch = self.slot_shells()[0].arch
53
+ # TODO these should not be platform properties
54
+ self.slot_ains = self.slot_shells()[0].num_ain
55
+ self.slot_aouts = self.slot_shells()[0].num_aout
56
+ self.slot_cbufs = self.slot_shells()[0].num_cbufs
57
+ self.sha = manifests[0]['plat_sha']
58
+
59
+ @cache
60
+ def slot_shells(self):
61
+ shells = [CheckpointShell(self, m) for m in self.manifests]
62
+ return sorted([s for s in shells if s.slot is not None], key=lambda s: s.slot)
63
+
64
+ async def do_step(self):
65
+ # manifest may be in one shared barfile, reduce to unique barfiles
66
+ barfiles = []
67
+ to_fetch = []
68
+ for m in self.manifests:
69
+ if m['location'] != 'local' and m['barfile'] not in barfiles:
70
+ to_fetch.append(m)
71
+ barfiles.append(m['barfile'])
72
+
73
+ # split downloads across mutliple threads (and unblock other BuildSteps)
74
+ tasks = []
75
+ loop = asyncio.get_running_loop()
76
+ with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
77
+ for m in to_fetch:
78
+ tasks.append(loop.run_in_executor(pool, _cache.download_barfile, m))
79
+ await asyncio.gather(*tasks)
80
+
81
+
82
+ class CheckpointShell(BuildStep):
83
+ def __init__(self, platform, manifest):
84
+ self.platform = platform
85
+ self.manifest = manifest
86
+ self.hwver = manifest['extra']['hwver']
87
+ self.cell_name = manifest['extra']['cell_name']
88
+ self.pblock_name = manifest['extra']['pblock_name']
89
+ self.part = manifest['extra']['part']
90
+ self.arch = manifest['extra']['arch']
91
+ self.num_ain = manifest['extra']['num_ain']
92
+ self.num_aout = manifest['extra']['num_aout']
93
+ self.num_cbufs = manifest['extra']['num_cbufs']
94
+
95
+ name = f'{platform.name}-{self.safe_cell_name}'
96
+ super().__init__(name=name)
97
+ self.depends_on(platform)
98
+
99
+ self.workdir = self.build_path / 'checkpoints' / platform.name
100
+ self.shell_dcp = SourceFile(self.workdir / f'{name}.dcp')
101
+ self.creates(self.shell_dcp)
102
+
103
+ @property
104
+ def plat_name(self):
105
+ return self.platform.name
106
+
107
+ @property
108
+ def path(self):
109
+ return next(iter(self.outputs())).path
110
+
111
+ @property
112
+ def safe_cell_name(self):
113
+ name = self.cell_name.translate(
114
+ str.maketrans({"[": "_", "]": None, "*": "_"}))
115
+ name = name.replace('_INSTR', '')
116
+ return name
117
+
118
+ @property
119
+ def slot(self):
120
+ """ This makes some big assumptions about cell naming conventions"""
121
+ if (m := re.search(r'INSTRS\[(\d+)\]', self.cell_name)) is None:
122
+ return None
123
+ return int(m.group(1))
124
+
125
+ def config_state(self):
126
+ sha = hashlib.sha256(json.dumps(self.manifest['extra']).encode())
127
+ sha.update(self.manifest['sha256'].encode())
128
+ sha.update(self.manifest['plat_sha'].encode())
129
+ return sha.digest()
130
+
131
+ async def do_step(self):
132
+ barfile_data = _cache.download_barfile(self.manifest)
133
+ barfile_data = io.BytesIO(barfile_data)
134
+
135
+ with tarfile.open(fileobj=barfile_data) as arc:
136
+ member = arc.getmember(self.manifest['filename'])
137
+ assert member.isfile()
138
+ data = arc.extractfile(member).read()
139
+
140
+ sha = hashlib.sha256(data)
141
+
142
+ assert sha.hexdigest() == self.manifest['sha256']
143
+
144
+ self.shell_dcp.path.parent.mkdir(exist_ok=True, parents=True)
145
+ self.shell_dcp.path.write_bytes(data)