visionsim 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. visionsim/__init__.py +0 -0
  2. visionsim/cli/__init__.py +128 -0
  3. visionsim/cli/blender.py +150 -0
  4. visionsim/cli/dataset.py +185 -0
  5. visionsim/cli/emulate.py +348 -0
  6. visionsim/cli/ffmpeg.py +360 -0
  7. visionsim/cli/interpolate.py +79 -0
  8. visionsim/cli/transforms.py +308 -0
  9. visionsim/dataset/__init__.py +9 -0
  10. visionsim/dataset/dataset.py +486 -0
  11. visionsim/dataset/schema.py +95 -0
  12. visionsim/emulate/__init__.py +0 -0
  13. visionsim/emulate/dvs/__init__.py +1 -0
  14. visionsim/emulate/dvs/v2e/emulator.py +797 -0
  15. visionsim/emulate/dvs/v2e/emulator_utils.py +348 -0
  16. visionsim/emulate/imu.py +197 -0
  17. visionsim/emulate/rgb.py +61 -0
  18. visionsim/emulate/spc.py +60 -0
  19. visionsim/interpolate/__init__.py +104 -0
  20. visionsim/interpolate/pose.py +78 -0
  21. visionsim/interpolate/rife/IFNet_HDv3.py +117 -0
  22. visionsim/interpolate/rife/RIFE_HDv3.py +33 -0
  23. visionsim/interpolate/rife/__init__.py +0 -0
  24. visionsim/interpolate/rife/inference_img.py +165 -0
  25. visionsim/interpolate/rife/warplayer.py +34 -0
  26. visionsim/simulate/__init__.py +0 -0
  27. visionsim/simulate/blender.py +1740 -0
  28. visionsim/simulate/nodes/__init__.py +3 -0
  29. visionsim/simulate/nodes/flow.py +339 -0
  30. visionsim/simulate/nodes/normals.py +207 -0
  31. visionsim/simulate/nodes/segmentation.py +87 -0
  32. visionsim/types.py +17 -0
  33. visionsim/utils/__init__.py +0 -0
  34. visionsim/utils/color.py +58 -0
  35. visionsim/utils/progress.py +160 -0
  36. visionsim-0.1.0a1.dist-info/METADATA +77 -0
  37. visionsim-0.1.0a1.dist-info/RECORD +40 -0
  38. visionsim-0.1.0a1.dist-info/WHEEL +4 -0
  39. visionsim-0.1.0a1.dist-info/entry_points.txt +2 -0
  40. visionsim-0.1.0a1.dist-info/licenses/LICENSE +21 -0
visionsim/__init__.py ADDED
File without changes
@@ -0,0 +1,128 @@
1
+ from __future__ import annotations
2
+
3
+ import glob
4
+ import inspect
5
+ import os
6
+ import re
7
+ import shlex
8
+ import subprocess
9
+ import sys
10
+ from pathlib import Path
11
+
12
+ import tyro
13
+ from natsort import natsorted
14
+ from typing_extensions import overload
15
+
16
+ from . import blender, dataset, emulate, ffmpeg, interpolate, transforms
17
+
18
+ # Exposed for tests
19
+ _cli_modules = [blender, dataset, emulate, ffmpeg, interpolate, transforms]
20
+
21
+
22
+ @overload
23
+ def _validate_directories(
24
+ input_dir: str | os.PathLike, output_dir: None = None, pattern: str = ""
25
+ ) -> tuple[Path, None, list[str]]: ...
26
+
27
+
28
+ @overload
29
+ def _validate_directories(
30
+ input_dir: str | os.PathLike, output_dir: str | os.PathLike = "", pattern: str = ""
31
+ ) -> tuple[Path, Path, list[str]]: ...
32
+
33
+
34
+ @overload
35
+ def _validate_directories(
36
+ input_dir: str | os.PathLike, output_dir: None = None, pattern: str | None = None
37
+ ) -> tuple[Path, None]: ...
38
+
39
+
40
+ @overload
41
+ def _validate_directories(
42
+ input_dir: str | os.PathLike, output_dir: str | os.PathLike = "", pattern: str | None = None
43
+ ) -> tuple[Path, Path]: ...
44
+
45
+
46
+ def _validate_directories(
47
+ input_dir: str | os.PathLike, output_dir: str | os.PathLike | None = None, pattern: str | None = None
48
+ ) -> tuple[Path, Path | None] | tuple[Path, Path | None, list[str]]:
49
+ input_path = Path(input_dir).resolve()
50
+
51
+ if output_dir is not None:
52
+ output_path = Path(output_dir).resolve()
53
+ output_path.mkdir(parents=True, exist_ok=True)
54
+ else:
55
+ output_path = None
56
+
57
+ if not input_path.exists():
58
+ raise RuntimeError(f"Input directory {input_path} does not exist.")
59
+
60
+ if pattern:
61
+ # Pattern might be ffmpeg-style like "frames_%06d.png", convert to "frames_*.png".
62
+ pattern = re.sub(r"(%\d+d)", "*", pattern)
63
+ if not (in_files := glob.glob(str(input_path / pattern))):
64
+ raise FileNotFoundError(f"No files matching {pattern} found in {input_path}.")
65
+ in_files = natsorted(in_files)
66
+ return input_path, output_path, in_files
67
+ return input_path, output_path
68
+
69
+
70
+ def _run(
71
+ command: list[str] | str,
72
+ shell: bool = False,
73
+ echo: bool = False,
74
+ log_path: str | os.PathLike | None = None,
75
+ text: bool = True,
76
+ check: bool = False,
77
+ ) -> subprocess.CompletedProcess:
78
+ """Execute a command and return an object with the result and failure status."""
79
+
80
+ if echo:
81
+ print(command)
82
+
83
+ # shlex the command if we don't want to run in shell
84
+ if not shell and isinstance(command, str):
85
+ command = shlex.split(command)
86
+
87
+ # Either Pipe output or save to a file
88
+ if log_path:
89
+ Path(log_path).mkdir(parents=True, exist_ok=True)
90
+ log_out = Path(log_path).resolve() / "out.log"
91
+ log_err = Path(log_path).resolve() / "err.log"
92
+
93
+ with open(str(log_out), "w") as f_out:
94
+ with open(str(log_err), "w") as f_err:
95
+ return subprocess.run(
96
+ command,
97
+ shell=shell,
98
+ check=check,
99
+ stdout=f_out,
100
+ stderr=f_err,
101
+ text=text,
102
+ )
103
+ else:
104
+ return subprocess.run(
105
+ command,
106
+ shell=shell,
107
+ check=check,
108
+ stdout=subprocess.PIPE,
109
+ stderr=subprocess.PIPE,
110
+ text=text,
111
+ )
112
+
113
+
114
+ def main():
115
+ cli_dict = {}
116
+
117
+ for module in _cli_modules:
118
+ current_module = sys.modules[module.__name__]
119
+ module_name = current_module.__name__.split(".")[-1]
120
+ cli_dict.update(
121
+ {
122
+ f"{module_name}.{func_name}": func
123
+ for func_name, func in inspect.getmembers(current_module, inspect.isfunction)
124
+ if func.__module__ == module.__name__ and not func_name.startswith("_")
125
+ }
126
+ )
127
+
128
+ tyro.extras.subcommand_cli_from_dict(cli_dict)
@@ -0,0 +1,150 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import sys
6
+ from functools import partial
7
+ from pathlib import Path
8
+ from typing import Literal
9
+
10
+
11
+ def render_animation(
12
+ blend_file: str | os.PathLike,
13
+ root_path: str | os.PathLike,
14
+ frame_start: int | None = None,
15
+ frame_end: int | None = None,
16
+ frame_step: int = 1,
17
+ height: int | None = None,
18
+ width: int | None = None,
19
+ bit_depth: int = 8,
20
+ device: Literal["cpu", "cuda", "optix", "metal"] | None = None,
21
+ dry_run: bool = False,
22
+ jobs: int = 1,
23
+ unbind_camera: bool = False,
24
+ use_animations: bool = True,
25
+ use_motion_blur: bool | None = None,
26
+ keyframe_multiplier: float = 1.0,
27
+ allow_skips: bool = True,
28
+ depths: bool = False,
29
+ normals: bool = False,
30
+ flows: bool = False,
31
+ segmentations: bool = False,
32
+ file_format: str = "PNG",
33
+ log_dir: str | None = None,
34
+ addons: list[str] | None = None,
35
+ adaptive_threshold: float | None = None,
36
+ autoexec: bool = False,
37
+ output_blend_file: str | os.PathLike | None = None,
38
+ executable: str | os.PathLike | None = None,
39
+ ):
40
+ """Render views of a .blend file while moving camera along an animated trajectory
41
+
42
+ Args:
43
+ blend_file: path to blender file to use
44
+ root_path: location at which to save dataset
45
+ frame_start: frame number to start capture at (inclusive)
46
+ frame_end: frame number to stop capture at (exclusive)
47
+ frame_step: step with which to capture frames
48
+ height: height of frame to capture
49
+ width: width of frame to capture
50
+ bit_depth: bit depth for frames, usually 8 for pngs
51
+ device: which device type to use, one of none (meaning no change), cpu, cuda, optix
52
+ dry_run: if enabled, do not render frames
53
+ jobs: number of blender instances to spawn and render from
54
+ unbind_camera: free the camera from it's parents, any constraints and animations it may have. Ensures it uses the world's coordinate frame and the provided camera trajectory
55
+ use_animations: allow any animations to play out, if false, scene will be static
56
+ use_motion_blur: enable realistic motion blur
57
+ keyframe_multiplier: slow down animations by this factor
58
+ allow_skips: whether or not to skip rendering a frame if it already exists
59
+ depths: whether or not to capture depth images
60
+ normals: whether or not to capture normals images
61
+ flows: whether or not to capture optical flow images
62
+ segmentations: whether or not to capture segmentation images
63
+ file_format: frame file format to use. Depth is always 'OPEN_EXR' thus is unaffected by this setting
64
+ log_dir: where to save log to
65
+ addons: list of extra addons to enable
66
+ adaptive_threshold: noise threshold of rendered images, for higher quality frames make this threshold smaller. The default value is intentionally a little high to speed up renders
67
+ autoexec: if enabled, allow any embedded python scripts to run
68
+ output_blend_file: if set, write the modified blend file to this path. Helpful for troubleshooting
69
+ executable: use a different blender executable that the one on PATH
70
+
71
+ Example:
72
+ visionsim blender.render_animation --blend-file=<blend-file> --root-path=<output-path>
73
+ """
74
+ from rich.progress import Progress
75
+
76
+ from visionsim.cli import _run
77
+ from visionsim.simulate.blender import BlenderClients
78
+
79
+ # Runtime checks and gard rails
80
+ if _run(f"{executable or 'blender'} --version", shell=True).returncode != 0:
81
+ raise RuntimeError("No blender installation found on path!")
82
+ if not (blend_file := Path(blend_file).resolve()).exists():
83
+ raise FileNotFoundError(f"Blender file {blend_file} not found.")
84
+ if "blender.render_animation" not in sys.argv[1]:
85
+ raise RuntimeError("Task `blender.render_animation` must run first if running multiple tasks simultaneously.")
86
+
87
+ with (
88
+ BlenderClients.spawn(
89
+ jobs=jobs, timeout=30, autoexec=autoexec, log_dir=log_dir, executable=executable
90
+ ) as clients,
91
+ Progress() as progress,
92
+ ):
93
+ clients.initialize(blend_file, Path(root_path).resolve())
94
+ clients.set_resolution(width=width, height=height)
95
+ clients.image_settings(file_format, bit_depth)
96
+ clients.use_animations(use_animations)
97
+ clients.load_addons(*(addons or []))
98
+
99
+ clients.cycles_settings(
100
+ device_type=device,
101
+ use_cpu=True,
102
+ adaptive_threshold=adaptive_threshold if adaptive_threshold is None else float(adaptive_threshold),
103
+ use_denoising=True,
104
+ )
105
+
106
+ if depths:
107
+ clients.include_depths()
108
+ if normals:
109
+ clients.include_normals()
110
+ if flows:
111
+ clients.include_flows(direction="both")
112
+ if segmentations:
113
+ clients.include_segmentations()
114
+
115
+ if unbind_camera:
116
+ clients.unbind_camera()
117
+ if use_motion_blur is not None:
118
+ clients.use_motion_blur(use_motion_blur)
119
+
120
+ clients.move_keyframes(scale=keyframe_multiplier)
121
+
122
+ if output_blend_file is not None:
123
+ output_blend_file = Path(output_blend_file).resolve()
124
+ print(f"Saving to {output_blend_file}...")
125
+ clients[0].save_file(output_blend_file)
126
+
127
+ task = progress.add_task(f"Rendering {Path(blend_file).name}...")
128
+ transforms = clients.render_animation(
129
+ frame_start=frame_start if frame_start is None else int(frame_start),
130
+ frame_end=frame_end if frame_end is None else int(frame_end),
131
+ frame_step=frame_step,
132
+ allow_skips=allow_skips,
133
+ dry_run=dry_run,
134
+ update_fn=partial(progress.update, task, advance=1),
135
+ )
136
+
137
+ with open(str(Path(root_path) / "transforms.json"), "w") as f:
138
+ json.dump(transforms, f, indent=2)
139
+
140
+
141
+ # TODO: enable using spline to move camera, i.e:
142
+ # "num_frames": "number of frame to capture, this argument is affected by `keyframe-multiplier`. default: 100",
143
+ # "location_points": (
144
+ # "points defining the spline the camera follows. Expected to be json-str or path "
145
+ # "to json file. Default is circular obit at Z=1 with radius=5."
146
+ # ),
147
+ # "viewing-points": (
148
+ # "points defining the spline the camera looks at. Expected to be json-str or path "
149
+ # "to json file. Default is static origin."
150
+ # ),
@@ -0,0 +1,185 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from pathlib import Path
5
+
6
+ import numpy as np
7
+
8
+
9
+ def imgs_to_npy(
10
+ input_dir: str | os.PathLike,
11
+ output_dir: str | os.PathLike,
12
+ bitpack: bool = False,
13
+ bitpack_dim: int | None = None,
14
+ batch_size: int = 4,
15
+ alpha_color: str = "(255, 255, 255)",
16
+ is_grayscale: bool = False,
17
+ force: bool = False,
18
+ ):
19
+ """Convert an image folder based dataset to a NPY dataset
20
+
21
+ Args:
22
+ input_dir: directory in which to look for frames
23
+ output_dir: directory in which to save npy file
24
+ bitpack: if true, each chunk of 8 binary pixels will by packed into a single byte. Only enable if data is binary valued
25
+ bitpack_dim: axis along which to pack bits (H=1, W=2)
26
+ batch_size: number of frames to write at once
27
+ alpha_color: if set, blend with this background color and do not store alpha channel
28
+ is_grayscale: If set, assume images are grayscale and only save first channel
29
+ force: if true, overwrite output file(s) if present
30
+ """
31
+ # TODO: Add data integrity check
32
+ # - If npy file is present, scan through it to check if any data is missing
33
+ # - If so, fill in those frames only and skip the rest (add allow-skips arg?)
34
+ # TODO: Add check to stop the user from bitpacking non-binary data
35
+ # TODO: Support more types of frames, depth, normals, etc...
36
+ import ast
37
+ import copy
38
+
39
+ import numpy as np
40
+ from rich.progress import Progress
41
+ from torch.utils.data import DataLoader
42
+
43
+ from visionsim.dataset import ImgDataset, NpyDatasetWriter, default_collate
44
+
45
+ from . import _validate_directories
46
+
47
+ input_path, output_path, *_ = _validate_directories(input_dir, output_dir)
48
+ dataset = ImgDataset(input_path)
49
+ transforms_new = copy.deepcopy(dataset.transforms or {})
50
+
51
+ if ".exr" in set(Path(p).suffix for p in dataset.paths):
52
+ # TODO: This is due to the alpha blending below, we need alpha in [0, 1] to blend.
53
+ raise NotImplementedError("Task does not yet support EXRs")
54
+ if dataset.transforms:
55
+ if any(any(k != "file_path" and "path" in k for k in f.keys()) for f in dataset.transforms["frames"]):
56
+ raise NotImplementedError(
57
+ "Only color frames are supported for now. Keys such as 'depth_file_path' "
58
+ "or 'mask_path' are not supported."
59
+ )
60
+ alpha_color = ast.literal_eval(alpha_color) if alpha_color else None
61
+ shape = np.array(dataset.full_shape)
62
+ shape[-1] = 1 if is_grayscale else (shape[-1] - int(alpha_color is not None))
63
+
64
+ # Bitpack if either is set, defaults to bitpacking width dimension
65
+ if bitpack or bitpack_dim is not None:
66
+ bitpack_dim = bitpack_dim if bitpack_dim is not None else 2
67
+ bitpack = True
68
+
69
+ if bitpack_dim == 0 or bitpack_dim >= 3:
70
+ raise NotImplementedError("Can only bitpack along H or W.")
71
+
72
+ transforms_new["bitpack"] = bitpack
73
+ transforms_new["bitpack_dim"] = bitpack_dim
74
+ shape[bitpack_dim] /= 8
75
+
76
+ loader = DataLoader(dataset, batch_size=batch_size, num_workers=os.cpu_count() or 1, collate_fn=default_collate)
77
+
78
+ with (
79
+ NpyDatasetWriter(
80
+ output_path, shape=np.ceil(shape).astype(int), transforms=transforms_new, force=force
81
+ ) as writer,
82
+ Progress() as progress,
83
+ ):
84
+ task1 = progress.add_task("Writing frames...", total=len(dataset))
85
+
86
+ for i, (idxs, imgs, poses) in enumerate(loader):
87
+ if alpha_color is not None and imgs.ndim == 4 and imgs.shape[-1] == 4:
88
+ alpha = imgs[..., -1][..., None] / 255.0
89
+ imgs = imgs[..., :-1] * alpha + alpha_color * (1 - alpha)
90
+ if bitpack:
91
+ imgs = imgs >= 128
92
+ imgs = np.packbits(imgs, axis=bitpack_dim)
93
+ if is_grayscale:
94
+ imgs = imgs[..., :1]
95
+ writer[idxs] = (imgs, poses)
96
+ progress.update(task1, advance=len(idxs))
97
+
98
+
99
+ def npy_to_imgs(
100
+ input_dir: str | os.PathLike,
101
+ output_dir: str | os.PathLike,
102
+ batch_size: int = 4,
103
+ pattern: str = "frame_{:06}.png",
104
+ step: int = 1,
105
+ force: bool = False,
106
+ ):
107
+ """Convert an NPY based dataset to an image-folder dataset
108
+
109
+ Args:
110
+ input_dir: directory in which to look for frames
111
+ output_dir: directory in which to save npy file
112
+ batch_size: number of frames to write at once
113
+ pattern: filenames of frames will match this
114
+ step: skip some frames when converting between formats
115
+ force: if true, overwrite output file(s) if present
116
+
117
+ """
118
+ import copy
119
+
120
+ from rich.progress import Progress
121
+ from torch.utils.data import DataLoader
122
+
123
+ from visionsim.dataset import ImgDatasetWriter, NpyDataset, default_collate
124
+
125
+ from . import _validate_directories
126
+
127
+ input_path, output_path, *_ = _validate_directories(input_dir, output_dir)
128
+ dataset = NpyDataset(input_path)
129
+
130
+ transforms_new = copy.deepcopy(dataset.transforms or {})
131
+ transforms_new.pop("file_path", None)
132
+ transforms_new.pop("bitpack", None)
133
+ transforms_new.pop("bitpack_dim", None)
134
+
135
+ sampler = range(0, len(dataset) - 1, step)
136
+ loader = DataLoader(
137
+ dataset, sampler=sampler, batch_size=batch_size, num_workers=os.cpu_count() or 1, collate_fn=default_collate
138
+ )
139
+
140
+ with (
141
+ ImgDatasetWriter(output_path, transforms=transforms_new, force=force, pattern=pattern) as writer,
142
+ Progress() as progress,
143
+ ):
144
+ task1 = progress.add_task("Writing frames...", total=len(sampler))
145
+
146
+ for idxs, imgs, poses in loader:
147
+ if np.issubdtype(imgs.dtype, np.uint8):
148
+ writer[idxs] = (imgs, poses)
149
+ else:
150
+ writer[idxs] = (np.repeat((imgs * 255).astype(np.uint8), 3, -1), poses)
151
+ progress.update(task1, advance=len(idxs))
152
+
153
+
154
+ def info(input_dir: str | os.PathLike, as_json: bool = False):
155
+ """Print information about the dataset
156
+
157
+ Args:
158
+ input_dir: directory in which to look for dataset
159
+ as_json: print the output in a json-formatted string
160
+ """
161
+ import json
162
+
163
+ from visionsim.dataset import Dataset
164
+
165
+ from . import _validate_directories
166
+
167
+ input_path, *_ = _validate_directories(input_dir=input_dir)
168
+ dataset = Dataset.from_path(input_path)
169
+
170
+ if as_json:
171
+ print(
172
+ json.dumps(
173
+ {
174
+ "arclength": dataset.arclength,
175
+ "full_shape": dataset.full_shape,
176
+ "paths": [str(p) for p in dataset.paths],
177
+ },
178
+ indent=2,
179
+ sort_keys=True,
180
+ )
181
+ )
182
+ else:
183
+ print(f"arclength: {dataset.arclength}")
184
+ print(f"full_shape: {dataset.full_shape}")
185
+ print(f"paths: {[str(p) for p in dataset.paths]}")