polyfile-weave 0.5.5__tar.gz

polyfile_weave-0.5.5/CODEOWNERS

@@ -0,0 +1 @@
+* @ESultanik

polyfile_weave-0.5.5/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

polyfile_weave-0.5.5/MANIFEST.in

@@ -0,0 +1,7 @@
+include compile_kaitai_parsers.py
+include CODEOWNERS
+graft kaitai_struct_formats
+graft file
+graft docs
+graft logo
+graft hooks

polyfile_weave-0.5.5/PKG-INFO

@@ -0,0 +1,173 @@
+Metadata-Version: 2.4
+Name: polyfile-weave
+Version: 0.5.5
+Summary: A utility to recursively map the structure of a file.
+Home-page: https://github.com/zbirenbaum/polyfile-weave
+Author: Trail of Bits
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: abnf~=2.2.0
+Requires-Dist: chardet>=5.0.0
+Requires-Dist: cint>=1.0.0
+Requires-Dist: fickling>=0.0.8
+Requires-Dist: graphviz>=0.20.1
+Requires-Dist: intervaltree>=2.4.0
+Requires-Dist: jinja2>=2.1.0
+Requires-Dist: kaitaistruct~=0.10
+Requires-Dist: networkx>=2.6.3
+Requires-Dist: pdfminer.six<=20240706,>=20220524
+Requires-Dist: Pillow>=5.0.0
+Requires-Dist: pyreadline3; platform_system == "Windows"
+Requires-Dist: pyyaml>=3.13
+Requires-Dist: setuptools>=80.9.0
+Provides-Extra: demangle
+Requires-Dist: cxxfilt; extra == "demangle"
+Provides-Extra: dev
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# PolyFile
+<p align="center">
+  <img src="logo/polyfile_name.png?raw=true" width="256" title="PolyFile">
+</p>
+<br />
+
+[![PyPI version](https://badge.fury.io/py/polyfile.svg)](https://badge.fury.io/py/polyfile)
+[![Tests](https://github.com/trailofbits/polyfile/workflows/Tests/badge.svg)](https://github.com/trailofbits/polyfile/actions)
+[![Slack Status](https://slack.empirehacking.nyc/badge.svg)](https://slack.empirehacking.nyc)
+
+A utility to identify and map the semantic and syntactic structure of files,
+including polyglots, chimeras, and schizophrenic files. It has [a pure-Python implementation of libmagic](#file-support) and can act as a drop-in replacement for the [`file` command](https://github.com/file/file). However, unlike `file`, PolyFile can recursively identify embedded files, like [binwalk](https://github.com/ReFirmLabs/binwalk).
+
+PolyFile can be used in conjunction with its sister tool
+[PolyTracker](https://github.com/trailofbits/polytracker) for
+_Automated Lexical Annotation and Navigation of Parsers_, a backronym
+devised solely for the purpose of collectively referring to the tools
+as _The ALAN Parsers Project_.
+
+## Quickstart
+
+You can install the latest stable version of PolyFile from PyPI:
+```
+pip3 install polyfile
+```
+
+To install PolyFile from source, in the same directory as this README, run:
+```
+pip3 install .
+```
+
+Important: Before installing from source, make sure Java is installed. Java is used to
+run the Kaitai Struct compiler, which compiles the file format definitions.
+
+This will automatically install the `polyfile` and `polymerge` executables in your path.
+
+## Usage
+
+Running `polyfile` on a file with no arguments will mimic the behavior of `file --keep-going`:
+```console
+$ polyfile png-polyglot.png
+PNG image data, 256 x 144, 8-bit/color RGB, non-interlaced
+Brainfu** Program
+Malformed PDF
+PDF document, version 1.3,  1 pages
+ZIP end of central directory record Java JAR archive 
+```
+To generate an interactive hex viewer for the file, use the `--html` option:
+```console
+$ polyfile --html output.html png-polyglot.png
+Found a file of type application/pdf at byte offset 0
+Found a file of type application/x-brainfuck at byte offset 0
+Found a file of type image/png at byte offset 0
+Found a file of type application/zip at byte offset 0
+Found a file of type application/java-archive at byte offset 0
+Saved HTML output to output.html
+```
+
+Run `polyfile --help` for full usage instructions.
+
+### Interactive Debugger
+
+PolyFile has an interactive debugger both for its file matching and parsing. It can be used to debug a libmagic pattern 
+definition, determine why a specific file fails to be classified as the expected MIME type, or step through a parser.
+You can run PolyFile with the debugger enabled using the `-db` option.
+
+### File Support
+
+PolyFile has a cleanroom, [pure Python implementation of the libmagic file classifier](#libmagic-implementation), and supports all 263 MIME types that it can identify.
+
+It currently has support for parsing and semantically mapping the following formats:
+* PDF, using an instrumented version of [Didier Stevens' public domain, permissive, forensic parser](https://blog.didierstevens.com/programs/pdf-tools/)
+* ZIP, including recursive identification of all ZIP contents
+* JPEG/JFIF, using its [Kaitai Struct grammar](https://formats.kaitai.io/jpeg/index.html)
+* [iNES](https://wiki.nesdev.com/w/index.php/INES)
+* [Any other format](https://formats.kaitai.io/index.html) specified in a [KSY grammar](https://doc.kaitai.io/user_guide.html)
+
+For an example that exercises all of these file formats, run:
+```bash
+curl -v --silent https://www.sultanik.com/files/ESultanikResume.pdf | polyfile --html ESultanikResume.html -
+```
+
+Prior to PolyFile version 0.3.0, it used the [TrID database](http://mark0.net/soft-trid-deflist.html) for file
+identification rather than the libmagic file definitions. This proved to be very slow (since TrID has many duplicate
+entries) and prone to false positives (since TrID's file definitions are much simpler than libmagic's). The original
+TrID matching code is still shipped with PolyFile and can be invoked programmatically, but it is not used by default.
+
+### Output Format
+
+PolyFile has several options for outputting its results, specified by its `--format` option. For computer-readable output, PolyFile has an extension of the [SBuD](https://github.com/corkami/sbud) JSON format described [in the documentation](docs/json_format.md). Prior to version 0.5.0 this was the default output format of PolyFile. However, now the default output format is to mimic the behavior of the `file` command. To maintain the original behavior, use the `--format sbud` option.
+
+### libmagic Implementation
+
+PolyFile has a cleanroom implementation of [libmagic (used in the `file` command)](https://github.com/file/file).
+It can be invoked programmatically by running:
+```python
+from polyfile.magic import MagicMatcher
+
+with open("file_to_test", "rb") as f:
+    # the default instance automatically loads all file definitions
+    for match in MagicMatcher.DEFAULT_INSTANCE.match(f.read()):
+        for mimetype in match.mimetypes:
+            print(f"Matched MIME: {mimetype}")
+        print(f"Match string: {match!s}")
+```
+To load a specific or custom file definition:
+```python
+list_of_paths_to_definitions = ["def1", "def2"]
+matcher = MagicMatcher.parse(*list_of_paths_to_definitions)
+with open("file_to_test", "rb") as f:
+    for match in matcher.match(f.read()):
+        ...
+```
+
+## Extending PolyFile
+
+Instructions on extending PolyFile to support more file formats with new matchers and parsers is described [in the documentation]([in the documentation](docs/extending_polyfile.md)).
+
+## License and Acknowledgements
+
+This research was developed by [Trail of
+Bits](https://www.trailofbits.com/) with funding from the Defense
+Advanced Research Projects Agency (DARPA) under the SafeDocs program
+as a subcontractor to [Galois](https://galois.com). It is licensed under the [Apache 2.0 license](LICENSE).
+© 2019, Trail of Bits.

polyfile_weave-0.5.5/README.md

@@ -0,0 +1,125 @@
+# PolyFile
+<p align="center">
+  <img src="logo/polyfile_name.png?raw=true" width="256" title="PolyFile">
+</p>
+<br />
+
+[![PyPI version](https://badge.fury.io/py/polyfile.svg)](https://badge.fury.io/py/polyfile)
+[![Tests](https://github.com/trailofbits/polyfile/workflows/Tests/badge.svg)](https://github.com/trailofbits/polyfile/actions)
+[![Slack Status](https://slack.empirehacking.nyc/badge.svg)](https://slack.empirehacking.nyc)
+
+A utility to identify and map the semantic and syntactic structure of files,
+including polyglots, chimeras, and schizophrenic files. It has [a pure-Python implementation of libmagic](#file-support) and can act as a drop-in replacement for the [`file` command](https://github.com/file/file). However, unlike `file`, PolyFile can recursively identify embedded files, like [binwalk](https://github.com/ReFirmLabs/binwalk).
+
+PolyFile can be used in conjunction with its sister tool
+[PolyTracker](https://github.com/trailofbits/polytracker) for
+_Automated Lexical Annotation and Navigation of Parsers_, a backronym
+devised solely for the purpose of collectively referring to the tools
+as _The ALAN Parsers Project_.
+
+## Quickstart
+
+You can install the latest stable version of PolyFile from PyPI:
+```
+pip3 install polyfile
+```
+
+To install PolyFile from source, in the same directory as this README, run:
+```
+pip3 install .
+```
+
+Important: Before installing from source, make sure Java is installed. Java is used to
+run the Kaitai Struct compiler, which compiles the file format definitions.
+
+This will automatically install the `polyfile` and `polymerge` executables in your path.
+
+## Usage
+
+Running `polyfile` on a file with no arguments will mimic the behavior of `file --keep-going`:
+```console
+$ polyfile png-polyglot.png
+PNG image data, 256 x 144, 8-bit/color RGB, non-interlaced
+Brainfu** Program
+Malformed PDF
+PDF document, version 1.3,  1 pages
+ZIP end of central directory record Java JAR archive 
+```
+To generate an interactive hex viewer for the file, use the `--html` option:
+```console
+$ polyfile --html output.html png-polyglot.png
+Found a file of type application/pdf at byte offset 0
+Found a file of type application/x-brainfuck at byte offset 0
+Found a file of type image/png at byte offset 0
+Found a file of type application/zip at byte offset 0
+Found a file of type application/java-archive at byte offset 0
+Saved HTML output to output.html
+```
+
+Run `polyfile --help` for full usage instructions.
+
+### Interactive Debugger
+
+PolyFile has an interactive debugger both for its file matching and parsing. It can be used to debug a libmagic pattern 
+definition, determine why a specific file fails to be classified as the expected MIME type, or step through a parser.
+You can run PolyFile with the debugger enabled using the `-db` option.
+
+### File Support
+
+PolyFile has a cleanroom, [pure Python implementation of the libmagic file classifier](#libmagic-implementation), and supports all 263 MIME types that it can identify.
+
+It currently has support for parsing and semantically mapping the following formats:
+* PDF, using an instrumented version of [Didier Stevens' public domain, permissive, forensic parser](https://blog.didierstevens.com/programs/pdf-tools/)
+* ZIP, including recursive identification of all ZIP contents
+* JPEG/JFIF, using its [Kaitai Struct grammar](https://formats.kaitai.io/jpeg/index.html)
+* [iNES](https://wiki.nesdev.com/w/index.php/INES)
+* [Any other format](https://formats.kaitai.io/index.html) specified in a [KSY grammar](https://doc.kaitai.io/user_guide.html)
+
+For an example that exercises all of these file formats, run:
+```bash
+curl -v --silent https://www.sultanik.com/files/ESultanikResume.pdf | polyfile --html ESultanikResume.html -
+```
+
+Prior to PolyFile version 0.3.0, it used the [TrID database](http://mark0.net/soft-trid-deflist.html) for file
+identification rather than the libmagic file definitions. This proved to be very slow (since TrID has many duplicate
+entries) and prone to false positives (since TrID's file definitions are much simpler than libmagic's). The original
+TrID matching code is still shipped with PolyFile and can be invoked programmatically, but it is not used by default.
+
+### Output Format
+
+PolyFile has several options for outputting its results, specified by its `--format` option. For computer-readable output, PolyFile has an extension of the [SBuD](https://github.com/corkami/sbud) JSON format described [in the documentation](docs/json_format.md). Prior to version 0.5.0 this was the default output format of PolyFile. However, now the default output format is to mimic the behavior of the `file` command. To maintain the original behavior, use the `--format sbud` option.
+
+### libmagic Implementation
+
+PolyFile has a cleanroom implementation of [libmagic (used in the `file` command)](https://github.com/file/file).
+It can be invoked programmatically by running:
+```python
+from polyfile.magic import MagicMatcher
+
+with open("file_to_test", "rb") as f:
+    # the default instance automatically loads all file definitions
+    for match in MagicMatcher.DEFAULT_INSTANCE.match(f.read()):
+        for mimetype in match.mimetypes:
+            print(f"Matched MIME: {mimetype}")
+        print(f"Match string: {match!s}")
+```
+To load a specific or custom file definition:
+```python
+list_of_paths_to_definitions = ["def1", "def2"]
+matcher = MagicMatcher.parse(*list_of_paths_to_definitions)
+with open("file_to_test", "rb") as f:
+    for match in matcher.match(f.read()):
+        ...
+```
+
+## Extending PolyFile
+
+Instructions on extending PolyFile to support more file formats with new matchers and parsers is described [in the documentation]([in the documentation](docs/extending_polyfile.md)).
+
+## License and Acknowledgements
+
+This research was developed by [Trail of
+Bits](https://www.trailofbits.com/) with funding from the Defense
+Advanced Research Projects Agency (DARPA) under the SafeDocs program
+as a subcontractor to [Galois](https://galois.com). It is licensed under the [Apache 2.0 license](LICENSE).
+© 2019, Trail of Bits.

polyfile_weave-0.5.5/compile_kaitai_parsers.py

@@ -0,0 +1,161 @@
+import concurrent.futures
+from datetime import datetime, timezone
+import json
+from multiprocessing import cpu_count
+from pathlib import Path
+import subprocess
+import sys
+from typing import Any, Dict, List, Optional, Tuple
+
+POLYFILE_DIR: Path = Path(__file__).absolute().parent
+COMPILE_SCRIPT: Path = POLYFILE_DIR / "polyfile" / "kaitai" / "compiler.py"
+KAITAI_FORMAT_LIBRARY: Path = POLYFILE_DIR / "kaitai_struct_formats"
+KAITAI_PARSERS_DIR: Path = POLYFILE_DIR / "polyfile" / "kaitai" / "parsers"
+MANIFEST_PATH: Path = KAITAI_PARSERS_DIR / "manifest.json"
+EXCLUDE = [
+    './kaitai_struct_formats/firmware/broadcom_trx.ksy',
+    './kaitai_struct_formats/filesystem/vdi.ksy',
+    './kaitai_struct_formats/filesystem/lvm2.ksy',
+    './kaitai_struct_formats/image/pif.ksy',
+    './kaitai_struct_formats/scientific/nt_mdt/nt_mdt.ksy'
+]
+
+
+
+# Make sure the ktaitai_struct_formats submodlue is cloned:
+if not (KAITAI_FORMAT_LIBRARY / "README.md").exists():
+    subprocess.check_call(["git", "submodule", "init"], cwd=str(POLYFILE_DIR))
+    subprocess.check_call(["git", "submodule", "update"], cwd=str(POLYFILE_DIR))
+
+
+def compile_ksy(path: Path) -> List[Tuple[str, str]]:
+    output = subprocess.check_output(
+        [sys.executable, str(COMPILE_SCRIPT), str(path), str(KAITAI_PARSERS_DIR)],
+        cwd=str(KAITAI_FORMAT_LIBRARY)
+    ).decode("utf-8")
+    return [  # type: ignore
+        tuple(line.split("\t")[:2])  # (class_name, python_path)
+        for line in output.split("\n") if line.strip()
+    ]
+
+
+def mtime(path: Path) -> datetime:
+    # has the file been modified?
+    was_modified = subprocess.call(
+        ["git", "diff", "--exit-code", str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+        cwd=str(KAITAI_FORMAT_LIBRARY)
+    ) != 0
+    if was_modified:
+        # the file was modified since the last commit, so use its filesystem mtime
+        return datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
+
+    # the file has not been modified since the last commit, so use the last commit time
+    last_commit_date = subprocess.check_output(["git", "log", "-1", "--format=\"%cd\"", str(path)],
+                                               cwd=str(KAITAI_FORMAT_LIBRARY)).decode("utf-8").strip()
+    return datetime.strptime(last_commit_date, "\"%a %b %d %H:%M:%S %Y %z\"")
+
+
+def rebuild(force: bool = False):
+    # Get the list of copyleft-licensed files to exclude
+    excluded_paths = {Path(f).absolute() for f in EXCLUDE}
+
+    # Remove the manifest file to force a rebuild:
+    if force or not MANIFEST_PATH.exists():
+        if MANIFEST_PATH.exists():
+            MANIFEST_PATH.unlink()
+        needs_rebuild = True
+    else:
+        # see if any of the files are out of date and need to be recompiled
+        newest_definition: Optional[datetime] = None
+        for definition in KAITAI_FORMAT_LIBRARY.glob("**/*.ksy"):
+            # Skip excluded files
+            if definition.absolute() in excluded_paths:
+                continue
+            modtime = mtime(definition)
+            if newest_definition is None or newest_definition < modtime:
+                newest_definition = modtime
+        needs_rebuild = newest_definition > mtime(MANIFEST_PATH)
+
+    if needs_rebuild:
+        # the definitions have been updated, so we need to recompile everything
+
+        if subprocess.call(
+            [sys.executable, str(COMPILE_SCRIPT), "--install"]
+        ) != 0:
+            sys.stderr.write("Error: You must have kaitai-struct-compiler installed\nSee https://kaitai.io/#download\n")
+            sys.exit(1)
+
+        # Count non-excluded files
+        all_ksy_files = list(KAITAI_FORMAT_LIBRARY.glob("**/*.ksy"))
+        ksy_files_to_compile = [f for f in all_ksy_files if f.absolute() not in excluded_paths]
+        num_excluded = len(all_ksy_files) - len(ksy_files_to_compile)
+        
+        if num_excluded > 0:
+            print(f"Excluding {num_excluded} copyleft-licensed KSY files from compilation")
+        
+        num_files = len(ksy_files_to_compile)
+
+        try:
+            from tqdm import tqdm
+        except ModuleNotFoundError:
+            def tqdm(*args, **kwargs):
+                class TQDM:
+                    def __enter__(self):
+                        return self
+
+                    def __exit__(self, exc_type, exc_val, exc_tb):
+                        pass
+
+                    def write(self, message, *_, **__):
+                        sys.stderr.write(message)
+                        sys.stderr.write("\n")
+                        sys.stderr.flush()
+
+                    def update(self, n: int):
+                        pass
+                return TQDM()
+
+        ksy_manifest: Dict[str, Dict[str, Any]] = {}
+
+        with tqdm(leave=False, desc="Compiling the Kaitai Struct Format Library", total=num_files) as t:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count()) as executor:
+                futures_to_path: Dict[concurrent.futures.Future, Path] = {
+                    executor.submit(compile_ksy, file): file
+                    for file in ksy_files_to_compile
+                }
+                for future in concurrent.futures.as_completed(futures_to_path):
+                    t.update(1)
+                    path = futures_to_path[future]
+                    relative_path = str(path.relative_to(KAITAI_FORMAT_LIBRARY))
+                    if relative_path in ksy_manifest:
+                        raise ValueError(f"{relative_path} appears twice in the Kaitai format library!")
+                    try:
+                        (first_spec_class_name, first_spec_python_path), *dependencies = future.result()
+                        ksy_manifest[relative_path] = {
+                            "class_name": first_spec_class_name,
+                            "python_path": str(Path(first_spec_python_path).relative_to(KAITAI_PARSERS_DIR)),
+                            "dependencies": [
+                                {
+                                    "class_name": class_name,
+                                    "python_path": str(Path(python_path).relative_to(KAITAI_PARSERS_DIR))
+                                }
+                                for class_name, python_path in dependencies
+                            ]
+                        }
+                        t.write(f"Compiled {path.name}")
+                    except Exception as e:
+                        t.write(f"Warning: Failed to compile {path}: {e}\n")
+
+        with open(MANIFEST_PATH, "w") as f:
+            json.dump(ksy_manifest, f)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rebuild-all", "-a", action="store_true", help="rebuilds all parsers from scratch")
+
+    args = parser.parse_args()
+
+    rebuild(force=args.rebuild_all)