frontmatter-format 0.0.0.dev0__tar.gz

frontmatter_format-0.0.0.dev0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Joshua Levy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

frontmatter_format-0.0.0.dev0/PKG-INFO

@@ -0,0 +1,289 @@
+Metadata-Version: 2.1
+Name: frontmatter-format
+Version: 0.0.0.dev0
+Summary: A format for YAML frontmatter on any file.
+Home-page: https://github.com/jlevy/frontmatter-format
+License: MIT
+Author: Joshua Levy
+Author-email: joshua@cal.berkeley.edu
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: ruamel-yaml (>=0.18.6,<0.19.0)
+Project-URL: Repository, https://github.com/jlevy/frontmatter-format
+Description-Content-Type: text/markdown
+
+# Frontmatter Format
+
+## Motivation
+
+Simple, readable metadata attached to files can be useful in numerous situations, such as
+recording title, author, source, copyright, or the provenance of a file.
+
+Unfortunately, it's often unclear how to format such metadata consistently across different
+file types while also not breaking interoperability with existing tools.
+
+**Frontmatter format** is a way to add metadata as frontmatter on any file.
+It is a simple set of conventions to put structured metadata as YAML at the top of a file in
+a syntax that is broadly compatible with programming languages, browsers, editors, and other
+tools.
+
+Frontmatter format specifies a syntax for the metadata as a comment block at the top of a
+file.
+This approach works while ensuring the file remains valid Markdown, HTML, CSS, Python,
+C/C++, Rust, SQL, or most other text formats.
+
+Frontmatter format is a generalization of the common format for frontmatter used by Jekyll
+and other CMSs for Markdown files.
+In that format, frontmatter is enclosed in lines containing `---` delimiters.
+
+In this generalized format, we allow several styles of frontmatter demarcation, with the
+first line of the file indicating the format and style.
+
+This is a description of the format and a simple reference implementation.
+The implementation is in Python but the format is very simple and easy to implement in any
+language.
+
+The purpose of this repository is to explain the idea of the format so anyone can use it,
+and encourage the adoption of the format, especially for workflows around text documents that
+are becoming increasingly common in AI tools and pipelines.
+
+## Examples
+
+```markdown
+---
+title: Sample Markdown File
+state: draft
+created_at: 2022-08-07 00:00:00
+tags:
+  - yaml
+  - examples
+# This is a YAML comment, so ignored.
+---
+Hello, *World*!
+```
+
+```html
+<!---
+title: Sample HTML File
+--->
+Hello, <i>World</i>!
+```
+
+```python
+#---
+# author: Jane Doe
+# description: A sample Python script
+#---
+print("Hello, World!")
+```
+
+```css
+/*---
+filename: styles.css
+---*/
+.hello {
+  color: green;
+}
+```
+
+```sql
+----
+-- title: Sample SQL Script
+----
+SELECT * FROM world;
+```
+
+## Advantages of this Approach
+
+- **Compatible with existing syntax:** By choosing a style for the metadata consistent with
+  any given file, it generally doesn't break existing tools.
+  Almost every language has a style for which frontmatter works as a comment.
+
+- **Auto-detectable format:** Frontmatter and its format can be recognized by the first few
+  bytes of the file.
+  That means it's possible to detect metadata and parse it automatically.
+
+- **Metadata is optional:** Files with or without metadata can be read with the same tools.
+  So it's easy to roll out metadata into files gracefully, as needed file by file.
+
+- **YAML syntax:** JSON, YAML, XML, and TOML are all used for metadata in some situatiohns.
+  YAML is the best choice here because it is already in widespread use with Markdown, is a
+  superset of JSON (in case an application wishes to use pure JSON), and is easy to read and
+  edit manually.
+
+## Format Definition
+
+A file is in frontmatter format if the first characters are one of the following:
+
+- `---`
+
+- `<!---`
+
+- `#---`
+
+- `//---`
+
+- `/*---`
+
+and if this prefix is followed by a newline (`\n`).
+
+The prefix determines the *style* of the frontmatter.
+The style specifies the matching terminating delimiter for the end of the frontmatter as
+well as an optional prefix (which is typically a comment character in some language).
+
+The supported frontmatter styles are:
+
+1. *YAML style*: delimiters `---` and `---` with no prefix on each line.
+   Useful for text or Markdown content.
+
+2. *HTML style*: delimiters `<!---` and `--->` with no prefix on each line.
+   Useful for HTML or XML or similar content.
+
+3. *Hash style*: delimiters `#---` and `#---` with `# ` prefix on each line.
+   Useful for Python or similar code content.
+   Also works for CSV files with many tools.
+
+4. *Rust style*: delimiters `//---` and `//---` with `// ` prefix on each line.
+   Useful for Rust or C++ or similar code content.
+
+5. *C style*: delimiters `/*---` and `---*/` with no prefix on each line.
+   Useful for JavaScript, TypeScript, CSS or C or similar code content.
+
+6. *Dash style*: delimiters `----` and `----` with `-- ` prefix on each line.
+   Useful for SQL or similar code content.
+
+The delimiters must be alone on their own lines, terminated with a newline.
+
+Any style is acceptable on any file as it can be automatically detected.
+When writing, you can specify the style.
+
+For all frontmatter styles, the content between the delimiters is YAML text in UTF-8
+encoding, with an optional prefix on each line that depends on the style.
+
+For some of the formats, each frontmatter line is prefixed with a prefix to make sure the
+entire file remains valid in a given syntax (Python, Rust, SQL, etc.). This prefix is
+stripped during parsing.
+
+It is recommended to use a prefix with a trailing space (such as `# ` or `// `) but a bare
+prefix without the trailing space (`#` or `##`) is also allowed.
+
+Other whitespace is preserved (before parsing with YAML).
+
+Note that YAML comments, which are lines beginning with `#` in the metadata, are allowed.
+For example, for hash style, this means there must be two hashes (`# #` or `##`) at the
+start of a comment line.
+
+There is no restriction on the content of the file after the frontmatter.
+It may even contain other content in frontmatter format, but this will not be parsed as
+frontmatter.
+Typically, it is text, but it could be binary as well.
+
+Frontmatter is optional.
+This means almost any text file can be read as frontmatter format.
+
+## Reference Implementation
+
+This is a simple Python reference implementation.
+It auto-detects all the frontmatter styles above.
+It supports reading small files easily into memory, but also allows extracting or changing
+frontmatter without reading an entire file.
+
+Both raw (string) parsed YAML frontmatter (using ruamel.yaml) are supported.
+For readability, there is also support for preferred sorting of YAML keys.
+
+## Installation
+
+```
+# Use pip
+pip install frontmatter-format
+# Or poetry
+poetry add frontmatter-format
+```
+
+## Usage
+
+```python
+from frontmatter_format import fmf_read, fmf_read_raw, fmf_write, FmStyle
+
+# Write some content:
+content = "Hello, World!"
+metadata = {"title": "Test Title", "author": "Test Author"}
+fmf_write("example.md", content, metadata, style=FmStyle.yaml)
+
+# Or any other desired style:
+html_content = "<p>Hello, World!</p>"
+fmf_write("example.html", content, metadata, style=FmStyle.html)
+
+# Read it back. Style is auto-detected:
+content, metadata = fmf_read("example.md")
+print(content)  # Outputs: Hello, World!
+print(metadata)  # Outputs: {'title': 'Test Title', 'author': 'Test Author'}
+
+# Read metadata without parsing:
+content, raw_metadata = fmf_read_raw("example.md")
+print(content)  # Outputs: Hello, World!
+print(raw_metadata)  # Outputs: 'title: Test Title\nauthor: Test Author\n'
+```
+
+The above is easiest for small files, but you can also operate more efficiently directly on
+files, without reading the file contents into memory.
+
+```python
+from frontmatter_format import fmf_strip_frontmatter, fmf_insert_frontmatter, fmf_read_frontmatter_raw
+
+# Strip and discard the metadata from a file:
+fmf_strip_frontmatter("example.md")
+
+# Insert the metadata at the top of an existing file:
+new_metadata = {"title": "New Title", "author": "New Author"}
+fmf_insert_frontmatter("example.md", new_metadata, fm_style=FmStyle.yaml)
+
+# Read the raw frontmatter metadata and get the offset for the rest of the content:
+raw_metadata, offset = fmf_read_frontmatter_raw("example.md")
+print(raw_metadata)  # Outputs: 'title: Test Title\nauthor: Test Author\n'
+print(offset)  # Outputs the byte offset where the content starts
+```
+
+## FAQ
+
+- **Is this mature?** This is the first draft of this format.
+  But I've been using this on my own projects for a couple months.
+  The flexibity of just having metadata on all your text files is great for workflows,
+  pipelines, etc.
+
+- **When should we use it?** All the time if you can!
+  It's especially important for command-line tools, AI agents, LLM workflows, since you
+  often want to store extra metadata is a consistent way on text inputs of various formats
+  like Markdown, HTML, CSS, and Python.
+
+- **Does this specify the format of the YAML itself?** No.
+  This is simply a format for attaching metadata.
+  What metadata you attach is up to your use case.
+  Standardizing headings like title, author, description, let alone other more
+  application-specific information is beyond the scope of this frontmatter format.
+
+- **Can this work with Pydantic?** Yes, definitely.
+  In fact, I think it's probably a good practice to define self-identifiable Pydantic (or
+  Zod) schemas for all your metadata, and then just serialize and deserialize them to
+  frontmatter everywhere.
+
+- **Isn't this the same as what some CMSs use, Markdown files and YAML at the top?** Yes!
+  But this generalizes that format, and removes the direct tie-in to Markdown or any CMS.
+  This can work with any tool.
+  For HTML and code, it works basically with no changes at all since the frontmatter is
+  considered a comment.
+
+- **Can this work with binary files?** No reason why not, if it makes sense for you!
+  You can use `fmf_insert_frontmatter()` to add metadata of any style to any file.
+  Whether this works for your application depends on the file format.
+
+- **Does this work for CSV files?** Sort of.
+  Some tools do properly honor hash style comments when parsing CSV files.
+  A few do not. Our recommendation is go ahead and use it, and find ways to strip the
+  metadata at the last minute if you really can't get a tool to work with the metadata.
+

frontmatter_format-0.0.0.dev0/README.md

@@ -0,0 +1,269 @@
+# Frontmatter Format
+
+## Motivation
+
+Simple, readable metadata attached to files can be useful in numerous situations, such as
+recording title, author, source, copyright, or the provenance of a file.
+
+Unfortunately, it's often unclear how to format such metadata consistently across different
+file types while also not breaking interoperability with existing tools.
+
+**Frontmatter format** is a way to add metadata as frontmatter on any file.
+It is a simple set of conventions to put structured metadata as YAML at the top of a file in
+a syntax that is broadly compatible with programming languages, browsers, editors, and other
+tools.
+
+Frontmatter format specifies a syntax for the metadata as a comment block at the top of a
+file.
+This approach works while ensuring the file remains valid Markdown, HTML, CSS, Python,
+C/C++, Rust, SQL, or most other text formats.
+
+Frontmatter format is a generalization of the common format for frontmatter used by Jekyll
+and other CMSs for Markdown files.
+In that format, frontmatter is enclosed in lines containing `---` delimiters.
+
+In this generalized format, we allow several styles of frontmatter demarcation, with the
+first line of the file indicating the format and style.
+
+This is a description of the format and a simple reference implementation.
+The implementation is in Python but the format is very simple and easy to implement in any
+language.
+
+The purpose of this repository is to explain the idea of the format so anyone can use it,
+and encourage the adoption of the format, especially for workflows around text documents that
+are becoming increasingly common in AI tools and pipelines.
+
+## Examples
+
+```markdown
+---
+title: Sample Markdown File
+state: draft
+created_at: 2022-08-07 00:00:00
+tags:
+  - yaml
+  - examples
+# This is a YAML comment, so ignored.
+---
+Hello, *World*!
+```
+
+```html
+<!---
+title: Sample HTML File
+--->
+Hello, <i>World</i>!
+```
+
+```python
+#---
+# author: Jane Doe
+# description: A sample Python script
+#---
+print("Hello, World!")
+```
+
+```css
+/*---
+filename: styles.css
+---*/
+.hello {
+  color: green;
+}
+```
+
+```sql
+----
+-- title: Sample SQL Script
+----
+SELECT * FROM world;
+```
+
+## Advantages of this Approach
+
+- **Compatible with existing syntax:** By choosing a style for the metadata consistent with
+  any given file, it generally doesn't break existing tools.
+  Almost every language has a style for which frontmatter works as a comment.
+
+- **Auto-detectable format:** Frontmatter and its format can be recognized by the first few
+  bytes of the file.
+  That means it's possible to detect metadata and parse it automatically.
+
+- **Metadata is optional:** Files with or without metadata can be read with the same tools.
+  So it's easy to roll out metadata into files gracefully, as needed file by file.
+
+- **YAML syntax:** JSON, YAML, XML, and TOML are all used for metadata in some situatiohns.
+  YAML is the best choice here because it is already in widespread use with Markdown, is a
+  superset of JSON (in case an application wishes to use pure JSON), and is easy to read and
+  edit manually.
+
+## Format Definition
+
+A file is in frontmatter format if the first characters are one of the following:
+
+- `---`
+
+- `<!---`
+
+- `#---`
+
+- `//---`
+
+- `/*---`
+
+and if this prefix is followed by a newline (`\n`).
+
+The prefix determines the *style* of the frontmatter.
+The style specifies the matching terminating delimiter for the end of the frontmatter as
+well as an optional prefix (which is typically a comment character in some language).
+
+The supported frontmatter styles are:
+
+1. *YAML style*: delimiters `---` and `---` with no prefix on each line.
+   Useful for text or Markdown content.
+
+2. *HTML style*: delimiters `<!---` and `--->` with no prefix on each line.
+   Useful for HTML or XML or similar content.
+
+3. *Hash style*: delimiters `#---` and `#---` with `# ` prefix on each line.
+   Useful for Python or similar code content.
+   Also works for CSV files with many tools.
+
+4. *Rust style*: delimiters `//---` and `//---` with `// ` prefix on each line.
+   Useful for Rust or C++ or similar code content.
+
+5. *C style*: delimiters `/*---` and `---*/` with no prefix on each line.
+   Useful for JavaScript, TypeScript, CSS or C or similar code content.
+
+6. *Dash style*: delimiters `----` and `----` with `-- ` prefix on each line.
+   Useful for SQL or similar code content.
+
+The delimiters must be alone on their own lines, terminated with a newline.
+
+Any style is acceptable on any file as it can be automatically detected.
+When writing, you can specify the style.
+
+For all frontmatter styles, the content between the delimiters is YAML text in UTF-8
+encoding, with an optional prefix on each line that depends on the style.
+
+For some of the formats, each frontmatter line is prefixed with a prefix to make sure the
+entire file remains valid in a given syntax (Python, Rust, SQL, etc.). This prefix is
+stripped during parsing.
+
+It is recommended to use a prefix with a trailing space (such as `# ` or `// `) but a bare
+prefix without the trailing space (`#` or `##`) is also allowed.
+
+Other whitespace is preserved (before parsing with YAML).
+
+Note that YAML comments, which are lines beginning with `#` in the metadata, are allowed.
+For example, for hash style, this means there must be two hashes (`# #` or `##`) at the
+start of a comment line.
+
+There is no restriction on the content of the file after the frontmatter.
+It may even contain other content in frontmatter format, but this will not be parsed as
+frontmatter.
+Typically, it is text, but it could be binary as well.
+
+Frontmatter is optional.
+This means almost any text file can be read as frontmatter format.
+
+## Reference Implementation
+
+This is a simple Python reference implementation.
+It auto-detects all the frontmatter styles above.
+It supports reading small files easily into memory, but also allows extracting or changing
+frontmatter without reading an entire file.
+
+Both raw (string) parsed YAML frontmatter (using ruamel.yaml) are supported.
+For readability, there is also support for preferred sorting of YAML keys.
+
+## Installation
+
+```
+# Use pip
+pip install frontmatter-format
+# Or poetry
+poetry add frontmatter-format
+```
+
+## Usage
+
+```python
+from frontmatter_format import fmf_read, fmf_read_raw, fmf_write, FmStyle
+
+# Write some content:
+content = "Hello, World!"
+metadata = {"title": "Test Title", "author": "Test Author"}
+fmf_write("example.md", content, metadata, style=FmStyle.yaml)
+
+# Or any other desired style:
+html_content = "<p>Hello, World!</p>"
+fmf_write("example.html", content, metadata, style=FmStyle.html)
+
+# Read it back. Style is auto-detected:
+content, metadata = fmf_read("example.md")
+print(content)  # Outputs: Hello, World!
+print(metadata)  # Outputs: {'title': 'Test Title', 'author': 'Test Author'}
+
+# Read metadata without parsing:
+content, raw_metadata = fmf_read_raw("example.md")
+print(content)  # Outputs: Hello, World!
+print(raw_metadata)  # Outputs: 'title: Test Title\nauthor: Test Author\n'
+```
+
+The above is easiest for small files, but you can also operate more efficiently directly on
+files, without reading the file contents into memory.
+
+```python
+from frontmatter_format import fmf_strip_frontmatter, fmf_insert_frontmatter, fmf_read_frontmatter_raw
+
+# Strip and discard the metadata from a file:
+fmf_strip_frontmatter("example.md")
+
+# Insert the metadata at the top of an existing file:
+new_metadata = {"title": "New Title", "author": "New Author"}
+fmf_insert_frontmatter("example.md", new_metadata, fm_style=FmStyle.yaml)
+
+# Read the raw frontmatter metadata and get the offset for the rest of the content:
+raw_metadata, offset = fmf_read_frontmatter_raw("example.md")
+print(raw_metadata)  # Outputs: 'title: Test Title\nauthor: Test Author\n'
+print(offset)  # Outputs the byte offset where the content starts
+```
+
+## FAQ
+
+- **Is this mature?** This is the first draft of this format.
+  But I've been using this on my own projects for a couple months.
+  The flexibity of just having metadata on all your text files is great for workflows,
+  pipelines, etc.
+
+- **When should we use it?** All the time if you can!
+  It's especially important for command-line tools, AI agents, LLM workflows, since you
+  often want to store extra metadata is a consistent way on text inputs of various formats
+  like Markdown, HTML, CSS, and Python.
+
+- **Does this specify the format of the YAML itself?** No.
+  This is simply a format for attaching metadata.
+  What metadata you attach is up to your use case.
+  Standardizing headings like title, author, description, let alone other more
+  application-specific information is beyond the scope of this frontmatter format.
+
+- **Can this work with Pydantic?** Yes, definitely.
+  In fact, I think it's probably a good practice to define self-identifiable Pydantic (or
+  Zod) schemas for all your metadata, and then just serialize and deserialize them to
+  frontmatter everywhere.
+
+- **Isn't this the same as what some CMSs use, Markdown files and YAML at the top?** Yes!
+  But this generalizes that format, and removes the direct tie-in to Markdown or any CMS.
+  This can work with any tool.
+  For HTML and code, it works basically with no changes at all since the frontmatter is
+  considered a comment.
+
+- **Can this work with binary files?** No reason why not, if it makes sense for you!
+  You can use `fmf_insert_frontmatter()` to add metadata of any style to any file.
+  Whether this works for your application depends on the file format.
+
+- **Does this work for CSV files?** Sort of.
+  Some tools do properly honor hash style comments when parsing CSV files.
+  A few do not. Our recommendation is go ahead and use it, and find ways to strip the
+  metadata at the last minute if you really can't get a tool to work with the metadata.

frontmatter_format-0.0.0.dev0/frontmatter_format/__init__.py

@@ -0,0 +1,41 @@
+from .frontmatter_format import (
+    fmf_insert_frontmatter,
+    fmf_read,
+    fmf_read_frontmatter_raw,
+    fmf_read_raw,
+    fmf_strip_frontmatter,
+    fmf_write,
+    FmFormatError,
+    FmStyle,
+    Metadata,
+)
+from .key_sort import custom_key_sort
+from .yaml_util import (
+    add_default_yaml_representer,
+    dump_yaml,
+    from_yaml_string,
+    new_yaml,
+    read_yaml_file,
+    to_yaml_string,
+    write_yaml_file,
+)
+
+__all__ = [
+    "FmStyle",
+    "FmFormatError",
+    "fmf_write",
+    "fmf_read",
+    "fmf_read_raw",
+    "fmf_read_frontmatter_raw",
+    "fmf_strip_frontmatter",
+    "fmf_insert_frontmatter",
+    "Metadata",
+    "add_default_yaml_representer",
+    "dump_yaml",
+    "from_yaml_string",
+    "new_yaml",
+    "read_yaml_file",
+    "to_yaml_string",
+    "write_yaml_file",
+    "custom_key_sort",
+]

frontmatter_format-0.0.0.dev0/frontmatter_format/frontmatter_format.py

@@ -0,0 +1,262 @@
+"""
+Python implementation of frontmatter format.
+"""
+
+import os
+import shutil
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any, cast, Dict, List, Optional, Tuple
+
+from ruamel.yaml.error import YAMLError
+
+from .yaml_util import from_yaml_string, KeySort, to_yaml_string
+
+
+class FmFormatError(ValueError):
+    """
+    Error for frontmatter file format issues.
+    """
+
+
+@dataclass(frozen=True)
+class FmDelimiters:
+    start: str
+    end: str
+    prefix: str
+    strip_prefixes: List[str]
+
+
+class FmStyle(Enum):
+    """
+    The style of frontmatter demarcation to use.
+    """
+
+    yaml = FmDelimiters("---", "---", "", [])
+    html = FmDelimiters("<!---", "--->", "", [])
+    hash = FmDelimiters("#---", "#---", "# ", ["# ", "#"])
+    slash = FmDelimiters("//---", "//---", "// ", ["// ", "//"])
+    slash_star = FmDelimiters("/*---", "---*/", "", [])
+    dash = FmDelimiters("----", "----", "-- ", ["-- ", "--"])
+
+    @property
+    def start(self) -> str:
+        return self.value.start
+
+    @property
+    def end(self) -> str:
+        return self.value.end
+
+    @property
+    def prefix(self) -> str:
+        return self.value.prefix
+
+    @property
+    def strip_prefixes(self) -> List[str]:
+        return self.value.strip_prefixes
+
+    def strip_prefix(self, line: str) -> str:
+        for prefix in self.strip_prefixes:
+            if line.startswith(prefix):
+                return line[len(prefix) :]
+        return line
+
+
+Metadata = Dict[str, Any]
+"""
+Parsed metadata from frontmatter.
+"""
+
+
+def fmf_write(
+    path: Path | str,
+    content: str,
+    metadata: Optional[Metadata | str],
+    style: FmStyle = FmStyle.yaml,
+    key_sort: Optional[KeySort] = None,
+    make_parents: bool = True,
+) -> None:
+    """
+    Write the given Markdown text content to a file, with associated YAML metadata, in a
+    generalized Jekyll-style frontmatter format. Metadata can be a raw string or a dict
+    that will be serialized to YAML.
+    """
+    if isinstance(metadata, str):
+        frontmatter_str = metadata
+    else:
+        frontmatter_str = to_yaml_string(metadata, key_sort=key_sort)
+
+    path = Path(path)
+    if make_parents and path.parent:
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+    tmp_path = f"{path}.fmf.write.tmp"
+    try:
+        with open(tmp_path, "w", encoding="utf-8") as f:
+            if metadata:
+                f.write(style.start)
+                f.write("\n")
+                for line in frontmatter_str.splitlines():
+                    f.write(style.prefix + line)
+                    f.write("\n")
+                f.write(style.end)
+                f.write("\n")
+
+            f.write(content)
+        os.replace(tmp_path, path)
+    except Exception as e:
+        try:
+            os.remove(tmp_path)
+        except FileNotFoundError:
+            pass
+        raise e
+
+
+def fmf_read(path: Path | str) -> Tuple[str, Optional[Metadata]]:
+    """
+    Read UTF-8 text content (typically Markdown) from a file with optional YAML metadata
+    in Jekyll-style frontmatter format. Auto-detects variant formats for HTML and code
+    (Python style) based on whether the prefix is `---` or `<!---` or `#---`.
+    Reads the entire file into memory. Parses the metadata as YAML.
+    """
+    content, metadata_str = fmf_read_raw(path)
+    metadata = None
+    if metadata_str:
+        try:
+            metadata = from_yaml_string(metadata_str)
+        except YAMLError as e:
+            raise FmFormatError(f"Error parsing YAML metadata: `{path}`: {e}") from e
+        if not isinstance(metadata, dict):
+            raise FmFormatError(f"Invalid metadata type: {type(metadata)}")
+        metadata = cast(Metadata, metadata)
+    return content, metadata
+
+
+def fmf_read_raw(path: Path | str) -> Tuple[str, Optional[str]]:
+    """
+    Reads the full content and raw (unparsed) metadata from the file, both as strings.
+    """
+    metadata_str, offset = fmf_read_frontmatter_raw(path)
+
+    with open(path, "r", encoding="utf-8") as f:
+        f.seek(offset)
+        content = f.read()
+
+    return content, metadata_str
+
+
+def fmf_read_frontmatter_raw(path: Path | str) -> Tuple[Optional[str], int]:
+    """
+    Reads the metadata frontmatter from the file and returns the metadata string and
+    the seek offset of the beginning of the content. Does not parse the metadata.
+    Does not read the body content into memory.
+    """
+    metadata_lines: List[str] = []
+    in_metadata = False
+
+    with open(path, "r", encoding="utf-8") as f:
+        first_line = f.readline().rstrip()
+
+        if first_line == FmStyle.yaml.start:
+            delimiters = FmStyle.yaml
+            in_metadata = True
+        elif first_line == FmStyle.html.start:
+            delimiters = FmStyle.html
+            in_metadata = True
+        elif first_line == FmStyle.hash.start:
+            delimiters = FmStyle.hash
+            in_metadata = True
+        else:
+            # Empty file or no recognized frontmatter.
+            return None, 0
+
+        while True:
+            line = f.readline()
+            if not line:
+                break
+
+            if line.rstrip() == delimiters.end and in_metadata:
+                metadata_str = "".join(delimiters.strip_prefix(mline) for mline in metadata_lines)
+                return metadata_str, f.tell()
+
+            if in_metadata:
+                metadata_lines.append(line)
+
+        if in_metadata:  # If still true, the end delimiter was never found
+            raise FmFormatError(
+                f"Delimiter `{delimiters.end}` for end of frontmatter not found: `{(path)}`"
+            )
+
+    return None, 0
+
+
+def fmf_strip_frontmatter(path: Path | str) -> None:
+    """
+    Strip the metadata frontmatter from the file, in place on the file.
+    Does not read the content (except to do a file copy) so should work fairly
+    quickly on large files. Does nothing if there is no frontmatter.
+    """
+    _, offset = fmf_read_frontmatter_raw(path)
+    if offset > 0:
+        tmp_path = f"{path}.fmf.strip.tmp"
+        try:
+            with open(path, "r", encoding="utf-8") as original_file, open(
+                tmp_path, "w", encoding="utf-8"
+            ) as temp_file:
+                original_file.seek(offset)
+                shutil.copyfileobj(original_file, temp_file)
+            os.replace(tmp_path, path)
+        except Exception as e:
+            try:
+                os.remove(tmp_path)
+            except FileNotFoundError:
+                pass
+            raise e
+
+
+def fmf_insert_frontmatter(
+    path: Path | str,
+    metadata: Optional[Metadata],
+    fm_style: FmStyle = FmStyle.yaml,
+    key_sort: Optional[KeySort] = None,
+) -> None:
+    """
+    Insert metadata as frontmatter into the given file, inserting at the top
+    and replacing any existing frontmatter.
+    """
+    if metadata is None:
+        return
+
+    if isinstance(metadata, str):
+        frontmatter_str = metadata
+    else:
+        frontmatter_str = to_yaml_string(metadata, key_sort=key_sort)
+
+    # Prepare the new frontmatter.
+    frontmatter_lines = [fm_style.start + "\n"]
+    if frontmatter_str:
+        for line in frontmatter_str.splitlines():
+            frontmatter_lines.append(fm_style.prefix + line + "\n")
+    frontmatter_lines.append(fm_style.end + "\n")
+
+    tmp_path = f"{path}.fmf.insert.tmp"
+
+    try:
+        # Determine where any existing frontmatter ends (offset).
+        _, offset = fmf_read_frontmatter_raw(path)
+
+        with open(tmp_path, "w", encoding="utf-8") as temp_file:
+            temp_file.writelines(frontmatter_lines)
+
+            with open(path, "r", encoding="utf-8") as original_file:
+                original_file.seek(offset)
+                shutil.copyfileobj(original_file, temp_file)
+
+        os.replace(tmp_path, path)
+    except Exception as e:
+        try:
+            os.remove(tmp_path)
+        except FileNotFoundError:
+            pass
+        raise e

frontmatter_format-0.0.0.dev0/frontmatter_format/key_sort.py

@@ -0,0 +1,19 @@
+from typing import Any, Callable, List, Tuple, TypeVar
+
+T = TypeVar("T")
+
+
+def custom_key_sort(priority_keys: List[T]) -> Callable[[T], Any]:
+    """
+    Custom sort function that prioritizes the specific keys in a certain order, followed
+    by all the other keys in natural order.
+    """
+
+    def sort_func(key: T) -> Tuple[float, T]:
+        try:
+            i = priority_keys.index(key)
+            return (float(i), key)
+        except ValueError:
+            return (float("inf"), key)
+
+    return sort_func

frontmatter_format-0.0.0.dev0/frontmatter_format/yaml_util.py

@@ -0,0 +1,132 @@
+"""
+YAML file storage. Wraps ruamel.yaml with a few extra features.
+"""
+
+import os
+from io import StringIO
+from typing import Any, Callable, Dict, Optional, TextIO, Type
+
+from ruamel.yaml import Representer, YAML
+
+KeySort = Callable[[str], Any]
+
+
+def none_or_empty_dict(val: Any) -> bool:
+    return val is None or val == {}
+
+
+_default_representers: Dict[Type[Any], Callable[[Representer, Any], Any]] = {}
+
+
+def add_default_yaml_representer(type: Type[Any], represent: Callable[[Representer, Any], Any]):
+    """
+    Add a default representer for a type.
+    """
+    global _default_representers
+    _default_representers[type] = represent
+
+
+def new_yaml(
+    key_sort: Optional[KeySort] = None,
+    suppress_vals: Optional[Callable[[Any], bool]] = none_or_empty_dict,
+    stringify_unknown: bool = False,
+    typ: str = "safe",
+) -> YAML:
+    """
+    Configure a new YAML instance with custom settings.
+
+    If just using this for pretty-printing values, can set `stringify_unknown` to avoid
+    RepresenterError for unexpected types.
+
+    For input, `typ="safe"` is safest. For output, consider using `typ="rt"` for better
+    control of string formatting (e.g. style of long strings).
+    """
+    yaml = YAML(typ=typ)
+    yaml.default_flow_style = False  # Block style dictionaries.
+
+    suppr = suppress_vals or (lambda v: False)
+
+    # Ignore None values in output. Sort keys if key_sort is provided.
+    def represent_dict(dumper, data):
+        if key_sort:
+            data = {k: data[k] for k in sorted(data.keys(), key=key_sort)}
+        return dumper.represent_dict({k: v for k, v in data.items() if not suppr(v)})
+
+    yaml.representer.add_representer(dict, represent_dict)
+
+    # Use YAML block style for strings with newlines.
+    def represent_str(dumper, data):
+        style = "|" if "\n" in data else None
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style)
+
+    yaml.representer.add_representer(str, represent_str)
+
+    # Add other default representers.
+    for type, representer in _default_representers.items():
+        yaml.representer.add_representer(type, representer)
+
+    if stringify_unknown:
+
+        def represent_unknown(dumper, data):
+            return dumper.represent_str(str(data))
+
+        yaml.representer.add_representer(None, represent_unknown)
+
+    if key_sort:
+        yaml.representer.sort_base_mapping_type_on_output = False
+
+    return yaml
+
+
+def from_yaml_string(yaml_string: str) -> Any:
+    """
+    Read a YAML string into a Python object.
+    """
+    return new_yaml().load(yaml_string)
+
+
+def read_yaml_file(filename: str) -> Any:
+    """
+    Read YAML file into a Python object.
+    """
+    with open(filename, "r") as f:
+        return new_yaml().load(f)
+
+
+def to_yaml_string(
+    value: Any, key_sort: Optional[KeySort] = None, stringify_unknown: bool = False
+) -> str:
+    """
+    Convert a Python object to a YAML string.
+    """
+    stream = StringIO()
+    new_yaml(key_sort=key_sort, stringify_unknown=stringify_unknown, typ="rt").dump(value, stream)
+    return stream.getvalue()
+
+
+def dump_yaml(
+    value: Any, stream: TextIO, key_sort: Optional[KeySort] = None, stringify_unknown: bool = False
+):
+    """
+    Write a Python object to a YAML stream.
+    """
+    new_yaml(key_sort=key_sort, stringify_unknown=stringify_unknown, typ="rt").dump(value, stream)
+
+
+def write_yaml_file(
+    value: Any, filename: str, key_sort: Optional[KeySort] = None, stringify_unknown: bool = False
+):
+    """
+    Write the given value to the YAML file, creating it atomically.
+    """
+    temp_filename = f"{filename}.yml.tmp"  # Same directory with a temporary suffix.
+    try:
+        with open(temp_filename, "w", encoding="utf-8") as f:
+            dump_yaml(value, f, key_sort, stringify_unknown=stringify_unknown)
+        os.replace(temp_filename, filename)
+    except Exception as e:
+        try:
+            os.remove(temp_filename)
+        except FileNotFoundError:
+            pass
+        raise e

frontmatter_format-0.0.0.dev0/pyproject.toml

@@ -0,0 +1,73 @@
+[tool.poetry]
+name = "frontmatter-format"
+# Keep this a dev version, as the dynamic versioning plugin is used for actual release versions:
+version = "0.0.0.dev"
+description = "A format for YAML frontmatter on any file."
+authors = ["Joshua Levy <joshua@cal.berkeley.edu>"]
+readme = "README.md"
+license = "MIT"
+repository = "https://github.com/jlevy/frontmatter-format"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+ruamel-yaml = "^0.18.6"
+
+[tool.poetry.group.dev.dependencies]
+black = "^24.10.0"
+pytest = "^8.3.3"
+ruff = "^0.4.10"
+usort = "^1.0.8.post1"
+mypy = "^1.13.0"
+codespell = "^2.3.0"
+rich = "^13.9.3"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# Auto-generated version in build, based on tag or commit.
+# https://sam.hooke.me/note/2023/08/poetry-automatically-generated-package-version-from-git-commit/
+[tool.poetry-dynamic-versioning]
+enable = true
+vcs = "git"
+pattern  = "^v?(?P<base>\\d+\\.\\d+\\.\\d+)(-?((?P<stage>[a-zA-Z]+)\\.?(?P<revision>\\d+)?))?"
+format-jinja = """
+    {%- if distance == 0 -%}
+        {{- base -}}
+    {%- else -%}
+        {{- base }}.dev{{ distance }}+{{commit}}
+    {%- endif -%}
+"""
+
+[tool.poetry.scripts]
+lint = "devtools.lint:main"
+test = "pytest:main"
+
+[tool.black]
+line-length = 100
+
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.lint]
+ignore = ["E402", "E731", "E712"]
+
+[tool.mypy]
+disable_error_code = [
+    "import-untyped",
+]
+
+[tool.codespell]
+# ignore-words-list = "foo,bar"
+# skip = "foo.py,bar.py"
+
+[tool.pytest.ini_options]
+python_files = ["*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+testpaths = [
+    "frontmatter_format",
+    "tests",
+]
+norecursedirs = []
+filterwarnings = []