pk3make 1.1__py3-none-any.whl

pk3make/__main__.py

@@ -0,0 +1,297 @@
+#!/bin/env python3
+
+
+def clean(workdir="build"):
+    import shutil
+    print("# Removing workdir '{}'".format(workdir))
+    try:
+        shutil.rmtree(workdir)
+    except FileNotFoundError:
+        pass
+    return
+
+def prepare(workdir="build"):
+    import os
+    print("# Creating WORKDIR '{}'".format(workdir))
+    os.makedirs(workdir, exist_ok=True)
+
+def cr_build_lump(lock, lumpdef, context):
+    import shutil,os,re
+    from modules import doompic
+
+    bytedump = None
+
+    print(f'## Building {lumpdef[1]} "{context["srcfile"]}"...')
+
+    match lumpdef[1]:
+        case "graphic":
+            pal = get_palette(lock, context["opts"]["palette"], context["opts"], context["pdict"])
+            print(f'Converting Picture "{context["srcfile"]}"...')
+            bytedump = doompic.Picture(context['srcfile'], pal, offset=lumpdef[2]).tobytes()
+        case "flat" | "fade":
+            pal = get_palette(lock, context["opts"]["palette"], context["opts"], context["pdict"])
+            print(f'Converting Flat "{context["srcfile"]}"...')
+            bytedump = doompic.Flat(context['srcfile'], pal).tobytes()
+        case "udmf":
+            print(f'UDMF lumps conversion is currently not supported.')
+        case "palette":
+            print(f'Loading palette "{context["srcfile"]}"')
+            pal = get_palette(lock, lumpdef[0], context["opts"], context["pdict"])
+            print(f'Dumping palette "{context["srcfile"]}"')
+            bytedump = pal.tobytes()
+        case "tinttab" | "colormap" as paltype:
+            palparams = re.match(r"\s*([\w]+)\s*([0-9]\.[0-9]f?)?", lumpdef[2])
+            pal = get_palette(lock, palparams.group(1), context["opts"], context["pdict"])
+            print(f'Generating {paltype} "{context["destfile"]}" with {palparams.group(1,2)}')
+            match paltype:
+                case "tinttab":
+                    palweight = float(palparams.group(2))
+                    bytedump = pal.tinttab_tobytes(palweight)
+                case "colormap":
+                    bytedump = pal.colormap_tobytes()
+        case "raw":
+            with open(context['srcfile'], mode='rb') as s:
+                bytedump = s.read()
+
+    if bytedump != None:
+        print(f'Writing {lumpdef[1]} "{context["destfile"]}"')
+        os.makedirs(os.path.dirname(context["destfile"]), exist_ok=True)
+        with lock:
+            with open(context["destfile"], "wb") as ofile:
+                ofile.write(bytedump)
+
+def get_palette(lock, lumpname, opts, pdict):
+    from modules import doompic, doomglob
+    import os
+
+    lock.acquire()
+    if not lumpname in pdict:
+
+        p_glob = doomglob.find_lump(opts["srcdir"], lumpname)
+        if len(p_glob) > 1:
+            globlist = []
+            for f in p_glob:
+                globlist.append(f[1])
+            raise doomglob.DuplicateLumpError(f"Color palette {lumpname} is not unique.\n{globlist}")
+        elif len(p_glob) < 1:
+            raise FileNotFoundError(f"Color palette {lumpname} not found.")
+
+        print(f'Caching Palette "{lumpname}"')
+        pdict[lumpname] = doompic.Palette(os.path.join(opts["srcdir"],p_glob[0][1]))
+    lock.release()
+    return pdict[lumpname] 
+
+
+def build(makefile):
+    from modules import doompic, doomglob
+    from natsort import natsorted, ns
+    import shutil, os, re
+    import asyncio, concurrent.futures, multiprocessing
+
+    opts = makefile.get_options()
+
+    print(f'# Building {opts["srcdir"]} => {opts["workdir"]}')
+
+    if opts["palette"] == None:
+        print("WARNING: Default color palette is not defined. Compiling graphics will lead to errors.")
+
+    ppx_man = multiprocessing.Manager()
+    ppx_lock = ppx_man.Lock()
+    palettes = ppx_man.dict()
+    ppx_futures = []
+
+    with concurrent.futures.ThreadPoolExecutor() as ppx:
+
+        for lumpdef in makefile.get_lumpdefs():
+            match lumpdef[1]:
+                case "colormap" | "tinttab" as ltype: # Hardcoded exceptions, eww
+                    lumpglob = doomglob.fake_lump(lumpdef[0])
+                case _:
+                    lumpglob = doomglob.find_lump(opts["srcdir"], lumpdef[0])
+
+
+
+            for lump in natsorted(lumpglob, alg=ns.PATH):
+                lump_dcheck = doomglob.find_lump(opts["srcdir"], lump[0])
+
+                srcfile = opts["srcdir"] + '/' + lump[1]
+                destfile = opts["workdir"] + '/' + lump[2]
+
+                params = re.match(r"\s*([\w]+)\s*", lumpdef[2] or '')
+                if params != None and "preserve_filename" in params.groups():
+                    destfile = opts["workdir"] +'/'+ lump[1]
+
+                # Out-Of-Date check
+                if lumpdef[1] in ["colormap", "tinttab"]:
+                    palbase_name = re.match(r"\s*([\w]+)\s*", lumpdef[2]).group(1)
+                    ood_target = doomglob.find_lump(opts["srcdir"],palbase_name)
+                    srcfile = opts["srcdir"] + '/' + ood_target[0][1]
+
+                if os.path.exists(destfile) and os.path.getmtime(srcfile) < os.path.getmtime(destfile):
+                    continue
+                fake_lumpdef = (lump[0],lumpdef[1],lumpdef[2])
+                ppx_context = {
+                    "srcfile" :  srcfile,
+                    "destfile" :  destfile,
+                    "opts" :  opts,
+                    "pdict": palettes,
+                    }
+                ppx_futures.append( ppx.submit(cr_build_lump, ppx_lock, fake_lumpdef, ppx_context ) )
+                #cr_build_lump(ppx_lock, fake_lumpdef, ppx_context ) # For testing single-threadedly
+
+        # Did anything actually work?
+        for f in ppx_futures:
+            result = f.result()
+    return
+
+def pack(makefile):
+    from modules import pk3zip, doomglob
+    from natsort import natsorted, ns
+    import io, os, hashlib, pathlib, re
+
+    opts = makefile.get_options()
+    if opts["destfile"] == None:
+        raise FileNotFoundError("destfile is not defined")
+
+    print("# Packing")
+    
+    # Keep PK3 file in memory to avoid Windows' file access locks
+    pk3buf = io.BytesIO()
+    
+    for lumpdef in makefile.get_lumpdefs():
+        
+        if args.verbose:
+            print(f'# Packing lumpdef {lumpdef}')
+            
+        match lumpdef[1]:
+            case "marker":
+                
+                if args.verbose:
+                    print(f"## Adding marker {lumpdef[0]}")
+                with pk3zip.PK3File(pk3buf, "a") as pk3:
+                    pk3.writestr(lumpdef[0], "")
+            case _:
+                params = re.match(r"\s*([\w]+)\s*", lumpdef[2] or '')
+                searchname = os.path.dirname(lumpdef[0])+'/'+pathlib.Path(lumpdef[0]).stem[:8]
+                if params != None and "preserve_filename" in params.groups():
+                    searchname = lumpdef[0]
+                
+                with pk3zip.PK3File(pk3buf, "a") as pk3:
+
+                    wf_glob = doomglob.find_lump(opts["workdir"], searchname)
+                    wf_glob = natsorted(wf_glob, alg=ns.PATH)
+
+                    #print(f'\nGLOB: {wf_glob}\n')
+                    #print(f'NAMELIST: {pk3.namelist()}\n')
+
+                    wf_unique = [x for x in wf_glob if x[2].lstrip('/').rstrip('/') not in pk3.namelist() ]
+                    if params != None and "preserve_filename" in params.groups():
+                        wf_unique = [x for x in wf_glob if x[1].lstrip('/').rstrip('/') not in pk3.namelist() ]
+
+                    #print(f'\nUNIQUE GLOB: {wf_unique}\n')
+
+                    for lump,srcfile,arcpath in wf_unique:
+                        wf_path = opts["workdir"] + '/' + srcfile
+
+                        if params != None and "preserve_filename" in params.groups():
+                            wf_path = opts["workdir"]+'/'+srcfile
+                            arcpath = os.path.dirname(arcpath)+'/'+os.path.basename(srcfile)
+
+                        
+                        if args.verbose:
+                            print(f'## Packing lump {arcpath}')    
+
+                        pk3.write(wf_path, arcpath)
+
+    
+    # Commit in-memory PK3 file to disk
+    
+    if not os.path.isdir(os.path.dirname(opts["destfile"])):
+        print(f'## Creating directory {os.path.dirname(opts["destfile"])}')
+        os.mkdir(os.path.dirname(opts["destfile"]))
+
+    if os.path.isfile(opts["destfile"]):
+        print(f'## Deleting {opts["destfile"]} for recreation')
+        os.remove(opts["destfile"])
+    
+    with open(opts["destfile"], "wb") as f:
+        print(f'## Writing {opts["destfile"]}')
+        f.write(pk3buf.getvalue())
+    
+    md5hash = hashlib.md5(pk3buf.getvalue())
+    print(f'\nMD5 Hash of {opts["destfile"]}: {md5hash.hexdigest()}')
+
+    return
+
+def main():
+    from modules import pk3makefile
+
+    # Step switches
+    step_prepare = False
+    step_build = False
+    step_pack = False
+
+    match args.verb:
+        case "prepare":
+            step_prepare = True
+        case "build":
+            step_build = True
+        case "pack":
+            step_pack = True
+        case None | "all":
+            step_prepare = True
+            step_build = True
+            step_pack = True
+
+    if args.verb == "clean":
+        clean()
+
+    pk3mf_name = "./PK3Makefile"
+    if args.makefile != None:
+        pk3mf_name = args.makefile
+    pk3mf = pk3makefile.PK3Makefile(pk3mf_name)
+
+    print(f"MAKEOPTS: = {pk3mf.get_options()}")
+
+    # TODO: Add resolve for missing dependencies
+    if step_prepare:
+        prepare(pk3mf.get_options("workdir"))
+    if step_build:
+        if args.verb == "build" and args.target != None:
+            pk3mf = pk3mf.filter_lumpdefs(args.target)
+        build(pk3mf)
+    if step_pack:
+        pack(pk3mf)
+
+    return
+
+if __name__ == "__main__":
+    import argparse
+    import pathlib
+
+    # Shell argument API
+    verbs = [
+        'clean', # Delete workdir
+        'prepare', # Make workdir tree etc.
+        'build', # Convert formats & copy files to workdir according to METAINFO
+        'pack', # Pack existing workdir into pk3. (May be used for music packs?)
+        'all', # Do everything
+    ]
+    ap_main = argparse.ArgumentParser(
+            prog='pk3make',
+            description='PK3Make - Make for (Weissblatt) PK3s',
+            epilog='Type `pk3make --help` for more info.')
+    ap_sub = ap_main.add_subparsers(title='Build steps', dest='verb', metavar="")
+    
+    ap_clean = ap_sub.add_parser('clean', help='Delete the build directory')
+    ap_build = ap_sub.add_parser('build', help='Compile assets into the build directory')
+    ap_pack = ap_sub.add_parser('pack', help='Assemble a PK3 file from the build directory')
+
+    ap_main.add_argument('-v', '--verbose' , action='store_true', help='Verbose log output')
+    ap_build.add_argument('target', nargs='?', help='Target LUMPDEF')
+    
+    ap_main.add_argument('makefile', nargs='?', const='./PK3Makefile', help='PK3Makefile to reference')
+    
+    args = ap_main.parse_args()
+
+    main()

pk3make/modules/doomglob.py

@@ -0,0 +1,34 @@
+class DuplicateLumpError(Exception):
+    """To be raised when a lump should really be unique"""
+    pass
+
+
+def find_lump(srcdir, lumpname):
+    import os, glob
+    import pathlib
+    out = list()
+
+    if srcdir == None:
+        raise FileNotFoundError(f'doomglob.find_lump(): No srcdir given')
+    if lumpname == None:
+        raise FileNotFoundError(f'doomglob.find_lump(): No lumpname given')
+
+    #for path in glob.glob(searchstr, root_dir=srcdir):
+    for path in glob.iglob('**/'+lumpname+'*', root_dir=srcdir, recursive=True):
+        posixpath = pathlib.Path(path).as_posix()
+        doomname = pathlib.Path(path).stem[:8]
+        arcpath = (os.path.dirname(posixpath)+'/'+doomname).lstrip('/').rstrip('/')
+        if  pathlib.Path(srcdir.rstrip('/')+'/'+posixpath).is_file(): # Filter out directories
+            out.append( (doomname, posixpath, arcpath) )
+
+    # Deduplicate out
+    out = [x for n,x in enumerate(out) if x not in out[:n] ]
+                    
+    return out # List of tuples (LUMPNAME, PATH, ARCPATH)
+
+def fake_lump(lumpname):
+    # Only for use with generated lumps, such as COLORMAPs or TINTTABs
+    from pathlib import Path
+    ln_short = Path(lumpname).stem[:8]
+    arcpath = '/'+ln_short.lstrip('/')
+    return [ (ln_short, lumpname, arcpath) ]

pk3make/modules/doompic.py

@@ -0,0 +1,354 @@
+class Palette:
+    def __init__(self, filename):
+        import os
+        from modules.doomglob import find_lump, DuplicateLumpError
+        from PIL import Image
+
+        self.colors = []
+        self.color_lookup = {} # Color LUT to speed up rgb2index (before estimate: 25:16,32)
+
+        # Colormath-based code commented out for future reference
+        # Euclidean distance is 50x faster
+        """
+        from colormath2.color_objects import sRGBColor, LabColor
+        from colormath2.color_conversions import convert_color
+        from colormath2.color_diff import delta_e_cie2000
+        """
+
+        with Image.open(filename).convert("RGB") as img:
+            # When it don't fit we make it fit
+            rez_i = img.resize( (16,16), Image.Resampling.NEAREST)
+
+            # Get pixels into self.colors
+            width, height = rez_i.size # should be (16,16)
+            for y in range(height):
+                for x in range(width):
+                    pixel = rez_i.getpixel((x,y))
+                    #self.colors.append(pixel) # Tuple (R,G,B)
+
+                    # Precalc color conversions to speed up rgb2index - UPDATE: No bc fuck colormath
+                    #px_srgb = sRGBColor(pixel[0], pixel[1], pixel[2], is_upscaled=True) # COLORMATH STUB
+                    #px_cielab = convert_color(px_srgb, LabColor) # COLORMATH STUB
+                    color_o = {
+                        "id": y*height+x,
+                        #"rgb": px_srgb, # COLORMATH STUB
+                        "r": pixel[0],
+                        "g": pixel[1],
+                        "b": pixel[2],
+                        #"cielab": px_cielab, # COLORMATH STUB
+                    }
+
+                    rgbcolor = (pixel[0] << 16) | (pixel[1] << 8) | (pixel[2])
+                    self.colors.append(color_o) # Tuple (R,G,B)
+                    self.color_lookup[rgbcolor] = color_o["id"]
+
+    def rgb2index(self, color: tuple):
+        # Colormath-based code commented out for future reference
+        # Euclidean distance is 50x faster
+        """
+        from colormath2.color_objects import sRGBColor, LabColor
+        from colormath2.color_conversions import convert_color
+        from colormath2.color_diff import delta_e_cie2000
+        """
+        # Hot path O(1): Color matches exactly (most common if you know what you're doing)
+        rgbcolor = (int(color[0]) << 16) | (int(color[1]) << 8) | int(color[2])
+        if rgbcolor in self.color_lookup.keys():
+            #print(f"Converting {color} => #{rgbcolor:X}")
+            return self.colors[self.color_lookup[rgbcolor]]["id"]
+
+        # Cold path: Linear search for the closest color
+        #color_lab = convert_color(sRGBColor(color[0], color[1], color[2], is_upscaled=True), LabColor)
+        min_delta_e = float('inf')
+        min_idx = -1
+        for icolor in self.colors:
+            """
+            #print(f"ICOLOR {index}: {icolor}")
+            #icolor_lab = convert_color(sRGBColor(icolor[0], icolor[1], icolor[2], is_upscaled=True), LabColor)
+            delta_e = delta_e_cie2000(color_lab, icolor["cielab"])
+            """
+            # Simple euclidean distance
+            delta_e = ( \
+                (color[0]-icolor['r'])**2 + \
+                (color[1]-icolor['g'])**2 + \
+                (color[2]-icolor['b'])**2 \
+            )**(1/2)
+            if delta_e < min_delta_e:
+                min_delta_e = delta_e
+                min_idx = icolor["id"]
+            if delta_e == 0: # Exact match, no need to continue
+                break
+        #print(f"Found color {min_idx}:{self.colors[min_idx]} for image color {color}")
+        return min_idx
+
+    def generate_colormap(self):
+        return
+
+    def tobytes(self):
+        # Convert self.colors to Doom Palette
+        # Return as IOBytes for saving
+        exbytes = bytearray()
+        for page in range(14):
+            for i,pcolor in enumerate(self.colors):
+                # Default unused palette: Grayscale
+                r = 255-i
+                g = 255-i
+                b = 255-i
+
+                if page == 0: # Regular palette
+                    r = pcolor["r"]
+                    g = pcolor["g"]
+                    b = pcolor["b"]
+                elif 0 < page < 4: # Whiteout palettes => 75% white tint
+                    r = pcolor["r"] + (255 - pcolor["r"]) * 0.75
+                    g = pcolor["g"] + (255 - pcolor["g"]) * 0.75
+                    b = pcolor["b"] + (255 - pcolor["b"]) * 0.75
+                elif page == 4: # Nuke palette => 75% white tint + g,b = 113
+                    r = pcolor["r"] + (255 - pcolor["r"]) * 0.75
+                    g = 113
+                    b = 113
+                elif page == 5: # Inverted palette at 75% brightness
+                    r = (255 - pcolor["r"]) * 0.75
+                    g = (255 - pcolor["g"]) * 0.75
+                    b = (255 - pcolor["b"]) * 0.75
+                # Add color idx.
+                # NOTE: the int() cast is janky but hopefully works
+                exbytes.append(int(r))
+                exbytes.append(int(g))
+                exbytes.append(int(b))
+        return bytes(exbytes)
+
+    def colormap_tobytes(self):
+        """
+        from colormath.color_objects import HSVColor, sRGBColor
+        from colormath.color_conversions import convert_color
+        """
+
+        out = bytearray()
+        # Y/X coordinate loop because loop order matters
+        for c,v in [(c,v) for v in range(32) for c in range(256)]:
+
+            """
+            input_hsv = convert_color( sRGBColor( \
+                        self.colors[c][0], \
+                        self.colors[c][1], \
+                        self.colors[c][2] \
+                    , HSVColor)
+            output_hsv = ( input_hsv[0], input_hsv[1], input_hsv[2] * (1-v/32) )
+            output_rgb = sRGBColor()
+
+            out += self.rgb2index(output_rgb.value_tuple())
+            """
+
+            # Simple RGB squash for now
+            # TODO: Add HSV/LAB-conversion after testing
+            brightness = ( \
+                            self.colors[c]["r"] * (1-(v/32)), \
+                            self.colors[c]["g"] * (1-(v/32)), \
+                            self.colors[c]["b"] * (1-(v/32)) \
+                        )
+            out += self.rgb2index(brightness).to_bytes(1)
+        return out
+
+    def tinttab_tobytes(self, factor:float):
+        if type(factor) != float or not (0 <= factor <= 1):
+            raise RuntimeError(f"Invalid TINTTAB factor {factor}")
+
+        out = bytearray()
+        for x,y in [(x,y) for x in range(256) for y in range(256)]:
+            tintcolor = ( \
+                            self.colors[x]["r"] * (1-factor) + self.colors[y]["r"] * factor, \
+                            self.colors[x]["g"] * (1-factor) + self.colors[y]["g"] * factor, \
+                            self.colors[x]["b"] * (1-factor) + self.colors[y]["b"] * factor \
+                        )
+            out += self.rgb2index(tintcolor).to_bytes(1)
+        return out
+
+
+class Flat():
+    def __init__(self, pngfile: str, palette: Palette):
+        from PIL import Image
+
+        self.pixelbuf = bytearray()
+        with Image.open(pngfile).convert("RGBA") as img:
+
+            # Get pixels into self.pixelbuf
+            self.width, self.height = img.size # should be
+
+            """# Removed to support FADEs
+            if self.width != self.height:
+                raise RuntimeError(f"Flat is not square. ({self.width},{self.height})")
+            """
+
+            for y in range(self.height):
+                for x in range(self.width):
+                    pixel = img.getpixel((x,y))
+                    # Flat = Raw paletted pixel dump
+                    self.pixelbuf += palette.rgb2index(pixel).to_bytes(1,"little")
+
+    def get_size(self):
+        return (self.width, self.height)
+
+    def tobytes(self):
+        return bytes(self.pixelbuf)
+
+class Picture():
+    def __init__(self, pngfile: str, palette: Palette, **kwargs):
+        from PIL import Image
+
+        self.palette = palette # Prolly unused but can't hurt
+
+        self.pixelbuf = []
+        with Image.open(pngfile).convert("RGBA") as img:
+
+            # Get pixels into self.pixelbuf
+            self.width, self.height = img.size # should be
+            for y in range(self.height):
+                for x in range(self.width):
+                    pixel = img.getpixel((x,y))
+                    # Save picture as indexed image (-1 = transparent)
+                    if pixel[3] == 0:
+                        self.pixelbuf.append( -1 )
+                    else:
+                        self.pixelbuf.append( palette.rgb2index(pixel) )
+
+        if "offset" in kwargs:
+            new_offset = self.set_offset(kwargs["offset"])
+
+
+    def set_offset(self, offset: str):
+        import re
+        if offset == None or re.match(r"^\s*$", offset):
+            self.offsetX = 0
+            self.offsetY = 0
+            return (self.offsetX, self.offsetY)
+
+        tokens = re.match(r"\s*(-?[0-9]+)\s+(-?[0-9]+)\s*", offset)
+        if tokens:
+            self.offsetX = int(tokens.group(1))
+            self.offsetY = int(tokens.group(2))
+            return (self.offsetX, self.offsetY)
+
+        tokens = re.match(r"\s*([^\s]+)\s*", offset)
+        if not tokens:
+            raise Exception(f'Offset "{offset}" not supported')
+
+        match tokens.group(1):
+            case "": # No offset given - default to "0 0"
+                self.offsetX = 0
+                self.offsetY = 0
+            case "center":
+                self.offsetX = int(self.width/2)
+                self.offsetY = int(self.height/2)
+            case "sprite":
+                self.offsetX = int(self.width/2)
+                self.offsetY = int(self.height-4)
+            case _:
+                raise Exception(f'Offset "{offset}" not supported')
+        return (self.offsetX, self.offsetY)
+
+    def tobytes(self):
+        # === Generate picture lump ===
+        #
+        # [HEADER]
+        # uint16_t LE width
+        # uint16_t LE height
+        # uint16_t LE offsetX
+        # uint16_t LE offsetY
+        # uint32_t[width] LE toc
+        # -----------------------------
+        # COLUMNS are arrays of POSTS separated by 0xFF
+        # [POSTS]
+        # uint8_t LE topdelta
+        # uint8_t LE length
+        # uint8_t LE padding
+        # uint8_t* LE pixels
+        # uint8_t LE padding
+
+
+        columns = bytearray()
+        # --- Create Header ---
+        # NOTE: All integers in a Picture header are LE uint16_t
+        out = bytearray( \
+            self.width.to_bytes(2, byteorder='little') + \
+            self.height.to_bytes(2, byteorder='little') + \
+            self.offsetX.to_bytes(2, byteorder='little', signed=True) + \
+            self.offsetY.to_bytes(2, byteorder='little', signed=True) \
+            )
+
+        # Iterate Column-wise. Yes, Doom picture are column-oriented
+        toc = bytearray() # Table of Columns
+        t_fseek = len(out) + 4 * self.width # whXY + column TOC
+        for x in range(self.width):
+            t_cdata = bytearray() # Column data
+            t_pdata = bytearray() # Setup/Reset Post data
+            t_insidepost = False
+            # Post offset markers
+            t_topdelta = -1
+            t_topoffset = -1
+            t_olddelta = -1
+            t_postheight = 0
+            dbg_postcounter = 0
+            for y in range(self.height):
+
+                ## Tall patch support ##
+
+                if  y == 254: # Tall patch border
+                    if  t_insidepost:
+                        # Abort post now, restart as usual
+                        t_cdata.extend(t_postheight.to_bytes(1, byteorder="little")) # Unused padding
+                        t_cdata.extend(b'\x00') # Unused padding
+                        t_cdata.extend(t_pdata) # Post data
+                        t_cdata.extend(b'\x00') # Unused padding
+                        t_pdata = bytearray() # Reset post data
+
+                    # Insert Fake post
+                    t_cdata.extend(b'\xfe\x00\x00\x00')
+                    t_topdelta = y # Flush topdelta
+                    t_postheight = 0
+                    t_insidepost = False
+
+                ## Actual algorithm ##
+
+                current_pixel = self.pixelbuf[y*self.width+x]
+                if (current_pixel == -1 or t_postheight == 254) and t_insidepost: # Post END
+                    t_cdata.extend(t_postheight.to_bytes(1, byteorder="little")) # Unused padding
+                    t_cdata.extend(b'\x00') # Unused padding
+                    t_cdata.extend(t_pdata) # Post data
+                    t_cdata.extend(b'\x00') # Unused padding
+                    t_pdata = bytearray() # Reset post data
+                    t_insidepost = False
+                if current_pixel != -1 and not t_insidepost: # Post START
+
+                    # Tall patch tracking
+                    t_olddelta = t_topdelta
+                    t_topdelta = y
+                    t_topoffset = y if y < 254 else t_topdelta - t_olddelta
+
+                    # Start new post
+                    t_postheight = 1
+                    t_cdata.extend((t_topoffset&0xFF).to_bytes(1, byteorder="little"))
+                    t_pdata.extend(current_pixel.to_bytes(1, byteorder="little"))
+                    t_insidepost = True
+                elif current_pixel != -1 and t_insidepost:
+                    t_pdata.extend(current_pixel.to_bytes(1, byteorder="little"))
+                    t_postheight = t_postheight + 1
+
+
+
+            if t_insidepost: # Finish last post if End Of Column
+                t_cdata.extend(t_postheight.to_bytes(1, byteorder="little")) # Unused padding
+                t_cdata.extend(b'\x00') # Unused padding
+                t_cdata.extend(t_pdata) # Post data
+                t_cdata.extend(b'\x00') # Unused padding
+            t_cdata.extend(b'\xff') # Column Terminator
+
+            columns.extend(t_cdata) # Save partitioned column whole
+
+            # Add TOC column offset
+            toc.extend(t_fseek.to_bytes(4, byteorder='little'))
+            t_fseek = t_fseek+len(t_cdata)
+
+        out.extend(toc) # Finish off header
+        out.extend(columns) # Write column data block
+
+        return bytes(out)

pk3make/modules/pk3makefile.py

@@ -0,0 +1,65 @@
+class PK3MakeConfigurationError(Exception):
+    """To be raised when a lump should really be unique"""
+    pass
+
+class PK3MakeDependencyError(Exception):
+    """To be raised when a lump should really be unique"""
+    pass
+
+class PK3Makefile():
+    #def __init__(self):
+        #pass
+
+    def __init__(self, filename):
+        import re
+
+        self.options = {
+            "srcdir": None,
+            "workdir": None,
+            "destfile": None,
+            "palette": None,
+        }
+
+        self.lumps = []
+
+        # List of tuples ( LUMPNAME, TYPE, OFFSET )
+        # OFFSET may either be an interger tuple or a string
+
+        with open(filename) as file:
+            for line in file:
+                re_buildopt = r"^\?([^\s]*): ([^\s]*)"
+                re_lumpdef = r"^([^\s]+)\s*([^\s]+)(?:\s*(.+))?"
+
+                workline = re.sub(r"#.*","", line) # Clean out comments
+                tokens = re.match(re_buildopt, workline)
+                if tokens: # Is it a Buildopt?
+                    match tokens.group(1):
+                        case "srcdir" | "workdir" | "destfile" | "palette" as cmd:
+                            self.options[cmd] = tokens.group(2).rstrip('/')
+                tokens = re.match(re_lumpdef, workline)
+                if tokens: # Is it a Lumpdef?
+                    match tokens.group(2):
+                        case "flat" | "fade" | "graphic" | "raw" | "colormap"| "tinttab" | "palette" | "marker" as cmd:
+                            self.lumps.append( tokens.group(1,2,3) )
+                        case "udmf":
+                            print(f'Lump type "udmf" is not supported yet. Ignored')
+                        case _ as lumptype:
+                            print(f'Invalid lumptype "{lumptype}". Ignored')
+
+    def get_options(self, option=None):
+        if option == None:
+            return self.options
+        else:
+            return self.options[option]
+
+    def get_lumpdefs(self):
+        return self.lumps
+    
+    def filter_lumpdefs(self, pattern):
+        import re,fnmatch
+
+        glob_re = re.compile(fnmatch.translate(pattern))
+
+        self.lumps = [x for x in self.lumps if glob_re.match(x[0])]
+
+        return self

pk3make/modules/pk3zip.py

@@ -0,0 +1,84 @@
+import os, io, re
+import zipfile, pathlib
+#from binaryornot.check import is_binary
+
+class PK3File(zipfile.ZipFile):
+    """This class is basically a deterministic ZIP file.
+    Four attributes need to be controlled:
+    1. Order of files follows programmatic order
+    2. Timestamp is set to 1980-01-01 00:00:00
+    3. All files are set to permissions (d)rw-rw-rw-
+    4. Create system is set to 03/Unix
+    """
+
+    ### Inherited/overwritten ZipFile functions ###
+    def mkdir(self, zinfo_or_directory, mode=511):
+        # Force metadata
+        if isinstance(zinfo_or_arcname, zipfile.ZipInfo):
+            zinfo = zinfo_or_directory
+        else:
+            zinfo = zipfile.ZipInfo(filename=(zinfo_or_directory.rstrip('/')+'/'))
+        
+        # Force deterministic metadata
+        zinfo.date_time = (1980, 1, 1, 0, 0, 0)
+        zinfo.create_system = 3
+        zinfo.external_attr = (0o40744 << 16) | 0x10  # Octal encoding for drwxr--r--
+            
+        # Mode is overwritten to achieve determinism
+        zipfile.ZipFile.mkdir(self, zinfo, 511)
+    
+    def write(self, filename, arcname, compress_type=None, compresslevel=None):
+        
+        nodename = pathlib.Path(filename).stem + pathlib.Path(filename).suffix
+        zinfo = zipfile.ZipInfo.from_file(filename, arcname)
+        
+        # Force deterministic metadata
+        zinfo.create_system = 3
+        zinfo.date_time = (1980, 1, 1, 0, 0, 0)
+        zinfo.external_attr = 0o0744 << 16  # Octal encoding for -rwxr--r--
+        if zinfo.is_dir():
+            zinfo.external_attr = (0o40744 << 16) | 0x10  # Octal encoding for drwxr--r--
+        
+        # Plain text file -> chain into writestr to convert line breaks
+        #if not is_binary(filename):
+        p = re.compile('(SOC_.*)|' \
+            '(.*\.soc)|' \
+            '(TEXTURES)|' \
+            '(ANIMDEFS)|' \
+            '(MUSICDEF)|' \
+            '(L_.*)|' \
+            '(lua_.*)|' \
+            '(.*\.lua)|' \
+            '(.*\.txt)|' \
+            '(S_SKIN)|' \
+            '(SPRTINFO)')
+        if p.match(nodename):
+            # Force LF line breaks for text files
+            with open(filename, mode='r', encoding='utf-8') as f:
+                raw_file = f.read().replace('\r\n','\n')
+        else:
+            with open(filename, mode='rb') as f:
+                raw_file = f.read()
+
+
+        self.writestr(arcname, raw_file,compress_type, compresslevel)
+        
+        return
+        
+        
+        #zipfile.ZipFile.write(self, filename, arcname, compress_type, compresslevel)
+
+    def writestr(self, zinfo_or_arcname, data, compress_type=None, compresslevel=None):
+        if isinstance(zinfo_or_arcname, zipfile.ZipInfo):
+            zinfo = zinfo_or_arcname
+        else:
+            zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname)
+            
+        # Force deterministic metadata
+        zinfo.filename = zinfo.filename.lstrip('/')
+        zinfo.date_time = (1980, 1, 1, 0, 0, 0)
+        zinfo.create_system = 3
+        zinfo.external_attr = 0o0744 << 16  # Octal encoding for -rwxr--r--
+        
+        # Force LF line breaks to guarantee determinism
+        zipfile.ZipFile.writestr(self, zinfo, data, compress_type, compresslevel)

pk3make-1.1.dist-info/METADATA

@@ -0,0 +1,138 @@
+Metadata-Version: 2.4
+Name: pk3make
+Version: 1.1
+Summary: Build system for Weissblatt PK3 files
+Project-URL: Homepage, https://github.com/liquidunderground/pk3make
+Project-URL: GitHub, https://github.com/liquidunderground/pk3make
+Project-URL: Issues, https://github.com/liquidunderground/pk3make/issues
+Author-email: Zibon Badi <zibonbadi@gmail.com>
+License-Expression: LGPL-3.0-or-later
+License-File: LICENSE
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Build Tools
+Requires-Python: >=3.12
+Requires-Dist: colormath2
+Requires-Dist: natsort
+Requires-Dist: pillow
+Description-Content-Type: text/markdown
+
+# PK3Make
+
+"Make" for Weissblatt PK3 files
+
+## Installation
+
+1. Set up a [virtual environment](https://docs.python.org/3/library/venv.html)
+2. Install dependencies `pip install -r requirements.txt`
+
+## How to use
+
+PK3Make supplies multiple subcommands. To get an overview type:
+
+    ./pk3make.py --help
+
+To fully build your project akin to a makefile, simply type:
+
+    ./pk3make.py make ./PK3Makefile # Default PK3Makefile
+
+### Notes, Tips and Caveats
+
+All text files are assumed to be UTF-8 encoded. PK3Make will automatically attempt to convert CRLF to LF newlines but using LF newlines is recommended.
+
+PK3Make will not find hidden files. To avoid `DuplicateLumpError`s, place
+  your workfiles in a hidden directory, for example `.local/`.
+
+Should your project contain custom palettes,  place it's corresponding
+  LUMPDEF before any `graphic`, `flat` or `fade`. That way, PK3Make can
+  cache your palettes and speed up build times by reducing thread idle.
+
+
+## But why?
+
+To put it bluntly: No other tools suited Weissblatt.
+
+Although the PK3 specification for Weissblatt's engine is based on
+[ZDoom PK3](https://zdoom.org/wiki/Using_ZIPs_as_WAD_replacement),
+it's directory namespaces are very different. This made Doom's usual
+autobuild toolkit [DoomTools](https://mtrop.github.io/DoomTools/) a
+poor fit for development. Due to the size of the Weissblatt project, manual
+assembly using SLADE was also out of the question.
+
+I chose Python as the basis for PK3Make because it is platform-independent,
+easy-to-read and ubiquitous and although some Doom/Weissblatt-specific
+modules needed to be written from scratch for PK3Make, Python's vast
+standard library and otherwise mature PyPI repository helped stem some
+of the heavy lifting for things such as image processing.
+
+# PK3Makefile reference
+
+PK3Make uses it's own build definition language called `PK3Makefile`, inspired by the `METAINFO` spec from
+[Matt Tropiano's dImgConv](https://mtrop.github.io/DoomTools/dimgconv.html).
+
+`PK3Makefile`s are processed line-by-line with everything following `#`
+being treated as a comment. Otherwise it is split into *Build Options*,
+which define PK3Make's general behavior and *LUMPDEFS*, which define what
+files to put into your PK3 and how to build them.
+
+## Build options
+
+Build options are specified per-line and follow the pattern
+
+    ?<OPTION>: <PARAM>
+
+PK3Make supports the following options:
+
+`?srcdir: <DIR>` specifies the directory to pull it's base assets from.
+PK3Make will attempt to find all defined lumps within this folder and
+mirror it's path within `?workdir` after compilation.
+
+`?workdir: <DIR>` specifies the temporary working directory. PK3Make will
+check the timestamps between this and `?srcdir` and rebuild/copy any
+outdated files into `?workdir` during the compilation process.
+
+`?palette:` defines the main color palette, by `LUMPNAME` (`PLAYPAL` by default)
+
+`?destfile:` describes a filepath to the destination PK3. This is where
+`?workdir` will get copied to during packing.
+
+
+## Lump definitions
+
+Lump definitions follow the following pattern:
+
+    <LUMPNAME> <TYPE> <OFFSET>
+
+`LUMPNAME` describes the filename as used in-engine. Just like the engine,
+it is matched against the first eight characters of the basename in a
+case-insensitive manner.  [Globbing] such as `D_*.mid` is allowed, in which
+case `TYPE` and `OFFSET` are applied to all matching lumps.  `LUMPNAME`s
+starting with a "/" are treated as explicit file paths and match against
+the full file path, starting at the source directory.
+
+[Globbing]: <https://en.wikipedia.org/wiki/Glob_(programming)>
+
+`TYPE` determines how the file is treated during compilation. It can be one
+of the following:
+
+- `colormap`: File is a Colormap. OFFSET specifies the lump name for the palette from which it is generated
+- `fade`|`flat`: File is an image and should be converted to a flat. Only PNG images are supported.
+- `graphic`: File is an image and should be converted to a Doom Picture using `OFFSET` (see below) as a picture offset. If missing, the offset is assumed to be `0 0`.
+- `marker`: File does not exist and is a 0-byte marker. Explicit path definition required.
+- `palette`: File is a graphic and should be converted to a color palette. Only PNG images supported.
+- `raw`: Copy the file over as-is. When `preserve_filename` is given in the offset, the original filename will be preserved.
+- `tinttab`: File is a TINTTAB. OFFSET is defined as `<PALETTE> <WEIGHT>`. Upon generation, `PALETTE` orthogonally maps each color index against one another, `WEIGHT` specifies a bias towards horizontal/vertical colors between 0 and 1.
+- `udmf`: (Not supported yet.) File is a UDMF TEXTMAP. PK3Make will generate a directory named LUMPNAME featuring:
+  - `<LUMPNAME>`: Marker
+  - `TEXTMAP`: Original TEXTMAP file (renamed)
+  - `ZNODES`: UDMF BSP tree generated by PK3Make
+  - `ENDMAP`: Marker
+
+`OFFSET` defines the offset of doom pictures. For convenience, these can be either:
+
+- `<x> <y>`: Explicit X/Y-coordinates
+- `center`: Sets the offset to the center of the image
+- `sprite`: Sets the offset to `width/2 (height-4)`. This is a very common
+  offset for sprites placed in the game world.

pk3make-1.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+pk3make/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pk3make/__main__.py,sha256=Ls3uIkNCE95ZY2HeR8DtWNRkSTKDjvAJ3uqSVQa4t-k,10859
+pk3make/modules/doomglob.py,sha256=tXRiFwScIP1KGmSJI-bfAeKbXzI-KGB5oc8LG0Yg7DE,1278
+pk3make/modules/doompic.py,sha256=KOZURZPrUcgL0yHhyloiCyKYl-6Y7LLnk-v9xQpuAmU,14208
+pk3make/modules/pk3makefile.py,sha256=YaNx0NoVwEZq1t1m1RTd3i5uRm3l-DikWsVp-HVdnfE,2199
+pk3make/modules/pk3zip.py,sha256=ZdaEa1wh4gTrWZrkE4Orf7IS_PyMidgII4yRK215Bpg,3209
+pk3make-1.1.dist-info/METADATA,sha256=0XIdDneGJ282VyHoOvWeISxTdg1uB8tmVfYO-z7jgKc,6000
+pk3make-1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pk3make-1.1.dist-info/entry_points.txt,sha256=I5d-GktPSJQuR7bdqW8rWo2Ta5usyFCVX3UlDmh-Duc,45
+pk3make-1.1.dist-info/licenses/LICENSE,sha256=zfXyi_cMnW2CRuY9eAQyCiSvFLG3n8OpQEnsj7x2Eyk,7626
+pk3make-1.1.dist-info/RECORD,,

pk3make-1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pk3make-1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pk3make = pk3make:__main__

pk3make-1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,157 @@
+# GNU LESSER GENERAL PUBLIC LICENSE
+
+Version 3, 29 June 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+<https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+This version of the GNU Lesser General Public License incorporates the
+terms and conditions of version 3 of the GNU General Public License,
+supplemented by the additional permissions listed below.
+
+## 0. Additional Definitions.
+
+As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the
+GNU General Public License.
+
+"The Library" refers to a covered work governed by this License, other
+than an Application or a Combined Work as defined below.
+
+An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+## 1. Exception to Section 3 of the GNU GPL.
+
+You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+## 2. Conveying Modified Versions.
+
+If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+-   a) under this License, provided that you make a good faith effort
+    to ensure that, in the event an Application does not supply the
+    function or data, the facility still operates, and performs
+    whatever part of its purpose remains meaningful, or
+-   b) under the GNU GPL, with none of the additional permissions of
+    this License applicable to that copy.
+
+## 3. Object Code Incorporating Material from Library Header Files.
+
+The object code form of an Application may incorporate material from a
+header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+-   a) Give prominent notice with each copy of the object code that
+    the Library is used in it and that the Library and its use are
+    covered by this License.
+-   b) Accompany the object code with a copy of the GNU GPL and this
+    license document.
+
+## 4. Combined Works.
+
+You may convey a Combined Work under terms of your choice that, taken
+together, effectively do not restrict modification of the portions of
+the Library contained in the Combined Work and reverse engineering for
+debugging such modifications, if you also do each of the following:
+
+-   a) Give prominent notice with each copy of the Combined Work that
+    the Library is used in it and that the Library and its use are
+    covered by this License.
+-   b) Accompany the Combined Work with a copy of the GNU GPL and this
+    license document.
+-   c) For a Combined Work that displays copyright notices during
+    execution, include the copyright notice for the Library among
+    these notices, as well as a reference directing the user to the
+    copies of the GNU GPL and this license document.
+-   d) Do one of the following:
+    -   0) Convey the Minimal Corresponding Source under the terms of
+        this License, and the Corresponding Application Code in a form
+        suitable for, and under terms that permit, the user to
+        recombine or relink the Application with a modified version of
+        the Linked Version to produce a modified Combined Work, in the
+        manner specified by section 6 of the GNU GPL for conveying
+        Corresponding Source.
+    -   1) Use a suitable shared library mechanism for linking with
+        the Library. A suitable mechanism is one that (a) uses at run
+        time a copy of the Library already present on the user's
+        computer system, and (b) will operate properly with a modified
+        version of the Library that is interface-compatible with the
+        Linked Version.
+-   e) Provide Installation Information, but only if you would
+    otherwise be required to provide such information under section 6
+    of the GNU GPL, and only to the extent that such information is
+    necessary to install and execute a modified version of the
+    Combined Work produced by recombining or relinking the Application
+    with a modified version of the Linked Version. (If you use option
+    4d0, the Installation Information must accompany the Minimal
+    Corresponding Source and Corresponding Application Code. If you
+    use option 4d1, you must provide the Installation Information in
+    the manner specified by section 6 of the GNU GPL for conveying
+    Corresponding Source.)
+
+## 5. Combined Libraries.
+
+You may place library facilities that are a work based on the Library
+side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+-   a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities, conveyed under the terms of this License.
+-   b) Give prominent notice with the combined library that part of it
+    is a work based on the Library, and explaining where to find the
+    accompanying uncombined form of the same work.
+
+## 6. Revised Versions of the GNU Lesser General Public License.
+
+The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Library
+as you received it specifies that a certain numbered version of the
+GNU Lesser General Public License "or any later version" applies to
+it, you have the option of following the terms and conditions either
+of that published version or of any later version published by the
+Free Software Foundation. If the Library as you received it does not
+specify a version number of the GNU Lesser General Public License, you
+may choose any version of the GNU Lesser General Public License ever
+published by the Free Software Foundation.
+
+If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.