fstdtools 0.0.1__py3-none-any.whl

fstdtools/__main__.py

@@ -0,0 +1,2 @@
+from .cli import cli
+cli()

fstdtools/cli.py

@@ -0,0 +1,130 @@
+import click
+import fstd
+from pathlib import Path
+from .convert import convert as converter
+from importlib.metadata import version, PackageNotFoundError
+
+
+def get_version():
+    try:
+        return version("fstdtools")
+    except PackageNotFoundError:
+        return "0.0.0-unknown"
+
+
+def print_version(ctx, param, value):
+    if not value or ctx.resilient_parsing:
+        return
+    ver = fstd.get_version()
+    click.echo(click.style(f"fstdtools v{get_version()} | fstd core v{ver}", fg="green"))
+    ctx.exit()
+
+
+def overwrite_confirm(ctx, file_path):
+    if Path(file_path).exists():
+        if not click.confirm(click.style(f"File {file_path} already exists. Overwrite?", fg="yellow"), default=False):
+            click.echo("Operation cancelled", err=True)
+            ctx.exit(code=1)
+            return False
+
+
+# ===================== Global options =====================
+
+@click.group(name="fstdtools", help="CLI tools for fstd dictionary to pack/unpack/list/info/convert.", context_settings={"help_option_names": ["-h", "--help"]})
+@click.option("-V", "--version", is_flag=True, callback=print_version, expose_value=False, is_eager=True, help="print version info and exit")
+@click.option("--verbose", "-v", count=True, help="log level, -v simple log, -vv debug log")
+@click.option("--log-level", "-l", type=click.IntRange(min=0, max=6), default=4, show_default=True, envvar="FSTDTOOLS_LOG_LEVEL", help=" log_level: 0-6, 0 is trace, 1 is debug, 2 is info, 3 is warn, 4 is error, 5 is critical, 6 is off")
+@click.pass_context
+def cli(ctx, verbose, log_level):
+    """global init, all subcommands share this context"""
+    ctx.ensure_object(dict)
+    ctx.obj["verbose"] = verbose
+    ctx.obj["log_level"] = log_level
+    fstd.set_log_level(log_level)
+
+
+# ===================== Subcommands write =====================
+
+@cli.command(name="write", help="from txt/fstdx/mdx to fstdx, from directory/mdd to fstdd")
+@click.argument("source_file", type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True))
+@click.argument("output_file", type=click.Path(file_okay=True, dir_okay=False, writable=True), required=False)
+@click.option("-c", "--compress-level", type=click.IntRange(min=0, max=22), default=5, help="compression level 0(fast) ~ 22(max compress)")
+@click.option("--compress-dict-size", type=click.IntRange(min=1, max=130), default=100, help="compression dict size, only for fstdx, 1~130, default 100")
+@click.option("-b", "--block-size", type=click.IntRange(min=4, max=512), default=4, help="block size, default 4, unit KB")
+@click.option("-t", "--thread", type=int, default=0, help="concurrency thread count, default 0, auto detect cpu count")
+@click.option("--substyle/--no-substyle", default=False, help="enable substyle, only for mdx/mdd to fstdx/fstdd, default False")
+@click.option("-y", "--yes", is_flag=True, help="overwrite output file, no confirm")
+@click.pass_context
+def convert_(
+    ctx, source_file, output_file, compress_level, compress_dict_size, block_size, thread, substyle, yes
+):
+    verbose = ctx.obj["verbose"]
+    src = Path(source_file)
+    out = Path(output_file) if output_file else None
+
+    if not src.exists():
+        click.echo(click.style(f"Source file {src} does not exist", fg="red"), err=True)
+        ctx.exit(code=1)
+
+    if out and out.exists() and not yes:
+        if not click.confirm(click.style(f"File {out} already exists, overwrite?", fg="yellow")):
+            click.echo("Operation cancelled", err=True)
+            ctx.exit(code=1)
+
+    def show_verbose():
+        if verbose >= 1:
+            click.echo(click.style(f"Source file: {src}", fg="cyan"))
+            click.echo(click.style(f"Output file: {output_file if output_file else 'default'}", fg="cyan"))
+            click.echo(click.style(f"Compression level: {compress_level}", fg="cyan"))
+            click.echo(click.style(f"Compression dict size: {compress_dict_size}", fg="cyan"))
+            click.echo(click.style(f"Block size: {block_size}", fg="cyan"))
+            click.echo(click.style(f"Concurrency threads: {thread if thread > 0 else 'auto detect cpu count'}", fg="cyan"))
+
+    if src.is_dir():
+        if out and not out.suffix == ".fstdd":
+            click.echo(click.style("For mdd source, output file must have .fstdd extension", fg="red"), err=True)
+            ctx.exit(code=1)
+        if not output_file:
+            output_file = str(src.with_suffix(".fstdd"))
+            overwrite_confirm(ctx, output_file)
+        writer = fstd.FstddWriter()
+        show_verbose()
+        writer.compile_fstdd(source_file, output_file, "{}", block_size, compress_level, thread, verbose >= 1)
+
+    elif src.suffix == ".mdx":
+        if out and not out.suffix == ".fstdx":
+            click.echo(click.style("For mdx source, output file must have .fstdx extension", fg="red"), err=True)
+            ctx.exit(code=1)
+        if not output_file:
+            output_file = src.with_suffix(".fstdx")
+            overwrite_confirm(ctx, output_file)
+        show_verbose()
+        converter(source_file, output_file, compress_level, compress_dict_size, block_size, thread, substyle, None)
+
+    elif src.suffix == ".mdd":
+        if out and not out.suffix == ".fstdd":
+            click.echo(click.style("For mdd source, output file must have .fstdd extension", fg="red"), err=True)
+            ctx.exit(code=1)
+        if not output_file:
+            output_file = str(src.with_suffix(".fstdd"))
+            overwrite_confirm(ctx, output_file)
+        show_verbose()
+        converter(source_file, output_file, compress_level, compress_dict_size, block_size, thread, substyle, None)
+
+    else:
+        # see src as txt or fstdx, then convert to fstdx
+        if out and not out.suffix == ".fstdx":
+            click.echo(click.style("For txt or fstdx source, output file must have .fstdx extension", fg="red"), err=True)
+            ctx.exit(code=1)
+        if not output_file:
+            output_file = src.with_suffix(".fstdx")
+            overwrite_confirm(ctx, output_file)
+        writer = fstd.FstdxWriter()
+        show_verbose()
+        writer.compile_fstdx(source_file, output_file, "{}", block_size, compress_level, compress_dict_size, thread, False, verbose >= 1)
+
+    click.echo(click.style(f"{output_file} written successfully", fg="bright_green"))
+
+
+if __name__ == "__main__":
+    cli()

fstdtools/convert.py

@@ -0,0 +1,77 @@
+import os.path
+import fstd
+import json
+from concurrent.futures import ThreadPoolExecutor
+
+from tqdm import tqdm
+
+from .mdict.readmdict import MDX, MDD
+
+
+def get_meta(source, substyle=False, passcode=None):
+    meta = {}
+    if source.endswith('.mdx'):
+        encoding = ''
+        md = MDX(source, encoding, substyle, passcode)
+    if source.endswith('.mdd'):
+        md = MDD(source, passcode)
+
+    for key, value in md.header.items():
+        # key has been decode from UTF-16 and encode again with UTF-8
+        key = key.decode('UTF-8').lower().title()
+        value = value.decode('UTF-8')
+        meta[key] = value
+        if value == 'Yes':
+            meta[key] = True
+        elif value == 'No':
+            meta[key] = False
+
+    return meta
+
+
+def convert(source, target, compress_level, compress_dict_size, block_size, thread=0, substyle=False, passcode=None):
+    meta_info = get_meta(source, substyle, passcode)
+    meta_json_str = json.dumps(meta_info, ensure_ascii=False)
+    if source.endswith('.mdx'):
+        encoding = ''
+        mdx = MDX(source, encoding, substyle, passcode)
+        item_count = 0
+        keys = []
+        values = []
+
+        print("Extracting raw data from mdx:")
+        bar = tqdm(total=len(mdx), unit='rec')
+        for key, value in mdx.items():
+            if not value.strip():
+                bar.write('Skip entry: %s' % key)
+                continue
+            item_count += 1
+            keys.append(key)
+            values.append(value)
+            bar.update(1)
+        bar.close()
+        writer = fstd.FstdxWriter()
+        writer.compile_fstdx(target, keys, values, meta_json_str, block_size, compress_level, compress_dict_size, thread, False, True)
+    elif source.endswith('.mdd'):
+        mdd = MDD(source, passcode)
+        writer = fstd.FstddWriter()
+        executor = ThreadPoolExecutor(max_workers=1)
+        future = executor.submit(
+            writer.compile_fstdd,
+            len(mdd),
+            target,
+            meta_json_str,
+            block_size,
+            compress_level,
+            thread,
+            False
+        )
+        for key, value in mdd.items():
+            fname = key.decode('UTF-8').replace('\\', os.path.sep)
+            if not writer.push_file_stream(fname, value):
+                break
+        ret = future.result()
+        if ret != 0:
+            print("Compile fstdd failed with error code: %d" % ret)
+    else:
+        raise ValueError("Unsupported source file type: %s" % source)

fstdtools/mdict/lzo.py

@@ -0,0 +1,246 @@
+import math
+
+
+class FlexBuffer():
+
+    def __init__(self):
+
+        self.blockSize = None
+        self.c = None
+        self.l = None
+        self.buf = None
+
+    def require(self, n):
+        
+        r = self.c - self.l + n
+        if r > 0:
+            self.l = self.l + self.blockSize * math.ceil(r / self.blockSize)
+            #tmp = bytearray(self.l)
+            #for i in len(self.buf):
+            #    tmp[i] = self.buf[i]
+            #self.buf = tmp
+            self.buf = self.buf + bytearray(self.l - len(self.buf))
+        self.c = self.c + n
+        return self.buf
+
+    def alloc(self, initSize, blockSize):
+        
+        if blockSize:
+            sz = blockSize
+        else:
+            sz = 4096
+        self.blockSize = self.roundUp(sz)
+        self.c = 0
+        self.l = self.roundUp(initSize) | 0
+        self.l += self.blockSize - (self.l % self.blockSize)
+        self.buf = bytearray(self.l)
+        return self.buf
+
+    def roundUp(self, n):
+        
+        r = n % 4
+        if r == 0:
+            return n
+        else:
+            return n + 4 - r
+
+    def reset(self):
+
+        self.c = 0
+        self.l = len(self.buf)
+
+    def pack(self, size):
+        
+        return self.buf[0:size]
+
+def _decompress(inBuf, outBuf):
+
+    c_top_loop = 1
+    c_first_literal_run = 2
+    c_match = 3
+    c_copy_match = 4
+    c_match_done = 5
+    c_match_next = 6
+
+    out = outBuf.buf
+    op = 0
+    ip = 0
+    t = inBuf[ip]
+    state = c_top_loop
+    m_pos = 0
+    ip_end = len(inBuf)
+
+    if t > 17:
+        ip = ip + 1
+        t = t - 17
+        if t < 4:
+            state = c_match_next
+        else:
+            out = outBuf.require(t)
+            while True:
+                out[op] = inBuf[ip]
+                op = op + 1
+                ip = ip + 1
+                t = t - 1
+                if not t > 0: break
+            state = c_first_literal_run
+
+    while True:
+        if_block = False
+
+        ##
+        if state == c_top_loop:
+            t = inBuf[ip]
+            ip = ip + 1
+            if t >= 16:
+                state = c_match
+                continue
+            if t == 0:
+                while inBuf[ip] == 0:
+                    t = t + 255
+                    ip = ip + 1
+                t = t + 15 + inBuf[ip]
+                ip = ip + 1
+
+            t = t + 3
+            out = outBuf.require(t)
+            while True:
+                out[op] = inBuf[ip]
+                op = op + 1
+                ip = ip + 1
+                t = t - 1
+                if not t > 0: break
+            # emulate c switch
+            state = c_first_literal_run
+
+        ##
+        if state == c_first_literal_run:
+            t = inBuf[ip]
+            ip = ip + 1
+            if t >= 16:
+                state = c_match
+                continue
+            m_pos = op - 0x801 - (t >> 2) - (inBuf[ip] << 2)
+            ip = ip + 1
+            out = outBuf.require(3)
+            out[op] = out[m_pos]
+            op = op + 1
+            m_pos = m_pos + 1
+            out[op] = out[m_pos]
+            op = op + 1
+            m_pos = m_pos + 1
+            out[op] = out[m_pos]
+            op = op + 1
+
+            state = c_match_done
+            continue
+
+        ##
+        if state == c_match:
+            if t >= 64:
+                m_pos = op - 1 - ((t >> 2) & 7) - (inBuf[ip] << 3)
+                ip = ip + 1
+                t = (t >> 5) - 1
+                state = c_copy_match
+                continue
+            elif t >= 32:
+                t = t & 31
+                if t == 0:
+                    while inBuf[ip] == 0:
+                        t = t + 255
+                        ip = ip + 1
+                    t = t + 31 + inBuf[ip]
+                    ip = ip + 1
+                m_pos = op - 1 - ((inBuf[ip] + (inBuf[ip + 1] << 8)) >> 2)
+                ip = ip + 2
+            elif t >= 16:
+                m_pos = op - ((t & 8) << 11)
+                t = t & 7
+                if t == 0:
+                    while inBuf[ip] == 0:
+                        t = t + 255
+                        ip = ip + 1
+                    t = t + 7 + inBuf[ip]
+                    ip = ip + 1
+                m_pos = m_pos - ((inBuf[ip] + (inBuf[ip + 1] << 8)) >> 2)
+                ip = ip + 2
+                if m_pos == op:
+                    break
+                m_pos = m_pos - 0x4000
+            else:
+                m_pos = op - 1 - (t >> 2) - (inBuf[ip] << 2);
+                ip = ip + 1
+                out = outBuf.require(2)
+                out[op] = out[m_pos]
+                op = op + 1
+                m_pos = m_pos + 1
+                out[op] = out[m_pos]
+                op = op + 1
+                state = c_match_done
+                continue
+
+            if t >= 6 and (op - m_pos) >= 4:
+                if_block = True
+                t += 2
+                out = outBuf.require(t)
+                while True:
+                    out[op] = out[m_pos]
+                    op += 1
+                    m_pos += 1
+                    t -= 1
+                    if not t > 0: break
+            #emulate c switch
+            state = c_copy_match
+        
+        ##
+        if state == c_copy_match:
+            if not if_block:
+                t += 2
+                out = outBuf.require(t)
+                while True:
+                    out[op] = out[m_pos]
+                    op += 1
+                    m_pos += 1
+                    t -= 1
+                    if not t > 0: break
+            #emulating c switch
+            state = c_match_done
+ 
+        ##
+        if state == c_match_done:
+            t = inBuf[ip - 2] & 3
+            if t == 0:
+                state = c_top_loop
+                continue
+            #emulate c switch
+            state = c_match_next
+
+        ##
+        if state == c_match_next:
+            out = outBuf.require(1)
+            out[op] = inBuf[ip]
+            op += 1
+            ip += 1
+            if t > 1:
+                out = outBuf.require(1)
+                out[op] = inBuf[ip]
+                op += 1
+                ip += 1
+                if t > 2:
+                    out = outBuf.require(1)
+                    out[op] = inBuf[ip]
+                    op += 1
+                    ip += 1
+            t = inBuf[ip]
+            ip += 1
+            state = c_match
+            continue
+
+    return bytes(outBuf.pack(op))
+
+def decompress(input, initSize = 16000, blockSize = 8192):
+    output = FlexBuffer()
+    output.alloc(initSize, blockSize)
+    return _decompress(bytearray(input), output)
+
+