zipFly64 1.0.0__tar.gz

zipfly64-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Pamparampam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

zipfly64-1.0.0/PKG-INFO

@@ -0,0 +1,125 @@
+Metadata-Version: 2.1
+Name: zipFly64
+Version: 1.0.0
+Summary: Stream zip64 archives on the fly.
+Author-email: Pamparampampam <reallpamparampam.pl@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2024 Pamparampam
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Project-URL: Homepage, https://github.com/pam-param-pam/ZipFly
+Keywords: zip64,zip,streaming,zipfly
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiofiles>=24.1.0
+
+# ZipFly
+
+<a href="http://forthebadge.com/"><img src="https://forthebadge.com/images/badges/0-percent-optimized.svg" alt="forthebadge"/></a>
+<a href="http://forthebadge.com/"><img src="https://forthebadge.com/images/badges/gluten-free.png" alt="forthebadge"/></a>
+ <a href="http://forthebadge.com/"><img src="https://web.archive.org/web/20230604002050/https://forthebadge.com/images/badges/mom-made-pizza-rolls.svg" alt="forthebadge"/></a>
+
+<img src="https://img.shields.io/badge/ZIP64-Certified-lightGreen" alt="Build Status"/>
+<img src="https://img.shields.io/badge/build-failing-red" alt="Build Status"/>
+<img src="https://img.shields.io/badge/made with-hate-orange" alt="Build Status"/>
+<img src="https://img.shields.io/badge/fuck-zip-green" alt="Build Status"/>
+
+### ZipFly is a library for creating & streaming ZIP64 archives "on the fly"
+
+**It allows to create/fetch file content dynamically while the archive is streamed.**
+
+- No temporary files, data is streamed directly
+- Ability to calculate archive size before streaming even begins
+- Supported `deflate` compression method
+- Small memory usage, streaming is done using yield statement
+- Archive structure is created on the fly, and all data can be created during stream
+- Files included into archive can be generated on the fly using Python generators
+- **Independent of the goofy 🤮🤮 python's standard ZipFile implementation**
+- No dependencies
+- Automatic detection and changing of duplicate names
+- `Zip64` format compatible files
+
+
+This library is based upon [this library](https://github.com/kbbdy/zipstream) <sub>_(this library was a piece of work...)_<sub>
+
+## Typical Usage
+
+```py
+from zipFly import ZipFly, LocalFile, consts
+# compression_method is optional, defaults to consts.NO_COMPRESSION
+file1 = LocalFile(file_path='files/lqbfa61deebf1.mp4', compression_method=consts.NO_COMPRESSION) #  or consts.COMPRESSION_DEFLATE 
+file2 = LocalFile(file_path='public/2ae9dcd01a3aa.mp4', name="files/my_file2.mp4")  # override the file name
+file3 = LocalFile(file_path='files/4shaw1dax4da.mp4', name="my_file3.mp4")  # you control the directory path by specifying it in name
+
+files = [file1, file2, file3]
+
+zipFly = ZipFly(files)
+
+# save to file, or do something else with the stream() generator
+with open("out/file.zip", 'wb') as f_out:
+    for chunk in zipFly.stream():
+        f_out.write(chunk)
+```
+
+### Supports dynamically created files
+```py
+from zipFly import ZipFly, GenFile, LocalFile, consts
+
+
+def file_generator():
+    yield b"uga buga"
+    yield b"a29jaGFtIGFsdGVybmF0eXdraQ=="
+    yield b"2137"
+    
+# size is optional, it allows to calculate the total size of the archive before any data is generated
+# modification_time in epoch time, defaults to time.time()
+file1 = GenFile(name="file.txt", generator=file_generator(), modification_time=time.time(), size=size, compression_method=consts.COMPRESSION_DEFLATE)
+file2 = LocalFile(file_path='files/as61aade2ebfd.mp4', compression_method=consts.NO_COMPRESSION) #  or consts.COMPRESSION_DEFLATE 
+
+files = [file1, file2]
+
+zipFly = ZipFly(files)
+archive_size = zipFly.calculate_archive_size() # raises ValueError if it can't calculate size
+
+# for example you can set as content length in http response
+response['Content-Length'] = archive_size
+
+for chunk in zipFly.stream():
+       # do something
+
+```
+### Other
+I created this library for my I Drive project.
+
+If you have a different case scenario, and LocalFile and GenFile are not enough, you can extend BaseFile and everything else should work out of the box.
+
+
+
+### PS
+
+I wholeheartedly hope everyone responsible for creating ZIP documentation gets slaughtered in the most gore and painful way 😊 (in game)
+
+(pls redo ur [docs](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT))
+
+

zipfly64-1.0.0/README.md

@@ -0,0 +1,89 @@
+# ZipFly
+
+<a href="http://forthebadge.com/"><img src="https://forthebadge.com/images/badges/0-percent-optimized.svg" alt="forthebadge"/></a>
+<a href="http://forthebadge.com/"><img src="https://forthebadge.com/images/badges/gluten-free.png" alt="forthebadge"/></a>
+ <a href="http://forthebadge.com/"><img src="https://web.archive.org/web/20230604002050/https://forthebadge.com/images/badges/mom-made-pizza-rolls.svg" alt="forthebadge"/></a>
+
+<img src="https://img.shields.io/badge/ZIP64-Certified-lightGreen" alt="Build Status"/>
+<img src="https://img.shields.io/badge/build-failing-red" alt="Build Status"/>
+<img src="https://img.shields.io/badge/made with-hate-orange" alt="Build Status"/>
+<img src="https://img.shields.io/badge/fuck-zip-green" alt="Build Status"/>
+
+### ZipFly is a library for creating & streaming ZIP64 archives "on the fly"
+
+**It allows to create/fetch file content dynamically while the archive is streamed.**
+
+- No temporary files, data is streamed directly
+- Ability to calculate archive size before streaming even begins
+- Supported `deflate` compression method
+- Small memory usage, streaming is done using yield statement
+- Archive structure is created on the fly, and all data can be created during stream
+- Files included into archive can be generated on the fly using Python generators
+- **Independent of the goofy 🤮🤮 python's standard ZipFile implementation**
+- No dependencies
+- Automatic detection and changing of duplicate names
+- `Zip64` format compatible files
+
+
+This library is based upon [this library](https://github.com/kbbdy/zipstream) <sub>_(this library was a piece of work...)_<sub>
+
+## Typical Usage
+
+```py
+from zipFly import ZipFly, LocalFile, consts
+# compression_method is optional, defaults to consts.NO_COMPRESSION
+file1 = LocalFile(file_path='files/lqbfa61deebf1.mp4', compression_method=consts.NO_COMPRESSION) #  or consts.COMPRESSION_DEFLATE 
+file2 = LocalFile(file_path='public/2ae9dcd01a3aa.mp4', name="files/my_file2.mp4")  # override the file name
+file3 = LocalFile(file_path='files/4shaw1dax4da.mp4', name="my_file3.mp4")  # you control the directory path by specifying it in name
+
+files = [file1, file2, file3]
+
+zipFly = ZipFly(files)
+
+# save to file, or do something else with the stream() generator
+with open("out/file.zip", 'wb') as f_out:
+    for chunk in zipFly.stream():
+        f_out.write(chunk)
+```
+
+### Supports dynamically created files
+```py
+from zipFly import ZipFly, GenFile, LocalFile, consts
+
+
+def file_generator():
+    yield b"uga buga"
+    yield b"a29jaGFtIGFsdGVybmF0eXdraQ=="
+    yield b"2137"
+    
+# size is optional, it allows to calculate the total size of the archive before any data is generated
+# modification_time in epoch time, defaults to time.time()
+file1 = GenFile(name="file.txt", generator=file_generator(), modification_time=time.time(), size=size, compression_method=consts.COMPRESSION_DEFLATE)
+file2 = LocalFile(file_path='files/as61aade2ebfd.mp4', compression_method=consts.NO_COMPRESSION) #  or consts.COMPRESSION_DEFLATE 
+
+files = [file1, file2]
+
+zipFly = ZipFly(files)
+archive_size = zipFly.calculate_archive_size() # raises ValueError if it can't calculate size
+
+# for example you can set as content length in http response
+response['Content-Length'] = archive_size
+
+for chunk in zipFly.stream():
+       # do something
+
+```
+### Other
+I created this library for my I Drive project.
+
+If you have a different case scenario, and LocalFile and GenFile are not enough, you can extend BaseFile and everything else should work out of the box.
+
+
+
+### PS
+
+I wholeheartedly hope everyone responsible for creating ZIP documentation gets slaughtered in the most gore and painful way 😊 (in game)
+
+(pls redo ur [docs](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT))
+
+

zipfly64-1.0.0/pyproject.toml

@@ -0,0 +1,27 @@
+# pyproject.toml
+
+[build-system]
+requires      = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "zipFly64"
+version = "1.0.0"
+description = "Stream zip64 archives on the fly."
+readme = "README.md"
+authors = [{ name = "Pamparampampam", email = "reallpamparampam.pl@gmail.com" }]
+license = { file = "LICENSE" }
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+]
+keywords = ["zip64", "zip", "streaming", "zipfly"]
+dependencies = [
+    "aiofiles >= 24.1.0",
+]
+requires-python = ">=3.7"
+
+[project.urls]
+Homepage = "https://github.com/pam-param-pam/ZipFly"
+

zipfly64-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

zipfly64-1.0.0/src/zipFly/BaseFile.py

@@ -0,0 +1,85 @@
+from abc import ABC, abstractmethod
+from typing import Generator, AsyncGenerator
+
+from zipFly import consts
+from zipFly.Compressor import Compressor
+
+
+class BaseFile(ABC):
+    def __init__(self, compression_method: int):
+        self.original_size = 0
+        self.compressed_size = 0
+        self.offset = 0  # Offset to local file header
+        self.crc = 0
+        self.flags = 0b00001000  # flag about using data descriptor is always on
+        self.compression_method = compression_method or consts.NO_COMPRESSION
+
+    def __str__(self):
+        return f"FILE[{self.name}]"
+
+    def generate_processed_file_data(self) -> Generator[bytes, None, None]:
+        compressor = Compressor(self)
+
+        """
+        Generates compressed file data
+        """
+        for chunk in self._generate_file_data():
+            chunk = compressor.process(chunk)
+            if len(chunk) > 0:
+                yield chunk
+            chunk = compressor.tail()
+            if len(chunk) > 0:
+                yield chunk
+
+    async def async_generate_processed_file_data(self) -> AsyncGenerator[bytes, None]:
+        compressor = Compressor(self)
+
+        """
+        Generates compressed file data
+        """
+        async for chunk in self._async_generate_file_data():
+            chunk = compressor.process(chunk)
+            if len(chunk) > 0:
+                yield chunk
+            chunk = compressor.tail()
+            if len(chunk) > 0:
+                yield chunk
+
+    def get_mod_time(self) -> int:
+        return int(self.modification_time) & 0xFFFF
+
+    def get_mod_date(self) -> int:
+        return int(self.modification_time / 86400 + 365 * 20) & 0xFFFF
+
+    @property
+    def file_path_bytes(self) -> bytes:
+        try:
+            return self.name.encode("ascii")
+        except UnicodeError:
+            self.flags |= consts.UTF8_FLAG
+            return self.name.encode("utf-8")
+
+    @abstractmethod
+    def _generate_file_data(self) -> Generator[bytes, None, None]:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def _async_generate_file_data(self) -> AsyncGenerator[bytes, None]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def set_file_name(self, new_name: str) -> None:
+        raise NotImplementedError
+
+    @property
+    def size(self) -> int:
+        raise NotImplementedError
+
+    @property
+    def modification_time(self) -> float:
+        raise NotImplementedError
+
+    @property
+    def name(self) -> str:
+        raise NotImplementedError
+

zipfly64-1.0.0/src/zipFly/Compressor.py

@@ -0,0 +1,37 @@
+import zlib
+
+
+class Compressor:
+    def __init__(self, file):
+        self.file = file
+
+        if file.compression_method == 0:
+            self.process = self._process_through
+            self.tail = self._no_tail
+        elif file.compression_method == 8:  # deflate compression
+            self.compr = zlib.compressobj(5, zlib.DEFLATED, -15)
+            self.process = self._process_deflate
+            self.tail = self._tail_deflate
+
+    # no compression
+    def _process_through(self, chunk):
+        self.file.original_size += len(chunk)
+        self.file.compressed_size += len(chunk)
+        self.file.crc = zlib.crc32(chunk, self.file.crc)
+        return chunk
+
+    def _no_tail(self):
+        return b''
+
+    # deflate compression
+    def _process_deflate(self, chunk):
+        self.file.original_size += len(chunk)
+        self.file.crc = zlib.crc32(chunk, self.file.crc)
+        chunk = self.compr.compress(chunk)
+        self.file.compressed_size += len(chunk)
+        return chunk
+
+    def _tail_deflate(self):
+        chunk = self.compr.flush(zlib.Z_FINISH)
+        self.file.compressed_size += len(chunk)
+        return chunk

zipfly64-1.0.0/src/zipFly/GenFile.py

@@ -0,0 +1,43 @@
+import time
+from typing import Generator, AsyncGenerator, Union
+from zipFly.BaseFile import BaseFile
+
+
+class GenFile(BaseFile):
+
+    def __init__(self, name: str, generator: Union[Generator[bytes, None, None], AsyncGenerator[bytes, None]], compression_method: int = None, modification_time: float = None, size: int = None):
+        super().__init__(compression_method)
+        self._name = name
+        self.generator = generator
+        self._size = size
+        self._modification_time = modification_time if modification_time else time.time()
+
+    def _generate_file_data(self) -> Generator[bytes, None, None]:
+        if isinstance(self.generator, Generator):
+            yield from self.generator
+        else:
+            raise ValueError("self.generator must be of type Generator")
+
+    async def _async_generate_file_data(self) -> AsyncGenerator[bytes, None]:
+        if isinstance(self.generator, AsyncGenerator):
+            async for chunk in self.generator:
+                yield chunk
+        else:
+            raise ValueError("self.generator must be of type AsyncIterator")
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def size(self) -> int:
+        if self._size is not None:
+            return self._size
+        raise ValueError("Archive size not known before streaming. Probably GenFile() is missing size attribute.")
+
+    @property
+    def modification_time(self) -> float:
+        return self._modification_time
+
+    def set_file_name(self, new_name: str) -> None:
+        self._name = new_name

zipfly64-1.0.0/src/zipFly/LocalFile.py

@@ -0,0 +1,59 @@
+import os
+import time
+from typing import Generator, AsyncGenerator
+from zipFly.BaseFile import BaseFile
+
+import aiofiles
+
+class LocalFile(BaseFile):
+
+    async def _async_generate_file_data(self) -> AsyncGenerator[bytes, None]:
+
+        async with aiofiles.open(self._file_path, "rb") as fh:
+            while True:
+                part = await fh.read(self.chunk_size)
+                if not part:
+                    break
+                yield part
+
+    def __init__(self, file_path: str, name: str = None, compression_method: int = None):
+        if not os.path.isfile(file_path):
+            raise ValueError(f"{file_path} is not a correct file path.")
+        self._file_path = file_path
+        self.chunk_size = 1048
+        self._name = name if name else file_path
+        super().__init__(compression_method)
+
+    def _generate_file_data(self) -> Generator[bytes, None, None]:
+        with open(self._file_path, 'rb') as file:
+            while True:
+                chunk = file.read(self.chunk_size)
+                if not chunk:
+                    break
+                yield chunk
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def size(self) -> int:
+        return os.path.getsize(self._file_path)
+
+    @property
+    def modification_time(self) -> float:
+        return os.path.getmtime(self._file_path)
+
+    def get_mod_time(self) -> int:
+        # Extract hours, minutes, and seconds from the modification time
+        t = time.localtime(self.modification_time)
+        return ((t.tm_hour << 11) | (t.tm_min << 5) | (t.tm_sec // 2)) & 0xFFFF
+
+    def get_mod_date(self) -> int:
+        # Extract year, month, and day from the modification time
+        t = time.localtime(self.modification_time)
+        year = t.tm_year - 1980  # ZIP format years start from 1980
+        return ((year << 9) | (t.tm_mon << 5) | t.tm_mday) & 0xFFFF
+
+    def set_file_name(self, new_name: str) -> None:
+        self._name = new_name

zipfly64-1.0.0/src/zipFly/ZipBase.py

@@ -0,0 +1,256 @@
+from collections import defaultdict
+from typing import List
+
+from zipFly import consts
+from zipFly.BaseFile import BaseFile
+
+"""
+Since the Official ZIP docs are terrible, here's a detailed structure of the zip this library builds. (pretty sure mine's just as bad lol)
+
+  [local file header 1]            |
+  [file data 1]                    |
+  [data descriptor 1]              |
+  .                                |
+  .                                } - This part of the of zip holds the file data. Local file headers are lowkey useless(nevertheless needed for zip to work).
+  .                                |        Data descriptors allow to stream the file(and create file headers) without knowing the size of file data. Instead of putting
+  [local file header n]            |        things like: CRC, uncompressed_size, compressed_size etc in file headers, you only put placeholder values there (0xFFFFFFFF),                                    
+  [file data n]                    |        and fill them later in data descriptor.
+  [data descriptor n]              |
+  
+  
+  =================This part is called Central Directory Structure==============                               <------------ TO HERE -------------------------------------------<
+                                                                                                                                                                                |
+  [central directory header 1]     |                                                                                                                                            |
+  [extra field 1]                  |                                                                                                                                            |
+  .                                } - I have no idea why it's called 'central directory header'. It should be called central directory file headers.                           |
+  .                                |        From now on, i will call central directory as just 'cdir'. Cdir headers are structures that again hold the file information         |   
+  .                                |        Cdir headers are retarded, and their compressed_size, uncompressed_size, offset values are in 4 bytes, meaning you can't put        |
+  [central directory header n]     |        values > 4GB. Hence the ZIP64 uses a special structure called 'extra field'. Just like before, in cdir headers we put               |
+  [extra field n]                  |        placeholder values, and fill them later in extra field.                                                                             |
+                                                                                                                                                                                |
+                                            An important thing to pay attention in cdir header, is **offset**. This offset is the amount of bytes from the                      |
+                                            beginning of the file, to the start of local file header. So for 1st file the offset is 0, for the 2nd it's length of               |
+                                            '[local file header 1]' + '[file data 1]' + '[data descriptor 1]'.                                                                  |
+                                                                                                                                                                                |
+  =================This part I call End of Central Directory Structure (It's still a part of cdir structure(I think))===============            <----- TO HERE -----------------|-----------<
+                                                                                                                                                                                |           |
+  [zip64 end of central directory record]    |                                                                                                                                  |           |
+  [zip64 end of central directory locator]   } - This is the actual end of the file. End of cdir directory record is a legacy(and kinda useless, nevertheless required).        |           |
+  [end of central directory record]          |      Size we work with files >4GB, the end of cdir directory record, again can't hold values like: compressed_size,              |           |
+                                                    uncompressed_size, offset. We again use placeholder values, and put the actual ones in                                      |           |
+                                                    'zip64 end of central directory record'. 'zip64 end of cdir locator' is used to locate the 'zip64 end of cdir record'.      |           |
+                                                                                                                                                                                |           |
+                                                    An important thing to pay attention to are **offsets*. There are two district ones here.                                    |           |
+                                                    1st offset is in 'zip64 end of cdir record'. It's the amount of bytes from the start of the file to start of   ------------->           |
+                                                    *Central Directory Structure*.                                                                                                          |
+                                                    2nd offset is in 'zip64 end of cdir locator'. It's the amount of bytes from the start of the file to start of -------------------------->
+                                                    'zip64 end of cdir record'.
+                                                    
+                                                    
+    Other goofy things are:
+        'extra_field_len' in 'central directory header' is 28, even tho in the 'extra field' it self, the 'extra_field_size' is 24. That's because the first
+        is the full length of the 'extra field' structure, while the second doesn't include 'extra_field_size' and
+        'signature' which are each 2 bytes, so together they are the 'missing' 4 bytes.
+        
+        'size_of_zip64_end_of_cdir_record' in zip64 end of cdir record is 44 bytes. Cuz: 'signature' - 4 bytes, 'size_of_zip64_end_of_central_dir_record' - 8 bytes,
+        'version_made_by' - 2 bytes, 'version_to_extract' - 2 bytes, 'number_of_this_disk' - 4 bytes, 'cd_start' - 4 bytes, 'cd_entries_this_disk' - 8 bytes, 
+        'cd_entries_total' - 8 bytes, 'cd_size' - 8 bytes, 'cd_offset' - 8 bytes.
+        So, 4 + 8 + 2 + 2 + 4 + 4 + 8 + 8 + 8 + 8 = 56, but we again don't include signature, and 'size_of_zip64_end_of_central_dir_record' so 56 - 4 - 8 = 44
+        
+I hope, that i made it a bit more clear to anyone reading, including future me.        
+"""
+
+
+def process_file_names(files):
+    name_counts = defaultdict(int)
+    for file in files:
+        # Split the name into base and extension
+        base, ext = file.name.rsplit('.', 1) if '.' in file.name else (file.name, '')
+
+        # Increment the count for this base name
+        name_counts[base] += 1
+
+        # Append the count to the base name if it's not the first occurrence
+        if name_counts[base] > 1:
+            new_base = f"{base} ({name_counts[base] - 1})"
+        else:
+            new_base = base
+
+        # Reassemble the filename
+        file.set_file_name(f"{new_base}.{ext}" if ext else new_base)
+
+    return files
+
+
+class ZipBase:
+
+    def __init__(self, files: List[BaseFile]):
+        self.__version_to_extract = 45
+
+        # process file names to make sure there are no duplicates
+        processed_files = process_file_names(files)
+        self.files = processed_files
+
+        self.__offset = 0  # Tracks the current offset within the ZIP archive
+        self._cdir_size = 0
+        self._offset_to_start_of_central_dir = 0
+        self.__version_made_by = 0x0345  # UNIX and ZIP version 45
+
+    def _make_local_file_header(self, file: BaseFile) -> bytes:
+        """
+        Create local file header for a ZIP64 archive   (4.3.7)
+        """
+
+        fields = {
+            "signature": consts.LOCAL_FILE_HEADER_SIGNATURE,
+            "version_to_extract": self.__version_to_extract,
+            "flags": file.flags,
+            "compression": file.compression_method,
+            "mod_time": file.get_mod_time(),
+            "mod_date": file.get_mod_date(),
+            "crc": 0xFFFFFFFF,  # Placeholder (will be updated in data descriptor)
+            "uncompressed_size": 0xFFFFFFFF,  # Placeholder (will be updated in data descriptor)
+            "compressed_size": 0xFFFFFFFF,  # Placeholder (will be updated in data descriptor)
+            "file_name_len": len(file.file_path_bytes),
+            "extra_field_len": 0  # 0 cuz no extra field is used with local file header
+        }
+
+        # Pack the local file header structure
+        header = consts.LOCAL_FILE_HEADER_TUPLE(**fields)
+        header = consts.LOCAL_FILE_HEADER_STRUCT.pack(*header)
+        header += file.file_path_bytes
+
+        return header
+
+    def _make_data_descriptor(self, file: BaseFile) -> bytes:
+        """
+        Create data descriptor.  (4.3.9)
+        """
+
+        fields = {
+            "signature": consts.ZIP64_DATA_DESCRIPTOR_SIGNATURE,
+            "crc": file.crc & 0xffffffff,  # hack for making CRC unsigned long
+            "uncompressed_size": file.original_size,
+            "compressed_size": file.compressed_size,
+        }
+
+        descriptor = consts.ZIP64_DATA_DESCRIPTOR_TUPLE(**fields)
+        descriptor = consts.ZIP64_DATA_DESCRIPTOR_STRUCT.pack(*descriptor)
+
+        return descriptor
+
+    def _make_cdir_file_header(self, file: BaseFile) -> bytes:
+        """
+        Create central directory file header for ZIP64 archive.  (4.3.12)
+        """
+        fields = {
+            "signature": consts.CENTRAL_DIR_FILE_HEADER_SIGNATURE,
+            "version_made_by": self.__version_made_by,
+            "version_to_extract": self.__version_to_extract,
+            "flags": file.flags,
+            "compression": file.compression_method,
+            "mod_time": file.get_mod_time(),
+            "mod_date": file.get_mod_date(),
+            "crc": file.crc,
+            "compressed_size": 0xFFFFFFFF,  # Placeholder (will be updated in zip64 extra field)
+            "uncompressed_size": 0xFFFFFFFF,  # Placeholder (will be updated in zip64 extra field)
+            "file_name_len": len(file.file_path_bytes),
+            "extra_field_len": 28,
+            "file_comment_len": 0,
+            "disk_start": 0,
+            "internal_file_attr": 0,
+            "external_file_attr": 0,
+            "offset": 0xFFFFFFFF  # Placeholder (will be updated in zip64 extra field)
+        }
+
+        cdfh = consts.CENTRAL_DIR_FILE_HEADER_TUPLE(**fields)
+        cdfh = consts.CENTRAL_DIR_FILE_HEADER_STRUCT.pack(*cdfh)
+        cdfh += file.file_path_bytes
+
+        return cdfh
+
+    def _make_zip64_extra_field(self, file: BaseFile) -> bytes:
+        """
+        Create the ZIP64 extra field.  (4.5.3)
+        """
+        fields = {
+            "signature": consts.ZIP64_EXTRA_FIELD_SIGNATURE,
+            "extra_field_size": 24,
+            "size": file.original_size,
+            "compressed_size": file.compressed_size,
+            "offset": file.offset,
+        }
+
+        extra = consts.ZIP64_EXTRA_FIELD_TUPLE(**fields)
+        extra = consts.ZIP64_EXTRA_FIELD_STRUCT.pack(*extra)
+
+        return extra
+
+    def _make_zip64_end_of_cdir_record(self) -> bytes:
+        """
+        Create the ZIP64 end of central directory record.  (4.3.14)
+        """
+        fields = {
+            "signature": consts.ZIP64_END_OF_CENTRAL_DIR_RECORD_SIGNATURE,
+            "size_of_zip64_end_of_cdir_record": 44,  # 44 bytes for the ZIP64 end of central directory record itself
+            "version_made_by": self.__version_made_by,
+            "version_to_extract": self.__version_to_extract,
+            "number_of_this_disk": 0,
+            "cd_start": 0,
+            "cd_entries_this_disk": len(self.files),
+            "cd_entries_total": len(self.files),
+            "cd_size": self._cdir_size,
+            "cd_offset": self._offset_to_start_of_central_dir
+        }
+
+        cdend = consts.ZIP64_END_OF_CENTRAL_DIR_RECORD_TUPLE(**fields)
+        cdend = consts.ZIP64_END_OF_CENTRAL_DIR_RECORD_STRUCT.pack(*cdend)
+
+        return cdend
+
+    def _make_zip64_end_of_cdir_locator(self) -> bytes:
+        """
+        Create the ZIP64 end of central directory locator.  (4.3.15)
+        """
+        fields = {
+            "signature": consts.ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIGNATURE,
+            "disk_with_zip64_end": 0,
+            "zip64_end_offset": self.__offset,
+            "total_disks": 1
+        }
+
+        locator = consts.ZIP64_END_OF_CENTRAL_DIR_LOCATOR_TUPLE(**fields)
+
+        locator = consts.ZIP64_END_OF_CENTRAL_DIR_LOCATOR_STRUCT.pack(*locator)
+
+        return locator
+
+    def _make_end_of_cdir_record(self) -> bytes:
+        """
+        Create the end of central directory record.  (4.3.16)
+        """
+        fields = {
+            "signature": consts.END_OF_CENTRAL_DIR_RECORD_SIGNATURE,
+            "number_of_this_disk": 0,
+            "number_of_disk_with_start_central_dir": 0,
+            "total_entries_on_this_disk": len(self.files),
+            "total_entries_total": len(self.files),
+            "central_directory_size": 0xFFFFFFFF,
+            "offset_of_central_directory": 0xFFFFFFFF,
+            "comment_length": 0  # No comment
+        }
+
+        eocd = consts.END_OF_CENTRAL_DIR_RECORD_TUPLE(**fields)
+        eocd = consts.END_OF_CENTRAL_DIR_RECORD_STRUCT.pack(*eocd)
+
+        return eocd
+
+    def _add_offset(self, value: int) -> None:
+        self.__offset += value
+
+    def _get_offset(self) -> int:
+        return self.__offset
+
+
+
+

zipfly64-1.0.0/src/zipFly/ZipFly.py

@@ -0,0 +1,110 @@
+from typing import Generator, AsyncGenerator
+
+from zipFly import BaseFile
+from zipFly.ZipBase import ZipBase
+
+class ZipFly(ZipBase):
+
+    def calculate_archive_size(self) -> int:
+        LOCAL_FILE_HEADER_SIZE = 30
+        DATA_DESCRIPTOR_SIZE = 24
+        CENTRAL_DIR_HEADER_SIZE = 46
+        ZIP64_EXTRA_FIELD_SIZE = 28
+        ZIP64_END_OF_CDIR_RECORD_SIZE = 56
+        ZIP64_END_OF_CDIR_LOCATOR_SIZE = 20
+        END_OF_CDIR_RECORD_CD_RECORD_SIZE = 22
+
+        total_size = 0
+
+        for file in self.files:
+            local_file_header_size = LOCAL_FILE_HEADER_SIZE + len(file.file_path_bytes)
+
+            total_size += local_file_header_size
+            total_size += file.size
+            total_size += DATA_DESCRIPTOR_SIZE
+
+            central_directory_header_size = CENTRAL_DIR_HEADER_SIZE + len(file.file_path_bytes) + ZIP64_EXTRA_FIELD_SIZE
+
+            total_size += central_directory_header_size
+
+        total_size += ZIP64_END_OF_CDIR_RECORD_SIZE
+        total_size += ZIP64_END_OF_CDIR_LOCATOR_SIZE
+        total_size += END_OF_CDIR_RECORD_CD_RECORD_SIZE
+
+        return total_size
+
+    def _make_end_structures(self) -> Generator[bytes, None, None]:
+        """
+        Make zip64 end structures, which include:
+            central directory file header for every file,
+            zip64 extra field for every file,
+            zip64 end of central dir record,
+            zip64 end of central dir locator
+            end of central dir record
+        """
+        # Save offset to start of central dir for zip64 end of cdir record
+        self._offset_to_start_of_central_dir = self._get_offset()
+
+        # Stream central directory entries
+        for file in self.files:
+            chunk = self._make_cdir_file_header(file)
+            chunk += self._make_zip64_extra_field(file)
+            self._cdir_size = len(chunk)
+            self._add_offset(len(chunk))
+
+            yield chunk
+
+        yield self._make_zip64_end_of_cdir_record()
+
+        yield self._make_zip64_end_of_cdir_locator()
+
+        yield self._make_end_of_cdir_record()
+
+    async def _async_stream_single_file(self, file: BaseFile) -> AsyncGenerator[bytes, None]:
+
+        yield self._make_local_file_header(file)
+
+        async for chunk in file.async_generate_processed_file_data():
+            yield chunk
+
+        yield self._make_data_descriptor(file)
+
+    async def async_stream(self) -> AsyncGenerator[bytes, None]:
+        # stream files
+        for file in self.files:
+
+            file.offset = self._get_offset()
+            async for chunk in self._async_stream_single_file(file):
+                self._add_offset(len(chunk))
+                yield chunk
+
+        # stream zip structures
+        for chunk in self._make_end_structures():
+            yield chunk
+
+    def stream(self) -> Generator[bytes, None, None]:
+        # stream files
+        for file in self.files:
+            print("offset")
+            print(self._get_offset())
+            file.offset = self._get_offset()
+            for chunk in self._stream_single_file(file):
+                self._add_offset(len(chunk))
+                yield chunk
+
+        # stream zip structures
+        for chunk in self._make_end_structures():
+            yield chunk
+
+    def _stream_single_file(self, file: BaseFile) -> Generator[bytes, None, None]:
+        """
+        stream single zip file with header and descriptor at the end.
+        """
+        yield self._make_local_file_header(file)
+
+        yield from file.generate_processed_file_data()
+
+        yield self._make_data_descriptor(file)
+
+
+

zipfly64-1.0.0/src/zipFly/__init__.py

@@ -0,0 +1,5 @@
+from zipFly.ZipFly import ZipFly
+from zipFly.LocalFile import LocalFile
+from zipFly.GenFile import GenFile
+from zipFly.BaseFile import BaseFile
+from zipFly import consts

zipfly64-1.0.0/src/zipFly/consts.py

@@ -0,0 +1,68 @@
+from collections import namedtuple
+import struct
+
+# ZIP CONSTANTS
+ZIP64_VERSION = 45
+UTF8_FLAG = 0x800  # utf-8 filename encoding flag
+
+# ZIP COMPRESSION METHODS
+NO_COMPRESSION = 0
+COMPRESSION_DEFLATE = 8
+# COMPRESSION_BZIP2 = 12
+# COMPRESSION_LZMA = 14
+
+# LOCAL FILE HEADER
+LOCAL_FILE_HEADER_SIGNATURE = b'\x50\x4b\x03\x04'
+LOCAL_FILE_HEADER_STRUCT = struct.Struct(b"<4sHHHHHLLLHH")
+LOCAL_FILE_HEADER_TUPLE = namedtuple("fileheader",
+                                     ("signature", "version_to_extract", "flags",
+                                      "compression", "mod_time", "mod_date",
+                                      "crc", "uncompressed_size", "compressed_size",
+                                      "file_name_len", "extra_field_len"))
+
+
+# FILE DESCRIPTOR
+ZIP64_DATA_DESCRIPTOR_SIGNATURE = b'\x50\x4b\x07\x08'
+ZIP64_DATA_DESCRIPTOR_STRUCT = struct.Struct(b"<4sLQQ")
+ZIP64_DATA_DESCRIPTOR_TUPLE = namedtuple("filecrc", ("signature", "crc", "compressed_size", "uncompressed_size"))
+
+
+# CENTRAL DIRECTORY FILE HEADER
+CENTRAL_DIR_FILE_HEADER_SIGNATURE = b'\x50\x4b\x01\x02'
+CENTRAL_DIR_FILE_HEADER_STRUCT = struct.Struct(b"<4sHHHHHHLLLHHHHHLL")
+CENTRAL_DIR_FILE_HEADER_TUPLE = namedtuple("cdfileheader",
+                                           ("signature", "version_made_by", "version_to_extract", "flags",
+                                            "compression", "mod_time", "mod_date", "crc",
+                                            "compressed_size", "uncompressed_size", "file_name_len", "extra_field_len",
+                                            "file_comment_len", "disk_start", "internal_file_attr", "external_file_attr", "offset"))
+
+
+# ZIP64 EXTRA FIELD
+ZIP64_EXTRA_FIELD_SIGNATURE = b'\x01\x00'
+ZIP64_EXTRA_FIELD_STRUCT = struct.Struct(b"<2sHQQQ")
+ZIP64_EXTRA_FIELD_TUPLE = namedtuple("extra", ("signature", "extra_field_size", "size", "compressed_size", "offset"))
+
+
+# ZIP64 END OF CENTRAL DIRECTORY RECORD
+ZIP64_END_OF_CENTRAL_DIR_RECORD_SIGNATURE = b'\x50\x4b\x06\x06'
+ZIP64_END_OF_CENTRAL_DIR_RECORD_STRUCT = struct.Struct(b"<4sQHHIIQQQQ")
+ZIP64_END_OF_CENTRAL_DIR_RECORD_TUPLE = namedtuple("zip64end",
+                                                   ("signature", "size_of_zip64_end_of_cdir_record", "version_made_by", "version_to_extract",
+                                                    "number_of_this_disk", "cd_start", "cd_entries_this_disk", "cd_entries_total",
+                                                    "cd_size", "cd_offset"))
+
+
+# END OF CENTRAL DIRECTORY LOCATOR
+ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIGNATURE = b'\x50\x4b\x06\x07'
+ZIP64_END_OF_CENTRAL_DIR_LOCATOR_STRUCT = struct.Struct(b"<4sLQL")
+ZIP64_END_OF_CENTRAL_DIR_LOCATOR_TUPLE = namedtuple("eocdlocator",
+                                                    ("signature", "disk_with_zip64_end", "zip64_end_offset", "total_disks"))
+
+
+# END OF CENTRAL DIRECTORY RECORD
+END_OF_CENTRAL_DIR_RECORD_SIGNATURE = b'P\x4b\x05\x06'
+END_OF_CENTRAL_DIR_RECORD_STRUCT = struct.Struct(b"<4sHHHHLLH")
+END_OF_CENTRAL_DIR_RECORD_TUPLE = namedtuple("eocdlocator", ("signature", "number_of_this_disk", "number_of_disk_with_start_central_dir",
+                                                             "total_entries_on_this_disk", "total_entries_total",
+                                                             "central_directory_size", "offset_of_central_directory",
+                                                             "comment_length"))

zipfly64-1.0.0/src/zipFly64.egg-info/PKG-INFO

@@ -0,0 +1,125 @@
+Metadata-Version: 2.1
+Name: zipFly64
+Version: 1.0.0
+Summary: Stream zip64 archives on the fly.
+Author-email: Pamparampampam <reallpamparampam.pl@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2024 Pamparampam
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Project-URL: Homepage, https://github.com/pam-param-pam/ZipFly
+Keywords: zip64,zip,streaming,zipfly
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiofiles>=24.1.0
+
+# ZipFly
+
+<a href="http://forthebadge.com/"><img src="https://forthebadge.com/images/badges/0-percent-optimized.svg" alt="forthebadge"/></a>
+<a href="http://forthebadge.com/"><img src="https://forthebadge.com/images/badges/gluten-free.png" alt="forthebadge"/></a>
+ <a href="http://forthebadge.com/"><img src="https://web.archive.org/web/20230604002050/https://forthebadge.com/images/badges/mom-made-pizza-rolls.svg" alt="forthebadge"/></a>
+
+<img src="https://img.shields.io/badge/ZIP64-Certified-lightGreen" alt="Build Status"/>
+<img src="https://img.shields.io/badge/build-failing-red" alt="Build Status"/>
+<img src="https://img.shields.io/badge/made with-hate-orange" alt="Build Status"/>
+<img src="https://img.shields.io/badge/fuck-zip-green" alt="Build Status"/>
+
+### ZipFly is a library for creating & streaming ZIP64 archives "on the fly"
+
+**It allows to create/fetch file content dynamically while the archive is streamed.**
+
+- No temporary files, data is streamed directly
+- Ability to calculate archive size before streaming even begins
+- Supported `deflate` compression method
+- Small memory usage, streaming is done using yield statement
+- Archive structure is created on the fly, and all data can be created during stream
+- Files included into archive can be generated on the fly using Python generators
+- **Independent of the goofy 🤮🤮 python's standard ZipFile implementation**
+- No dependencies
+- Automatic detection and changing of duplicate names
+- `Zip64` format compatible files
+
+
+This library is based upon [this library](https://github.com/kbbdy/zipstream) <sub>_(this library was a piece of work...)_<sub>
+
+## Typical Usage
+
+```py
+from zipFly import ZipFly, LocalFile, consts
+# compression_method is optional, defaults to consts.NO_COMPRESSION
+file1 = LocalFile(file_path='files/lqbfa61deebf1.mp4', compression_method=consts.NO_COMPRESSION) #  or consts.COMPRESSION_DEFLATE 
+file2 = LocalFile(file_path='public/2ae9dcd01a3aa.mp4', name="files/my_file2.mp4")  # override the file name
+file3 = LocalFile(file_path='files/4shaw1dax4da.mp4', name="my_file3.mp4")  # you control the directory path by specifying it in name
+
+files = [file1, file2, file3]
+
+zipFly = ZipFly(files)
+
+# save to file, or do something else with the stream() generator
+with open("out/file.zip", 'wb') as f_out:
+    for chunk in zipFly.stream():
+        f_out.write(chunk)
+```
+
+### Supports dynamically created files
+```py
+from zipFly import ZipFly, GenFile, LocalFile, consts
+
+
+def file_generator():
+    yield b"uga buga"
+    yield b"a29jaGFtIGFsdGVybmF0eXdraQ=="
+    yield b"2137"
+    
+# size is optional, it allows to calculate the total size of the archive before any data is generated
+# modification_time in epoch time, defaults to time.time()
+file1 = GenFile(name="file.txt", generator=file_generator(), modification_time=time.time(), size=size, compression_method=consts.COMPRESSION_DEFLATE)
+file2 = LocalFile(file_path='files/as61aade2ebfd.mp4', compression_method=consts.NO_COMPRESSION) #  or consts.COMPRESSION_DEFLATE 
+
+files = [file1, file2]
+
+zipFly = ZipFly(files)
+archive_size = zipFly.calculate_archive_size() # raises ValueError if it can't calculate size
+
+# for example you can set as content length in http response
+response['Content-Length'] = archive_size
+
+for chunk in zipFly.stream():
+       # do something
+
+```
+### Other
+I created this library for my I Drive project.
+
+If you have a different case scenario, and LocalFile and GenFile are not enough, you can extend BaseFile and everything else should work out of the box.
+
+
+
+### PS
+
+I wholeheartedly hope everyone responsible for creating ZIP documentation gets slaughtered in the most gore and painful way 😊 (in game)
+
+(pls redo ur [docs](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT))
+
+

zipfly64-1.0.0/src/zipFly64.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+README.md
+pyproject.toml
+src/zipFly/BaseFile.py
+src/zipFly/Compressor.py
+src/zipFly/GenFile.py
+src/zipFly/LocalFile.py
+src/zipFly/ZipBase.py
+src/zipFly/ZipFly.py
+src/zipFly/__init__.py
+src/zipFly/consts.py
+src/zipFly64.egg-info/PKG-INFO
+src/zipFly64.egg-info/SOURCES.txt
+src/zipFly64.egg-info/dependency_links.txt
+src/zipFly64.egg-info/requires.txt
+src/zipFly64.egg-info/top_level.txt

zipfly64-1.0.0/src/zipFly64.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

zipfly64-1.0.0/src/zipFly64.egg-info/requires.txt

@@ -0,0 +1 @@
+aiofiles>=24.1.0

zipfly64-1.0.0/src/zipFly64.egg-info/top_level.txt

@@ -0,0 +1 @@
+zipFly