omdev 0.0.0.dev290__tar.gz → 0.0.0.dev291__tar.gz

{omdev-0.0.0.dev290/omdev.egg-info → omdev-0.0.0.dev291}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omdev
-Version: 0.0.0.dev290
+Version: 0.0.0.dev291
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev290
+Requires-Dist: omlish==0.0.0.dev291
 Provides-Extra: all
 Requires-Dist: black~=25.1; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

omdev-0.0.0.dev291/README.md

@@ -0,0 +1,31 @@
+It's like my previous python monorepo-ey thing [`omnibus`](https://github.com/wrmsr/omnibus/tree/wrmsr_exp_split)... ish.
+
+Core packages begin with `om`, scratch app is in `app`, temp / dump code is in `x`.
+
+----
+
+The core packages are:
+
+- **omlish**: core foundational code
+- **omdev**: development utilities
+- **omserv**: production web server
+- **ominfra**: infrastructure and cloud code
+- **ommlx**: ml / ai code
+
+----
+
+Core packages installable from pypi, or from git via:
+
+```bash
+pip install 'git+https://github.com/wrmsr/omlish@master#subdirectory=.pkg/<pkg>'
+```
+
+Core packages have no required dependencies, but numerous optional ones - see their respective `pyproject.toml` files for details.
+
+The CLI is installable through uvx or pipx via:
+
+```bash
+curl -LsSf 'https://raw.githubusercontent.com/wrmsr/omlish/master/omdev/cli/install.py' | python3 -
+```
+
+Additional deps to be injected may be appended to the command.

{omdev-0.0.0.dev290 → omdev-0.0.0.dev291}/omdev/ci/github/api/clients.py

@@ -430,7 +430,7 @@ class BaseGithubCacheClient(GithubCacheClient, abc.ABC):
         ):
             await self._upload_file_chunk_(chunk)
 
-    async def _upload_file_chunks(
+    def _generate_file_upload_chunks(
             self,
             *,
             in_file: str,
@@ -438,7 +438,7 @@ class BaseGithubCacheClient(GithubCacheClient, abc.ABC):
             key: str,
 
             file_size: ta.Optional[int] = None,
-    ) -> None:
+    ) -> ta.List[_UploadChunk]:
         check.state(os.path.isfile(in_file))
 
         if file_size is None:
@@ -446,17 +446,37 @@ class BaseGithubCacheClient(GithubCacheClient, abc.ABC):
 
         #
 
-        upload_tasks = []
+        upload_chunks: ta.List[BaseGithubCacheClient._UploadChunk] = []
         chunk_size = self._chunk_size
         for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
             offset = i * chunk_size
             size = min(chunk_size, file_size - offset)
-            upload_tasks.append(self._upload_file_chunk(self._UploadChunk(
+            upload_chunks.append(self._UploadChunk(
                 url=url,
                 key=key,
                 in_file=in_file,
                 offset=offset,
                 size=size,
-            )))
+            ))
+
+        return upload_chunks
+
+    async def _upload_file_chunks(
+            self,
+            *,
+            in_file: str,
+            url: str,
+            key: str,
+
+            file_size: ta.Optional[int] = None,
+    ) -> None:
+        upload_tasks = []
+        for chunk in self._generate_file_upload_chunks(
+            in_file=in_file,
+            url=url,
+            key=key,
+            file_size=file_size,
+        ):
+            upload_tasks.append(self._upload_file_chunk(chunk))
 
         await asyncio_wait_concurrent(upload_tasks, self._concurrency)

omdev-0.0.0.dev291/omdev/ci/github/api/v2/azure.py

@@ -0,0 +1,185 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - ominfra? no, circdep
+"""
+import base64
+import dataclasses as dc
+import datetime
+import typing as ta
+import urllib.parse
+import xml.etree.ElementTree as ET
+
+from omlish.asyncs.asyncio.utils import asyncio_wait_concurrent
+from omlish.lite.check import check
+from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
+
+
+##
+
+
+class AzureBlockBlobUploader:
+    """
+    https://learn.microsoft.com/en-us/rest/api/storageservices/put-block
+    https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list
+    """
+
+    DEFAULT_CONCURRENCY = 4
+
+    @dc.dataclass(frozen=True)
+    class Request:
+        method: str
+        url: str
+        headers: ta.Optional[ta.Dict[str, str]] = None
+        body: ta.Optional[bytes] = None
+
+    @dc.dataclass(frozen=True)
+    class Response:
+        status: int
+        headers: ta.Optional[ta.Mapping[str, str]] = None
+        data: ta.Optional[bytes] = None
+
+        def get_header(self, name: str) -> ta.Optional[str]:
+            for k, v in (self.headers or {}).items():
+                if k.lower() == name.lower():
+                    return v
+            return None
+
+    def __init__(
+            self,
+            blob_url_with_sas: str,
+            make_request: ta.Callable[[Request], ta.Awaitable[Response]],
+            *,
+            api_version: str = '2020-10-02',
+            concurrency: int = DEFAULT_CONCURRENCY,
+    ) -> None:
+        """
+        blob_url_with_sas should be of the form:
+           https://<account>.blob.core.windows.net/<container>/<blob>?<SAS-token>
+        """
+
+        super().__init__()
+
+        self._make_request = make_request
+        self._api_version = api_version
+        check.arg(concurrency >= 1)
+        self._concurrency = concurrency
+
+        parsed = urllib.parse.urlparse(blob_url_with_sas)
+        self._base_url = f'{parsed.scheme}://{parsed.netloc}'
+        parts = parsed.path.lstrip('/').split('/', 1)
+        self._container = parts[0]
+        self._blob_name = parts[1]
+        self._sas = parsed.query
+
+    def _headers(self) -> ta.Dict[str, str]:
+        """Standard headers for Azure Blob REST calls."""
+
+        now = datetime.datetime.now(datetime.UTC).strftime('%a, %d %b %Y %H:%M:%S GMT')
+        return {
+            'x-ms-date': now,
+            'x-ms-version': self._api_version,
+        }
+
+    @dc.dataclass(frozen=True)
+    class FileChunk:
+        in_file: str
+        offset: int
+        size: int
+
+    async def _upload_file_chunk_(
+            self,
+            block_id: str,
+            chunk: FileChunk,
+    ) -> None:
+        with open(chunk.in_file, 'rb') as f:  # noqa
+            f.seek(chunk.offset)
+            data = f.read(chunk.size)
+
+        check.equal(len(data), chunk.size)
+
+        params = {
+            'comp': 'block',
+            'blockid': block_id,
+        }
+        query = self._sas + '&' + urllib.parse.urlencode(params)
+        url = f'{self._base_url}/{self._container}/{self._blob_name}?{query}'
+
+        log.debug(f'Uploading azure blob chunk {chunk} with block id {block_id}')  # noqa
+
+        resp = await self._make_request(self.Request(
+            'PUT',
+            url,
+            headers=self._headers(),
+            body=data,
+        ))
+        if resp.status not in (201, 202):
+            raise RuntimeError(f'Put Block failed: {block_id=} {resp.status=}')
+
+    async def _upload_file_chunk(
+            self,
+            block_id: str,
+            chunk: FileChunk,
+    ) -> None:
+        with log_timing_context(f'Uploading azure blob chunk {chunk} with block id {block_id}'):
+            await self._upload_file_chunk_(
+                block_id,
+                chunk,
+            )
+
+    async def upload_file(
+            self,
+            chunks: ta.List[FileChunk],
+    ) -> ta.Dict[str, ta.Any]:
+        block_ids = []
+
+        # 1) Stage each block
+        upload_tasks = []
+        for idx, chunk in enumerate(chunks):
+            # Generate a predictable block ID (must be URL-safe base64)
+            raw_id = f'{idx:08d}'.encode()
+            block_id = base64.b64encode(raw_id).decode('utf-8')
+            block_ids.append(block_id)
+
+            upload_tasks.append(self._upload_file_chunk(
+                block_id,
+                chunk,
+            ))
+
+        await asyncio_wait_concurrent(upload_tasks, self._concurrency)
+
+        # 2) Commit block list
+        root = ET.Element('BlockList')
+        for bid in block_ids:
+            elm = ET.SubElement(root, 'Latest')
+            elm.text = bid
+        body = ET.tostring(root, encoding='utf-8', method='xml')
+
+        params = {'comp': 'blocklist'}
+        query = self._sas + '&' + urllib.parse.urlencode(params)
+        url = f'{self._base_url}/{self._container}/{self._blob_name}?{query}'
+
+        log.debug(f'Putting azure blob chunk list block ids {block_ids}')  # noqa
+
+        resp = await self._make_request(self.Request(
+            'PUT',
+            url,
+            headers={
+                **self._headers(),
+                'Content-Type': 'application/xml',
+            },
+            body=body,
+        ))
+        if resp.status not in (200, 201):
+            raise RuntimeError(f'Put Block List failed: {resp.status} {resp.data!r}')
+
+        ret = {
+            'status_code': resp.status,
+            'etag': resp.get_header('ETag'),
+        }
+
+        log.debug(f'Uploaded azure blob chunk {ret}')  # noqa
+
+        return ret

{omdev-0.0.0.dev290 → omdev-0.0.0.dev291}/omdev/ci/github/api/v2/client.py

@@ -2,6 +2,7 @@
 import dataclasses as dc
 import os
 import typing as ta
+import urllib.request
 
 from omlish.lite.check import check
 from omlish.lite.logs import log
@@ -13,6 +14,7 @@ from ..clients import GithubCacheClient
 from .api import GithubCacheServiceV2
 from .api import GithubCacheServiceV2RequestT
 from .api import GithubCacheServiceV2ResponseT
+from .azure import AzureBlockBlobUploader
 
 
 ##
@@ -137,19 +139,52 @@ class GithubCacheServiceV2Client(BaseGithubCacheClient):
 
         #
 
-        await self._upload_file_chunks(
+        upload_chunks = self._generate_file_upload_chunks(
             in_file=in_file,
             url=reserve_resp.signed_upload_url,
             key=fixed_key,
             file_size=file_size,
         )
 
+        az_chunks = [
+            AzureBlockBlobUploader.FileChunk(
+                in_file=in_file,
+                offset=c.offset,
+                size=c.size,
+            )
+            for c in upload_chunks
+        ]
+
+        async def az_make_request(req: AzureBlockBlobUploader.Request) -> AzureBlockBlobUploader.Response:
+            u_req = urllib.request.Request(  # noqa
+                req.url,
+                method=req.method,
+                headers=req.headers or {},
+                data=req.body,
+            )
+
+            u_resp, u_body = await self._send_urllib_request(u_req)
+
+            return AzureBlockBlobUploader.Response(
+                status=u_resp.status,
+                headers=dict(u_resp.headers),
+                data=u_body,
+            )
+
+        az_uploader = AzureBlockBlobUploader(
+            reserve_resp.signed_upload_url,
+            az_make_request,
+            concurrency=self._concurrency,
+        )
+
+        await az_uploader.upload_file(az_chunks)
+
         #
 
         commit_resp = check.not_none(await self._send_method_request(
             GithubCacheServiceV2.FINALIZE_CACHE_ENTRY_METHOD,  # type: ignore[arg-type]
             GithubCacheServiceV2.FinalizeCacheEntryUploadRequest(
-                key=key,
+                key=fixed_key,
                 size_bytes=file_size,
                 version=version,
             ),

{omdev-0.0.0.dev290 → omdev-0.0.0.dev291}/omdev/pyproject/pkg.py

@@ -175,8 +175,11 @@ class BasePyprojectPackageGenerator(abc.ABC):
     #
 
     _STANDARD_FILES: ta.Sequence[str] = [
-        'LICENSE',
-        'README.rst',
+        *[
+            ''.join([n, x])
+            for n in ('LICENSE', 'README')
+            for x in ('', '.txt', '.md', '.rst')
+        ],
     ]
 
     def _symlink_standard_files(self) -> None:

{omdev-0.0.0.dev290 → omdev-0.0.0.dev291}/omdev/scripts/ci.py

@@ -71,6 +71,7 @@ import urllib.parse
 import urllib.request
 import uuid
 import weakref
+import xml.etree.ElementTree as ET
 
 
 ########################################
@@ -6478,7 +6479,7 @@ class BaseGithubCacheClient(GithubCacheClient, abc.ABC):
         ):
             await self._upload_file_chunk_(chunk)
 
-    async def _upload_file_chunks(
+    def _generate_file_upload_chunks(
             self,
             *,
             in_file: str,
@@ -6486,7 +6487,7 @@ class BaseGithubCacheClient(GithubCacheClient, abc.ABC):
             key: str,
 
             file_size: ta.Optional[int] = None,
-    ) -> None:
+    ) -> ta.List[_UploadChunk]:
         check.state(os.path.isfile(in_file))
 
         if file_size is None:
@@ -6494,22 +6495,218 @@ class BaseGithubCacheClient(GithubCacheClient, abc.ABC):
 
         #
 
-        upload_tasks = []
+        upload_chunks: ta.List[BaseGithubCacheClient._UploadChunk] = []
         chunk_size = self._chunk_size
         for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
             offset = i * chunk_size
             size = min(chunk_size, file_size - offset)
-            upload_tasks.append(self._upload_file_chunk(self._UploadChunk(
+            upload_chunks.append(self._UploadChunk(
                 url=url,
                 key=key,
                 in_file=in_file,
                 offset=offset,
                 size=size,
-            )))
+            ))
+
+        return upload_chunks
+
+    async def _upload_file_chunks(
+            self,
+            *,
+            in_file: str,
+            url: str,
+            key: str,
+
+            file_size: ta.Optional[int] = None,
+    ) -> None:
+        upload_tasks = []
+        for chunk in self._generate_file_upload_chunks(
+            in_file=in_file,
+            url=url,
+            key=key,
+            file_size=file_size,
+        ):
+            upload_tasks.append(self._upload_file_chunk(chunk))
 
         await asyncio_wait_concurrent(upload_tasks, self._concurrency)
 
 
+########################################
+# ../github/api/v2/azure.py
+"""
+TODO:
+ - ominfra? no, circdep
+"""
+
+
+##
+
+
+class AzureBlockBlobUploader:
+    """
+    https://learn.microsoft.com/en-us/rest/api/storageservices/put-block
+    https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list
+    """
+
+    DEFAULT_CONCURRENCY = 4
+
+    @dc.dataclass(frozen=True)
+    class Request:
+        method: str
+        url: str
+        headers: ta.Optional[ta.Dict[str, str]] = None
+        body: ta.Optional[bytes] = None
+
+    @dc.dataclass(frozen=True)
+    class Response:
+        status: int
+        headers: ta.Optional[ta.Mapping[str, str]] = None
+        data: ta.Optional[bytes] = None
+
+        def get_header(self, name: str) -> ta.Optional[str]:
+            for k, v in (self.headers or {}).items():
+                if k.lower() == name.lower():
+                    return v
+            return None
+
+    def __init__(
+            self,
+            blob_url_with_sas: str,
+            make_request: ta.Callable[[Request], ta.Awaitable[Response]],
+            *,
+            api_version: str = '2020-10-02',
+            concurrency: int = DEFAULT_CONCURRENCY,
+    ) -> None:
+        """
+        blob_url_with_sas should be of the form:
+           https://<account>.blob.core.windows.net/<container>/<blob>?<SAS-token>
+        """
+
+        super().__init__()
+
+        self._make_request = make_request
+        self._api_version = api_version
+        check.arg(concurrency >= 1)
+        self._concurrency = concurrency
+
+        parsed = urllib.parse.urlparse(blob_url_with_sas)
+        self._base_url = f'{parsed.scheme}://{parsed.netloc}'
+        parts = parsed.path.lstrip('/').split('/', 1)
+        self._container = parts[0]
+        self._blob_name = parts[1]
+        self._sas = parsed.query
+
+    def _headers(self) -> ta.Dict[str, str]:
+        """Standard headers for Azure Blob REST calls."""
+
+        now = datetime.datetime.now(datetime.UTC).strftime('%a, %d %b %Y %H:%M:%S GMT')
+        return {
+            'x-ms-date': now,
+            'x-ms-version': self._api_version,
+        }
+
+    @dc.dataclass(frozen=True)
+    class FileChunk:
+        in_file: str
+        offset: int
+        size: int
+
+    async def _upload_file_chunk_(
+            self,
+            block_id: str,
+            chunk: FileChunk,
+    ) -> None:
+        with open(chunk.in_file, 'rb') as f:  # noqa
+            f.seek(chunk.offset)
+            data = f.read(chunk.size)
+
+        check.equal(len(data), chunk.size)
+
+        params = {
+            'comp': 'block',
+            'blockid': block_id,
+        }
+        query = self._sas + '&' + urllib.parse.urlencode(params)
+        url = f'{self._base_url}/{self._container}/{self._blob_name}?{query}'
+
+        log.debug(f'Uploading azure blob chunk {chunk} with block id {block_id}')  # noqa
+
+        resp = await self._make_request(self.Request(
+            'PUT',
+            url,
+            headers=self._headers(),
+            body=data,
+        ))
+        if resp.status not in (201, 202):
+            raise RuntimeError(f'Put Block failed: {block_id=} {resp.status=}')
+
+    async def _upload_file_chunk(
+            self,
+            block_id: str,
+            chunk: FileChunk,
+    ) -> None:
+        with log_timing_context(f'Uploading azure blob chunk {chunk} with block id {block_id}'):
+            await self._upload_file_chunk_(
+                block_id,
+                chunk,
+            )
+
+    async def upload_file(
+            self,
+            chunks: ta.List[FileChunk],
+    ) -> ta.Dict[str, ta.Any]:
+        block_ids = []
+
+        # 1) Stage each block
+        upload_tasks = []
+        for idx, chunk in enumerate(chunks):
+            # Generate a predictable block ID (must be URL-safe base64)
+            raw_id = f'{idx:08d}'.encode()
+            block_id = base64.b64encode(raw_id).decode('utf-8')
+            block_ids.append(block_id)
+
+            upload_tasks.append(self._upload_file_chunk(
+                block_id,
+                chunk,
+            ))
+
+        await asyncio_wait_concurrent(upload_tasks, self._concurrency)
+
+        # 2) Commit block list
+        root = ET.Element('BlockList')
+        for bid in block_ids:
+            elm = ET.SubElement(root, 'Latest')
+            elm.text = bid
+        body = ET.tostring(root, encoding='utf-8', method='xml')
+
+        params = {'comp': 'blocklist'}
+        query = self._sas + '&' + urllib.parse.urlencode(params)
+        url = f'{self._base_url}/{self._container}/{self._blob_name}?{query}'
+
+        log.debug(f'Putting azure blob chunk list block ids {block_ids}')  # noqa
+
+        resp = await self._make_request(self.Request(
+            'PUT',
+            url,
+            headers={
+                **self._headers(),
+                'Content-Type': 'application/xml',
+            },
+            body=body,
+        ))
+        if resp.status not in (200, 201):
+            raise RuntimeError(f'Put Block List failed: {resp.status} {resp.data!r}')
+
+        ret = {
+            'status_code': resp.status,
+            'etag': resp.get_header('ETag'),
+        }
+
+        log.debug(f'Uploaded azure blob chunk {ret}')  # noqa
+
+        return ret
+
+
 ########################################
 # ../../dataserver/targets.py
 
@@ -8043,19 +8240,52 @@ class GithubCacheServiceV2Client(BaseGithubCacheClient):
 
         #
 
-        await self._upload_file_chunks(
+        upload_chunks = self._generate_file_upload_chunks(
             in_file=in_file,
             url=reserve_resp.signed_upload_url,
             key=fixed_key,
             file_size=file_size,
         )
 
+        az_chunks = [
+            AzureBlockBlobUploader.FileChunk(
+                in_file=in_file,
+                offset=c.offset,
+                size=c.size,
+            )
+            for c in upload_chunks
+        ]
+
+        async def az_make_request(req: AzureBlockBlobUploader.Request) -> AzureBlockBlobUploader.Response:
+            u_req = urllib.request.Request(  # noqa
+                req.url,
+                method=req.method,
+                headers=req.headers or {},
+                data=req.body,
+            )
+
+            u_resp, u_body = await self._send_urllib_request(u_req)
+
+            return AzureBlockBlobUploader.Response(
+                status=u_resp.status,
+                headers=dict(u_resp.headers),
+                data=u_body,
+            )
+
+        az_uploader = AzureBlockBlobUploader(
+            reserve_resp.signed_upload_url,
+            az_make_request,
+            concurrency=self._concurrency,
+        )
+
+        await az_uploader.upload_file(az_chunks)
+
         #
 
         commit_resp = check.not_none(await self._send_method_request(
             GithubCacheServiceV2.FINALIZE_CACHE_ENTRY_METHOD,  # type: ignore[arg-type]
             GithubCacheServiceV2.FinalizeCacheEntryUploadRequest(
-                key=key,
+                key=fixed_key,
                 size_bytes=file_size,
                 version=version,
             ),

{omdev-0.0.0.dev290 → omdev-0.0.0.dev291}/omdev/scripts/pyproject.py

@@ -7972,8 +7972,11 @@ class BasePyprojectPackageGenerator(abc.ABC):
     #
 
     _STANDARD_FILES: ta.Sequence[str] = [
-        'LICENSE',
-        'README.rst',
+        *[
+            ''.join([n, x])
+            for n in ('LICENSE', 'README')
+            for x in ('', '.txt', '.md', '.rst')
+        ],
     ]
 
     def _symlink_standard_files(self) -> None: