magic-pocket-cli 0.2.0__py3-none-any.whl

pocket_cli/resources/aws/ecr.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from functools import cached_property
+from typing import TYPE_CHECKING
+
+import boto3
+from pydantic import BaseModel, Field, computed_field
+
+if TYPE_CHECKING:
+    from pocket_cli.resources.aws.builders import Builder
+
+
+class RepositoryDetail(BaseModel):
+    uri: str | None = Field(alias="repositoryUri", default=None)
+    arn: str | None = Field(alias="repositoryArn", default=None)
+
+
+class ImageDetail(BaseModel):
+    image_digest: str | None = Field(alias="imageDigest", default=None)
+
+    @computed_field
+    def hash(self) -> str | None:
+        if self.image_digest:
+            alg, hash = self.image_digest.split(":")
+            if alg == "sha256":
+                return hash
+            raise ValueError("unsupported algorithm")
+
+
+class Ecr:
+    def __init__(
+        self,
+        region_name: str,
+        name: str,
+        tag: str,
+        dockerfile_path: str,
+        platform: str,
+        builder: Builder | None = None,
+    ):
+        self.client = boto3.client("ecr", region_name=region_name)
+        self.name = name
+        self.tag = tag
+        self.dockerfile_path = dockerfile_path
+        self.platform = platform
+        self._builder = builder
+
+    @cached_property
+    def info(self) -> RepositoryDetail:
+        for repository in self.client.describe_repositories()["repositories"]:
+            if repository["repositoryName"] == self.name:
+                return RepositoryDetail(**repository)
+        return RepositoryDetail()
+
+    @property
+    def uri(self):
+        return self.info.uri
+
+    @property
+    def arn(self):
+        return self.info.arn
+
+    @property
+    def target(self):
+        if self.uri:
+            return self.uri + ":" + self.tag
+
+    @property
+    def image_detail(self):
+        data = self.client.describe_images(repositoryName=self.name)
+        for detail in data["imageDetails"]:
+            if image_tags := detail.get("imageTags"):
+                if self.tag in image_tags:
+                    return ImageDetail(**detail)
+        return ImageDetail()
+
+    def create(self):
+        print("Creating repository ...")
+        print("  %s" % self.name)
+        self.client.create_repository(repositoryName=self.name)
+        if hasattr(self, "info"):
+            del self.info
+        print("  %s" % self.uri)
+
+    def ensure_exists(self):
+        if not self.info.uri:
+            self.create()
+
+    def build_and_push(self, tag: str | None = None):
+        if self.uri is None:
+            raise ValueError("target is not defined")
+        # tag 省略時は self.tag (= stage) を使う。build once 用に commit hash 等を
+        # 焼きたい場合は tag を明示する。
+        target = self.uri + ":" + (tag or self.tag)
+        if self._builder is None:
+            raise ValueError("builder is not configured")
+        self._builder.build_and_push(
+            target=target,
+            dockerfile_path=self.dockerfile_path,
+            platform=self.platform,
+        )
+
+    def retag(self, source_tag: str, dest_tag: str):
+        """source_tag の image に dest_tag を付与する (タグの付け替え。build しない)。
+
+        build once の昇格用: `:<commit hash>` の image へ `:<stage>` タグを移す。
+        source_tag の image が存在しない場合は ValueError。
+        """
+        try:
+            images = self.client.batch_get_image(
+                repositoryName=self.name,
+                imageIds=[{"imageTag": source_tag}],
+            )["images"]
+        except self.client.exceptions.RepositoryNotFoundException:
+            raise ValueError(
+                "ECR repository '%s' が存在しません。"
+                "先に `pocket django build` を実行してください。" % self.name
+            )
+        if not images:
+            raise ValueError(
+                "image :%s が ECR repository '%s' に存在しません。"
+                "先に `pocket django build` を実行してください。"
+                % (source_tag, self.name)
+            )
+        try:
+            self.client.put_image(
+                repositoryName=self.name,
+                imageManifest=images[0]["imageManifest"],
+                imageTag=dest_tag,
+            )
+        except self.client.exceptions.ImageAlreadyExistsException:
+            pass  # dest_tag が既に同じ digest を指している (再昇格の冪等性)
+
+    def exists(self) -> bool:
+        return self.info.uri is not None
+
+    def delete(self):
+        if not self.exists():
+            return
+        self.client.delete_repository(repositoryName=self.name, force=True)
+        if hasattr(self, "info"):
+            del self.info
+
+    def sync(self):
+        self.ensure_exists()
+        self.build_and_push()

pocket_cli/resources/aws/efs.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import time
+import uuid
+from functools import cached_property
+from typing import TYPE_CHECKING
+
+import boto3
+
+from pocket.resources.base import ResourceStatus
+
+if TYPE_CHECKING:
+    from pocket.general_context import EfsContext
+
+
+class Efs:
+    context: EfsContext
+
+    def __init__(self, context: EfsContext) -> None:
+        self.context = context
+        self.client = boto3.client("efs", region_name=context.region)
+
+    @cached_property
+    def description(self):
+        for fs in self._iter_all_efs():
+            for tag in fs["Tags"]:
+                if tag["Key"] == "Name" and tag["Value"] == self.context.name:
+                    return fs
+        return None
+
+    @cached_property
+    def lifecycle_policies(self):
+        res = self.client.describe_lifecycle_configuration(
+            FileSystemId=self.filesystem_id
+        )
+        return res["LifecyclePolicies"]
+
+    @property
+    def lifecycle_policies_should_be(self):
+        return [
+            {
+                "TransitionToIA": "AFTER_30_DAYS",
+            },
+            {
+                "TransitionToPrimaryStorageClass": "AFTER_1_ACCESS",
+            },
+        ]
+
+    @property
+    def filesystem_id(self):
+        if self.description:
+            return self.description["FileSystemId"]
+
+    def clear_status(self):
+        if hasattr(self, "description"):
+            del self.description
+        if hasattr(self, "lifecycle_policies"):
+            del self.lifecycle_policies
+
+    def wait_status(self, status: ResourceStatus, timeout=60):
+        max_iter = 100
+        interval = 3
+        if (timeout < 0) or ((max_iter * interval) < timeout):
+            raise Exception("timeout value is out of range")
+        for i in range(max_iter):
+            self.clear_status()
+            if self.status == status:
+                print("")
+                return
+            if i == 0:
+                print("Waiting for efs status to be %s" % status, end="", flush=True)
+            print(".", end="", flush=True)
+            time.sleep(interval)
+
+    def create(self):
+        res = self.client.create_file_system(
+            CreationToken=str(uuid.uuid4()),
+            Encrypted=True,
+            Backup=True,
+            Tags=[
+                {
+                    "Key": "Name",
+                    "Value": self.context.name,
+                },
+            ],
+        )
+        filesystem_id = res["FileSystemId"]
+        print(res)
+        print(filesystem_id)
+        self.wait_status("REQUIRE_UPDATE")
+        self.ensure_lifecycle_policies()
+
+    def update(self):
+        self.client.put_lifecycle_configuration(
+            FileSystemId=self.filesystem_id,
+            LifecyclePolicies=self.lifecycle_policies_should_be,
+        )
+
+    def ensure_lifecycle_policies(self):
+        if self.lifecycle_policies != self.lifecycle_policies_should_be:
+            self.update()
+
+    def delete(self):
+        self.client.delete_file_system(FileSystemId=self.filesystem_id)
+
+    def _iter_all_efs(self):
+        res = self.client.describe_file_systems()
+        if res.get("NextMarker"):
+            raise Exception("Your efs number is over 100. Please implement here.")
+        return res["FileSystems"]
+
+    def exists(self):
+        if self.description:
+            return True
+        return False
+
+    def ensure_exists(self):
+        if self.exists():
+            return
+        self.create()
+
+    @property
+    def status(self) -> ResourceStatus:
+        if not self.exists():
+            return "NOEXIST"
+        assert self.description
+        if self.description["LifeCycleState"] != "available":
+            return "PROGRESS"
+        if self.lifecycle_policies != [
+            {
+                "TransitionToIA": "AFTER_30_DAYS",
+            },
+            {
+                "TransitionToPrimaryStorageClass": "AFTER_1_ACCESS",
+            },
+        ]:
+            return "REQUIRE_UPDATE"
+        return "COMPLETED"

pocket_cli/resources/aws/lambdahandler.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import datetime
+import time
+from email.utils import parsedate_to_datetime
+from functools import cached_property
+from typing import TYPE_CHECKING
+
+import boto3
+from botocore.exceptions import ClientError
+from pydantic import BaseModel, Field
+
+from pocket.resources.base import ResourceStatus
+
+if TYPE_CHECKING:
+    from pocket.context import LambdaHandlerContext
+
+
+class Configuration(BaseModel):
+    hash: str | None = Field(alias="CodeSha256", default=None)
+    last_update_status: str | None = Field(alias="LastUpdateStatus", default=None)
+
+
+class LambdaHandler:
+    context: LambdaHandlerContext
+
+    def __init__(self, context: LambdaHandlerContext) -> None:
+        self.context = context
+        self.client = boto3.client("lambda", region_name=context.region)
+        self.logs_client = boto3.client("logs", region_name=self.context.region)
+
+    @property
+    def name(self):
+        return self.context.function_name
+
+    @cached_property
+    def configuration(self):
+        try:
+            data = self.client.get_function(FunctionName=self.name)["Configuration"]
+            return Configuration(**data)
+        except ClientError:
+            return Configuration()
+
+    def refresh(self):
+        try:
+            del self.configuration
+        except AttributeError:
+            pass
+
+    @property
+    def status(self) -> ResourceStatus:
+        match self.configuration.last_update_status:
+            case None:
+                return "NOEXIST"
+            case "InProgress":
+                return "PROGRESS"
+            case "Failed":
+                return "FAILED"
+            case "Successful":
+                return "COMPLETED"
+            case _:
+                raise Exception("unexpected status")
+
+    @property
+    def queueurl(self) -> str | None:
+        if self.context.sqs:
+            res = boto3.client("sqs").get_queue_url(QueueName=self.context.sqs.name)
+            return res["QueueUrl"]
+
+    def update(self, image_uri, wait=False):
+        res = self.client.update_function_code(
+            FunctionName=self.name, ImageUri=image_uri
+        )
+        print(f"lambda function {self.name} was updated.")
+        show_keys = ["FunctionName", "Timeout", "MemorySize", "Version", "Environment"]
+        for key, value in res.items():
+            if key in show_keys:
+                print(f"  - {key}: {value}")
+        if wait:
+            self.wait_update()
+
+    def wait_update(self, interval=3, limit=60):
+        print(f"waiting lambda fanction {self.name} update.", end="", flush=True)
+        for _i in range(limit // interval):
+            time.sleep(interval)
+            self.refresh()
+            if self.status == "PROGRESS":
+                print(".", end="", flush=True)
+            else:
+                print(f"\nlambda function {self.name} was updated.")
+                break
+        else:
+            raise Exception("Lambda couldn't stop updating. Please check.")
+
+    def invoke(self, payload: str):
+        return self.client.invoke(
+            FunctionName=self.name, InvocationType="Event", Payload=payload
+        )
+
+    def show_logs(self, invoke_http_response, timeout_seconds=120):
+        res = invoke_http_response
+        request_id = res["ResponseMetadata"]["RequestId"]
+        created_at_rfc1123 = res["ResponseMetadata"]["HTTPHeaders"]["date"]
+        created_at = parsedate_to_datetime(created_at_rfc1123)
+        print("lambda request_id:", request_id)
+        print("lambda created_at:", created_at)
+        start_pattern = '"START RequestId: %s"' % request_id
+        report_prefix = "REPORT RequestId: %s" % request_id
+        events = self._find_events(start_pattern, created_at)
+        print("Log stream found: %s" % events[0]["logStreamName"])
+        printed = []
+        sleep_seconds = 5
+        for _i in range(timeout_seconds // sleep_seconds):
+            res = self.logs_client.filter_log_events(
+                logGroupName=self.context.log_group_name,
+                logStreamNames=[events[0]["logStreamName"]],
+                startTime=events[0]["timestamp"],
+            )
+            messages = [event["message"] for event in res["events"]]
+            if messages[: len(printed)] != printed:
+                raise Exception("log stream changed")
+            for message in messages[len(printed) :]:
+                print(message.strip())
+                printed.append(message)
+                time.sleep(0.05)
+                if message.startswith(report_prefix):
+                    return
+            time.sleep(sleep_seconds)
+        print("Timeout %s seconds. Please check logs in cloudwatch." % timeout_seconds)
+
+    def _get_recent_log_stream_names(self, limit: int):
+        try:
+            res = self.logs_client.describe_log_streams(
+                logGroupName=self.context.log_group_name,
+                orderBy="LastEventTime",
+                descending=True,
+                limit=limit,
+            )
+        except self.logs_client.exceptions.ResourceNotFoundException:
+            return []
+        return [s["logStreamName"] for s in res["logStreams"]]
+
+    def _find_events(
+        self,
+        filter_pattern: str,
+        created_at: datetime.datetime,
+        log_stream_names: list[str] | None = None,
+        log_stream_limit: int = 3,
+    ):
+        for i in range(20):
+            if i != 0:
+                msg = "Waiting for log stream." if i == 1 else "."
+                print(msg, end="", flush=True)
+                time.sleep(3)
+            target_log_stream_names = (
+                log_stream_names or self._get_recent_log_stream_names(log_stream_limit)
+            )
+            if not target_log_stream_names:
+                continue
+            kwargs = {
+                "logGroupName": self.context.log_group_name,
+                "logStreamNames": target_log_stream_names,
+                "filterPattern": filter_pattern,
+                "startTime": int(created_at.timestamp() * 1000),
+            }
+            events = []
+            for _j in range(10):
+                res = self.logs_client.filter_log_events(**kwargs)
+                if res["events"]:
+                    events += res["events"]
+                if "nextToken" in res:
+                    kwargs["nextToken"] = res["nextToken"]
+                else:
+                    break
+            if events:
+                return events
+        print(
+            "Searched %s log stream below..." % len(target_log_stream_names)  # pyright: ignore
+        )
+        for s in target_log_stream_names:  # pyright: ignore
+            print("  - %s" % s)
+        raise Exception("log stream not found")

pocket_cli/resources/aws/s3_utils.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from botocore.exceptions import ClientError
+
+
+def bucket_exists(client, bucket_name: str) -> bool:
+    """バケットの存在確認。404以外のエラーはClientErrorとして再送出"""
+    try:
+        client.head_bucket(Bucket=bucket_name)
+        return True
+    except ClientError as e:
+        if e.response.get("Error", {}).get("Code") == "404":
+            return False
+        raise
+
+
+def create_bucket(client, bucket_name: str, region: str):
+    """リージョンを考慮してバケットを作成"""
+    if region == "us-east-1":
+        client.create_bucket(Bucket=bucket_name)
+    else:
+        client.create_bucket(
+            Bucket=bucket_name,
+            CreateBucketConfiguration={"LocationConstraint": region},
+        )
+
+
+def empty_bucket(client, bucket_name: str):
+    """バケット内の全オブジェクト（バージョン含む）を削除"""
+    # 通常オブジェクトの削除
+    paginator = client.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=bucket_name):
+        objects = page.get("Contents", [])
+        if not objects:
+            continue
+        delete_keys = [{"Key": obj["Key"]} for obj in objects]
+        client.delete_objects(Bucket=bucket_name, Delete={"Objects": delete_keys})
+
+    # バージョニングが有効な場合のバージョン・DeleteMarker削除
+    paginator = client.get_paginator("list_object_versions")
+    for page in paginator.paginate(Bucket=bucket_name):
+        delete_keys = []
+        for version in page.get("Versions", []):
+            delete_keys.append(
+                {"Key": version["Key"], "VersionId": version["VersionId"]}
+            )
+        for marker in page.get("DeleteMarkers", []):
+            delete_keys.append({"Key": marker["Key"], "VersionId": marker["VersionId"]})
+        if delete_keys:
+            client.delete_objects(Bucket=bucket_name, Delete={"Objects": delete_keys})
+
+
+def delete_bucket_with_contents(client, bucket_name: str):
+    """バケットを中身ごと削除（存在しなければ no-op）"""
+    if not bucket_exists(client, bucket_name):
+        return
+    empty_bucket(client, bucket_name)
+    client.delete_bucket(Bucket=bucket_name)

pocket_cli/resources/aws/state.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+
+import boto3
+from botocore.exceptions import ClientError
+
+from pocket.settings import _deep_merge
+from pocket_cli.resources.aws.s3_utils import (
+    bucket_exists,
+    create_bucket,
+    delete_bucket_with_contents,
+)
+
+
+class StateStore:
+    STATE_KEY = "resources.json"
+
+    def __init__(self, bucket_name: str, region: str):
+        self.bucket_name = bucket_name
+        self.region = region
+        self.client = boto3.client("s3", region_name=region)
+        self._state: dict | None = None
+
+    def ensure_bucket(self):
+        """state バケットが存在しなければ作成（全公開ブロック）"""
+        if bucket_exists(self.client, self.bucket_name):
+            return
+        create_bucket(self.client, self.bucket_name, self.region)
+        self.client.put_public_access_block(
+            Bucket=self.bucket_name,
+            PublicAccessBlockConfiguration={
+                "BlockPublicAcls": True,
+                "IgnorePublicAcls": True,
+                "BlockPublicPolicy": True,
+                "RestrictPublicBuckets": True,
+            },
+        )
+
+    def load(self) -> dict:
+        """S3から resources.json を読み込み。存在しなければ空stateを返す"""
+        if self._state is not None:
+            return self._state
+        try:
+            response = self.client.get_object(
+                Bucket=self.bucket_name, Key=self.STATE_KEY
+            )
+            self._state = json.loads(response["Body"].read().decode("utf-8"))
+        except ClientError as e:
+            if e.response.get("Error", {}).get("Code") == "NoSuchKey":
+                self._state = {"version": 1, "resources": {}}
+            else:
+                raise
+        return self._state  # type: ignore[return-value]
+
+    def save(self):
+        """現在のstateをS3に書き込み"""
+        state = self.load()
+        self.client.put_object(
+            Bucket=self.bucket_name,
+            Key=self.STATE_KEY,
+            Body=json.dumps(state, indent=2).encode("utf-8"),
+            ContentType="application/json",
+        )
+
+    def record(self, info: dict):
+        """リソース情報を deep merge して保存"""
+        state = self.load()
+        _deep_merge(state["resources"], info)
+        self.save()
+
+    def delete_bucket(self):
+        """ステートバケットを中身ごと削除"""
+        delete_bucket_with_contents(self.client, self.bucket_name)