aimodelshare 0.1.55__tar.gz → 0.1.59__tar.gz

aimodelshare-0.1.59/.github/workflows/aimodelshare-tests.yml

@@ -0,0 +1,84 @@
+name: Test latest master branch aimodelshare
+
+on:
+  workflow_dispatch: # Manual trigger only
+
+permissions:
+  contents: read
+
+concurrency:
+  group: master-aimodelshare-tests
+  cancel-in-progress: true
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    env:
+      USERNAME: ${{ secrets.AIMODELSHARE_USERNAME }}
+      PASSWORD: ${{ secrets.AIMODELSHARE_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.DATA_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DATA_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: us-east-1
+
+    steps:
+      - name: Checkout repository (master)
+        uses: actions/checkout@v4
+        with:
+          # Uncomment the next line if you want to ALWAYS test master even when dispatching from another ref
+          # ref: master
+          fetch-depth: 0
+
+      - name: Show commit being tested
+        run: |
+          echo "Testing commit:"
+          git --no-pager log -1 --oneline
+          echo "Current branch (if detached this may be a commit SHA):"
+          git branch --show-current || true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Upgrade pip
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: Install local aimodelshare (editable) + test dependencies
+        run: |
+          pip install boto3 onnx onnxmltools onnxruntime Pympler scikeras shortuuid skl2onnx tf2onnx wget
+          pip install -e .
+          pip install pytest scikit-learn pandas numpy opencv-python-headless tensorflow pydot regex psutil IPython "PyJWT<2.0" matplotlib seaborn importlib_resources onnxscript
+          pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision
+          python -c "import aimodelshare, os; print('aimodelshare imported from:', aimodelshare.__file__)"
+
+      - name: Verify import now points to repository source (not PyPI)
+        run: |
+          python - <<'PYCODE'
+          import aimodelshare, os
+          path = aimodelshare.__file__
+          print("Resolved aimodelshare.__file__:", path)
+          repo_root = os.getcwd()
+          assert repo_root in os.path.abspath(path), f"aimodelshare does not appear to come from the checked-out repository root: {repo_root}"
+          assert "site-packages" not in path, "aimodelshare should NOT be the PyPI site-packages build."
+          print("Verification passed: using local master branch source.")
+          PYCODE
+
+      - name: Run targeted tests against local master branch
+        run: |
+          echo "Starting pytest for tests/test_playgrounds_nodataimport.py"
+          pytest -vv tests/test_aimsonnx.py
+          pytest -vv tests/test_playgrounds_nodataimport.py
+
+      - name: Post-run diagnostics (always runs)
+        if: always()
+        run: |
+          echo "Job conclusion: ${{ job.status }}"
+          python - <<'PYCODE'
+          import aimodelshare
+          print("aimodelshare version attribute (may reflect local dev state):", getattr(aimodelshare, "__version__", "unknown"))
+          print("Module file path:", aimodelshare.__file__)
+          PYCODE

aimodelshare-0.1.59/.github/workflows/bootstrap-terraform.yml

@@ -0,0 +1,149 @@
+name: Bootstrap Terraform State Resources
+
+on:
+  workflow_dispatch:
+  workflow_call:
+    outputs:
+      s3_bucket_name:
+        description: "Name of the created S3 bucket"
+        value: ${{ jobs.bootstrap.outputs.s3_bucket_name }}
+      dynamodb_table_name:
+        description: "Name of the created DynamoDB table"
+        value: ${{ jobs.bootstrap.outputs.dynamodb_table_name }}
+      github_actions_role_arn:
+        description: "ARN of the created GitHub Actions IAM role"
+        value: ${{ jobs.bootstrap.outputs.github_actions_role_arn }}
+
+jobs:
+  bootstrap:
+    runs-on: ubuntu-latest
+    
+    outputs:
+      s3_bucket_name: ${{ steps.terraform_output.outputs.s3_bucket_name }}
+      dynamodb_table_name: ${{ steps.terraform_output.outputs.dynamodb_table_name }}
+      github_actions_role_arn: ${{ steps.terraform_output.outputs.github_actions_role_arn }}
+    
+    permissions:
+      contents: read
+
+    env:
+      AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+      TF_IN_AUTOMATION: "true"
+      
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.9.5
+          terraform_wrapper: false
+
+      # Configure AWS credentials using access keys per company policy
+      # Note: Requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY repository secrets
+      # Security: Ensure access keys follow principle of least privilege and key rotation policies
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.DATA_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DATA_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Check if resources already exist
+        id: check_resources
+        working-directory: infra/bootstrap
+        run: |
+          # Check if S3 bucket exists
+          if aws s3api head-bucket --bucket "aimodelshare-tfstate-prod-copilot-2024" 2>/dev/null; then
+            echo "s3_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "s3_exists=false" >> $GITHUB_OUTPUT
+          fi
+          
+          # Check if DynamoDB table exists
+          if aws dynamodb describe-table --table-name "aimodelshare-tf-locks" 2>/dev/null; then
+            echo "dynamodb_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "dynamodb_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+          # Check if GitHub OIDC provider exists
+          ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+          if aws iam get-open-id-connect-provider --open-id-connect-provider-arn "arn:aws:iam::${ACCOUNT_ID}:oidc-provider/token.actions.githubusercontent.com" 2>/dev/null; then
+            echo "oidc_provider_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "oidc_provider_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+          # Check if GitHub Actions IAM role exists
+          if aws iam get-role --role-name "aimodelshare-github-oidc-deployer" 2>/dev/null; then
+            echo "github_role_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "github_role_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Terraform Init
+        working-directory: infra/bootstrap
+        run: terraform init -input=false
+
+      - name: Terraform Validate
+        working-directory: infra/bootstrap
+        run: terraform validate
+
+      - name: Import existing resources (if they exist)
+        working-directory: infra/bootstrap
+        if: steps.check_resources.outputs.s3_exists == 'true' || steps.check_resources.outputs.dynamodb_exists == 'true' || steps.check_resources.outputs.oidc_provider_exists == 'true' || steps.check_resources.outputs.github_role_exists == 'true'
+        run: |
+          ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+          
+          if [ "${{ steps.check_resources.outputs.s3_exists }}" = "true" ]; then
+            echo "Importing existing S3 bucket..."
+            terraform import aws_s3_bucket.terraform_state aimodelshare-tfstate-prod-copilot-2024 || true
+            terraform import aws_s3_bucket_versioning.terraform_state aimodelshare-tfstate-prod-copilot-2024 || true
+            terraform import aws_s3_bucket_server_side_encryption_configuration.terraform_state aimodelshare-tfstate-prod-copilot-2024 || true
+            terraform import aws_s3_bucket_public_access_block.terraform_state aimodelshare-tfstate-prod-copilot-2024 || true
+          fi
+          
+          if [ "${{ steps.check_resources.outputs.dynamodb_exists }}" = "true" ]; then
+            echo "Importing existing DynamoDB table..."
+            terraform import aws_dynamodb_table.terraform_locks aimodelshare-tf-locks || true
+          fi
+
+          if [ "${{ steps.check_resources.outputs.oidc_provider_exists }}" = "true" ]; then
+            echo "Importing existing GitHub OIDC provider..."
+            terraform import aws_iam_openid_connect_provider.github "arn:aws:iam::${ACCOUNT_ID}:oidc-provider/token.actions.githubusercontent.com" || true
+          fi
+
+          if [ "${{ steps.check_resources.outputs.github_role_exists }}" = "true" ]; then
+            echo "Importing existing GitHub Actions IAM role..."
+            terraform import aws_iam_role.github_actions aimodelshare-github-oidc-deployer || true
+            # Also try to import the policy and attachment if they exist
+            terraform import aws_iam_policy.github_actions_deployment "arn:aws:iam::${ACCOUNT_ID}:policy/aimodelshare-github-actions-deployment" || true
+            terraform import aws_iam_role_policy_attachment.github_actions_deployment aimodelshare-github-oidc-deployer/arn:aws:iam::${ACCOUNT_ID}:policy/aimodelshare-github-actions-deployment || true
+          fi
+
+      - name: Terraform Plan
+        id: plan
+        working-directory: infra/bootstrap
+        run: |
+          terraform plan -input=false -out=tfplan
+          
+      - name: Terraform Apply
+        working-directory: infra/bootstrap
+        run: terraform apply -input=false -auto-approve tfplan
+
+      - name: Get Terraform Outputs
+        id: terraform_output
+        working-directory: infra/bootstrap
+        run: |
+          S3_BUCKET=$(terraform output -raw s3_bucket_name)
+          DYNAMODB_TABLE=$(terraform output -raw dynamodb_table_name)
+          GITHUB_ROLE_ARN=$(terraform output -raw github_actions_role_arn)
+          echo "s3_bucket_name=$S3_BUCKET" >> $GITHUB_OUTPUT
+          echo "dynamodb_table_name=$DYNAMODB_TABLE" >> $GITHUB_OUTPUT
+          echo "github_actions_role_arn=$GITHUB_ROLE_ARN" >> $GITHUB_OUTPUT
+          echo "✅ Bootstrap completed successfully!"
+          echo "  S3 Bucket: $S3_BUCKET"
+          echo "  DynamoDB Table: $DYNAMODB_TABLE"
+          echo "  GitHub Actions Role ARN: $GITHUB_ROLE_ARN"

aimodelshare-0.1.59/.github/workflows/deploy-infra.yml

@@ -0,0 +1,224 @@
+name: Deploy Infra (Terraform Workspaces)
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'infra/**'
+      - 'aimodelshare/moral_compass/**'
+      - '.github/workflows/deploy-infra.yml'
+      - '.github/workflows/bootstrap-terraform.yml'
+  workflow_dispatch:
+
+jobs:
+  bootstrap:
+    uses: ./.github/workflows/bootstrap-terraform.yml
+    secrets: inherit
+
+  deploy:
+    needs: bootstrap
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        env: [dev]
+
+    permissions:
+      contents: read
+
+    env:
+      AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+      TF_IN_AUTOMATION: "true"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.9.5
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.DATA_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DATA_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: (Optional) Build Lambda Layer
+        working-directory: infra/layer
+        run: |
+          if [ -f requirements.txt ]; then
+            bash build_layer.sh || true
+          fi
+
+      - name: Terraform Init
+        working-directory: infra
+        run: terraform init -input=false
+
+      - name: Select/Create Workspace
+        working-directory: infra
+        run: |
+          set -euo pipefail
+
+          WS="${{ matrix.env }}"
+            echo "Ensuring Terraform workspace '${WS}' exists..."
+          if terraform workspace list | sed 's/^[* ]*//' | grep -Fxq "$WS"; then
+            echo "Selecting existing workspace: $WS"
+            terraform workspace select "$WS"
+          else
+            echo "Creating new workspace: $WS"
+            terraform workspace new "$WS"
+          fi
+          echo "Current workspace: $(terraform workspace show)"
+
+          if [ ! -d .terraform ]; then
+            echo "Terraform not initialized yet; running 'terraform init'..."
+            terraform init -input=false
+          fi
+
+          STATE_KEY="env:/${WS}/aimodelshare/infra/terraform.tfstate"
+          echo "Checking if remote state object exists: ${STATE_KEY}"
+          if aws s3api head-object --bucket "aimodelshare-tfstate-prod-copilot-2024" --key "${STATE_KEY}" >/dev/null 2>&1; then
+            echo "Remote state object exists for workspace '${WS}'"
+          else
+            echo "Remote state object missing for workspace '${WS}' - creating initial state"
+            terraform apply -auto-approve -target=null_resource.state_seed
+          fi
+
+          echo "Workspace '$WS' ready; proceeding to plan/apply steps."
+
+      - name: Terraform Validate
+        working-directory: infra
+        run: terraform validate
+
+      - name: Terraform Plan
+        id: plan
+        working-directory: infra
+        run: terraform plan -input=false -out=tfplan
+
+      - name: Terraform Apply
+        if: github.ref == 'refs/heads/master'
+        working-directory: infra
+        run: terraform apply -input=false -auto-approve tfplan
+
+      - name: Show Outputs
+        working-directory: infra
+        run: terraform output -json
+
+      - name: Wait for DynamoDB GSI readiness
+        if: github.ref == 'refs/heads/master'
+        working-directory: infra
+        run: |
+          TABLE_NAME=$(terraform output -raw dynamodb_table_name)
+          echo "Waiting for GSI 'byUser' on table $TABLE_NAME to become ACTIVE..."
+          ATTEMPTS=0
+          MAX_ATTEMPTS=30
+          SLEEP_SECONDS=5
+          while true; do
+            STATUS=$(aws dynamodb describe-table --table-name "$TABLE_NAME" \
+              --query "Table.GlobalSecondaryIndexes[?IndexName=='byUser'].IndexStatus" --output text 2>/dev/null || echo "UNKNOWN")
+            if [ "$STATUS" = "ACTIVE" ]; then
+              echo "GSI 'byUser' is ACTIVE."
+              break
+            fi
+            ATTEMPTS=$((ATTEMPTS+1))
+            if [ $ATTEMPTS -ge $MAX_ATTEMPTS ]; then
+              echo "Timeout waiting for GSI 'byUser' to become ACTIVE (last status: $STATUS)"
+              exit 1
+            fi
+            echo "Attempt $ATTEMPTS/$MAX_ATTEMPTS: status=$STATUS; sleeping $SLEEP_SECONDS s..."
+            sleep $SLEEP_SECONDS
+          done
+
+      - name: Cache Terraform Outputs
+        if: github.ref == 'refs/heads/master'
+        working-directory: .
+        run: |
+          bash scripts/cache_terraform_outputs.sh
+
+      - name: Verify API Health Endpoint
+        if: github.ref == 'refs/heads/master'
+        working-directory: .
+        run: |
+          bash scripts/verify_api_reachable.sh
+
+      - name: Install Package in Editable Mode
+        if: github.ref == 'refs/heads/master'
+        working-directory: .
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+
+      - name: Run moral_compass Integration Tests
+        if: github.ref == 'refs/heads/master'
+        working-directory: .
+        run: |
+          python -m pip install pytest
+          pytest -m integration tests/test_moral_compass_client_minimal.py -v
+
+      - name: Run API Integration Tests
+        if: github.ref == 'refs/heads/master'
+        working-directory: .
+        run: |
+          API_BASE_URL=$(cd infra && terraform output -raw api_base_url)
+          echo "API Base URL: $API_BASE_URL"
+
+          if [[ -z "$API_BASE_URL" || "$API_BASE_URL" == "null" ]]; then
+            echo "❌ Failed to get API base URL from Terraform outputs"
+            exit 1
+          fi
+
+          python -m pip install --upgrade pip
+          python -m pip install requests
+
+          echo "⏳ Extra stabilization wait (30s) after GSI readiness..."
+          sleep 30
+
+          echo "🚀 Starting API Integration Tests..."
+          python tests/test_api_integration.py "$API_BASE_URL"
+
+      - name: Run Pagination Tests
+        if: github.ref == 'refs/heads/master'
+        working-directory: .
+        run: |
+          API_BASE_URL=$(cd infra && terraform output -raw api_base_url)
+          echo "API Base URL: $API_BASE_URL"
+          python -m pip install requests
+          python tests/test_api_pagination.py "$API_BASE_URL"
+
+      - name: Install Load Test Dependencies
+        if: github.ref == 'refs/heads/master' && (vars.RUN_LOAD_TESTS != 'false')
+        working-directory: .
+        run: |
+          python -m pip install aiohttp rich
+
+      - name: Run Load Tests - Single Table
+        if: github.ref == 'refs/heads/master' && (vars.RUN_LOAD_TESTS != 'false')
+        working-directory: .
+        run: |
+          API_BASE_URL=$(cd infra && terraform output -raw api_base_url)
+          export API_BASE_URL
+          echo "🚀 Running Single Table Load Test..."
+          python tests/load_single_table.py
+
+      - name: Run Load Tests - Multi Table
+        if: github.ref == 'refs/heads/master' && (vars.RUN_LOAD_TESTS != 'false')
+        working-directory: .
+        run: |
+          API_BASE_URL=$(cd infra && terraform output -raw api_base_url)
+          export API_BASE_URL
+          echo "🚀 Running Multi Table Load Test..."
+          python tests/load_multi_table.py
+
+      - name: Run Load Tests - Mixed Duration
+        if: github.ref == 'refs/heads/master' && (vars.RUN_LOAD_TESTS != 'false')
+        working-directory: .
+        env:
+          LOAD_DURATION_SECONDS: 20
+        run: |
+          API_BASE_URL=$(cd infra && terraform output -raw api_base_url)
+          export API_BASE_URL
+          echo "🚀 Running Mixed Duration Load Test (20s for CI)..."
+          python tests/load_mixed_duration.py

aimodelshare-0.1.59/.github/workflows/destroy-infra.yml

@@ -0,0 +1,114 @@
+name: Destroy Infra (Terraform, OIDC, Workspaces)
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Workspace to destroy (dev|stage|prod)'
+        required: true
+        default: 'dev'
+      destroy_bootstrap:
+        description: 'Also destroy bootstrap resources (S3 bucket and DynamoDB table) - USE WITH EXTREME CAUTION'
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  destroy:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+
+    env:
+      AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+      TF_IN_AUTOMATION: "true"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.9.5
+          terraform_wrapper: false
+
+      # Configure AWS credentials using access keys per company policy
+      # Note: Requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY repository secrets
+      # Security: Ensure access keys follow principle of least privilege and key rotation policies
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.DATA_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DATA_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Terraform Init (Main Infrastructure)
+        working-directory: infra
+        run: terraform init -input=false
+
+      - name: Select Workspace and Validate
+        working-directory: infra
+        run: |
+          set -euo pipefail
+
+          WS="${{ github.event.inputs.environment }}"
+          echo "Checking for existing Terraform workspace '${WS}'..."
+
+          # Verify backend is initialized
+          if [ ! -d .terraform ]; then
+            echo "Terraform not initialized yet; running 'terraform init'..."
+            terraform init -input=false
+          fi
+
+          # Check if workspace exists - abort if not (don't create for destroy)
+          if terraform workspace list | sed 's/^[* ]*//' | grep -Fxq "$WS"; then
+            echo "Selecting existing workspace: $WS"
+            terraform workspace select "$WS"
+          else
+            echo "❌ ERROR: Workspace '$WS' does not exist. Cannot destroy non-existent workspace."
+            echo "Available workspaces:"
+            terraform workspace list
+            exit 1
+          fi
+
+          echo "✅ Workspace '$WS' validated and selected."
+
+      - name: Terraform Destroy Plan
+        working-directory: infra
+        run: |
+          echo "🔍 Generating destroy plan for audit..."
+          terraform plan -destroy -input=false -out=destroy.tfplan
+          echo "✅ Destroy plan generated. Proceeding with destroy..."
+
+      - name: Terraform Destroy (Main Infrastructure)
+        working-directory: infra
+        run: terraform destroy -input=false -auto-approve
+
+      - name: Destroy Bootstrap Resources
+        if: github.event.inputs.destroy_bootstrap == 'true'
+        working-directory: infra/bootstrap
+        run: |
+          echo "⚠️  WARNING: Destroying bootstrap resources (S3 bucket and DynamoDB table)"
+          echo "This will remove the Terraform state storage and could cause data loss!"
+          
+          # Initialize bootstrap Terraform
+          terraform init -input=false
+          
+          # Check if state bucket is empty (except for our state files)
+          BUCKET_NAME="aimodelshare-tfstate-prod-copilot-2024"
+          OBJECT_COUNT=$(aws s3api list-objects-v2 --bucket "$BUCKET_NAME" --query 'Contents | length(@)' --output text 2>/dev/null || echo "0")
+          
+          # AWS CLI returns 'None' for empty buckets, 'null' if bucket doesn't exist, or a number for non-empty buckets
+          if [[ "$OBJECT_COUNT" =~ ^[0-9]+$ ]] && [ "$OBJECT_COUNT" -gt 0 ]; then
+            echo "⚠️  S3 bucket contains $OBJECT_COUNT objects. Emptying bucket first..."
+            aws s3 rm s3://$BUCKET_NAME --recursive
+          else
+            echo "✅ S3 bucket is empty or doesn't exist (count: $OBJECT_COUNT), proceeding with destroy..."
+          fi
+          
+          # Destroy bootstrap resources
+          terraform destroy -input=false -auto-approve
+          
+          echo "✅ Bootstrap resources destroyed"