Resolve fork

.github/workflows/cd-deploy.yml

@@ -1,99 +1,128 @@
-name: CD-Deploy
-on:
-  push:
-    branches:
-      - 'develop'
-#    paths:
-#      - '06-best-practices/code/**'
+# name: CD_DEPLOY
+# on:
+#   push:
+#     branches:
+#       - 'main'
+#     paths:
+#       - '06-best-practices/code-practice/**'
+# env:
+#   AWS_DEFAULT_REGION: 'us-east-2'
+#   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+#   RUN_ID: '20c7e3f3b3584b769bf6cacd4643d43d'
 
-jobs:
-  build-push-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out repo
-        uses: actions/checkout@v3
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: "eu-west-1"
-      - uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_wrapper: false
+# jobs:
+#   deploy:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Checkout Repo
+#         uses: actions/checkout@v2
+
+#       - name: Setup terraform
+#         uses: hashicorp/setup-terraform@v2
+#         with:
+#           terraform_version: 1.0.0
+#           terraform_wrapper: false
 
-      # Define the infrastructure
-      - name: TF plan
-        id: tf-plan
-        working-directory: '06-best-practices/code/infrastructure'
-        run: |
-          terraform init -backend-config="key=mlops-zoomcamp-prod.tfstate" -reconfigure && terraform plan -var-file=vars/prod.tfvars
+#       - name: Configure AWS Credentials
+#         uses: aws-actions/configure-aws-credentials@v1
+#         with:
+#           aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+#           aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+#           aws-region: ${{ env.AWS_DEFAULT_REGION }}
+
+#       - name: Login to Amazon ECR
+#         id: login-ecr
+#         uses: aws-actions/amazon-ecr-login@v1
 
-      - name: TF Apply
-        id: tf-apply
-        working-directory: '06-best-practices/code/infrastructure'
-        if: ${{ steps.tf-plan.outcome }} == 'success'
-        run: |
-          terraform apply -auto-approve -var-file=vars/prod.tfvars
-          echo "::set-output name=ecr_repo::$(terraform output ecr_repo | xargs)"
-          echo "::set-output name=predictions_stream_name::$(terraform output predictions_stream_name | xargs)"
-          echo "::set-output name=model_bucket::$(terraform output model_bucket | xargs)"
-          echo "::set-output name=lambda_function::$(terraform output lambda_function | xargs)"
+#       - name: TF plan-stage
+#         id: tf-plan-stage
+#         working-directory: '06-best-practices/code-practice'
+#         run: make plan_stage
 
-      # Build-Push
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
+#       - name: TF apply-stage
+#         id: tf-apply-stage
+#         working-directory: '06-best-practices/code-practice/deploy'
+#         if: ${{ steps.tf-plan-stage.outcome }} == 'success'
+#         env:
+#           ENVIRONMENT: "stage"
+#           TEST_RUN: "False"
+#         run: | 
+#           echo "Initialing terraform"
+#           terraform init -backend-config="key=mlops-zoomcamp-${{ env.ENVIRONMENT }}.tfstate" --reconfigure
 
-      - name: Build, tag, and push image to Amazon ECR
-        id: build-image-step
-        working-directory: "06-best-practices/code"
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: ${{ steps.tf-apply.outputs.ecr_repo }}
-          IMAGE_TAG: "latest"   # ${{ github.sha }}
-        run: |
-          docker build -t ${ECR_REGISTRY}/${ECR_REPOSITORY}:${IMAGE_TAG} .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-          echo "::set-output name=image_uri::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
+#           echo "Running terraform apply" 
+#           TF_VAR_run_id=${{ env.RUN_ID }} TF_VAR_test_run=${{ env.TEST_RUN }} terraform apply -auto-approve -var-file="vars/${{ env.ENVIRONMENT }}.tfvars"
 
-      # Deploy
-      - name: Get model artifacts
-      # The steps here are not suited for production.
-      # In practice, retrieving the latest model version or RUN_ID from a service like MLflow or DVC can also be integrated into a CI/CD pipeline.
-      # But due to the limited scope of this workshop, we would be keeping things simple.
-      # In practice, you would also have a separate training pipeline to write new model artifacts to your Model Bucket in Prod.
+#           echo "::set-output name=ride_prediction_lambda::$(terraform output ride_prediction_lambda)"
+#           echo "::set-output name=ride_prediction_bucket::$(terraform output ride_prediction_bucket)"
+#           echo "::set-output name=kinesis_source_stream::$(terraform output kinesis_source_stream_name)"
+#           echo "::set-output name=kinesis_output_stream::$(terraform output kinesis_output_stream_name)"
+#           echo "::set-output name=ecr_repo::$(terraform output ecr_repo)"
+#           echo "::set-output name=run_id::$(terraform output run_id)"
 
-        id: get-model-artifacts
-        working-directory: "06-best-practices/code"
-        env:
-          MODEL_BUCKET_DEV: "mlflow-models-alexey"
-          MODEL_BUCKET_PROD: ${{ steps.tf-apply.outputs.model_bucket }}
-        run: |
-          export RUN_ID=$(aws s3api list-objects-v2 --bucket ${MODEL_BUCKET_DEV} \
-          --query 'sort_by(Contents, &LastModified)[-1].Key' --output=text | cut -f2 -d/)
-          aws s3 sync s3://${MODEL_BUCKET_DEV} s3://${MODEL_BUCKET_PROD}
-          echo "::set-output name=run_id::${RUN_ID}"
+#           echo "Image name deployed is $(terraform output image_uri)"
 
-      - name: Update Lambda
-        env:
-          LAMBDA_FUNCTION: ${{ steps.tf-apply.outputs.lambda_function }}
-          PREDICTIONS_STREAM_NAME: ${{ steps.tf-apply.outputs.predictions_stream_name }}
-          MODEL_BUCKET: ${{ steps.tf-apply.outputs.model_bucket }}
-          RUN_ID: ${{ steps.get-model-artifacts.outputs.run_id }}
-        run: |
-          variables="{ \
-                    PREDICTIONS_STREAM_NAME=$PREDICTIONS_STREAM_NAME, MODEL_BUCKET=$MODEL_BUCKET, RUN_ID=$RUN_ID \
-                    }"
+#       - name: TF plan-prod
+#         id: tf-plan-prod
+#         working-directory: '06-best-practices/code-practice'
+#         if: ${{ steps.tf-apply-stage.outcome }} == 'success'
+#         run: make plan_prod
 
-          STATE=$(aws lambda get-function --function-name $LAMBDA_FUNCTION --region "eu-west-1" --query 'Configuration.LastUpdateStatus' --output text)
-              while [[ "$STATE" == "InProgress" ]]
-              do
-                  echo "sleep 5sec ...."
-                  sleep 5s
-                  STATE=$(aws lambda get-function --function-name $LAMBDA_FUNCTION --region "eu-west-1" --query 'Configuration.LastUpdateStatus' --output text)
-                  echo $STATE
-              done
+#       - name: TF apply-prod
+#         id: tf-apply-prod
+#         working-directory: '06-best-practices/code-practice/deploy'
+#         if: ${{ steps.tf-plan-prod.outcome }} == 'success'
+#         env:
+#           ENVIRONMENT: "prod"
+#           TEST_RUN: "False"
+#         run: | 
+#           echo "Initialing terraform"
+#           terraform init -backend-config="key=mlops-zoomcamp-${{ env.ENVIRONMENT }}.tfstate" --reconfigure
 
-          aws lambda update-function-configuration --function-name $LAMBDA_FUNCTION \
-                    --environment "Variables=${variables}"
+#           echo "Running terraform apply" 
+#           TF_VAR_run_id=${{ env.RUN_ID }} TF_VAR_test_run=${{ env.TEST_RUN }} terraform apply -auto-approve -var-file="vars/${{ env.ENVIRONMENT }}.tfvars"
+
+#           echo "::set-output name=ride_prediction_lambda::$(terraform output ride_prediction_lambda)"
+#           echo "::set-output name=ride_prediction_bucket::$(terraform output ride_prediction_bucket)"
+#           echo "::set-output name=kinesis_source_stream::$(terraform output kinesis_source_stream_name)"
+#           echo "::set-output name=kinesis_output_stream::$(terraform output kinesis_output_stream_name)"
+#           echo "Image  name deployed is $(terraform output image_uri)"
+#           echo "::set-output name=ecr_repo::$(terraform output ecr_repo)"
+#           echo "::set-output name=run_id::$(terraform output run_id)"
+
+#       - name: Build, tag and push image stage
+#         working-directory: '06-best-practices/code-practice'
+#         env:
+#           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+#           REPOSITORY: ${{ steps.tf-apply-stage.outputs.ecr_repo }}
+#           IMAGE_TAG: "latest"
+#           TEST_RUN: "False"
+#         run: |
+#           docker build --build-arg STREAM_NAME_ARG=${{ steps.tf-apply-stage.outputs.kinesis_output_stream }}  --build-arg RUN_ID_ARG=${{ steps.tf-apply-stage.outputs.run_id }} --build-arg TEST_RUN_ARG=${{ env.TEST_RUN }} --build-arg MODEL_BUCKET_ARG=${{ steps.tf-apply-stage.outputs.ride_prediction_bucket }} -t ${REGISTRY}/${{ env.REPOSITORY }}:${IMAGE_TAG} .
+#           docker push ${REGISTRY}/${{ env.REPOSITORY }}:${IMAGE_TAG}
+
+#       - name: Build, tag and push image prod
+#         working-directory: '06-best-practices/code-practice'
+#         env:
+#           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+#           REPOSITORY: ${{ steps.tf-apply-prod.outputs.ecr_repo }}
+#           IMAGE_TAG: "latest"
+#           TEST_RUN: "False"
+#         run: |
+#           docker build --build-arg STREAM_NAME_ARG=${{ steps.tf-apply-prod.outputs.kinesis_output_stream }}  --build-arg RUN_ID_ARG=${{ steps.tf-apply-prod.outputs.run_id }} --build-arg TEST_RUN_ARG=${{ env.TEST_RUN }} --build-arg MODEL_BUCKET_ARG=${{ steps.tf-apply-prod.outputs.ride_prediction_bucket }} -t ${REGISTRY}/${{ env.REPOSITORY }}:${IMAGE_TAG} .
+#           docker push ${REGISTRY}/${{ env.REPOSITORY }}:${IMAGE_TAG}
+
+#       - name: Update bucket artifacts stage
+#         env:
+#           MODEL_BUCKET_DEV: "mlops-zoomcamp-nakul-dev"
+#           MODEL_BUCKET: ${{ steps.tf-apply-stage.outputs.ride_prediction_bucket }}
+#         run: |
+#           aws s3 sync s3://${{ env.MODEL_BUCKET_DEV }} s3://${{ env.MODEL_BUCKET }}
+
+#       - name: Update bucket artifacts prod
+#         env:
+#           MODEL_BUCKET_DEV: "mlops-zoomcamp-nakul-dev"
+#           MODEL_BUCKET: ${{ steps.tf-apply-prod.outputs.ride_prediction_bucket }}
+#         run: |
+#           aws s3 sync s3://${{ env.MODEL_BUCKET_DEV }} s3://${{ env.MODEL_BUCKET }}

.github/workflows/ci-tests.yml

@@ -1,65 +1,60 @@
-name: CI-Tests
-on:
-  pull_request:
-    branches:
-      - 'develop'
-    paths:
-      - '06-best-practices/code/**'
-
-env:
-  AWS_DEFAULT_REGION: 'eu-west-1'
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        working-directory: "06-best-practices/code"
-        run: pip install pipenv && pipenv install --dev
-
-      - name: Run Unit tests
-        working-directory: "06-best-practices/code"
-        run: pipenv run pytest tests/
-
-      - name: Lint
-        working-directory: "06-best-practices/code"
-        run: pipenv run pylint --recursive=y .
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_DEFAULT_REGION }}
-
-      - name: Integration Test
-        working-directory: '06-best-practices/code/integraton-test'
-        run: |
-          . run.sh
-
-  tf-plan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_DEFAULT_REGION }}
-
-      - uses: hashicorp/setup-terraform@v2
-
-      - name: TF plan
-        id: plan
-        working-directory: '06-best-practices/code/infrastructure'
-        run: |
-          terraform init -backend-config="key=mlops-zoomcamp-prod.tfstate" --reconfigure && terraform plan --var-file vars/prod.tfvars
+# name: CI_TESTS
+# on:
+#   pull_request:
+#     branches:
+#       - main
+#     paths:
+#       - '06-best-practices/code-practice/**'
+# env:
+#   AWS_DEFAULT_REGION: 'us-east-2'
+#   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+# jobs:
+#   test:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Checkout Repo
+#         uses: actions/checkout@v2
+#       - name: Setup Python 3.9
+#         uses: actions/setup-python@v2
+#         with:
+#           python-version: 3.9
+
+#       - name: Install dependencies
+#         working-directory: '06-best-practices/code-practice'
+#         run: pip install pipenv==2022.8.5 && pipenv install --dev
+
+#       - name: Run Quality tests
+#         working-directory: '06-best-practices/code-practice'
+#         run: make quality_checks 
+
+#       - name: Run Integration tests
+#         working-directory: '06-best-practices/code-practice'
+#         run: make integration_tests
+
+#   tf-plan:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Checkout Repo
+#         uses: actions/checkout@v2
+
+#       - name: Setup terraform
+#         uses: hashicorp/setup-terraform@v2
+#         with:
+#           terraform_version: 1.0.0
+
+#       - name: Configure AWS Credentials
+#         uses: aws-actions/configure-aws-credentials@v1
+#         with:
+#           aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+#           aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+#           aws-region: ${{ env.AWS_DEFAULT_REGION }}
+
+#       - name: TF plan-stage
+#         working-directory: '06-best-practices/code-practice'
+#         run: make plan_stage
+
+#       - name: TF plan-prod
+#         working-directory: '06-best-practices/code-practice'
+#         run: make plan_prod

.gitignore

@@ -9,6 +9,7 @@ data/
 
 .venv
 venv
+
 .idea
 **/artifacts/
 **/models/