Deploy and Keep Active #270
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy and Keep Active | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - 'Dockerfile' | |
| - 'docker-compose.yml' | |
| - '2fa_bot.py' | |
| - 'requirements.txt' | |
| - 'scripts/**' | |
| - 'container_overrides/**' | |
| # Runs automatically every day at 0:00 UTC (8:00 AM Beijing Time) | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| target: | |
| description: gateway target name from IB_GATEWAY_TARGETS_JSON, or all | |
| required: false | |
| default: all | |
| type: string | |
| deploy_mode: | |
| description: keepalive skips docker build; full rebuilds the container | |
| required: false | |
| default: keepalive | |
| type: choice | |
| options: | |
| - keepalive | |
| - full | |
| sync_github_secrets_to_secret_manager: | |
| description: write current GitHub secrets into Secret Manager before deploy | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: false | |
| jobs: | |
| select-targets: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| matrix: ${{ steps.targets.outputs.matrix }} | |
| env: | |
| TARGETS_JSON: ${{ vars.IB_GATEWAY_TARGETS_JSON }} | |
| SELECTED_TARGET: ${{ github.event.inputs.target }} | |
| LEGACY_GCP_PROJECT_ID: ${{ vars.IB_GATEWAY_GCP_PROJECT_ID }} | |
| LEGACY_GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.IB_GATEWAY_GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| LEGACY_GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ${{ vars.IB_GATEWAY_GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} | |
| LEGACY_GCE_USER: ${{ vars.IB_GATEWAY_GCE_USER }} | |
| LEGACY_GCE_INSTANCE_NAME: ${{ vars.IB_GATEWAY_INSTANCE_NAME }} | |
| LEGACY_GCE_ZONE: ${{ vars.IB_GATEWAY_ZONE }} | |
| LEGACY_DEPLOY_PATH: ${{ vars.IB_GATEWAY_DEPLOY_PATH }} | |
| LEGACY_MODE: ${{ vars.IB_GATEWAY_MODE }} | |
| LEGACY_CONTAINER_NAME: ${{ vars.IB_GATEWAY_CONTAINER_NAME }} | |
| LEGACY_COMPOSE_PROJECT_NAME: ${{ vars.IB_GATEWAY_COMPOSE_PROJECT_NAME }} | |
| LEGACY_COMPOSE_SERVICE_NAME: ${{ vars.IB_GATEWAY_COMPOSE_SERVICE_NAME }} | |
| LEGACY_UNIT_SUFFIX: ${{ vars.IB_GATEWAY_UNIT_SUFFIX }} | |
| LEGACY_LIVE_HOST_PORT: ${{ vars.IB_GATEWAY_LIVE_HOST_PORT }} | |
| LEGACY_PAPER_HOST_PORT: ${{ vars.IB_GATEWAY_PAPER_HOST_PORT }} | |
| LEGACY_VNC_HOST_ADDRESS: ${{ vars.IB_GATEWAY_VNC_HOST_ADDRESS }} | |
| LEGACY_VNC_HOST_PORT: ${{ vars.IB_GATEWAY_VNC_HOST_PORT }} | |
| LEGACY_CLOUD_RUN_EGRESS_CIDR: ${{ vars.IB_GATEWAY_CLOUD_RUN_EGRESS_CIDR }} | |
| LEGACY_ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY: ${{ vars.IB_GATEWAY_ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY }} | |
| LEGACY_TWS_ACCEPT_INCOMING: ${{ vars.IB_GATEWAY_TWS_ACCEPT_INCOMING }} | |
| LEGACY_READ_ONLY_API: ${{ vars.IB_GATEWAY_READ_ONLY_API }} | |
| LEGACY_TWOFA_DEVICE: ${{ vars.IB_GATEWAY_TWOFA_DEVICE }} | |
| LEGACY_2FA_AUTOFILL: ${{ vars.IB_GATEWAY_2FA_AUTOFILL }} | |
| LEGACY_2FA_MAX_SUBMISSIONS: ${{ vars.IB_GATEWAY_2FA_MAX_SUBMISSIONS }} | |
| LEGACY_2FA_MAX_SUBMISSIONS_PER_WINDOW: ${{ vars.IB_GATEWAY_2FA_MAX_SUBMISSIONS_PER_WINDOW }} | |
| LEGACY_2FA_SUBMISSION_RESET_SECONDS: ${{ vars.IB_GATEWAY_2FA_SUBMISSION_RESET_SECONDS }} | |
| LEGACY_SSH_PRIVATE_KEY_SECRET_NAME: ${{ vars.IB_GATEWAY_SSH_PRIVATE_KEY_SECRET_NAME }} | |
| LEGACY_TWS_USERID_SECRET_NAME: ${{ vars.IB_GATEWAY_TWS_USERID_SECRET_NAME }} | |
| LEGACY_TWS_PASSWORD_SECRET_NAME: ${{ vars.IB_GATEWAY_TWS_PASSWORD_SECRET_NAME }} | |
| LEGACY_TOTP_SECRET_SECRET_NAME: ${{ vars.IB_GATEWAY_TOTP_SECRET_SECRET_NAME }} | |
| LEGACY_VNC_SERVER_PASSWORD_SECRET_NAME: ${{ vars.IB_GATEWAY_VNC_SERVER_PASSWORD_SECRET_NAME }} | |
| steps: | |
| - id: targets | |
| name: Select gateway targets | |
| run: | | |
| set -euo pipefail | |
| python3 - <<'PY' | |
| import json | |
| import os | |
| def env(name: str, default: str = "") -> str: | |
| return os.environ.get(name, "") or default | |
| raw_targets = env("TARGETS_JSON").strip() | |
| if raw_targets: | |
| loaded = json.loads(raw_targets) | |
| if isinstance(loaded, dict): | |
| targets = [] | |
| for name, config in loaded.items(): | |
| if not isinstance(config, dict): | |
| raise SystemExit(f"Target {name} must be an object") | |
| target = dict(config) | |
| target.setdefault("name", name) | |
| targets.append(target) | |
| elif isinstance(loaded, list): | |
| targets = loaded | |
| else: | |
| raise SystemExit("IB_GATEWAY_TARGETS_JSON must be an object or a list") | |
| else: | |
| targets = [{ | |
| "name": "default", | |
| "gcp_project_id": env("LEGACY_GCP_PROJECT_ID", "interactivebrokersquant"), | |
| "gcp_workload_identity_provider": env( | |
| "LEGACY_GCP_WORKLOAD_IDENTITY_PROVIDER", | |
| "projects/303168642265/locations/global/workloadIdentityPools/github-actions/providers/github-ibkr-gateway-main", | |
| ), | |
| "gcp_workload_identity_service_account": env( | |
| "LEGACY_GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT", | |
| "ibkr-gateway-deploy@interactivebrokersquant.iam.gserviceaccount.com", | |
| ), | |
| "gce_user": env("LEGACY_GCE_USER"), | |
| "gce_instance_name": env("LEGACY_GCE_INSTANCE_NAME"), | |
| "gce_zone": env("LEGACY_GCE_ZONE"), | |
| "deploy_path": env("LEGACY_DEPLOY_PATH"), | |
| "mode": env("LEGACY_MODE"), | |
| "container_name": env("LEGACY_CONTAINER_NAME"), | |
| "compose_project_name": env("LEGACY_COMPOSE_PROJECT_NAME"), | |
| "compose_service_name": env("LEGACY_COMPOSE_SERVICE_NAME"), | |
| "unit_suffix": env("LEGACY_UNIT_SUFFIX"), | |
| "live_host_port": env("LEGACY_LIVE_HOST_PORT"), | |
| "paper_host_port": env("LEGACY_PAPER_HOST_PORT"), | |
| "vnc_host_address": env("LEGACY_VNC_HOST_ADDRESS"), | |
| "vnc_host_port": env("LEGACY_VNC_HOST_PORT"), | |
| "cloud_run_egress_cidr": env("LEGACY_CLOUD_RUN_EGRESS_CIDR"), | |
| "allow_connections_from_localhost_only": env("LEGACY_ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY"), | |
| "tws_accept_incoming": env("LEGACY_TWS_ACCEPT_INCOMING"), | |
| "read_only_api": env("LEGACY_READ_ONLY_API"), | |
| "twofa_device": env("LEGACY_TWOFA_DEVICE"), | |
| "ibkr_2fa_autofill": env("LEGACY_2FA_AUTOFILL"), | |
| "ibkr_2fa_max_submissions": env("LEGACY_2FA_MAX_SUBMISSIONS"), | |
| "ibkr_2fa_max_submissions_per_window": env("LEGACY_2FA_MAX_SUBMISSIONS_PER_WINDOW"), | |
| "ibkr_2fa_submission_reset_seconds": env("LEGACY_2FA_SUBMISSION_RESET_SECONDS"), | |
| "ssh_private_key_secret_name": env("LEGACY_SSH_PRIVATE_KEY_SECRET_NAME"), | |
| "tws_userid_secret_name": env("LEGACY_TWS_USERID_SECRET_NAME"), | |
| "tws_password_secret_name": env("LEGACY_TWS_PASSWORD_SECRET_NAME"), | |
| "totp_secret_secret_name": env("LEGACY_TOTP_SECRET_SECRET_NAME"), | |
| "vnc_server_password_secret_name": env("LEGACY_VNC_SERVER_PASSWORD_SECRET_NAME"), | |
| }] | |
| normalized = [] | |
| seen = set() | |
| for target in targets: | |
| if not isinstance(target, dict): | |
| raise SystemExit("Each gateway target must be an object") | |
| name = target.get("name") | |
| if not name: | |
| raise SystemExit("Each gateway target requires a name") | |
| if name in seen: | |
| raise SystemExit(f"Duplicate gateway target: {name}") | |
| seen.add(name) | |
| normalized.append(target) | |
| selected_name = env("SELECTED_TARGET") or "all" | |
| if selected_name == "all": | |
| selected = normalized | |
| else: | |
| selected = [target for target in normalized if target["name"] == selected_name] | |
| if not selected: | |
| known = ", ".join(target["name"] for target in normalized) | |
| raise SystemExit(f"Unknown gateway target {selected_name!r}. Known targets: {known}") | |
| with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: | |
| output.write("matrix=" + json.dumps({"target": selected}, separators=(",", ":")) + "\n") | |
| print("Selected targets: " + ", ".join(target["name"] for target in selected)) | |
| PY | |
| deploy: | |
| needs: select-targets | |
| name: deploy (${{ matrix.target.name }}) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: ${{ fromJSON(needs.select-targets.outputs.matrix) }} | |
| concurrency: | |
| group: ibkr-gateway-${{ matrix.target.name }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| TARGET_NAME: ${{ matrix.target.name }} | |
| DEPLOY_EVENT_NAME: ${{ github.event_name }} | |
| GCP_PROJECT_ID: ${{ matrix.target.gcp_project_id }} | |
| GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ matrix.target.gcp_workload_identity_provider }} | |
| GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ${{ matrix.target.gcp_workload_identity_service_account }} | |
| GCE_USER: ${{ matrix.target.gce_user }} | |
| GCE_INSTANCE_NAME: ${{ matrix.target.gce_instance_name }} | |
| GCE_ZONE: ${{ matrix.target.gce_zone }} | |
| DEPLOY_PATH: ${{ matrix.target.deploy_path }} | |
| IB_GATEWAY_MODE: ${{ matrix.target.mode }} | |
| IB_GATEWAY_CONTAINER_NAME: ${{ matrix.target.container_name }} | |
| IB_GATEWAY_COMPOSE_PROJECT_NAME: ${{ matrix.target.compose_project_name }} | |
| IB_GATEWAY_COMPOSE_SERVICE_NAME: ${{ matrix.target.compose_service_name }} | |
| IB_GATEWAY_UNIT_SUFFIX: ${{ matrix.target.unit_suffix }} | |
| IB_GATEWAY_LIVE_HOST_PORT: ${{ matrix.target.live_host_port }} | |
| IB_GATEWAY_PAPER_HOST_PORT: ${{ matrix.target.paper_host_port }} | |
| IB_GATEWAY_VNC_HOST_ADDRESS: ${{ matrix.target.vnc_host_address }} | |
| IB_GATEWAY_VNC_HOST_PORT: ${{ matrix.target.vnc_host_port }} | |
| CLOUD_RUN_EGRESS_CIDR: ${{ matrix.target.cloud_run_egress_cidr }} | |
| ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY: ${{ matrix.target.allow_connections_from_localhost_only }} | |
| TWS_ACCEPT_INCOMING: ${{ matrix.target.tws_accept_incoming }} | |
| READ_ONLY_API: ${{ matrix.target.read_only_api }} | |
| TWOFA_DEVICE: ${{ matrix.target.twofa_device }} | |
| IBKR_2FA_AUTOFILL: ${{ matrix.target.ibkr_2fa_autofill }} | |
| IBKR_2FA_MAX_SUBMISSIONS: ${{ matrix.target.ibkr_2fa_max_submissions }} | |
| IBKR_2FA_MAX_SUBMISSIONS_PER_WINDOW: ${{ matrix.target.ibkr_2fa_max_submissions_per_window }} | |
| IBKR_2FA_SUBMISSION_RESET_SECONDS: ${{ matrix.target.ibkr_2fa_submission_reset_seconds }} | |
| SSH_PRIVATE_KEY_SECRET_NAME: ${{ matrix.target.ssh_private_key_secret_name }} | |
| TWS_USERID_SECRET_NAME: ${{ matrix.target.tws_userid_secret_name }} | |
| TWS_PASSWORD_SECRET_NAME: ${{ matrix.target.tws_password_secret_name }} | |
| TOTP_SECRET_SECRET_NAME: ${{ matrix.target.totp_secret_secret_name }} | |
| VNC_SERVER_PASSWORD_SECRET_NAME: ${{ matrix.target.vnc_server_password_secret_name }} | |
| WORKFLOW_DISPATCH_MODE: ${{ github.event.inputs.deploy_mode }} | |
| WORKFLOW_DISPATCH_TARGET: ${{ github.event.inputs.target }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Check whether deployment config is complete | |
| env: | |
| SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| TWS_USERID: ${{ secrets.TWS_USERID }} | |
| TWS_PASSWORD: ${{ secrets.TWS_PASSWORD }} | |
| TOTP_SECRET: ${{ secrets.TOTP_SECRET }} | |
| VNC_SERVER_PASSWORD: ${{ secrets.VNC_SERVER_PASSWORD }} | |
| run: | | |
| set -euo pipefail | |
| required_vars=( | |
| GCP_PROJECT_ID | |
| GCP_WORKLOAD_IDENTITY_PROVIDER | |
| GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT | |
| GCE_USER | |
| GCE_INSTANCE_NAME | |
| GCE_ZONE | |
| DEPLOY_PATH | |
| IB_GATEWAY_MODE | |
| CLOUD_RUN_EGRESS_CIDR | |
| ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY | |
| ) | |
| for var_name in "${required_vars[@]}"; do | |
| if [ -z "${!var_name:-}" ]; then | |
| echo "${var_name} is required. Set it in GitHub Actions variables before running this workflow." >&2 | |
| exit 1 | |
| fi | |
| done | |
| require_secret_source() { | |
| local secret_name_var="$1" | |
| local fallback_var="$2" | |
| local label="$3" | |
| if [ -n "${!secret_name_var:-}" ] || [ -n "${!fallback_var:-}" ]; then | |
| return 0 | |
| fi | |
| echo "${label} is required. Set ${secret_name_var} as a GitHub variable pointing to Secret Manager, or set ${fallback_var} as a GitHub Actions secret." >&2 | |
| exit 1 | |
| } | |
| require_secret_source SSH_PRIVATE_KEY_SECRET_NAME SSH_PRIVATE_KEY SSH_PRIVATE_KEY | |
| require_secret_source TWS_USERID_SECRET_NAME TWS_USERID TWS_USERID | |
| require_secret_source TWS_PASSWORD_SECRET_NAME TWS_PASSWORD TWS_PASSWORD | |
| require_secret_source VNC_SERVER_PASSWORD_SECRET_NAME VNC_SERVER_PASSWORD VNC_SERVER_PASSWORD | |
| twofa_autofill="$(printf '%s' "${IBKR_2FA_AUTOFILL:-}" | tr '[:upper:]' '[:lower:]')" | |
| if [ "${twofa_autofill}" = "yes" ] || [ "${twofa_autofill}" = "true" ] || [ "${twofa_autofill}" = "1" ]; then | |
| require_secret_source TOTP_SECRET_SECRET_NAME TOTP_SECRET TOTP_SECRET | |
| fi | |
| - name: Authenticate to Google Cloud | |
| id: auth | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} | |
| - name: Set up gcloud | |
| uses: google-github-actions/setup-gcloud@v3 | |
| with: | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| version: '>= 416.0.0' | |
| - name: Sync GitHub secrets to Secret Manager | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.sync_github_secrets_to_secret_manager }} | |
| env: | |
| SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| TWS_USERID: ${{ secrets.TWS_USERID }} | |
| TWS_PASSWORD: ${{ secrets.TWS_PASSWORD }} | |
| TOTP_SECRET: ${{ secrets.TOTP_SECRET }} | |
| VNC_SERVER_PASSWORD: ${{ secrets.VNC_SERVER_PASSWORD }} | |
| run: | | |
| set -euo pipefail | |
| if [ "${WORKFLOW_DISPATCH_TARGET:-all}" = "all" ]; then | |
| echo "sync_github_secrets_to_secret_manager requires one explicit target to avoid writing the same GitHub secrets to multiple gateways." >&2 | |
| exit 1 | |
| fi | |
| sync_secret() { | |
| local secret_name="$1" | |
| local secret_value="$2" | |
| local label="$3" | |
| if [ -z "${secret_name:-}" ]; then | |
| echo "${label} secret name variable is required before syncing to Secret Manager." >&2 | |
| exit 1 | |
| fi | |
| if [ -z "${secret_value:-}" ]; then | |
| echo "${label} GitHub secret is empty; cannot sync to Secret Manager." >&2 | |
| exit 1 | |
| fi | |
| printf '%s' "${secret_value}" | gcloud secrets versions add "${secret_name}" --project "${GCP_PROJECT_ID}" --data-file=- >/dev/null | |
| cleanup_secret_versions "${secret_name}" | |
| } | |
| cleanup_secret_versions() { | |
| local secret_name="$1" | |
| local old_versions | |
| old_versions="$( | |
| gcloud secrets versions list "${secret_name}" \ | |
| --project "${GCP_PROJECT_ID}" \ | |
| --format=json \ | |
| | python3 -c 'import json, sys; rows = json.load(sys.stdin); active = [int(row["name"].rsplit("/", 1)[-1]) for row in rows if row.get("state") in {"ENABLED", "DISABLED"}]; keep = max(active) if active else None; print("\n".join(str(version) for version in active if version != keep))' | |
| )" | |
| while IFS= read -r version; do | |
| if [ -z "${version}" ]; then | |
| continue | |
| fi | |
| gcloud secrets versions destroy "${version}" \ | |
| --secret "${secret_name}" \ | |
| --project "${GCP_PROJECT_ID}" \ | |
| --quiet >/dev/null | |
| done <<< "${old_versions}" | |
| } | |
| mask_if_present() { | |
| if [ -n "${1:-}" ]; then | |
| echo "::add-mask::$1" | |
| fi | |
| } | |
| mask_if_present "${TWS_USERID:-}" | |
| mask_if_present "${TWS_PASSWORD:-}" | |
| mask_if_present "${TOTP_SECRET:-}" | |
| mask_if_present "${VNC_SERVER_PASSWORD:-}" | |
| sync_secret "${SSH_PRIVATE_KEY_SECRET_NAME:-}" "${SSH_PRIVATE_KEY:-}" "SSH_PRIVATE_KEY" | |
| sync_secret "${TWS_USERID_SECRET_NAME:-}" "${TWS_USERID:-}" "TWS_USERID" | |
| sync_secret "${TWS_PASSWORD_SECRET_NAME:-}" "${TWS_PASSWORD:-}" "TWS_PASSWORD" | |
| sync_secret "${VNC_SERVER_PASSWORD_SECRET_NAME:-}" "${VNC_SERVER_PASSWORD:-}" "VNC_SERVER_PASSWORD" | |
| twofa_autofill="$(printf '%s' "${IBKR_2FA_AUTOFILL:-}" | tr '[:upper:]' '[:lower:]')" | |
| if [ "${twofa_autofill}" = "yes" ] || [ "${twofa_autofill}" = "true" ] || [ "${twofa_autofill}" = "1" ] || [ -n "${TOTP_SECRET:-}" ]; then | |
| sync_secret "${TOTP_SECRET_SECRET_NAME:-}" "${TOTP_SECRET:-}" "TOTP_SECRET" | |
| fi | |
| - name: Prepare SSH key and deployment files | |
| env: | |
| SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} | |
| TWS_USERID: ${{ secrets.TWS_USERID }} | |
| TWS_PASSWORD: ${{ secrets.TWS_PASSWORD }} | |
| TOTP_SECRET: ${{ secrets.TOTP_SECRET }} | |
| VNC_SERVER_PASSWORD: ${{ secrets.VNC_SERVER_PASSWORD }} | |
| run: | | |
| set -euo pipefail | |
| resolve_secret() { | |
| local secret_name="$1" | |
| local fallback_value="$2" | |
| if [ -n "${secret_name:-}" ]; then | |
| gcloud secrets versions access latest --project "${GCP_PROJECT_ID}" --secret "${secret_name}" | |
| else | |
| printf '%s' "${fallback_value}" | |
| fi | |
| } | |
| ssh_private_key="$(resolve_secret "${SSH_PRIVATE_KEY_SECRET_NAME:-}" "${SSH_PRIVATE_KEY:-}")" | |
| tws_userid="$(resolve_secret "${TWS_USERID_SECRET_NAME:-}" "${TWS_USERID:-}")" | |
| tws_password="$(resolve_secret "${TWS_PASSWORD_SECRET_NAME:-}" "${TWS_PASSWORD:-}")" | |
| totp_secret="" | |
| if [ -n "${TOTP_SECRET_SECRET_NAME:-}" ] || [ -n "${TOTP_SECRET:-}" ]; then | |
| totp_secret="$(resolve_secret "${TOTP_SECRET_SECRET_NAME:-}" "${TOTP_SECRET:-}")" | |
| fi | |
| vnc_server_password="$(resolve_secret "${VNC_SERVER_PASSWORD_SECRET_NAME:-}" "${VNC_SERVER_PASSWORD:-}")" | |
| required_values=(ssh_private_key tws_userid tws_password vnc_server_password) | |
| twofa_autofill="$(printf '%s' "${IBKR_2FA_AUTOFILL:-}" | tr '[:upper:]' '[:lower:]')" | |
| if [ "${twofa_autofill}" = "yes" ] || [ "${twofa_autofill}" = "true" ] || [ "${twofa_autofill}" = "1" ]; then | |
| required_values+=(totp_secret) | |
| fi | |
| for required_value in "${required_values[@]}"; do | |
| if [ -z "${!required_value:-}" ]; then | |
| echo "Resolved ${required_value} is empty." >&2 | |
| exit 1 | |
| fi | |
| done | |
| mask_if_present() { | |
| if [ -n "${1:-}" ]; then | |
| echo "::add-mask::$1" | |
| fi | |
| } | |
| mask_if_present "${tws_userid:-}" | |
| mask_if_present "${tws_password:-}" | |
| mask_if_present "${totp_secret:-}" | |
| mask_if_present "${vnc_server_password:-}" | |
| install -d -m 700 "$RUNNER_TEMP/ssh" | |
| SSH_KEY_FILE="$RUNNER_TEMP/ssh/google_compute_engine" | |
| printf '%s\n' "$ssh_private_key" | tr -d '\r' > "$SSH_KEY_FILE" | |
| chmod 600 "$SSH_KEY_FILE" | |
| if ! ssh-keygen -y -f "$SSH_KEY_FILE" > "$SSH_KEY_FILE.pub"; then | |
| echo "Resolved SSH private key is invalid or passphrase-protected; provide an unencrypted private key that matches the VM authorized_keys entry." >&2 | |
| exit 1 | |
| fi | |
| chmod 644 "$SSH_KEY_FILE.pub" | |
| echo "SSH_KEY_FILE=$SSH_KEY_FILE" >> "$GITHUB_ENV" | |
| ENV_FILE="$RUNNER_TEMP/ibkr-gateway.env" | |
| export ENV_FILE | |
| export RESOLVED_TWS_USERID="$tws_userid" | |
| export RESOLVED_TWS_PASSWORD="$tws_password" | |
| export RESOLVED_TOTP_SECRET="$totp_secret" | |
| export RESOLVED_VNC_SERVER_PASSWORD="$vnc_server_password" | |
| python3 - <<'PY' | |
| import os | |
| env_file = os.environ["ENV_FILE"] | |
| values = { | |
| "TWS_USERID": os.environ["RESOLVED_TWS_USERID"], | |
| "TWS_PASSWORD": os.environ["RESOLVED_TWS_PASSWORD"], | |
| "TOTP_SECRET": os.environ["RESOLVED_TOTP_SECRET"], | |
| "VNC_SERVER_PASSWORD": os.environ["RESOLVED_VNC_SERVER_PASSWORD"], | |
| "TRADING_MODE": os.environ["IB_GATEWAY_MODE"], | |
| "ACCEPT_API_FROM_IP": os.environ["CLOUD_RUN_EGRESS_CIDR"], | |
| "ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY": os.environ["ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY"], | |
| "TWS_ACCEPT_INCOMING": os.environ["TWS_ACCEPT_INCOMING"], | |
| "READ_ONLY_API": os.environ["READ_ONLY_API"], | |
| "TWOFA_DEVICE": os.environ.get("TWOFA_DEVICE", ""), | |
| "IBKR_2FA_AUTOFILL": os.environ.get("IBKR_2FA_AUTOFILL", ""), | |
| "IBKR_2FA_MAX_SUBMISSIONS": os.environ.get("IBKR_2FA_MAX_SUBMISSIONS") or "3", | |
| "IBKR_2FA_MAX_SUBMISSIONS_PER_WINDOW": os.environ.get("IBKR_2FA_MAX_SUBMISSIONS_PER_WINDOW") or "1", | |
| "IBKR_2FA_SUBMISSION_RESET_SECONDS": os.environ.get("IBKR_2FA_SUBMISSION_RESET_SECONDS") or "0", | |
| } | |
| optional_values = { | |
| "IB_GATEWAY_CONTAINER_NAME": os.environ.get("IB_GATEWAY_CONTAINER_NAME", ""), | |
| "COMPOSE_PROJECT_NAME": os.environ.get("IB_GATEWAY_COMPOSE_PROJECT_NAME", ""), | |
| "IB_GATEWAY_COMPOSE_SERVICE_NAME": os.environ.get("IB_GATEWAY_COMPOSE_SERVICE_NAME", ""), | |
| "IB_GATEWAY_UNIT_SUFFIX": os.environ.get("IB_GATEWAY_UNIT_SUFFIX", ""), | |
| "IB_GATEWAY_LIVE_HOST_PORT": os.environ.get("IB_GATEWAY_LIVE_HOST_PORT", ""), | |
| "IB_GATEWAY_PAPER_HOST_PORT": os.environ.get("IB_GATEWAY_PAPER_HOST_PORT", ""), | |
| "IB_GATEWAY_VNC_HOST_ADDRESS": os.environ.get("IB_GATEWAY_VNC_HOST_ADDRESS", ""), | |
| "IB_GATEWAY_VNC_HOST_PORT": os.environ.get("IB_GATEWAY_VNC_HOST_PORT", ""), | |
| } | |
| values.update({key: value for key, value in optional_values.items() if value}) | |
| def quote(value: str) -> str: | |
| return "'" + value.replace("'", "'\"'\"'") + "'" | |
| with open(env_file, "w", encoding="utf-8") as fh: | |
| for key, value in values.items(): | |
| fh.write(f"{key}={quote(value)}\n") | |
| PY | |
| echo "ENV_FILE=$ENV_FILE" >> "$GITHUB_ENV" | |
| - name: Deploy and Log Setup | |
| run: | | |
| set -euo pipefail | |
| log_step() { | |
| echo "[$(date -u +%FT%TZ)] $*" | |
| } | |
| REMOTE_TARGET="${GCE_USER}@${GCE_INSTANCE_NAME}" | |
| REMOTE_COMMON_FLAGS=( | |
| --project "${GCP_PROJECT_ID}" | |
| --zone "${GCE_ZONE}" | |
| --quiet | |
| --tunnel-through-iap | |
| --ssh-key-file "${SSH_KEY_FILE}" | |
| ) | |
| SSH_FLAGS=( | |
| "${REMOTE_COMMON_FLAGS[@]}" | |
| --ssh-flag="-o ServerAliveInterval=30" | |
| --ssh-flag="-o ServerAliveCountMax=10" | |
| --ssh-flag="-o TCPKeepAlive=yes" | |
| ) | |
| SCP_FLAGS=( | |
| "${REMOTE_COMMON_FLAGS[@]}" | |
| --scp-flag="-o ServerAliveInterval=30" | |
| --scp-flag="-o ServerAliveCountMax=10" | |
| --scp-flag="-o TCPKeepAlive=yes" | |
| ) | |
| reset_instance_and_wait_for_ssh() { | |
| log_step "Resetting ${GCE_INSTANCE_NAME} after SSH/SCP failure" | |
| gcloud compute instances reset "${GCE_INSTANCE_NAME}" \ | |
| --project "${GCP_PROJECT_ID}" \ | |
| --zone "${GCE_ZONE}" \ | |
| --quiet | |
| for attempt in {1..36}; do | |
| if gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "true" >/dev/null 2>&1; then | |
| log_step "SSH recovered after VM reset on attempt ${attempt}" | |
| return 0 | |
| fi | |
| sleep 10 | |
| done | |
| echo "SSH did not recover after VM reset." >&2 | |
| return 1 | |
| } | |
| run_remote_ssh() { | |
| local label="$1" | |
| local remote_command="$2" | |
| if gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "${remote_command}"; then | |
| return 0 | |
| fi | |
| log_step "${label} failed; resetting VM and retrying once" | |
| reset_instance_and_wait_for_ssh | |
| gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "${remote_command}" | |
| } | |
| copy_remote_file() { | |
| local source_file="$1" | |
| local dest_file="$2" | |
| if gcloud compute scp "${source_file}" "${REMOTE_TARGET}:${dest_file}" "${SCP_FLAGS[@]}"; then | |
| return 0 | |
| fi | |
| log_step "SCP upload failed; resetting VM and retrying once" | |
| reset_instance_and_wait_for_ssh | |
| gcloud compute scp "${source_file}" "${REMOTE_TARGET}:${dest_file}" "${SCP_FLAGS[@]}" | |
| } | |
| SOURCE_ARCHIVE="$RUNNER_TEMP/ibkr-gateway-source.tar.gz" | |
| REMOTE_ARCHIVE="/tmp/ibkr-gateway-${TARGET_NAME}-${GITHUB_SHA}.tar.gz" | |
| git archive --format=tar.gz --output="${SOURCE_ARCHIVE}" HEAD | |
| log_step "Uploading source archive to ${REMOTE_ARCHIVE}" | |
| copy_remote_file "${SOURCE_ARCHIVE}" "${REMOTE_ARCHIVE}" | |
| REMOTE_SYNC_COMMAND=$(cat <<EOF | |
| set -euo pipefail | |
| case '${DEPLOY_PATH}' in | |
| ''|'/'|'/home'|'/opt') | |
| echo "Refusing unsafe DEPLOY_PATH: '${DEPLOY_PATH}'" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| echo "[\$(date -u +%FT%TZ)] Syncing source archive" | |
| sudo rm -rf '${DEPLOY_PATH}' | |
| sudo mkdir -p '${DEPLOY_PATH}' | |
| sudo chown -R '${GCE_USER}':'${GCE_USER}' '${DEPLOY_PATH}' | |
| tar -xzf '${REMOTE_ARCHIVE}' -C '${DEPLOY_PATH}' | |
| rm -f '${REMOTE_ARCHIVE}' | |
| EOF | |
| ) | |
| log_step "Syncing repository to ${DEPLOY_PATH}" | |
| run_remote_ssh "Repository sync" "${REMOTE_SYNC_COMMAND}" | |
| log_step "Uploading runtime env file" | |
| copy_remote_file "${ENV_FILE}" "${DEPLOY_PATH}/.env" | |
| DEPLOY_MODE="full" | |
| if [ "${DEPLOY_EVENT_NAME}" = "schedule" ]; then | |
| DEPLOY_MODE="keepalive" | |
| elif [ "${DEPLOY_EVENT_NAME}" = "workflow_dispatch" ]; then | |
| DEPLOY_MODE="${WORKFLOW_DISPATCH_MODE:-keepalive}" | |
| fi | |
| if [ "${DEPLOY_MODE}" = "keepalive" ]; then | |
| REMOTE_DEPLOY_COMMAND=$(cat <<EOF | |
| set -euo pipefail | |
| cd '${DEPLOY_PATH}' | |
| echo "[\$(date -u +%FT%TZ)] Scheduled keepalive mode: skip docker build" | |
| set -a | |
| . ./.env | |
| set +a | |
| . ./scripts/ibkr_gateway_units.sh | |
| container_name="\${IB_GATEWAY_CONTAINER_NAME:-ib-gateway}" | |
| compose_service_name="\${IB_GATEWAY_COMPOSE_SERVICE_NAME:-ib-gateway}" | |
| resolve_ibkr_gateway_unit_names "\${container_name}" "\${IB_GATEWAY_UNIT_SUFFIX:-}" | |
| sudo systemctl stop "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" "\${IBKR_GATEWAY_HEALTHCHECK_SERVICE}" 2>/dev/null || true | |
| sudo systemctl stop "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" "\${IBKR_GATEWAY_DAILY_RESTART_SERVICE}" 2>/dev/null || true | |
| sudo bash ./scripts/ensure_host_swap.sh | |
| sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_2fa_bot_watcher.sh | |
| sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}' | |
| sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_gateway_health_watcher.sh '${IB_GATEWAY_MODE}' | |
| sudo docker compose ps | |
| sudo systemctl status "\${IBKR_2FA_BOT_TIMER}" --no-pager | |
| sudo systemctl status "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" --no-pager | |
| sudo systemctl status "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" --no-pager | |
| echo "[\$(date -u +%FT%TZ)] Keepalive complete." | |
| EOF | |
| ) | |
| else | |
| REMOTE_DEPLOY_COMMAND=$(cat <<EOF | |
| set -euo pipefail | |
| cd '${DEPLOY_PATH}' | |
| echo "[\$(date -u +%FT%TZ)] Full deploy mode: rebuilding container" | |
| set -a | |
| . ./.env | |
| set +a | |
| . ./scripts/ibkr_gateway_units.sh | |
| container_name="\${IB_GATEWAY_CONTAINER_NAME:-ib-gateway}" | |
| compose_service_name="\${IB_GATEWAY_COMPOSE_SERVICE_NAME:-ib-gateway}" | |
| resolve_ibkr_gateway_unit_names "\${container_name}" "\${IB_GATEWAY_UNIT_SUFFIX:-}" | |
| sudo systemctl stop "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" "\${IBKR_GATEWAY_HEALTHCHECK_SERVICE}" 2>/dev/null || true | |
| sudo systemctl stop "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" "\${IBKR_GATEWAY_DAILY_RESTART_SERVICE}" 2>/dev/null || true | |
| sudo bash ./scripts/ensure_host_swap.sh | |
| sudo docker compose build "\${compose_service_name}" | |
| sudo docker compose down | |
| sudo docker compose up -d "\${compose_service_name}" | |
| sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_2fa_bot_watcher.sh | |
| sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}' | |
| sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" IB_GATEWAY_UNIT_SUFFIX="\${IB_GATEWAY_UNIT_SUFFIX:-}" bash ./scripts/install_gateway_health_watcher.sh '${IB_GATEWAY_MODE}' | |
| sudo docker compose ps | |
| sudo systemctl status "\${IBKR_2FA_BOT_TIMER}" --no-pager | |
| sudo systemctl status "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" --no-pager | |
| sudo systemctl status "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" --no-pager | |
| echo "Deployment complete. Bot is monitoring the login window." | |
| echo "Run 'docker exec \${container_name} tail -f /home/ibgateway/2fa.log' for real-time status." | |
| EOF | |
| ) | |
| fi | |
| log_step "Running remote deploy commands for event ${DEPLOY_EVENT_NAME} in mode ${DEPLOY_MODE}" | |
| run_remote_ssh "Remote deploy" "${REMOTE_DEPLOY_COMMAND}" |