Skip to content

Commit a2cfd15

Browse files
snreddygopusnreddygopu
committed
Create connections using Airflow cli instead of using environment variables
1 parent c075f0d commit a2cfd15

File tree

1 file changed

+69
-34
lines changed

1 file changed

+69
-34
lines changed

‎.github/workflows/ci-teradata-main-systemtest-dashboard.yml‎

Lines changed: 69 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -85,43 +85,78 @@ jobs:
8585
echo "Teradata Server Name QVCI: '${{ steps.create-csae-environment.outputs.teradata-server-name-qvci }}'"
8686
echo "=== Connection String Preview ==="
8787
echo "AIRFLOW_CONN_TERADATA_DEFAULT will use server: ${{ steps.create-csae-environment.outputs.teradata-server-name }}"
88-
- name: "Install Airflow Teradata Provider and Run System tests"
88+
- name: "Setup Airflow and Add Connections"
8989
if: always()
9090
run: |
91-
export AIRFLOW_HOME=~/airflow
92-
mkdir -p $AIRFLOW_HOME
93-
curl -LsSf https://astral.sh/uv/install.sh | sh
94-
uv venv
95-
source .venv/bin/activate
96-
sudo apt update
91+
export AIRFLOW_HOME=~/airflow
92+
mkdir -p $AIRFLOW_HOME
93+
curl -LsSf https://astral.sh/uv/install.sh | sh
94+
uv venv
95+
source .venv/bin/activate
96+
97+
sudo apt update
98+
sudo apt-get install -y apt-transport-https apt-utils build-essential ca-certificates dirmngr \
99+
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \
100+
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \
101+
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \
102+
software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev python3-venv python3-pip pipx default-libmysqlclient-dev \
103+
default-libmysqlclient-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl libtool libkrb5-dev krb5-multidev gcc g++
97104
98-
sudo apt-get install -y apt-transport-https apt-utils build-essential ca-certificates dirmngr \
99-
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \
100-
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \
101-
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \
102-
software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev python3-venv python3-pip pipx default-libmysqlclient-dev \
103-
default-libmysqlclient-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl libtool libkrb5-dev krb5-multidev gcc g++
104-
105-
sudo pip install pytest time-machine pytest-asyncio pytest-md-report cryptography pytest-html tailer pandas jinja2 sqlalchemy hatch
106-
107-
uv sync --extra teradata --extra microsoft.azure --extra amazon
108-
109-
export SYSTEM_TESTS_ENV_ID=teradatasystemtest
110-
export AIRFLOW_CONN_TERADATA_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}'
111-
export AIRFLOW_CONN_SSH_DEFAULT='ssh://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}'
112-
export AIRFLOW_CONN_TERADATA_LAKE='teradata://${{ vars.LAKE_ENV_USERNAME }}:${{ secrets.LAKE_ENV_PASSWORD }}@${{ vars.LAKE_SERVER_NAME }}:/${{ vars.LAKE_ENV_USERNAME }}'
113-
export AIRFLOW_CONN_TERADATA_SSL_DEFAULT='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?sslmode=allow'
114-
export AIRFLOW_CONN_TERADATA_SP_CALL='teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?tmode=TERA'
115-
export AIRFLOW_CONN_WASB_DEFAULT='wasb://${{ vars.AZURE_BLOB_ACCOUNTNAME }}:${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY_URI }}@'
116-
export AIRFLOW_CONN_AWS_DEFAULT=aws://${{ vars.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}@/?aws_session_token=${{ secrets.AWS_SESSION_TOKEN }}
117-
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNTNAME=${{ vars.AZURE_BLOB_ACCOUNTNAME }}
118-
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNT_SECRET_KEY=${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY }}
119-
export AIRFLOW_VAR_AWS_ACCESS_KEY_ID=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_ID }}
120-
export AIRFLOW_VAR_AWS_SECRET_ACCESS_KEY=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_KEY }}
121-
122-
123-
uv run pytest --keep-env-variables --system -W ignore::pytest.PytestRemovedIn9Warning --junitxml=report_test.xml providers/teradata/tests/system/teradata/*.py
124-
cat report_test.xml
105+
sudo pip install pytest time-machine pytest-asyncio pytest-md-report cryptography pytest-html tailer pandas jinja2 sqlalchemy hatch
106+
107+
uv sync --extra teradata --extra microsoft.azure --extra amazon
108+
109+
# Initialize Airflow DB
110+
airflow db migrate
111+
112+
# Add Airflow connections using CLI
113+
echo "Adding Teradata connections to Airflow metadata database..."
114+
airflow connections delete teradata_default 2>/dev/null || true
115+
airflow connections add 'teradata_default' \
116+
--conn-uri "teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}"
117+
118+
airflow connections delete ssh_default 2>/dev/null || true
119+
airflow connections add 'ssh_default' \
120+
--conn-uri "ssh://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}"
121+
122+
airflow connections delete teradata_lake 2>/dev/null || true
123+
airflow connections add 'teradata_lake' \
124+
--conn-uri "teradata://${{ vars.LAKE_ENV_USERNAME }}:${{ secrets.LAKE_ENV_PASSWORD }}@${{ vars.LAKE_SERVER_NAME }}:/${{ vars.LAKE_ENV_USERNAME }}"
125+
126+
airflow connections delete teradata_ssl_default 2>/dev/null || true
127+
airflow connections add 'teradata_ssl_default' \
128+
--conn-uri "teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?sslmode=allow"
129+
130+
airflow connections delete teradata_sp_call 2>/dev/null || true
131+
airflow connections add 'teradata_sp_call' \
132+
--conn-uri "teradata://${{ vars.CSAE_AIRFLOW_ENV_USERNAME }}:${{ secrets.CSAE_AIRFLOW_ENV_PASSWORD }}@${{ steps.create-csae-environment.outputs.teradata-server-name }}:/${{ vars.CSAE_AIRFLOW_ENV_DATABASE }}?tmode=TERA"
133+
134+
airflow connections delete wasb_default 2>/dev/null || true
135+
airflow connections add 'wasb_default' \
136+
--conn-uri "wasb://${{ vars.AZURE_BLOB_ACCOUNTNAME }}:${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY_URI }}@"
137+
138+
airflow connections delete aws_default 2>/dev/null || true
139+
airflow connections add 'aws_default' \
140+
--conn-uri "aws://${{ vars.AWS_ACCESS_KEY_ID }}:${{ secrets.AWS_SECRET_ACCESS_KEY }}@/?aws_session_token=${{ secrets.AWS_SESSION_TOKEN }}"
141+
142+
# List connections to verify
143+
echo "=== Verifying Airflow Connections ==="
144+
airflow connections list
145+
- name: "Run System Tests"
146+
if: always()
147+
run: |
148+
export AIRFLOW_HOME=~/airflow
149+
source .venv/bin/activate
150+
151+
export SYSTEM_TESTS_ENV_ID=teradatasystemtest
152+
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNTNAME=${{ vars.AZURE_BLOB_ACCOUNTNAME }}
153+
export AIRFLOW_VAR_AZURE_BLOB_ACCOUNT_SECRET_KEY=${{ secrets.AZURE_BLOB_ACCOUNT_SECRET_KEY }}
154+
export AIRFLOW_VAR_AWS_ACCESS_KEY_ID=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_ID }}
155+
export AIRFLOW_VAR_AWS_SECRET_ACCESS_KEY=${{ secrets.TERADATA_DOWNLOAD_AWS_SECRET_ACCESS_KEY }}
156+
157+
echo "=== Running Teradata System Tests ==="
158+
uv run pytest --system -W ignore::pytest.PytestRemovedIn9Warning --junitxml=report_test.xml providers/teradata/tests/system/teradata/*.py
159+
cat report_test.xml
125160
- name: "Delete CSAE environment"
126161
if: always()
127162
run: |

0 commit comments

Comments
 (0)