Running on amazonlinux2023 + Java 17 Added support for json logging layout

CHANGELOG.md

@@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.13.1] - 2025-07-28
+### Changed
+- Upgraded the base docker image to amazonlinux:2023
+- Upgraded to Java 17 Amazon Corretto
+- Switched from xml to yaml Log4j config for better readability
+- Renamed env vars controlling Log Levels to have consistent naming
+### 
+- Added an option to use the default JSON layout in Log4j
+- Added the datadog java agent for optional profiling
+
 ## [1.13.0] - 2025-07-22
 ### Changed
 - Upgrade `WAGGLE_DANCE_VERSION` to `4.0.2` (was `3.13.8`). See [Changelog](https://github.com/ExpediaGroup/waggle-dance/blob/main/CHANGELOG.md)

Dockerfile

@@ -1,29 +1,39 @@
 # Copyright (C) 2018 Expedia Inc.
 # Licensed under the Apache License, Version 2.0 (the "License");
 
-FROM amazonlinux:2
+FROM amazonlinux:2023
 
-ARG JAVA_VERSION=1.8.0
+ARG JAVA_VERSION=java-17-amazon-corretto
 ARG WAGGLE_DANCE_VERSION=4.0.3
 ARG APIARY_EXTENSIONS_VERSION=7.3.2
+ARG DD_JAVA_AGENT_VERSION=1.51.2
 
 ENV WAGGLE_DANCE_HOME=/opt/waggle-dance
 
-RUN yum -y update && \
-  yum install -y java-${JAVA_VERSION}-openjdk \
-    procps \
-    awscli \
-    wget \
-    util-linux \
-    jq \
+RUN dnf install -y ${JAVA_VERSION} procps wget util-linux jq zip \
     https://repo1.maven.org/maven2/com/expediagroup/waggle-dance-rpm/${WAGGLE_DANCE_VERSION}/waggle-dance-rpm-${WAGGLE_DANCE_VERSION}.rpm \
-  && yum clean all \
+  && dnf clean all \
   && rm -rf /var/cache/yum
 
+RUN ARCH=$(uname -m) && \
+  if [ "$ARCH" = "x86_64" ]; then \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip ; \
+  elif [ "$ARCH" = "aarch64" ]; then \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o /tmp/awscliv2.zip ; \
+  else \
+    echo "Unsupported architecture: $ARCH" && exit 1 ; \
+  fi \
+  && unzip /tmp/awscliv2.zip -d /tmp \
+  && /tmp/aws/install -i /opt/aws-cli -b /usr/local/bin \
+  && rm -rf /tmp/awscliv2.zip /tmp/aws  
+
 ADD https://repo1.maven.org/maven2/com/expediagroup/apiary/hive-hooks/${APIARY_EXTENSIONS_VERSION}/hive-hooks-${APIARY_EXTENSIONS_VERSION}.jar "${WAGGLE_DANCE_HOME}"/jars/
+ADD https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${DD_JAVA_AGENT_VERSION}/dd-java-agent-${DD_JAVA_AGENT_VERSION}.jar "${WAGGLE_DANCE_HOME}"/jars/dd-java-agent.jar
+ADD https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-layout-template-json/2.17.1/log4j-layout-template-json-2.17.1.jar /tmp
 
 COPY files/waggle-dance-server.yml "${WAGGLE_DANCE_HOME}"/conf/
 COPY files/waggle-dance-federation.yml "${WAGGLE_DANCE_HOME}"/conf/
+COPY files/log4j2.yml "${WAGGLE_DANCE_HOME}"/conf/
 COPY files/waggle-dance-core-latest-exec.conf "${WAGGLE_DANCE_HOME}"/service/
 COPY scripts/startup.sh "${WAGGLE_DANCE_HOME}"
 

files/log4j2.yml

@@ -0,0 +1,55 @@
+# Copyright (C) 2016-2024 Expedia, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Configuration:
+  monitorInterval: 30
+
+  Appenders:
+    Console:
+      - name: STDOUT-TEXT
+        target: SYSTEM_OUT
+        PatternLayout:
+          pattern: "%d{ISO8601} [%tn] %-5p %c:%L - %m%n"
+
+      - name: STDOUT-JSON
+        target: SYSTEM_OUT
+        JsonLayout:
+          compact: false
+          eventEol: true
+          includeStacktrace: true
+          properties: true
+          includeTimeMillis: true
+          includeSource: true
+
+  Loggers:
+    Logger:
+      - name: com.hotels.bdp.waggledance
+        level: ${env:LOGGING_LEVEL_WAGGLEDANCE:-info}
+      - name: com.hotels.bdp.waggledance.server.invocation-log
+        level: ${env:LOGGING_LEVEL_INVOCATION:-info}        
+      - name: com.amazonaws.glue.catalog.metastore.AWSGlueClientFactory
+        level: ${env:LOGGING_LEVEL_AWS_GLUE:-warn}
+      - name: com.amazonaws
+        level: ${env:LOGGING_LEVEL_AWS:-warn}
+      - name: org.apache.hadoop
+        level: ${env:LOGGING_LEVEL_APACHE:-warn}
+      - name: org.apache.hive
+        level: ${env:LOGGING_LEVEL_APACHE:-warn}
+      - name: org.springframework
+        level: ${env:LOGGING_LEVEL_SPRING:-warn}
+
+    Root:
+      level: ${env:LOGLEVEL:-info}
+      AppenderRef:
+        - ref: ${env:LOG4J_APPENDER:-STDOUT-TEXT}

files/waggle-dance-core-latest-exec.conf

@@ -4,5 +4,5 @@
 # Spring boot service conf: see http://docs.spring.io/spring-boot/docs/current/reference/html/deployment-install.html#deployment-script-customization-conf-file
 
 LOG_FOLDER=/var/log/waggle-dance/
-JAVA_OPTS="-Dloader.path=/opt/waggle-dance/jars -XshowSettings:vm -Xms${HEAPSIZE}m -Xmx${HEAPSIZE}m -XX:+UseG1GC -XX:G1ReservePercent=15 -Dlog4j.configurationFile=/opt/waggle-dance/conf/log4j2.xml -Dlogging.config=/opt/waggle-dance/conf/log4j2.xml"
+JAVA_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED -Dloader.path=/opt/waggle-dance/jars -XshowSettings:vm -Xms${HEAPSIZE}m -Xmx${HEAPSIZE}m -XX:+UseG1GC -XX:G1ReservePercent=15 -Dlog4j.configurationFile=/opt/waggle-dance/conf/log4j2.yml -Dlogging.config=/opt/waggle-dance/conf/log4j2.yml"
 RUN_ARGS="--server-config=/opt/waggle-dance/conf/waggle-dance-server.yml --federation-config=/opt/waggle-dance/conf/waggle-dance-federation.yml"

scripts/startup.sh

@@ -25,12 +25,29 @@ source "${WAGGLE_DANCE_HOME}"/service/waggle-dance-core-latest-exec.conf
 
 [[ -n $HIVE_SITE_XML ]] && echo $HIVE_SITE_XML|base64 -d > ${WAGGLE_DANCE_HOME}/jars/hive-site.xml
 
-[[ -z "${INVOCATIONLOGLEVEL}" ]] && INVOCATIONLOGLEVEL=${LOGLEVEL:-info}
-[[ -z "${WDLOGLEVEL}" ]] && WDLOGLEVEL=${LOGLEVEL:-info}
+# Configure logging format based on environment variable
+if [[ "${LOGGING_STRUCTURED_FORMAT_CONSOLE}" == "json" ]]; then
+    export LOG4J_APPENDER="STDOUT-JSON"
+    echo "Using JSON structured logging"
+else
+    export LOG4J_APPENDER="STDOUT-TEXT"
+    echo "Using text pattern logging"
+fi
+
+if [[ "${LOG4J_DEBUG:-false}" == "true" ]]; then
+    JAVA_OPTS="$JAVA_OPTS -Dlog4j2.debug=true"
+    echo "Log4j2 debug mode enabled"
+fi
+
+if [[ -n $DD_PROFILING_ENABLED && "$DD_PROFILING_ENABLED" = "true" ]]; then
+  # To enable the Datadog Java Agent, the following environment variables must be set at runtime:
+  # DD_PROFILING_ENABLED=true
+  # DD_SERVICE=service-name
+  # DD_ENV=env-name
+  # DD_VERSION=version
+  export JAVA_OPTS="$JAVA_OPTS -javaagent:/opt/waggle-dance/jars/dd-java-agent.jar -XX:FlightRecorderOptions=stackdepth=256 -Ddd.profiling.enabled=${DD_PROFILING_ENABLED} -Ddd.service=${DD_SERVICE} -Ddd.env=${DD_ENV} -Ddd.version=${DD_VERSION}"
+fi  
 
-sed "/invocation-log/!s/level=\".*\"/level=\"${LOGLEVEL:-info}\"/"    -i /opt/waggle-dance/conf/log4j2.xml
-sed "/invocation-log/s/level=\".*\"/level=\"${INVOCATIONLOGLEVEL}\"/" -i /opt/waggle-dance/conf/log4j2.xml
-# Allow override of WaggleDance package logs.
-sed "/<Logger name=\"com.hotels.bdp.waggledance\"/s/level=\".*\"/level=\"${WDLOGLEVEL}\"/" -i /opt/waggle-dance/conf/log4j2.xml
+echo "=== Starting Waggle Dance ==="
 
 exec java $JAVA_OPTS -jar "${WAGGLE_DANCE_HOME}"/service/waggle-dance-core-latest-exec.jar $RUN_ARGS