Merge pull request #168 from OpenVoxProject/fix_rake_tasks

Refactor Vox rake tasks

Author: nmburgan

Rakefile

@@ -1,32 +1,4 @@
 require 'rake'
-require 'open3'
-
-RED = "\033[31m"
-GREEN = "\033[32m"
-RESET = "\033[0m"
-
-def run_command(cmd, silent: true, print_command: false, report_status: false, allowed_exit_codes: [0])
-  puts "#{GREEN}Running #{cmd}#{RESET}" if print_command
-  output = ''
-  Open3.popen2e(cmd) do |_stdin, stdout_stderr, thread|
-    stdout_stderr.each do |line|
-      puts line unless silent
-      output += line
-    end
-    exitcode = thread.value.exitstatus
-    unless allowed_exit_codes.include?(exitcode)
-      err = "#{RED}Command failed! Command: #{cmd}, Exit code: #{exitcode}"
-      # Print details if we were running silent
-      err += "\nOutput:\n#{output}" if silent
-      err += RESET
-      abort err
-    end
-    puts "#{GREEN}Command finished with status #{exitcode}#{RESET}" if report_status
-  end
-  output.chomp
-end
-
-Dir.glob(File.join('tasks/**/*.rake')).each { |file| load file }
 
 ### puppetlabs stuff ###
 def run_beaker(test_files)
@@ -211,30 +183,3 @@ namespace :release do
     end
   end
 end
-
-begin
-  require 'github_changelog_generator/task'
-rescue LoadError
-  task :changelog do
-    abort('Run `bundle install --with release` to install the `github_changelog_generator` gem.')
-  end
-else
-  GitHubChangelogGenerator::RakeTask.new :changelog do |config|
-    config.header = <<~HEADER.chomp
-      # Changelog
-
-      All notable changes to this project will be documented in this file.
-    HEADER
-    config.user = 'openvoxproject'
-    config.project = 'openvoxdb'
-    config.exclude_labels = %w[dependencies duplicate question invalid wontfix wont-fix modulesync skip-changelog]
-    # this is probably the worst way to do this
-    # ideally there would be a VERSION file and clojure and Rake would read it
-    config.future_release = File.readlines('project.clj').first.scan(/".*"/).first.gsub('"', '')
-    # we limit the changelog to all new openvox releases, to skip perforce onces
-    # otherwise the changelog generate takes a lot amount of time
-    config.since_tag = '8.9.1'
-    #config.exclude_tags_regex = /\A7\./
-    config.release_branch = 'main'
-  end
-end

rakelib/build.rake

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+# Builds packages using a Docker container
+# Env var options:
+#  NO_TEARDOWN - If set, do not stop the docker container after the build for debugging purposes.
+#  EZBAKE_BRANCH - If set, use this ezbake branch for building. If used, must also set EZBAKE_VERSION accordingly.
+#  EZBAKE_REPO - If EZBAKE_BRANCH is set, use this repo URL for ezbake (default: https://github.com/openvoxproject/ezbake)
+#  EZBAKE_VERSION - If set, use this version string in project.clj for ezbake. Must correspond to the version currently in defproject
+#                   in EZBAKE_BRANCH.
+#  FULL_DEP_REBUILD_BRANCH - If set, rebuild all dependencies, where all dependency repos have this branch present with your
+#                            desired changes. Overrides any other DEP_ settings.
+#  DEP_REBUILD - Comma-separated list of dependencies to rebuild from source. Should be the repo names.
+#  DEP_REBUILD_BRANCH - If DEP_REBUILD is set, use this branch for the specified dependencies (default: main)
+#  DEP_REBUILD_ORG - If DEP_REBUILD is set, use this GitHub org for the specified dependencies to build the repo URL (default: openvoxproject)
+#  DEB_PLATFORMS - Comma-separated list of debian/ubuntu platforms to build for
+#  RPM_PLATFORMS - Comma-separated list of rpm platforms to build for
+#  FIPS - If set, build specified platforms with the appropriate 'fips' lein profile(s) enabled.
+
+require 'fileutils'
+require 'tmpdir'
+require_relative 'utils/docker_runner'
+
+module Vox
+  class Build
+    def platform_targets
+      # It seems like these are special files/names that, when you want to add a new one, require
+      # changes in some other component.  But no, it seems to only really look at the parts of
+      # the text in the string, as long as it looks like "base-<whatever you want to call the platform>-i386.cow"
+      # and "<doesn't matter>-<os>-<osver>-<arch which doesn't matter because it's actually noarch>".
+      # I think it just treats all debs like Debian these days. And all rpms are similar.
+      # So do whatever you want I guess. We really don't need separate packages for each platform.
+      # To be fixed one of these days. Relevant stuff:
+      #   https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/tasks/build.rake
+      #   https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/ext/fpm.rb
+      deb_platforms = ENV['DEB_PLATFORMS'] || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,ubuntu-25.04,debian-11,debian-12,debian-13'
+      rpm_platforms = ENV['RPM_PLATFORMS'] || 'el-8,el-9,el-10,sles-15,sles-16,amazon-2,amazon-2023,fedora-42,fedora-43'
+
+      debs = deb_platforms.split(',').map { |p| "base-#{p.split("-").join}-i386.cow" }.join(" ")
+      rpms = rpm_platforms.split(',').map { |p| "pl-#{p}-x86_64" }.join(" ")
+
+      [debs, rpms]
+    end
+    # The deps must be built in this order due to dependencies between them.
+    # There is a circular dependency between clj-http-client and trapperkeeper-webserver-jetty10,
+    # but only for tests, so the build *should* work.
+    DEP_BUILD_ORDER = [
+      'clj-parent',
+      'clj-kitchensink',
+      'clj-i18n',
+      'comidi',
+      'jvm-ssl-utils',
+      'trapperkeeper',
+      'trapperkeeper-filesystem-watcher',
+      'trapperkeeper-webserver-jetty10',
+      'trapperkeeper-authorization',
+      'trapperkeeper-metrics',
+      'trapperkeeper-status',
+      'stockpile',
+      'structured-logging',
+    ].freeze
+
+    def initialize(tag:)
+      @tag = tag
+      @runner = Vox::DockerRunner.new(container_name: 'openvoxdb-builder', image: 'ezbake-builder')
+      @deps_tmp = Dir.mktmpdir('deps')
+    end
+
+    def build
+      checkout_tag_if_requested
+      build_image_unless_present
+      start_container
+      build_and_install_libs(prepare_deps)
+      build_project
+      postprocess_output
+    ensure
+      @runner.teardown unless ENV['NO_TEARDOWN']
+      FileUtils.rm_rf(@deps_tmp)
+    end
+
+    private
+
+    def checkout_tag_if_requested
+      if @tag.nil? || @tag.empty?
+        puts 'Running build with current branch'
+      else
+        puts "Running build on #{@tag}"
+        Vox::Shell.run("git fetch --tags && git checkout #{@tag}")
+      end
+    end
+
+    def build_image_unless_present
+      return if @runner.image_exists?
+
+      # If the Dockerfile has changed since this was last built,
+      # delete all containers and do `docker rmi ezbake-builder`
+      puts 'Building ezbake-builder image'
+      Vox::Shell.run("docker build -t ezbake-builder .", silent: false, print_command: true)
+    end
+
+    def prepare_deps
+      libs = {}
+
+      # Manage ezbake override
+      ezbake_branch = ENV.fetch('EZBAKE_BRANCH', '').strip
+      unless ezbake_branch.empty?
+        libs['ezbake'] = {
+          repo: ENV.fetch('EZBAKE_REPO', 'https://github.com/openvoxproject/ezbake'),
+          branch: ENV.fetch('EZBAKE_BRANCH', 'main'),
+        }
+      end
+
+      # Decide if we're rebuilding everything or a subset
+      full_rebuild_branch = ENV.fetch('FULL_DEP_REBUILD_BRANCH', '').strip
+      subset_list = ENV.fetch('DEP_REBUILD', '').split(',').map(&:strip).reject(&:empty?)
+      subset_branch = ENV.fetch('DEP_REBUILD_BRANCH', 'main').strip
+      rebuild_org = ENV.fetch('DEP_REBUILD_ORG', 'openvoxproject').strip
+
+      selected_libs = []
+      selected_branch = nil
+
+      if !full_rebuild_branch.empty?
+        selected_branch = full_rebuild_branch
+        selected_libs = DEP_BUILD_ORDER.dup
+      elsif !subset_list.empty?
+        selected_branch = subset_branch
+        unknown = subset_list - DEP_BUILD_ORDER
+        puts "WARNING: Unknown deps in DEP_REBUILD (will be ignored): #{unknown.join(', ')}" unless unknown.empty?
+        selected_libs = DEP_BUILD_ORDER & subset_list # Keeps DEP_BUILD_ORDER ordering
+      end
+
+      selected_libs.each do |lib|
+        libs[lib] = { repo: "https://github.com/#{rebuild_org}/#{lib}", branch: selected_branch }
+      end
+
+      libs.each do |lib, config|
+        puts "Checking out #{lib}"
+        Vox::Shell.run(
+          "git clone --revision #{config[:branch]} #{config[:repo]} #{@deps_tmp}/#{lib}",
+          silent: false,
+          print_command: true
+        )
+      end
+
+      libs
+    end
+
+    def start_container
+      @runner.teardown if @runner.container_exists?
+      volumes = [[Dir.pwd, '/code'], [@deps_tmp, '/deps']]
+      @runner.start(volumes: volumes)
+    end
+
+    def build_and_install_libs(libs)
+      libs.each_key do |lib|
+        puts "Building and installing #{lib} from source"
+        @runner.exec("cd /deps/#{lib} && lein install")
+      end
+    end
+
+    def build_project
+      debs, rpms = platform_targets
+      fips = !ENV['FIPS'].nil?
+      ezbake_version_var = ENV['EZBAKE_VERSION'] ? "EZBAKE_VERSION=#{ENV['EZBAKE_VERSION']}" : ""
+
+      puts 'Building openvoxdb'
+      @runner.exec('cd /code && rm -rf ruby output && bundle install --without test && lein install')
+      @runner.exec(
+        "cd /code && COW=\"#{debs}\" MOCK=\"#{rpms}\" GEM_SOURCE='https://rubygems.org' #{ezbake_version_var} " \
+        "EZBAKE_ALLOW_UNREPRODUCIBLE_BUILDS=true EZBAKE_NODEPLOY=true LEIN_PROFILES=ezbake " \
+        "lein with-profile #{fips ? "fips," : ""}user,ezbake,provided,internal ezbake local-build"
+      )
+    end
+
+    def postprocess_output
+      Vox::Shell.run("sudo chown -R $USER output", print_command: true)
+      Dir.glob("output/**/*i386*").each { |f| FileUtils.rm_rf(f) }
+      Dir.glob("output/puppetdb-*.tar.gz").each { |f| FileUtils.mv(f, f.sub('puppetdb', 'openvoxdb')) }
+    end
+  end
+end
+
+namespace :vox do
+  desc 'Build openvoxdb packages with Docker'
+  task :build, [:tag] do |_, args|
+    Vox::Build.new(tag: args[:tag]).build
+  end
+end

rakelib/changelog.rake

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+begin
+  require 'github_changelog_generator/task'
+rescue LoadError
+  task :changelog do
+    abort('Run `bundle install --with release` to install the `github_changelog_generator` gem.')
+  end
+else
+  GitHubChangelogGenerator::RakeTask.new :changelog do |config|
+    config.header = <<~HEADER.chomp
+      # Changelog
+
+      All notable changes to this project will be documented in this file.
+    HEADER
+    config.user = 'openvoxproject'
+    config.project = 'openvoxdb'
+    config.exclude_labels = %w[dependencies duplicate question invalid wontfix wont-fix modulesync skip-changelog]
+    # this is probably the worst way to do this
+    # ideally there would be a VERSION file and clojure and Rake would read it
+    config.future_release = File.readlines('project.clj').first.scan(/".*"/).first.gsub('"', '')
+    # we limit the changelog to all new openvox releases, to skip perforce onces
+    # otherwise the changelog generate takes a lot amount of time
+    config.since_tag = '8.9.1'
+    #config.exclude_tags_regex = /\A7\./
+    config.release_branch = 'main'
+  end
+end

rakelib/test.rake

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Tests OpenVoxDB in the same manner as we do in PR tests.
+# Only tested with a handful of the cells we use in CI. May need adjustment for others.
+# 
+# Env var options:
+#   NO_TEARDOWN - If set, do not stop the docker container after the tests for debugging purposes.
+
+require_relative 'utils/docker_runner'
+
+module Vox
+  class Test
+    def initialize(spec:)
+      @spec = spec
+      @runner = Vox::DockerRunner.new(container_name: 'openvoxdb-test', image: select_image(@spec))
+    end
+
+    def run_tests
+      @runner.teardown if @runner.container_exists?
+      @runner.start(volumes: [[Dir.pwd, '/code']])
+
+      # The tests make tons of temp dirs in the current working directory, and trying to do this
+      # inside the volume mount dir results in permissions issues. So we copy all of the code to /tmp/code
+      # and run from there. Not ideal, but we have to since it curls down pdbbox and we don't control that.
+      @runner.exec('cp -r /code /tmp && chown -R root:root /tmp/code')
+      @runner.exec('apt update && apt install -y leiningen curl python3 procps')
+      @runner.exec("cd /tmp/code && rm -rf ci/local && ext/bin/prep-debianish-root --for #{@spec} --install ci/local")
+      @runner.exec('echo "postgres ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers')
+      # There is a non-fatal error when running on an arm64 host, so we can ignore exit 255.
+      @runner.exec("cd /tmp/code && ci/bin/prep-and-run-in local #{@spec}", allowed_exit_codes: [0, 255])
+      @runner.exec('chown -R postgres:postgres /tmp/code')
+      @runner.exec("cd /tmp/code && NO_ACCEPTANCE=true ci/bin/run #{@spec}", user: 'postgres')
+    ensure
+      @runner.teardown unless ENV['NO_TEARDOWN']
+    end
+
+    private
+
+    def select_image(spec)
+      _suite, java, _pg = spec.split('/')
+      java =~ /17/ ? 'ruby:3.2-bookworm' : 'ruby:3.2-trixie'
+    end
+  end
+end
+
+namespace :vox do
+  desc 'Run lein test locally in the same way that PR checks run with a properly configured postgres and other artifacts.'
+  task :test, [:spec] do |_, args|
+    Vox::Test.new(spec: (args[:spec] || 'core+ext/openjdk17/pg-17')).run_tests
+  end
+end

tasks/upload.rake rakelib/upload.raketasks/upload.rake renamed to rakelib/upload.rake

@@ -1,15 +1,17 @@
+# frozen_string_literal: true
+
+require_relative 'utils/shell'
+
 namespace :vox do
   desc 'Upload artifacts from the output directory to S3. Requires the AWS CLI to be installed and configured appropriately.'
   task :upload, [:platform] do |_, args|
     endpoint = ENV.fetch('ENDPOINT_URL')
     bucket = ENV.fetch('BUCKET_NAME')
     component = 'openvoxdb'
     os = nil
-    arch = nil
     if args[:platform]
       parts = args[:platform].split('-')
       os = parts[0].gsub('fedora','fc') + parts[1]
-      arch = parts[2]
     end
 
     abort 'You must set the ENDPOINT_URL environment variable to the S3 server you want to upload to.' if endpoint.nil? || endpoint.empty?
@@ -18,7 +20,7 @@ namespace :vox do
     s3 = "aws s3 --endpoint-url=#{endpoint}"
 
     # Ensure the AWS CLI isn't going to fail with the given parameters
-    run_command("#{s3} ls s3://#{bucket}/")
+    Vox::Shell.run("#{s3} ls s3://#{bucket}/")
 
     config = File.expand_path("../target/staging/ezbake.rb", __dir__)
     abort "Could not find ezbake config from the build at #{config}" unless File.exist?(config)
@@ -40,7 +42,7 @@ namespace :vox do
 
     path = "s3://#{bucket}/#{component}/#{tag}"
     files.each do |f|
-      run_command("#{s3} cp #{f} #{path}/#{File.basename(f)}", silent: false)
+      Vox::Shell.run("#{s3} cp #{f} #{path}/#{File.basename(f)}", silent: false)
     end
   end
 end