Add --cachedir

This caches downloaded files and git repos so when we are working on changes to packaging, we don't have to download them every time we run the build again.

Author: nmburgan

lib/vanagon/cli/build.rb

@@ -22,6 +22,7 @@ class Build < Vanagon::CLI
                                              managed automatically based on `keepwork` option
           -k, --keepwork RULE              Rule for preserving local `workdir`: [Default: never]
                                              always, on-success, on-failure, never
+          --cachedir DIRECTORY             Directory to cache source files and git repos
           -v, --verbose                    Only here for backwards compatibility. Does nothing.
 
         Engines:
@@ -64,6 +65,7 @@ def options_translate(docopt_options)
         translations = {
           '--verbose' => :verbose,
           '--workdir' => :workdir,
+          '--cachedir' => :cachedir,
           '--remote-workdir' => :'remote-workdir',
           '--configdir' => :configdir,
           '--engine' => :engine,

lib/vanagon/component.rb

@@ -313,8 +313,8 @@ def deprecated_rewrite_url
     # makefile template
     #
     # @param workdir [String] working directory to put the source into
-    def get_source(workdir) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
-      opts = options.merge({ workdir: workdir, dirname: dirname })
+    def get_source(workdir, cachedir = nil) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
+      opts = options.merge({ workdir: workdir, cachedir: cachedir, dirname: dirname })
       if url || !mirrors.empty?
         if %w[y yes true 1].include? ENV.fetch('VANAGON_USE_MIRRORS', 'n').downcase
           fetch_mirrors(opts) || fetch_url(opts)
@@ -358,10 +358,10 @@ def get_dependency_hash
     # Fetches secondary sources for the component. These are just dumped into the workdir currently.
     #
     # @param workdir [String] working directory to put the source into
-    def get_sources(workdir) # rubocop:disable Metrics/AbcSize
+    def get_sources(workdir, cachedir) # rubocop:disable Metrics/AbcSize
       sources.each do |source|
         src = Vanagon::Component::Source.source(
-          source.url, workdir: workdir, ref: source.ref, sum: source.sum
+          source.url, workdir: workdir, cachedir: cachedir, ref: source.ref, sum: source.sum
         )
         src.fetch
         src.verify

lib/vanagon/component/source.rb

@@ -17,6 +17,7 @@ class << self
         # @param uri_instance [#to_s] URI of the source file (includes git@... style links)
         # @param options [Hash] hash of the options needed for the subtype
         # @param workdir [String] working directory to fetch the source into
+        # @param cacheedir [String] directory to cache source files and git repos
         # @return [Vanagon::Component::Source] the correct subtype for the given source
         def source(uri_instance, **options) # rubocop:disable Metrics/AbcSize
           # Sometimes the uri comes in as a string, but sometimes it's already been
@@ -39,6 +40,7 @@ def source(uri_instance, **options) # rubocop:disable Metrics/AbcSize
               sum: options[:sum],
               ref: options[:ref],
               workdir: options[:workdir],
+              cachedir: options[:cachedir],
               dirname: options[:dirname],
               clone_options: options[:clone_options]
           end
@@ -47,6 +49,7 @@ def source(uri_instance, **options) # rubocop:disable Metrics/AbcSize
             return Vanagon::Component::Source::Http.new uri,
               sum: options[:sum],
               workdir: options[:workdir],
+              cachedir: options[:cachedir],
               # Default sum_type is md5 if unspecified:
               sum_type: options[:sum_type] || "md5"
           end

lib/vanagon/component/source/git.rb

@@ -13,7 +13,7 @@ class Vanagon
   class Component
     class Source
       class Git
-        attr_accessor :url, :log_url, :ref, :workdir, :clone_options
+        attr_accessor :url, :log_url, :ref, :workdir, :cachedir, :clone_options
         attr_reader :version, :default_options, :repo
 
         class << self
@@ -94,7 +94,7 @@ def default_options # rubocop:disable Lint/DuplicateMethods
         # @param url [String] url of git repo to use as source
         # @param ref [String] ref to checkout from git repo
         # @param workdir [String] working directory to clone into
-        def initialize(url, workdir:, **options) # rubocop:disable Metrics/AbcSize
+        def initialize(url, workdir:, cachedir: nil, **options) # rubocop:disable Metrics/AbcSize
           opts = default_options.merge(options.reject { |k, v| v.nil? })
 
           # Ensure that #url returns a URI object
@@ -103,6 +103,7 @@ def initialize(url, workdir:, **options) # rubocop:disable Metrics/AbcSize
           @ref = opts[:ref]
           @dirname = opts[:dirname]
           @workdir = File.realpath(workdir)
+          @cachedir = cachedir ? File.realpath(cachedir) : nil
           @clone_options = opts[:clone_options] ||= {}
 
           # We can test for Repo existence without cloning
@@ -114,7 +115,13 @@ def initialize(url, workdir:, **options) # rubocop:disable Metrics/AbcSize
         # a side effect.
         def fetch
           begin
-            @clone ||= ::Git.open(File.join(workdir, dirname))
+            if @cachedir && File.directory?("#{@cachedir}/#{dirname}")
+              VanagonLogger.info "Using cached copy of Git repo '#{dirname}' in #{cachedir}"
+              FileUtils.cp_r("#{@cachedir}/#{dirname}", workdir)
+              @clone = ::Git.init(File.join(workdir, dirname))
+            else
+              @clone ||= ::Git.open(File.join(workdir, dirname))
+            end
             @clone.fetch
           rescue ::Git::Error, ArgumentError
             clone!

lib/vanagon/component/source/http.rb

@@ -12,7 +12,7 @@ class Http < Vanagon::Component::Source::Local
         include Vanagon::Utilities
 
         # Accessors :url, :file, :extension, :workdir, :cleanup are inherited from Local
-        attr_accessor :sum, :sum_type
+        attr_accessor :sum, :sum_type, :cachedir
 
         # Allowed checksum algorithms to use when validating files
         CHECKSUM_TYPES = %w[md5 sha1 sha256 sha512].freeze
@@ -49,7 +49,7 @@ def valid_url?(target_url)
         # @param workdir [String] working directory to download into
         # @param sum_type [String] type of sum we are verifying
         # @raise [RuntimeError] an exception is raised is sum is nil
-        def initialize(url, sum:, workdir:, sum_type:, **options)
+        def initialize(url, sum:, workdir:, sum_type:, cachedir: nil, **options)
           unless sum
             fail "sum is required to validate the http source"
           end
@@ -63,6 +63,7 @@ def initialize(url, sum:, workdir:, sum_type:, **options)
           @url = url
           @sum = sum
           @workdir = workdir
+          @cachedir = cachedir ? File.realpath(cachedir) : nil
           @sum_type = sum_type
 
           if Vanagon::Component::Source::Http.valid_url?(@sum)
@@ -84,6 +85,10 @@ def initialize(url, sum:, workdir:, sum_type:, **options)
         # file as @file and the @extension for the file as a side effect.
         def fetch
           @file = File.basename(URI.parse(@url).path)
+          if @cachedir && File.exist?(File.join(@cachedir, @file))
+            VanagonLogger.info "Using cached copy of #{@file} in #{@cachedir}"
+            FileUtils.cp(File.join(@cachedir, @file), workdir)
+          end
           if File.exist?(File.join(workdir, file))
             begin
               return if verify
@@ -138,9 +143,13 @@ def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "id
                 location = URI.parse(response.header['location'])
                 download(uri + location, target_file)
               when Net::HTTPSuccess
-                File.open(File.join(@workdir, target_file), 'w') do |io|
+                dir = @cachedir || @workdir
+                File.open(File.join(dir, target_file), 'w') do |io|
                   response.read_body { |chunk| io.write(chunk) }
                 end
+                if @cachedir
+                  FileUtils.cp(File.join(@cachedir, target_file), File.join(@workdir, target_file))
+                end
               else
                 fail "Error: #{response.code.to_s}. Unable to get source from #{target_url}"
               end

lib/vanagon/driver.rb

@@ -12,7 +12,7 @@ class Vanagon
   class Driver
     include Vanagon::Utilities
 
-    attr_accessor :platform, :project, :target, :workdir, :remote_workdir, :verbose, :preserve, :keepwork
+    attr_accessor :platform, :project, :target, :workdir, :cachedir, :remote_workdir, :verbose, :preserve, :keepwork
 
     def timeout
       @timeout ||= @project.timeout || ENV["VANAGON_TIMEOUT"] || 7200
@@ -27,6 +27,7 @@ def initialize(platform, project, options = {}) # rubocop:disable Metrics/AbcSiz
       @verbose = options[:verbose] || false
       @preserve = options[:preserve] || :'on-failure'
       @workdir = options[:workdir] || Dir.mktmpdir
+      @cachedir = options[:cachedir]
       @keepwork = options[:keepwork] || :never
 
       @@configdir = options[:configdir] || File.join(Dir.pwd, "configs")
@@ -144,7 +145,7 @@ def run # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
       Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
         install_build_dependencies
       end
-      @project.fetch_sources(workdir, retry_count, timeout)
+      @project.fetch_sources(workdir, retry_count, timeout, cachedir)
 
       @project.make_makefile(workdir)
       @project.make_bill_of_materials(workdir)

lib/vanagon/project.rb

@@ -227,17 +227,18 @@ def vendor_email_only
     # Collects all sources and patches into the provided workdir
     #
     # @param workdir [String] directory to stage sources into
+    # @param cachedir [String] directory to cache source files and git repos
     # @param retry_count [Integer] number of times to retry each fetch
     # @param timeout [Integer] How long to wait (in seconds) for each
     #   fetch before aborting
-    def fetch_sources(workdir, retry_count = 1, timeout = 7200)
+    def fetch_sources(workdir, retry_count = 1, timeout = 7200, cachedir = nil)
       @components.each do |component|
         Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
-          component.get_source(workdir)
+          component.get_source(workdir, cachedir)
         end
         # Fetch secondary sources
         Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
-          component.get_sources(workdir)
+          component.get_sources(workdir, cachedir)
         end
         Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
           component.get_patches(workdir)