Add --cachedir

This caches downloaded files and git repos so when we are working on changes to packaging, we don't have to download them every time we run the build again.

Author: nmburgan

lib/vanagon/cli/build.rb

@@ -22,6 +22,7 @@ class Build < Vanagon::CLI
                                              managed automatically based on `keepwork` option
           -k, --keepwork RULE              Rule for preserving local `workdir`: [Default: never]
                                              always, on-success, on-failure, never
+          --cachedir DIRECTORY             Directory to cache source files and git repos
           -v, --verbose                    Only here for backwards compatibility. Does nothing.
 
         Engines:
@@ -64,6 +65,7 @@ def options_translate(docopt_options)
         translations = {
           '--verbose' => :verbose,
           '--workdir' => :workdir,
+          '--cachedir' => :cachedir,
           '--remote-workdir' => :'remote-workdir',
           '--configdir' => :configdir,
           '--engine' => :engine,

lib/vanagon/component.rb

@@ -261,7 +261,7 @@ def mirrors # rubocop:disable Lint/DuplicateMethods
     #   if #fetch is successful.
     def fetch_mirrors(options)
       mirrors.to_a.shuffle.each do |mirror|
-        VanagonLogger.info %(Attempting to fetch from mirror URL "#{mirror}")
+        VanagonLogger.info %(Attempting to fetch from mirror URL "#{mirror}"#{' unless already cached' if options[:cachedir]})
         @source = Vanagon::Component::Source.source(mirror, **options)
         return true if source.fetch
       rescue Vanagon::InvalidSource
@@ -290,7 +290,7 @@ def fetch_mirrors(options)
     # @return [Boolean] return True if the source can be retrieved,
     #   or False otherwise
     def fetch_url(options)
-      VanagonLogger.info %(Attempting to fetch from canonical URL "#{url}")
+      VanagonLogger.info %(Attempting to fetch from canonical URL "#{url}"#{' unless already cached' if options[:cachedir]})
       @source = Vanagon::Component::Source.source(url, **options)
       # Explicitly coerce the return value of #source.fetch,
       # because each subclass of Vanagon::Component::Source returns
@@ -313,8 +313,9 @@ def deprecated_rewrite_url
     # makefile template
     #
     # @param workdir [String] working directory to put the source into
-    def get_source(workdir)
-      opts = options.merge({ workdir: workdir, dirname: dirname })
+    # @param cachedir [String] directory to cache downloaded sources
+    def get_source(workdir, cachedir = nil)
+      opts = options.merge({ workdir: workdir, cachedir: cachedir, dirname: dirname })
       if url || !mirrors.empty?
         if %w[y yes true 1].include? ENV.fetch('VANAGON_USE_MIRRORS', 'n').downcase
           fetch_mirrors(opts) || fetch_url(opts)
@@ -358,10 +359,11 @@ def get_dependency_hash
     # Fetches secondary sources for the component. These are just dumped into the workdir currently.
     #
     # @param workdir [String] working directory to put the source into
-    def get_sources(workdir) # rubocop:disable Metrics/AbcSize
+    # @param cachedir [String] directory to cache downloaded sources
+    def get_sources(workdir, cachedir = nil)
       sources.each do |source|
         src = Vanagon::Component::Source.source(
-          source.url, workdir: workdir, ref: source.ref, sum: source.sum
+          source.url, workdir: workdir, cachedir: cachedir, ref: source.ref, sum: source.sum
         )
         src.fetch
         src.verify

lib/vanagon/component/source.rb

@@ -17,6 +17,7 @@ class << self
         # @param uri_instance [#to_s] URI of the source file (includes git@... style links)
         # @param options [Hash] hash of the options needed for the subtype
         # @param workdir [String] working directory to fetch the source into
+        # @param cacheedir [String] directory to cache source files and git repos
         # @return [Vanagon::Component::Source] the correct subtype for the given source
         def source(uri_instance, **options)
           # Sometimes the uri comes in as a string, but sometimes it's already been
@@ -39,6 +40,7 @@ def source(uri_instance, **options)
               sum: options[:sum],
               ref: options[:ref],
               workdir: options[:workdir],
+              cachedir: options[:cachedir],
               dirname: options[:dirname],
               clone_options: options[:clone_options]
           end
@@ -47,6 +49,7 @@ def source(uri_instance, **options)
             return Vanagon::Component::Source::Http.new uri,
               sum: options[:sum],
               workdir: options[:workdir],
+              cachedir: options[:cachedir],
               # Default sum_type is md5 if unspecified:
               sum_type: options[:sum_type] || "md5"
           end

lib/vanagon/component/source/git.rb

@@ -13,7 +13,7 @@ class Vanagon
   class Component
     class Source
       class Git
-        attr_accessor :url, :log_url, :ref, :workdir, :clone_options
+        attr_accessor :url, :log_url, :ref, :workdir, :cachedir, :clone_options
         attr_reader :version, :default_options, :repo
 
         class << self
@@ -94,7 +94,8 @@ def default_options # rubocop:disable Lint/DuplicateMethods
         # @param url [String] url of git repo to use as source
         # @param ref [String] ref to checkout from git repo
         # @param workdir [String] working directory to clone into
-        def initialize(url, workdir:, **options)
+        # @param cachedir [String] directory to cache cloned repos
+        def initialize(url, workdir:, cachedir: nil, **options)
           opts = default_options.merge(options.reject { |k, v| v.nil? })
 
           # Ensure that #url returns a URI object
@@ -103,6 +104,7 @@ def initialize(url, workdir:, **options)
           @ref = opts[:ref]
           @dirname = opts[:dirname]
           @workdir = File.realpath(workdir)
+          @cachedir = cachedir ? File.realpath(cachedir) : nil
           @clone_options = opts[:clone_options] ||= {}
 
           # We can test for Repo existence without cloning
@@ -114,6 +116,11 @@ def initialize(url, workdir:, **options)
         # a side effect.
         def fetch
           begin
+            if @cachedir && File.directory?("#{@cachedir}/#{dirname}")
+              VanagonLogger.info "Using cached copy of Git repo '#{dirname}' in #{cachedir}"
+              FileUtils.cp_r("#{@cachedir}/#{dirname}", workdir)
+              @wascached = true
+            end
             @clone ||= ::Git.open(File.join(workdir, dirname))
             @clone.fetch
           rescue ::Git::Error, ArgumentError
@@ -156,6 +163,11 @@ def clone
           else
             @clone ||= ::Git.clone(url, dirname, path: workdir, **clone_options)
           end
+          if @cachedir && !@wascached
+            FileUtils.cp_r(File.join(workdir, dirname), cachedir)
+            @wascached = true
+          end
+          @clone
         end
 
         # Attempt to connect to whatever URL is provided and
@@ -183,8 +195,10 @@ def refs
         # Clone a remote repo, make noise about it, and fail entirely
         # if we're unable to retrieve the remote repo
         def clone!
-          VanagonLogger.info "Cloning Git repo '#{log_url}'"
-          VanagonLogger.info "Successfully cloned '#{dirname}'" if clone
+          unless @wascached
+            VanagonLogger.info "Cloning Git repo '#{log_url}'"
+            VanagonLogger.info "Successfully cloned '#{dirname}'" if clone
+          end
         rescue ::Git::Error
           raise Vanagon::InvalidRepo, "Unable to clone from '#{log_url}'"
         end

lib/vanagon/component/source/http.rb

@@ -12,7 +12,7 @@ class Http < Vanagon::Component::Source::Local
         include Vanagon::Utilities
 
         # Accessors :url, :file, :extension, :workdir, :cleanup are inherited from Local
-        attr_accessor :sum, :sum_type
+        attr_accessor :sum, :sum_type, :cachedir
 
         # Allowed checksum algorithms to use when validating files
         CHECKSUM_TYPES = %w[md5 sha1 sha256 sha512].freeze
@@ -48,8 +48,9 @@ def valid_url?(target_url)
         #                     sum from
         # @param workdir [String] working directory to download into
         # @param sum_type [String] type of sum we are verifying
+        # @param cachedir [String] directory to cache downloaded sources
         # @raise [RuntimeError] an exception is raised is sum is nil
-        def initialize(url, sum:, workdir:, sum_type:, **options)
+        def initialize(url, sum:, workdir:, sum_type:, cachedir: nil, **options)
           unless sum
             fail "sum is required to validate the http source"
           end
@@ -63,6 +64,7 @@ def initialize(url, sum:, workdir:, sum_type:, **options)
           @url = url
           @sum = sum
           @workdir = workdir
+          @cachedir = cachedir ? File.realpath(cachedir) : nil
           @sum_type = sum_type
 
           if Vanagon::Component::Source::Http.valid_url?(@sum)
@@ -81,9 +83,14 @@ def initialize(url, sum:, workdir:, sum_type:, **options)
         end
 
         # Download the source from the url specified. Sets the full path to the
-        # file as @file and the @extension for the file as a side effect.
+        # file as @file and the @extension for the file as a side effect. Use
+        # the cached copy if it exists and is valid.
         def fetch
           @file = File.basename(URI.parse(@url).path)
+          if @cachedir && File.exist?(File.join(@cachedir, @file))
+            VanagonLogger.info "Using cached copy of #{@file} in #{@cachedir}"
+            FileUtils.cp(File.join(@cachedir, @file), workdir)
+          end
           if File.exist?(File.join(workdir, file))
             begin
               return if verify
@@ -138,9 +145,13 @@ def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "id
                 location = URI.parse(response.header['location'])
                 download(uri + location, target_file)
               when Net::HTTPSuccess
-                File.open(File.join(@workdir, target_file), 'w') do |io|
+                dir = @cachedir || @workdir
+                File.open(File.join(dir, target_file), 'w') do |io|
                   response.read_body { |chunk| io.write(chunk) }
                 end
+                if @cachedir
+                  FileUtils.cp(File.join(@cachedir, target_file), File.join(@workdir, target_file))
+                end
               else
                 fail "Error: #{response.code.to_s}. Unable to get source from #{target_url}"
               end

lib/vanagon/driver.rb

@@ -12,7 +12,7 @@ class Vanagon
   class Driver
     include Vanagon::Utilities
 
-    attr_accessor :platform, :project, :target, :workdir, :remote_workdir, :verbose, :preserve, :keepwork
+    attr_accessor :platform, :project, :target, :workdir, :cachedir, :remote_workdir, :verbose, :preserve, :keepwork
 
     def timeout
       @timeout ||= @project.timeout || ENV["VANAGON_TIMEOUT"] || 7200
@@ -27,6 +27,7 @@ def initialize(platform, project, options = {})
       @verbose = options[:verbose] || false
       @preserve = options[:preserve] || :'on-failure'
       @workdir = options[:workdir] || Dir.mktmpdir
+      @cachedir = options[:cachedir]
       @keepwork = options[:keepwork] || :never
 
       @@configdir = options[:configdir] || File.join(Dir.pwd, "configs")
@@ -144,7 +145,7 @@ def run
       Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
         install_build_dependencies
       end
-      @project.fetch_sources(workdir, retry_count, timeout)
+      @project.fetch_sources(workdir, retry_count, timeout, cachedir)
 
       @project.make_makefile(workdir)
       @project.make_bill_of_materials(workdir)

lib/vanagon/project.rb

@@ -227,17 +227,18 @@ def vendor_email_only
     # Collects all sources and patches into the provided workdir
     #
     # @param workdir [String] directory to stage sources into
+    # @param cachedir [String] directory to cache source files and git repos
     # @param retry_count [Integer] number of times to retry each fetch
     # @param timeout [Integer] How long to wait (in seconds) for each
     #   fetch before aborting
-    def fetch_sources(workdir, retry_count = 1, timeout = 7200)
+    def fetch_sources(workdir, retry_count = 1, timeout = 7200, cachedir = nil)
       @components.each do |component|
         Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
-          component.get_source(workdir)
+          component.get_source(workdir, cachedir)
         end
         # Fetch secondary sources
         Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
-          component.get_sources(workdir)
+          component.get_sources(workdir, cachedir)
         end
         Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
           component.get_patches(workdir)