Merge branch 'master' into releases/1.6.x

Author: mmetc

debian/migrate-hub.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env sh
+
+# This script is provided (only in the source distribution) as an ad-hoc solution
+# to migrate an installation from the crowdsec package maintained in the debian or ubuntu repositories
+# to the official crowdsec repository.
+
+set -eu
+
+if [ ! -d /var/lib/crowdsec/hub/ ]; then
+    echo "You don't have a hub directory to migrate."
+    echo
+    echo "Use this script only if you upgrade from the crowdsec package included in the debian or ubuntu repositories."
+    exit 1
+fi
+
+# Download everything on the new hub but don't install anything yet
+
+echo "Downloading Hub content..."
+
+for itemtype in $(cscli hub types -o raw); do
+    ALL_ITEMS=$(cscli "$itemtype" list -a -o raw | tail +2 | cut -d, -f1)
+    if [ -n "${ALL_ITEMS}" ]; then
+        # shellcheck disable=SC2086
+        cscli "$itemtype" install \
+            $ALL_ITEMS \
+            --download-only -y
+    fi
+done
+
+# Fix links
+
+BASEDIR=/etc/crowdsec/
+OLD_PATH=/var/lib/crowdsec/hub/
+NEW_PATH=/etc/crowdsec/hub/
+
+find "$BASEDIR" -type l 2>/dev/null | while IFS= read -r link
+do
+    target="$(readlink "$link")" || continue
+
+    case "$target" in
+        "$OLD_PATH"*)
+            suffix="${target#"$OLD_PATH"}"
+            new_target="${NEW_PATH}${suffix}"
+
+            if [ -e "$target" ]; then
+                continue
+            fi
+
+            if [ ! -e "$new_target" ]; then
+                continue
+            fi
+
+            echo "Update symlink: $link"
+            ln -sf "$new_target" "$link"
+            ;;
+        *)
+            ;;
+    esac
+done
+
+# upgrade tainted collections
+
+cscli hub upgrade --force

pkg/hubops/colorize.go

@@ -15,8 +15,10 @@ func colorizeItemName(fullname string) string {
 		bold := color.New(color.Bold)
 		author := parts[0]
 		name := parts[1]
+
 		return author + "/" + bold.Sprint(name)
 	}
+
 	return fullname
 }
 

pkg/hubops/download.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"time"
 
@@ -118,6 +119,13 @@ func downloadDataSet(ctx context.Context, dataFolder string, force bool, reader
 				continue
 			}
 
+			// twopenny validation
+			if u, err := url.Parse(dataS.SourceURL); err != nil {
+				return false, err
+			} else if u.Scheme == "" {
+				return false, fmt.Errorf("a valid URL was expected (note: local items can download data too): %s", dataS.SourceURL)
+			}
+
 			// XXX: check context cancellation
 			destPath, err := cwhub.SafePath(dataFolder, dataS.DestPath)
 			if err != nil {

test/bats/20_hub_items.bats

@@ -151,6 +151,29 @@ teardown() {
     assert_output --partial "Nothing to do."
 }
 
+@test "when upgrading the hub, a local item's data will be downloaded" {
+    rune -0 mkdir -p "$CONFIG_DIR/collections"
+    cat >"$CONFIG_DIR"/collections/foobar.yaml <<-EOT
+        data:
+          - source_url: https://localhost:1234/database.mmdb
+            dest_file: database.mmdb
+	EOT
+    rune -1 cscli hub upgrade
+    assert_line "downloading https://localhost:1234/database.mmdb"
+    assert_stderr --partial 'Get "https://localhost:1234/database.mmdb":'
+    assert_stderr --partial 'connect: connection refused'
+
+    # bad link, or local path
+    cat >"$CONFIG_DIR"/collections/foobar.yaml <<-EOT
+        data:
+          - source_url: /tmp/meh
+            dest_file: database.mmdb
+	EOT
+    rune -1 cscli hub upgrade
+    refute_line "downloading /tmp/meh"
+    assert_stderr --partial 'a valid URL was expected (note: local items can download data too): /tmp/meh'
+}
+
 @test "a local item cannot be removed by cscli" {
     rune -0 mkdir -p "$CONFIG_DIR/scenarios"
     rune -0 touch "$CONFIG_DIR/scenarios/foobar.yaml"