Skip to content

Commit ac5e6f9

Browse files
blotusblotus
authored
cscli allowlists: add import command (#4378)
1 parent 30ca8b2 commit ac5e6f9

File tree

3 files changed

+263
-0
lines changed

3 files changed

+263
-0
lines changed

‎cmd/crowdsec-cli/cliallowlists/allowlists.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ func (cli *cliAllowLists) NewCommand() *cobra.Command {
239239
cmd.AddCommand(cli.newListCmd())
240240
cmd.AddCommand(cli.newDeleteCmd())
241241
cmd.AddCommand(cli.newAddCmd())
242+
cmd.AddCommand(cli.newImportCmd())
242243
cmd.AddCommand(cli.newRemoveCmd())
243244
cmd.AddCommand(cli.newInspectCmd())
244245
cmd.AddCommand(cli.newCheckCmd())

‎cmd/crowdsec-cli/cliallowlists/import.go‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
package cliallowlists
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"io"
8+
"os"
9+
"time"
10+
11+
"github.com/go-openapi/strfmt"
12+
"github.com/jszwec/csvutil"
13+
log "github.com/sirupsen/logrus"
14+
"github.com/spf13/cobra"
15+
16+
"github.com/crowdsecurity/go-cs-lib/cstime"
17+
18+
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/args"
19+
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/core/require"
20+
"github.com/crowdsecurity/crowdsec/pkg/database"
21+
"github.com/crowdsecurity/crowdsec/pkg/models"
22+
)
23+
24+
type allowlistItemRaw struct {
25+
Value string `csv:"value"`
26+
Expiration string `csv:"expiration,omitempty"`
27+
Comment string `csv:"comment,omitempty"`
28+
}
29+
30+
func (cli *cliAllowLists) newImportCmd() *cobra.Command {
31+
var input string
32+
33+
cmd := &cobra.Command{
34+
Use: "import [allowlist_name] -i <file>",
35+
Short: "Import values to an allowlist from a CSV file",
36+
Long: "Import values to an allowlist from a CSV file.\n\n" +
37+
"The CSV file must have a header line with at least a 'value' column.\n" +
38+
"Optional columns: 'expiration' (duration like 1h, 1d), 'comment'.",
39+
Example: `csv file:
40+
value,expiration,comment
41+
1.2.3.4,24h,my comment
42+
2.3.4.5,,another comment
43+
10.0.0.0/8,1d,
44+
45+
$ cscli allowlists import my_allowlist -i allowlist.csv
46+
47+
From standard input:
48+
49+
$ cat allowlist.csv | cscli allowlists import my_allowlist -i -`,
50+
Args: args.ExactArgs(1),
51+
ValidArgsFunction: cli.validAllowlists,
52+
RunE: func(cmd *cobra.Command, args []string) error {
53+
cfg := cli.cfg()
54+
55+
if err := require.LAPI(cfg); err != nil {
56+
return err
57+
}
58+
59+
ctx := cmd.Context()
60+
61+
db, err := require.DBClient(ctx, cfg.DbConfig)
62+
if err != nil {
63+
return err
64+
}
65+
66+
name := args[0]
67+
68+
return cli.import_allowlist(ctx, db, name, input)
69+
},
70+
}
71+
72+
flags := cmd.Flags()
73+
flags.StringVarP(&input, "input", "i", "", "Input file (use - for stdin)")
74+
75+
_ = cmd.MarkFlagRequired("input")
76+
77+
return cmd
78+
}
79+
80+
func (*cliAllowLists) import_allowlist(ctx context.Context, db *database.Client, name string, input string) error {
81+
var (
82+
fin *os.File
83+
err error
84+
)
85+
86+
if input == "-" {
87+
fin = os.Stdin
88+
} else {
89+
fin, err = os.Open(input)
90+
if err != nil {
91+
return fmt.Errorf("unable to open %s: %w", input, err)
92+
}
93+
defer fin.Close()
94+
}
95+
96+
content, err := io.ReadAll(fin)
97+
if err != nil {
98+
return fmt.Errorf("unable to read from %s: %w", input, err)
99+
}
100+
101+
var items []allowlistItemRaw
102+
103+
if err := csvutil.Unmarshal(content, &items); err != nil {
104+
return fmt.Errorf("unable to parse CSV: %w", err)
105+
}
106+
107+
if len(items) == 0 {
108+
return errors.New("no values to import")
109+
}
110+
111+
allowlist, err := db.GetAllowList(ctx, name, true)
112+
if err != nil {
113+
return err
114+
}
115+
116+
if allowlist.FromConsole {
117+
return fmt.Errorf("allowlist %s is managed by console, cannot update with cscli. Please visit https://app.crowdsec.net/allowlists/%s to update", name, allowlist.AllowlistID)
118+
}
119+
120+
toAdd := make([]*models.AllowlistItem, 0)
121+
122+
for i, item := range items {
123+
if item.Value == "" {
124+
return fmt.Errorf("row %d: missing 'value'", i+1)
125+
}
126+
127+
found := false
128+
129+
for _, existing := range allowlist.Edges.AllowlistItems {
130+
if existing.Value == item.Value {
131+
found = true
132+
133+
log.Warnf("value %s already in allowlist", item.Value)
134+
135+
break
136+
}
137+
}
138+
139+
if found {
140+
continue
141+
}
142+
143+
expTS := time.Time{}
144+
145+
if item.Expiration != "" {
146+
duration, err := cstime.ParseDurationWithDays(item.Expiration)
147+
if err != nil {
148+
return fmt.Errorf("row %d: invalid expiration %q: %w", i+1, item.Expiration, err)
149+
}
150+
151+
expTS = time.Now().UTC().Add(duration)
152+
}
153+
154+
toAdd = append(toAdd, &models.AllowlistItem{
155+
Value: item.Value,
156+
Description: item.Comment,
157+
Expiration: strfmt.DateTime(expTS),
158+
})
159+
}
160+
161+
if len(toAdd) == 0 {
162+
fmt.Fprintln(os.Stdout, "no new values for allowlist")
163+
return nil
164+
}
165+
166+
added, err := db.AddToAllowlist(ctx, allowlist, toAdd)
167+
if err != nil {
168+
return fmt.Errorf("unable to add values to allowlist: %w", err)
169+
}
170+
171+
if added > 0 {
172+
fmt.Fprintf(os.Stdout, "added %d values to allowlist %s\n", added, name)
173+
}
174+
175+
deleted, err := db.ApplyAllowlistsToExistingDecisions(ctx)
176+
if err != nil {
177+
return fmt.Errorf("unable to apply allowlists to existing decisions: %w", err)
178+
}
179+
180+
if deleted > 0 {
181+
fmt.Fprintf(os.Stdout, "%d decisions deleted by allowlists\n", deleted)
182+
}
183+
184+
return nil
185+
}

‎test/bats/cscli-allowlists.bats‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,83 @@ teardown() {
281281
assert_json '{"description":"a foo","items":[],"name":"foo"}'
282282
}
283283

284+
@test "cscli allowlists import" {
285+
rune -1 cscli allowlist import
286+
assert_stderr 'Error: cscli allowlists import: accepts 1 arg(s), received 0'
287+
288+
rune -1 cscli allowlist import foo
289+
assert_stderr 'Error: cscli allowlists import: required flag(s) "input" not set'
290+
291+
rune -0 cscli allowlist create foo -d 'a foo'
292+
293+
# empty file
294+
rune -1 cscli allowlist import foo -i /dev/null
295+
assert_stderr --partial 'Error: cscli allowlists import: no values to import'
296+
297+
# nonexistent allowlist
298+
tmpfile=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp)
299+
cat > "$tmpfile" <<-EOT
300+
value
301+
1.2.3.4
302+
EOT
303+
rune -1 cscli allowlist import does-not-exist -i "$tmpfile"
304+
assert_stderr "Error: cscli allowlists import: allowlist 'does-not-exist' not found"
305+
cat > "$tmpfile" <<-EOT
306+
value,expiration,comment
307+
1.2.3.4,24h,my comment
308+
5.6.7.8,,another comment
309+
10.0.0.0/8,1d,
310+
EOT
311+
rune -0 cscli allowlist import foo -i "$tmpfile"
312+
assert_output 'added 3 values to allowlist foo'
313+
refute_stderr
314+
315+
# deduplication: 1.2.3.4 already exists
316+
cat > "$tmpfile" <<-EOT
317+
value,expiration,comment
318+
1.2.3.4,,duplicate
319+
9.9.9.9,,new one
320+
EOT
321+
rune -0 cscli allowlist import foo -i "$tmpfile"
322+
assert_stderr --partial 'level=warning msg="value 1.2.3.4 already in allowlist"'
323+
assert_output 'added 1 values to allowlist foo'
324+
325+
# csv with only value column
326+
cat > "$tmpfile" <<-EOT
327+
value
328+
100.100.100.100
329+
200.200.200.200
330+
EOT
331+
rune -0 cscli allowlist import foo -i "$tmpfile"
332+
assert_output 'added 2 values to allowlist foo'
333+
refute_stderr
334+
335+
# missing value in a row
336+
cat > "$tmpfile" <<-EOT
337+
value,comment
338+
1.1.1.1,ok
339+
,missing value
340+
EOT
341+
rune -1 cscli allowlist import foo -i "$tmpfile"
342+
assert_stderr --partial "row 2: missing 'value'"
343+
344+
# invalid expiration
345+
cat > "$tmpfile" <<-EOT
346+
value,expiration
347+
2.2.2.2,toto
348+
EOT
349+
rune -1 cscli allowlist import foo -i "$tmpfile"
350+
assert_stderr --partial 'row 1: invalid expiration "toto"'
351+
352+
# stdin support
353+
rune -0 cscli allowlist import foo -i - <<-EOT
354+
value,comment
355+
30.30.30.30,from stdin
356+
EOT
357+
assert_output 'added 1 values to allowlist foo'
358+
refute_stderr
359+
}
360+
284361
@test "allowlists expire active decisions" {
285362
rune -0 cscli decisions add -i 1.2.3.4
286363
rune -0 cscli decisions add -r 2.3.4.0/24

0 commit comments

Comments
 (0)