Skip to content

Commit d47c6bc

Browse files
authored
Ver2.1.0 (#27)
* add ID for plugin * update schema * add input/output metrics * update: dns plugin * fix promauto * fix: metrics registry * add reload * update golang.org/x/net * add api.md
1 parent d6d7f10 commit d47c6bc

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+838
-315
lines changed

README.md

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,10 @@
1010
go install github.com/mimuret/dtap/v2@latest
1111
```
1212

13-
## Configuration
13+
## Documents
1414

1515
- [configuration](./doc/)
1616
- [JSON schema](./schemas/schema.json)
17-
18-
## design
19-
20-
[design](./doc/design.md)
17+
- [API](./doc/api.md)
18+
- [design](./doc/design.md)
2119

doc/api.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# api
2+
3+
## metrics
4+
5+
DTAP provides metrics.
6+
7+
```
8+
GET: /metrics
9+
```
10+
11+
12+
## Reload
13+
14+
DTAP can reload the confog file at runtime.
15+
After the config file is successfully loaded,
16+
all currently running plugin processes and API servers are terminated.
17+
Then, start Plugin and the API server based on the reloaded configuration.
18+
If the Plugin fails to start, the program terminates.
19+
20+
```
21+
POST: /reload
22+
```
23+

example/stdout.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Inputs:
2+
- Name: "unix"
3+
ID: "input_unix_0"
4+
Path: "dnstap.sock"
5+
OutputGroups:
6+
- Name: "og0"
7+
Outputs:
8+
- Name: "stdout"
9+
ID: "og0_stdout_0"

go.mod

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ require (
2727
github.com/spf13/cobra v1.7.0
2828
go.uber.org/zap v1.25.0
2929
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
30-
golang.org/x/net v0.14.0
30+
golang.org/x/net v0.17.0
3131
golang.org/x/sync v0.3.0
3232
google.golang.org/protobuf v1.31.0
3333
gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -157,11 +157,11 @@ require (
157157
go.uber.org/atomic v1.10.0 // indirect
158158
go.uber.org/goleak v1.2.0 // indirect
159159
go.uber.org/multierr v1.10.0 // indirect
160-
golang.org/x/crypto v0.12.0 // indirect
160+
golang.org/x/crypto v0.14.0 // indirect
161161
golang.org/x/mod v0.11.0 // indirect
162162
golang.org/x/oauth2 v0.8.0 // indirect
163-
golang.org/x/sys v0.11.0 // indirect
164-
golang.org/x/text v0.12.0 // indirect
163+
golang.org/x/sys v0.13.0 // indirect
164+
golang.org/x/text v0.13.0 // indirect
165165
golang.org/x/time v0.3.0 // indirect
166166
golang.org/x/tools v0.9.3 // indirect
167167
google.golang.org/appengine v1.6.7 // indirect

go.sum

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -886,8 +886,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
886886
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
887887
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
888888
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
889-
golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
890-
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
889+
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
890+
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
891891
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
892892
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
893893
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -991,8 +991,8 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug
991991
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
992992
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
993993
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
994-
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
995-
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
994+
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
995+
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
996996
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
997997
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
998998
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1127,13 +1127,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
11271127
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11281128
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11291129
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1130-
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
1131-
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1130+
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
1131+
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11321132
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
11331133
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
11341134
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
1135-
golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
1136-
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
1135+
golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
1136+
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
11371137
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
11381138
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
11391139
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1144,8 +1144,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
11441144
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
11451145
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
11461146
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
1147-
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
1148-
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
1147+
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
1148+
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
11491149
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
11501150
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
11511151
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/cmd/root.go

Lines changed: 74 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,26 @@
1717
package cmd
1818

1919
import (
20-
"github.com/mimuret/dtap/v2/pkg/config"
20+
"context"
21+
"os"
22+
"os/signal"
23+
"sync"
24+
"syscall"
25+
2126
"github.com/mimuret/dtap/v2/pkg/core"
2227
_ "github.com/mimuret/dtap/v2/pkg/core/plugins"
23-
"github.com/mimuret/dtap/v2/pkg/logger"
24-
"github.com/spf13/afero"
28+
"github.com/mimuret/dtap/v2/pkg/promauto"
29+
"github.com/prometheus/client_golang/prometheus"
2530
"github.com/spf13/cobra"
2631
"go.uber.org/zap"
2732
)
2833

34+
type Runner interface {
35+
Run(context.Context) error
36+
}
37+
2938
var cfgFile string
39+
var runnerCh chan Runner
3040

3141
// rootCmd represents the base command when called without any subcommands
3242
var rootCmd = &cobra.Command{
@@ -36,25 +46,74 @@ var rootCmd = &cobra.Command{
3646
// Uncomment the following line if your bare application
3747
// has an action associated with it:
3848
RunE: func(cmd *cobra.Command, args []string) error {
39-
c, err := config.LoadConfig(afero.NewOsFs(), cfgFile)
40-
if err != nil {
41-
return err
42-
}
43-
l, err := logger.New(c.LogLevel)
49+
var (
50+
reloadCh = make(chan struct{}, 1)
51+
sigHUP = make(chan os.Signal, 1)
52+
sigSTOP = make(chan os.Signal, 1)
53+
wg = &sync.WaitGroup{}
54+
cctx context.Context
55+
cancelFunc context.CancelFunc
56+
)
57+
signal.Notify(sigHUP, syscall.SIGHUP)
58+
signal.Notify(sigSTOP, syscall.SIGINT, syscall.SIGTERM)
59+
defer close(sigHUP)
60+
defer close(sigSTOP)
61+
logger, err := makeRunner(cfgFile, reloadCh)
4462
if err != nil {
4563
return err
4664
}
4765

48-
ctl := core.NewController(c, l)
49-
if err := ctl.Setup(); err != nil {
50-
l.Fatal("failed to setup", zap.Error(err))
66+
LOOP:
67+
for {
68+
select {
69+
case r := <-runnerCh:
70+
wg.Add(1)
71+
cctx, cancelFunc = context.WithCancel(cmd.Context())
72+
go func(cctx context.Context) {
73+
logger.Info("Start runner.")
74+
if err := r.Run(cctx); err != nil {
75+
logger.Fatal("runtime error", zap.Error(err))
76+
}
77+
wg.Done()
78+
}(cctx)
79+
case <-sigSTOP:
80+
logger.Info("KILL signal recieved.")
81+
cancelFunc()
82+
wg.Wait()
83+
break LOOP
84+
case <-sigHUP:
85+
reloadCh <- struct{}{}
86+
case <-reloadCh:
87+
if len(runnerCh) != 0 {
88+
continue
89+
}
90+
logger.Info("SIGHUP recieved.")
91+
newLogger, err := makeRunner(cfgFile, reloadCh)
92+
if err != nil {
93+
logger.Error("failed to reload", zap.Error(err))
94+
} else {
95+
logger = newLogger
96+
logger.Info("Stop the current runner to start reloading.")
97+
cancelFunc()
98+
wg.Wait()
99+
}
100+
}
51101
}
52-
go ctl.PrometheusListen(cmd.Context())
53-
54-
return ctl.Run(cmd.Context())
102+
return nil
55103
},
56104
}
57105

106+
func makeRunner(cfgFile string, reloadCh chan struct{}) (*zap.Logger, error) {
107+
registery := prometheus.NewRegistry()
108+
promauto.Set(registery)
109+
runner, l, err := core.NewRunner(context.Background(), cfgFile, registery, reloadCh)
110+
if err != nil {
111+
return nil, err
112+
}
113+
runnerCh <- runner
114+
return l, nil
115+
}
116+
58117
// Execute adds all child commands to the root command and sets flags appropriately.
59118
// This is called by main.main(). It only needs to happen once to the rootCmd.
60119
func Execute() {
@@ -67,4 +126,5 @@ func init() {
67126
// will be global for your application.
68127

69128
rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file")
129+
runnerCh = make(chan Runner, 1)
70130
}

pkg/config/config.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ type Config struct {
8989
// default value is 'info'
9090
LogLevel string
9191
// Listen IP and port to output metrics
92-
MetricsListen string
92+
ManageHTTPSServer string
9393
// Input buffer settings
9494
InputBufferConfig *BufferConfig
9595
// Input plugin settings. Must not be empty.
@@ -148,14 +148,14 @@ func (c *Config) UnmarshalJSON(bs []byte) error {
148148
cfg := struct {
149149
InputFilterWorkerNum uint
150150
LogLevel string
151-
MetricsListen string
151+
ManageHTTPSServer string
152152
InputBufferConfig *BufferConfig
153153
Inputs json.RawMessage
154154
Filters json.RawMessage
155155
OutputGroups []json.RawMessage
156156
}{
157-
MetricsListen: ":9520",
158-
LogLevel: "info",
157+
ManageHTTPSServer: ":9520",
158+
LogLevel: "info",
159159
InputBufferConfig: &BufferConfig{
160160
Name: "input",
161161
Size: DefaultInputBufferSize,
@@ -168,7 +168,7 @@ func (c *Config) UnmarshalJSON(bs []byte) error {
168168
}
169169
c.InputFilterWorkerNum = cfg.InputFilterWorkerNum
170170
c.LogLevel = cfg.LogLevel
171-
c.MetricsListen = cfg.MetricsListen
171+
c.ManageHTTPSServer = cfg.ManageHTTPSServer
172172
c.InputBufferConfig = cfg.InputBufferConfig
173173

174174
var results error

pkg/config/config_test.go

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ var _ = Describe("config", func() {
9090
It("returns Config", func() {
9191
Expect(err).To(Succeed())
9292
Expect(cfg).NotTo(BeNil())
93-
ip, err := registry.CreateInputPlugin("file", json.RawMessage(`{"Name": "file","Path":"/var/tmp/hoge"}`))
93+
ip, err := registry.CreateInputPlugin("file", json.RawMessage(`{"Name": "file","ID":"input_file_1","Path":"/var/tmp/hoge"}`))
9494
Expect(err).To(Succeed())
95-
filter1, err := registry.CreateFilterPlugin("matcher", json.RawMessage(`{"Name": "matcher","Rule": {
95+
filter1, err := registry.CreateFilterPlugin("matcher", json.RawMessage(`{"Name": "matcher","ID":"filter_matcher_1","Rule": {
9696
"Op": "AND",
9797
"Matchers": [
9898
{
@@ -103,7 +103,7 @@ var _ = Describe("config", func() {
103103
]
104104
}}`))
105105
Expect(err).To(Succeed())
106-
filter2, err := registry.CreateFilterPlugin("matcher", json.RawMessage(`{"Name": "matcher","Rule": {
106+
filter2, err := registry.CreateFilterPlugin("matcher", json.RawMessage(`{"Name": "matcher","ID":"og1_filter_matcher_1","Rule": {
107107
"Op": "AND",
108108
"Matchers": [
109109
{
@@ -117,7 +117,7 @@ var _ = Describe("config", func() {
117117
c := &config.Config{
118118
InputFilterWorkerNum: 10,
119119
LogLevel: "trace",
120-
MetricsListen: ":19520",
120+
ManageHTTPSServer: ":19520",
121121
InputBufferConfig: &config.BufferConfig{
122122
Name: "input_common",
123123
Size: 100,
@@ -136,7 +136,18 @@ var _ = Describe("config", func() {
136136
},
137137
},
138138
}
139-
Expect(cfg).To(Equal(c))
139+
Expect(cfg.InputBufferConfig).To(Equal(c.InputBufferConfig))
140+
Expect(cfg.InputFilterWorkerNum).To(Equal(c.InputFilterWorkerNum))
141+
Expect(cfg.LogLevel).To(Equal(c.LogLevel))
142+
Expect(cfg.ManageHTTPSServer).To(Equal(c.ManageHTTPSServer))
143+
Expect(cfg.InputBufferConfig).To(Equal(c.InputBufferConfig))
144+
Expect(len(cfg.Filters)).To(Equal(len(c.Filters)))
145+
Expect(len(cfg.Inputs)).To(Equal(len(c.Inputs)))
146+
Expect(len(cfg.OutputGroups)).To(Equal(len(c.OutputGroups)))
147+
for i := range cfg.OutputGroups {
148+
Expect(len(cfg.OutputGroups[i].Filters)).To(Equal(len(c.OutputGroups[i].Filters)))
149+
Expect(len(cfg.OutputGroups[i].Outputs)).To(Equal(len(c.OutputGroups[i].Filters)))
150+
}
140151
})
141152
})
142153
})

pkg/config/testdata/valid_config.json

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
{
22
"InputFilterWorkerNum": 1,
33
"LogLevel": "info",
4-
"MetricsListen": ":9520",
4+
"ManageHTTPSServer": ":9520",
55
"InputBufferConfig": {
66
"Name": "input_common",
77
"Size": 100
88
},
99
"Input": [
1010
{
1111
"Name": "unix",
12+
"ID": "input_unix_1",
1213
"Path": "/var/run/unbound/dnstap.sock",
1314
"User": "unbound"
1415
}
1516
],
1617
"Filter": [
1718
{
1819
"Name": "grep",
20+
"ID": "filter_grep_1",
1921
"Rules": {
2022
"Op": "AND",
2123
"Matchers": [
@@ -33,6 +35,7 @@
3335
"Filter": [
3436
{
3537
"Name": "grep",
38+
"ID": "og1_filter_grep_1",
3639
"Rules": {
3740
"Op": "AND",
3841
"Matchers": [
@@ -47,10 +50,12 @@
4750
],
4851
"Outputs": [
4952
{
50-
"Name": "nop"
53+
"Name": "nop",
54+
"ID": "og1_nop_1"
5155
},
5256
{
53-
"Name": "nop"
57+
"Name": "nop",
58+
"ID": "og1_nop_2"
5459
}
5560
]
5661
}

0 commit comments

Comments
 (0)