refactor: add new config file and split nodes setup for create cluster (#487)

* chore: add new config file and split nodes setup for create cluster

* chore: update config to use latest docker image

Author: gacevicljubisa

cmd/beekeeper/cmd/cluster.go

@@ -49,7 +49,7 @@ func (c *command) deleteCluster(ctx context.Context, clusterName string, cfg *co
 				return fmt.Errorf("get node group: %w", err)
 			}
 
-			for i := 0; i < len(v.Nodes); i++ {
+			for i := range v.Nodes {
 				nName := fmt.Sprintf("%s-%d", ngName, i)
 				if len(v.Nodes[i].Name) > 0 {
 					nName = v.Nodes[i].Name
@@ -76,7 +76,7 @@ func (c *command) deleteCluster(ctx context.Context, clusterName string, cfg *co
 			}
 
 			if len(v.Nodes) > 0 {
-				for i := 0; i < len(v.Nodes); i++ {
+				for i := range v.Nodes {
 					nName := fmt.Sprintf("%s-%d", ngName, i)
 					if len(v.Nodes[i].Name) > 0 {
 						nName = v.Nodes[i].Name
@@ -94,7 +94,7 @@ func (c *command) deleteCluster(ctx context.Context, clusterName string, cfg *co
 					}
 				}
 			} else {
-				for i := 0; i < v.Count; i++ {
+				for i := range v.Count {
 					nName := fmt.Sprintf("%s-%d", ngName, i)
 					if err := ng.DeleteNode(ctx, nName); err != nil {
 						return fmt.Errorf("deleting node %s from the node group %s: %w", nName, ngName, err)
@@ -146,27 +146,23 @@ func (c *command) setupCluster(ctx context.Context, clusterName string, startClu
 
 	cluster = orchestrationK8S.NewCluster(clusterConfig.GetName(), clusterConfig.Export(), c.k8sClient, c.swapClient, c.log)
 
-	nodeResultChan := make(chan nodeResult)
-	defer close(nodeResultChan)
-
 	inCluster := c.globalConfig.GetBool(optionNameInCluster)
 
-	// setup bootnode node group
-	fundAddresses, bootnodes, err := setupNodes(ctx, clusterConfig, c.config, true, cluster, startCluster, inCluster, "", nodeResultChan)
+	fundAddresses, bootnodes, err := setupBootnodes(ctx, clusterConfig, c.config, cluster, startCluster, inCluster)
 	if err != nil {
 		return nil, fmt.Errorf("setup node group bootnode: %w", err)
 	}
 
-	// fund bootnode node group if cluster is started
-	if startCluster {
+	// fund bootnode node group if cluster is started and bootnode is defined in config
+	if startCluster && len(fundAddresses) > 0 {
 		if err = fund(ctx, fundAddresses, chainNodeEndpoint, walletKey, fundOpts, c.log); err != nil {
 			return nil, fmt.Errorf("funding node group bootnode: %w", err)
 		}
 		c.log.Infof("bootnode node group funded")
 	}
 
 	// setup other node groups
-	fundAddresses, _, err = setupNodes(ctx, clusterConfig, c.config, false, cluster, startCluster, inCluster, bootnodes, nodeResultChan)
+	fundAddresses, err = setupNodes(ctx, clusterConfig, c.config, cluster, startCluster, inCluster, bootnodes)
 	if err != nil {
 		return nil, fmt.Errorf("setup other node groups: %w", err)
 	}
@@ -197,21 +193,19 @@ func ensureFundingDefaults(fundOpts orchestration.FundingOptions, log logging.Lo
 	return fundOpts
 }
 
-func setupNodes(ctx context.Context,
+func setupBootnodes(ctx context.Context,
 	clusterConfig config.Cluster,
 	cfg *config.Config,
-	bootnode bool,
 	cluster orchestration.Cluster,
 	startCluster bool,
 	inCluster bool,
-	bootnodesIn string,
-	nodeResultCh chan nodeResult,
 ) (fundAddresses []string, bootnodesOut string, err error) {
+	nodeResultChan := make(chan nodeResult)
+	defer close(nodeResultChan)
 	var nodeCount uint32
 
 	for ngName, v := range clusterConfig.GetNodeGroups() {
-
-		if (v.Mode != bootnodeMode && bootnode) || (v.Mode == bootnodeMode && !bootnode) {
+		if v.Mode != bootnodeMode {
 			continue
 		}
 
@@ -227,17 +221,95 @@ func setupNodes(ctx context.Context,
 		}
 		bConfig := beeConfig.Export()
 
-		if !bootnode {
+		cluster.AddNodeGroup(ngName, ngOptions)
+
+		// start nodes in the node group
+		ng, err := cluster.NodeGroup(ngName)
+		if err != nil {
+			return nil, "", fmt.Errorf("get node group: %w", err)
+		}
+
+		if clusterConfig.IsUsingStaticEndpoints() {
+			for nodeName, endpoint := range v.GetEndpoints() {
+				beeOpt := orchestration.WithURL(endpoint.APIURL)
+				nodeCount++
+				go setupOrAddNode(ctx, false, inCluster, ng, nodeName, orchestration.NodeOptions{
+					Config: &bConfig,
+				}, nodeResultChan, beeOpt)
+			}
+			continue
+		}
+
+		for i, node := range v.Nodes {
+			// set node name
+			nodeName := fmt.Sprintf("%s-%d", ngName, i)
+			if len(node.Name) > 0 {
+				nodeName = node.Name
+			}
+
+			bConfig.Bootnodes = fmt.Sprintf(node.Bootnodes, clusterConfig.GetNamespace()) // TODO: improve bootnode management, support more than 2 bootnodes
+			bootnodesOut = bConfig.Bootnodes
+			nodeOpts := setupNodeOptions(node, &bConfig)
+			nodeCount++
+			go setupOrAddNode(ctx, startCluster, inCluster, ng, nodeName, nodeOpts, nodeResultChan, orchestration.WithNoOptions())
+		}
+	}
+
+	// wait for nodes to be setup and get their eth addresses
+	// or wait for nodes to be added and check for errors
+	for range nodeCount {
+		nodeResult := <-nodeResultChan
+		if nodeResult.err != nil {
+			return nil, "", fmt.Errorf("setup or add node result: %w", nodeResult.err)
+		}
+		if nodeResult.ethAddress != "" {
+			fundAddresses = append(fundAddresses, nodeResult.ethAddress)
+		}
+	}
+
+	return fundAddresses, bootnodesOut, nil
+}
+
+func setupNodes(ctx context.Context,
+	clusterConfig config.Cluster,
+	cfg *config.Config,
+	cluster orchestration.Cluster,
+	startCluster bool,
+	inCluster bool,
+	bootnodesIn string,
+) (fundAddresses []string, err error) {
+	nodeResultChan := make(chan nodeResult)
+	defer close(nodeResultChan)
+	var nodeCount uint32
+
+	for ngName, v := range clusterConfig.GetNodeGroups() {
+		if v.Mode == bootnodeMode {
+			continue
+		}
+
+		ngConfig, ok := cfg.NodeGroups[v.Config]
+		if !ok {
+			return nil, fmt.Errorf("node group profile %s not defined", v.Config)
+		}
+		ngOptions := ngConfig.Export()
+
+		beeConfig, ok := cfg.BeeConfigs[v.BeeConfig]
+		if !ok {
+			return nil, fmt.Errorf("bee profile %s not defined", v.BeeConfig)
+		}
+		bConfig := beeConfig.Export()
+
+		if bConfig.Bootnodes == "" {
 			bConfig.Bootnodes = bootnodesIn
-			ngOptions.BeeConfig = &bConfig
 		}
+		ngOptions.BeeConfig = &bConfig
 
 		cluster.AddNodeGroup(ngName, ngOptions)
 
 		// start nodes in the node group
 		ng, err := cluster.NodeGroup(ngName)
 		if err != nil {
-			return nil, "", fmt.Errorf("get node group: %w", err)
+			return nil, fmt.Errorf("get node group: %w", err)
 		}
 
 		if clusterConfig.IsUsingStaticEndpoints() {
@@ -246,7 +318,7 @@ func setupNodes(ctx context.Context,
 				nodeCount++
 				go setupOrAddNode(ctx, false, inCluster, ng, nodeName, orchestration.NodeOptions{
 					Config: &bConfig,
-				}, nodeResultCh, beeOpt)
+				}, nodeResultChan, beeOpt)
 			}
 			continue
 		}
@@ -258,42 +330,34 @@ func setupNodes(ctx context.Context,
 				nodeName = node.Name
 			}
 
-			var nodeOpts orchestration.NodeOptions
-			if bootnode {
-				// set bootnodes
-				bConfig.Bootnodes = fmt.Sprintf(node.Bootnodes, clusterConfig.GetNamespace()) // TODO: improve bootnode management, support more than 2 bootnodes
-				bootnodesOut += bootnodesIn + bConfig.Bootnodes + " "
-				nodeOpts = setupNodeOptions(node, &bConfig)
-			} else {
-				nodeOpts = setupNodeOptions(node, nil)
-			}
+			nodeOpts := setupNodeOptions(node, nil)
 			nodeCount++
-			go setupOrAddNode(ctx, startCluster, inCluster, ng, nodeName, nodeOpts, nodeResultCh, orchestration.WithNoOptions())
+			go setupOrAddNode(ctx, startCluster, inCluster, ng, nodeName, nodeOpts, nodeResultChan, orchestration.WithNoOptions())
 		}
 
-		if len(v.Nodes) == 0 && !bootnode {
-			for i := 0; i < v.Count; i++ {
+		if len(v.Nodes) == 0 {
+			for i := range v.Count {
 				// set node name
 				nodeName := fmt.Sprintf("%s-%d", ngName, i)
 				nodeCount++
-				go setupOrAddNode(ctx, startCluster, inCluster, ng, nodeName, orchestration.NodeOptions{}, nodeResultCh, orchestration.WithNoOptions())
+				go setupOrAddNode(ctx, startCluster, inCluster, ng, nodeName, orchestration.NodeOptions{}, nodeResultChan, orchestration.WithNoOptions())
 			}
 		}
 	}
 
 	// wait for nodes to be setup and get their eth addresses
 	// or wait for nodes to be added and check for errors
-	for i := uint32(0); i < nodeCount; i++ {
-		nodeResult := <-nodeResultCh
+	for range nodeCount {
+		nodeResult := <-nodeResultChan
 		if nodeResult.err != nil {
-			return nil, "", fmt.Errorf("setup or add node result: %w", nodeResult.err)
+			return nil, fmt.Errorf("setup or add node result: %w", nodeResult.err)
 		}
 		if nodeResult.ethAddress != "" {
 			fundAddresses = append(fundAddresses, nodeResult.ethAddress)
 		}
 	}
 
-	return fundAddresses, bootnodesOut, nil
+	return fundAddresses, nil
 }
 
 func setupOrAddNode(ctx context.Context,

config/config.yaml

@@ -54,7 +54,7 @@ clusters:
 node-groups:
   default:
     _inherit: ""
-    image: ethersphere/bee:2.3.0
+    image: ethersphere/bee:latest
     image-pull-policy: Always
     ingress-annotations:
       nginx.ingress.kubernetes.io/affinity: "cookie"

config/light-node.yaml

@@ -8,6 +8,6 @@ bee-configs:
 node-groups:
   light-node:
     _inherit: default
-    image: ethersphere/bee:2.3.0
+    image: ethersphere/bee:latest
     image-pull-policy: Always
     persistence-enabled: false

config/testnet-bee-playground.yaml

@@ -0,0 +1,67 @@
+clusters:
+  bee-playground:
+    _inherit: ""
+    name: bee
+    namespace: bee-playground
+    disable-namespace: true
+    api-domain: bee-playground.testnet.internal
+    api-domain-internal: bee-playground.svc.swarm1.local:1633
+    api-insecure-tls: true
+    api-scheme: http
+    funding:
+      eth: 0.1
+      bzz: 10.0
+    node-groups:
+      bee:
+        mode: node
+        bee-config: sepolia-playground
+        config: ng-bee-playground
+        count: 5
+
+# node-groups defines node groups that can be registered in the cluster
+# node-groups may inherit it's configuration from already defined node-group and override specific fields from it
+node-groups:
+  ng-bee-playground:
+    _inherit: default
+    persistence-enabled: true
+    image: ethersphere/bee:latest
+
+# bee-configs defines Bee configuration that can be assigned to node-groups
+# bee-configs may inherit it's configuration from already defined bee-config and override specific fields from it
+bee-configs:
+  sepolia-playground:
+    _inherit: ""
+    api-addr: :1633
+    blockchain-rpc-endpoint: http://rpc-sepolia-haproxy.default.svc.swarm1.local
+    bootnodes: /dnsaddr/testnet.ethswarm.org
+    full-node: true
+    mainnet: false
+    network-id: 10
+    p2p-addr: :1634
+    password: "beekeeper"
+    storage-incentives-enable: true
+    swap-enable: true
+    swap-initial-deposit: 0
+    tracing-enabled: true
+    tracing-endpoint: "10.10.11.199:6831"
+    tracing-service-name: "bee-playground"
+    verbosity: 5
+    welcome-message: Welcome to the bee-playground Testnet!
+    warmup-time: 0s
+    payment-early-percent: 50
+    payment-threshold: 13500000
+    payment-tolerance-percent: 25
+    allow-private-cidrs: false
+    block-time: 12
+    bootnode-mode: false
+    cache-capacity: 1000000
+    chequebook-enable: true
+    cors-allowed-origins: ""
+    data-dir: "/home/bee/.bee"
+    db-block-cache-capacity: 33554432
+    db-disable-seeks-compaction: true
+    db-open-files-limit: 200
+    db-write-buffer-size: 33554432
+    nat-addr: ""
+    p2p-ws-enable: true
+    postage-stamp-start-block: 0

pkg/check/pss/pss.go

@@ -151,7 +151,7 @@ func (c *Check) testPss(nodeAName, nodeBName string, clients map[string]*bee.Cli
 				c.metrics.SendAndReceiveGauge.WithLabelValues(nodeAName, nodeBName).Set(time.Since(tStart).Seconds())
 				return nil
 			}
-			c.logger.Infof("pss: received incorrect message. trying again. want %s, got %s", string(testData), msg)
+			c.logger.Infof("pss: received incorrect message. trying again. want: '%s', got: '%s'", string(testData), msg)
 		}
 	}
 }

pkg/orchestration/k8s/nodegroup.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"net/http"
 	"net/url"
 	"slices"
@@ -487,9 +488,7 @@ func (g *NodeGroup) HasChunkStream(ctx context.Context, a swarm.Address) (<-chan
 // Nodes returns map of nodes in the node group
 func (g *NodeGroup) Nodes() map[string]orchestration.Node {
 	nodes := make(map[string]orchestration.Node)
-	for k, v := range g.getNodes() {
-		nodes[k] = v
-	}
+	maps.Copy(nodes, g.getNodes())
 	return nodes
 }
 
@@ -700,6 +699,7 @@ func (g *NodeGroup) pregenerateSwarmKey(ctx context.Context, name string) (err e
 		time.Sleep(10 * time.Second)
 		g.log.Infof("overlay Ethereum address %s for node %s attested successfully: transaction: %s", key.Address, name, txHash)
 	}
+
 	return
 }
 

pkg/wslistener/wslistener.go

@@ -55,7 +55,7 @@ func ListenWebSocket(ctx context.Context, client *bee.Client, endpoint string, l
 		for {
 			select {
 			case data := <-readCh:
-				logger.WithField("node", client.Name()).Infof("websocket received message: %s", string(data))
+				logger.WithField("node", client.Name()).Infof("websocket received message: `%s`", string(data))
 				select {
 				case ch <- string(data):
 				case <-done: