Merge branch 'ash2k/job-router' into 'main'

Job Router

See merge request https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/5945

Merged-by: Mikhail Mazurskiy <mmazurskiy@gitlab.com>
Approved-by: Timo Furrer <tfurrer@gitlab.com>
Approved-by: Roshni Sarangadharan <rsarangadharan@gitlab.com>
Approved-by: Arran Walker <ajwalker@gitlab.com>
Reviewed-by: Timo Furrer <tfurrer@gitlab.com>

Author: ash2k

common/mocks.go

@@ -1199,6 +1199,11 @@ type ArtifactsOptions struct {
 	LogResponseDetails bool
 }
 
+type RouterDiscovery struct {
+	ServerURL string  `json:"server_url"`
+	TLSData   TLSData `json:"-"`
+}
+
 type FailuresCollector interface {
 	RecordFailure(reason JobFailureReason, runnerConfig RunnerConfig)
 }

common/network.go

@@ -93,6 +93,7 @@ The flags are defined in `./helpers/featureflags/flags.go` file.
 | `FF_USE_GITALY_CORRELATION_ID` | `true` | {{< icon name="dotted-circle" >}} No |  | When enabled, the `X-Gitaly-Correlation-ID` header is added to all Git HTTP requests. When disabled, the Git operations execute without Gitaly Correlation ID headers. |
 | `FF_HASH_CACHE_KEYS` | `false` | {{< icon name="dotted-circle" >}} No |  | When GitLab Runner creates or extracts caches, it hashes the cache keys (SHA256) before using them, both for local and distributed caches (for example, S3). For more information, see [cache key handling](advanced-configuration.md#cache-key-handling). |
 | `FF_ENABLE_JOB_INPUTS_INTERPOLATION` | `false` | {{< icon name="dotted-circle" >}} No |  | When enabled, job inputs are interpolated. For more information, see [&17833](https://gitlab.com/groups/gitlab-org/-/epics/17833). |
+| `FF_USE_JOB_ROUTER` | `false` | {{< icon name="dotted-circle" >}} No |  | Makes GitLab Runner fetch jobs by connecting to Job Router rather than GitLab directly. |
 
 <!-- feature_flags_list_end -->
 

docs/configuration/feature-flags.md

@@ -90,6 +90,7 @@ require (
 	golang.org/x/text v0.32.0
 	google.golang.org/api v0.191.0
 	google.golang.org/grpc v1.75.1
+	google.golang.org/protobuf v1.36.9
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.32.10
 	k8s.io/apimachinery v0.32.10
@@ -267,7 +268,6 @@ require (
 	google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090 // indirect
-	google.golang.org/protobuf v1.36.9 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

go.mod

@@ -1,37 +1,83 @@
 package certificate
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
-	"errors"
-	"math/big"
+	"fmt"
 	"net"
 	"time"
 )
 
 const (
 	x509CertificatePrivateKeyBits = 2048
 	x509CertificateExpiryInYears  = 2
-	x509CertificateSerialNumber   = 1
 	x509CertificateOrganization   = "GitLab Runner"
 )
 
 type X509Generator struct{}
 
+func (c X509Generator) GenerateCA() ([]byte, []byte, *x509.Certificate, *ecdsa.PrivateKey, error) {
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	publicKey := privateKey.Public()
+
+	tpl := &x509.Certificate{
+		Subject: pkix.Name{
+			Organization:       []string{"GitLab test CA"},
+			OrganizationalUnit: []string{"group::runner core"},
+			CommonName:         "test CA cert",
+		},
+
+		NotAfter:  time.Now().AddDate(x509CertificateExpiryInYears, 0, 0),
+		NotBefore: time.Now(),
+
+		KeyUsage: x509.KeyUsageCertSign,
+
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		MaxPathLenZero:        true,
+	}
+	caCert, err := x509.CreateCertificate(rand.Reader, tpl, tpl, publicKey, privateKey)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	certTyped, err := x509.ParseCertificate(caCert)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	caCertPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCert})
+	caKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER})
+
+	return caCertPEM, caKeyPEM, certTyped, privateKey, nil
+}
+
 func (c X509Generator) Generate(host string) (tls.Certificate, []byte, error) {
+	return c.GenerateWithCA(host, nil, nil)
+}
+
+func (c X509Generator) GenerateWithCA(host string, caCert *x509.Certificate, caPrivateKey any) (tls.Certificate, []byte, error) {
 	priv, err := rsa.GenerateKey(rand.Reader, x509CertificatePrivateKeyBits)
 	if err != nil {
-		return tls.Certificate{}, []byte{}, err
+		return tls.Certificate{}, nil, err
 	}
 
 	template := x509.Certificate{
-		SerialNumber: big.NewInt(x509CertificateSerialNumber),
-		NotBefore:    time.Now(),
-		NotAfter:     time.Now().AddDate(x509CertificateExpiryInYears, 0, 0),
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().AddDate(x509CertificateExpiryInYears, 0, 0),
 		Subject: pkix.Name{
 			Organization: []string{x509CertificateOrganization},
 		},
@@ -48,17 +94,21 @@ func (c X509Generator) Generate(host string) (tls.Certificate, []byte, error) {
 		template.DNSNames = append(template.DNSNames, host)
 	}
 
-	publicKeyBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv)
+	if caCert == nil {
+		caCert = &template
+		caPrivateKey = priv
+	}
+	publicKeyBytes, err := x509.CreateCertificate(rand.Reader, &template, caCert, priv.Public(), caPrivateKey)
 	if err != nil {
-		return tls.Certificate{}, []byte{}, errors.New("failed to create certificate")
+		return tls.Certificate{}, nil, fmt.Errorf("failed to create certificate: %w", err)
 	}
 
 	publicKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: publicKeyBytes})
 	privateKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
 
 	parsedCertificate, err := tls.X509KeyPair(publicKeyPEM, privateKeyPEM)
 	if err != nil {
-		return tls.Certificate{}, []byte{}, err
+		return tls.Certificate{}, nil, err
 	}
 
 	return parsedCertificate, publicKeyPEM, nil

helpers/certificate/x509.go

@@ -52,6 +52,7 @@ const (
 	UseGitalyCorrelationId               string = "FF_USE_GITALY_CORRELATION_ID"
 	HashCacheKeys                        string = "FF_HASH_CACHE_KEYS"
 	EnableJobInputsInterpolation         string = "FF_ENABLE_JOB_INPUTS_INTERPOLATION"
+	UseJobRouter                         string = "FF_USE_JOB_ROUTER"
 )
 
 type FeatureFlag struct {
@@ -429,6 +430,12 @@ var flags = []FeatureFlag{
 		Deprecated:   false,
 		Description:  "When enabled, job inputs are interpolated. For more information, see [&17833](https://gitlab.com/groups/gitlab-org/-/epics/17833).",
 	},
+	{
+		Name:         UseJobRouter,
+		DefaultValue: false,
+		Deprecated:   false,
+		Description:  "Makes GitLab Runner fetch jobs by connecting to Job Router rather than GitLab directly.",
+	},
 }
 
 func GetAll() []FeatureFlag {

helpers/featureflags/flags.go

@@ -15,8 +15,10 @@ import (
 	"gitlab.com/gitlab-org/gitlab-runner/commands/steps"
 	"gitlab.com/gitlab-org/gitlab-runner/common"
 	cli_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/cli"
+	"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
 	"gitlab.com/gitlab-org/gitlab-runner/log"
 	"gitlab.com/gitlab-org/gitlab-runner/network"
+	"gitlab.com/gitlab-org/gitlab-runner/router"
 	"gitlab.com/gitlab-org/labkit/fips"
 
 	_ "gitlab.com/gitlab-org/gitlab-runner/cache/azure"
@@ -59,6 +61,8 @@ func main() {
 	}()
 
 	fips.Check()
+	gitLabClient, clientShutdown, apiRequestsCollector := newClient()
+	defer clientShutdown()
 
 	app := cli.NewApp()
 	app.Name = filepath.Base(os.Args[0])
@@ -71,7 +75,7 @@ func main() {
 			Email: "support@gitlab.com",
 		},
 	}
-	app.Commands = newCommands()
+	app.Commands = newCommands(gitLabClient, apiRequestsCollector)
 	app.CommandNotFound = func(context *cli.Context, command string) {
 		logrus.Fatalln("Command", command, "not found.")
 	}
@@ -89,12 +93,7 @@ func main() {
 	}
 }
 
-func newCommands() []cli.Command {
-	apiRequestsCollector := network.NewAPIRequestsCollector()
-	n := network.NewGitLabClient(
-		network.WithAPIRequestsCollector(apiRequestsCollector),
-		network.WithCertificateDirectory(commands.GetDefaultCertificateDirectory()),
-	)
+func newCommands(n common.Network, apiRequestsCollector *network.APIRequestsCollector) []cli.Command {
 	cmds := []cli.Command{
 		commands.NewListCommand(),
 		commands.NewRegisterCommand(n),
@@ -118,3 +117,22 @@ func newCommands() []cli.Command {
 	cmds = append(cmds, commands.NewServiceCommands()...)
 	return cmds
 }
+
+func newClient() (common.Network, func(), *network.APIRequestsCollector) {
+	apiRequestsCollector := network.NewAPIRequestsCollector()
+	certDir := commands.GetDefaultCertificateDirectory()
+
+	mainClient := network.NewGitLabClient(
+		network.WithAPIRequestsCollector(apiRequestsCollector),
+		network.WithCertificateDirectory(certDir),
+	)
+	if os.Getenv(featureflags.UseJobRouter) != "true" {
+		return mainClient, func() {}, apiRequestsCollector
+	}
+	rc := router.NewClient(
+		mainClient,
+		certDir,
+		common.AppVersion.UserAgent(),
+	)
+	return rc, rc.Shutdown, apiRequestsCollector
+}

main.go

@@ -18,6 +18,7 @@ const (
 	apiEndpointRequestJob apiEndpoint = "request_job"
 	apiEndpointUpdateJob  apiEndpoint = "update_job"
 	apiEndpointPatchTrace apiEndpoint = "patch_trace"
+	apiEndpointDiscovery  apiEndpoint = "discovery"
 )
 
 var (