diff --git a/docs-mintlify/cube-core/deployment.mdx b/docs-mintlify/cube-core/deployment.mdx index 5926b670abeed..62f18304d24d0 100644 --- a/docs-mintlify/cube-core/deployment.mdx +++ b/docs-mintlify/cube-core/deployment.mdx @@ -463,6 +463,16 @@ monitoring service of choice to use the [`/readyz`][ref-api-readyz] and [`/livez`][ref-api-livez] API endpoints so you can check on the Cube deployment's health and be alerted to any issues. +For rolling deploys where a broken data model should not be promoted to +production, set the +[`CUBEJS_READINESS_CHECK_DATA_MODEL`](/reference/configuration/environment-variables#cubejs_readiness_check_data_model) +environment variable to `true`. The `/readyz` probe will then compile the +data model for every tenant, fail with a `500` response if any tenant fails +to compile, and let the previous (working) build continue serving traffic +until the issue is resolved. Compiled schemas are cached by version, and +probe responses are coalesced for one second, so steady-state cost is +minimal even with many tenants. + ### Appropriate cluster sizing There's no one-size-fits-all when it comes to sizing a Cube cluster and its diff --git a/docs-mintlify/reference/configuration/environment-variables.mdx b/docs-mintlify/reference/configuration/environment-variables.mdx index d90e2fdea7651..a6a4bc06255b0 100644 --- a/docs-mintlify/reference/configuration/environment-variables.mdx +++ b/docs-mintlify/reference/configuration/environment-variables.mdx @@ -1431,6 +1431,20 @@ For example, if set to `/`, a nested folder structure like the `Customer Informa folder with the `Personal Details` subfolder will be flattened to `Customer Information / Personal Details` at the root level. +## `CUBEJS_READINESS_CHECK_DATA_MODEL` + +If `true`, the [`/readyz`](/reference/rest-api/reference#readyz) readiness +probe additionally compiles the data model for every tenant returned by +[`scheduled_refresh_contexts`](/reference/configuration/config#scheduled_refresh_contexts) +(or once with an empty security context in single-tenant deployments). The +probe returns a `500` response if compilation fails for any tenant, allowing +container orchestrators to keep the previous build serving traffic when a +new build ships with a broken data model. + +| Possible Values | Default in Development | Default in Production | +| --------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + ## `CUBEJS_TELEMETRY` If `true`, then send telemetry to Cube. diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 049c4e30391bc..5e9d664ecae98 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -2783,9 +2783,54 @@ class ApiGateway { } } + if (health === 'HEALTH' && getEnv('readinessCheckDataModel')) { + health = await this.checkDataModelCompiles(); + } + return this.healthResponse(res, health); }; + // Returns 'DOWN' on the first per-tenant compile failure so the probe + // fails fast and the deployment platform refuses to promote a broken + // build. The previous (working) version keeps serving traffic. + protected async checkDataModelCompiles(): Promise<'HEALTH' | 'DOWN'> { + let backgroundContexts: UserBackgroundContext[]; + try { + backgroundContexts = this.scheduledRefreshContexts + ? await this.scheduledRefreshContexts() + : []; + } catch (e: any) { + this.logProbeError(e, 'Internal Server Error on readiness probe (scheduledRefreshContexts)'); + return 'DOWN'; + } + + if (backgroundContexts.length === 0 && this.scheduledRefreshContexts) { + this.log({ + type: 'Readiness probe data model check', + warning: 'scheduledRefreshContexts returned no contexts; checking with empty security context', + }); + } + + const contexts = backgroundContexts.length > 0 ? backgroundContexts : [{ securityContext: {} }]; + + for (const backgroundContext of contexts) { + const requestContext: RequestContext = { + securityContext: backgroundContext?.securityContext || backgroundContext?.authInfo || {}, + requestId: `readiness-${uuidv4()}`, + }; + + try { + const compilerApi = await this.getCompilerApi(requestContext); + await compilerApi.metaConfig(requestContext, { requestId: requestContext.requestId }); + } catch (e: any) { + this.logProbeError(e, 'Internal Server Error on readiness probe (data model compilation)'); + return 'DOWN'; + } + } + + return 'HEALTH'; + } + protected liveness: RequestHandler = async (req, res) => { let health: 'HEALTH' | 'DOWN' = 'HEALTH'; diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index 26c1c88a18ef0..6cfc6db0d3687 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -1186,6 +1186,135 @@ describe('API Gateway', () => { expect(dataSourceStorage.$testConnectionsDone).toEqual(true); expect(dataSourceStorage.$testOrchestratorConnectionsDone).toEqual(false); }); + + describe('readyz with CUBEJS_READINESS_CHECK_DATA_MODEL', () => { + const originalEnv = process.env.CUBEJS_READINESS_CHECK_DATA_MODEL; + const originalCompilerApiImpl = compilerApi.getMockImplementation(); + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.CUBEJS_READINESS_CHECK_DATA_MODEL; + } else { + process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = originalEnv; + } + if (originalCompilerApiImpl) { + compilerApi.mockImplementation(originalCompilerApiImpl); + } + compilerApi.mockClear(); + }); + + test('disabled by default — /readyz does not invoke compilerApi', async () => { + delete process.env.CUBEJS_READINESS_CHECK_DATA_MODEL; + + const metaConfigMock = jest.fn(); + compilerApi.mockImplementation(async () => ({ metaConfig: metaConfigMock })); + + const { app } = await createApiGateway(); + const res = await request(app) + .get('/readyz') + .set('Content-type', 'application/json') + .expect(200); + + expect(res.body).toMatchObject({ health: 'HEALTH' }); + expect(metaConfigMock).not.toHaveBeenCalled(); + }); + + test('enabled, single tenant, healthy compile', async () => { + process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true'; + + const metaConfigMock = jest.fn().mockResolvedValue([]); + compilerApi.mockImplementation(async () => ({ metaConfig: metaConfigMock })); + + const { app } = await createApiGateway(); + const res = await request(app) + .get('/readyz') + .set('Content-type', 'application/json') + .expect(200); + + expect(res.body).toMatchObject({ health: 'HEALTH' }); + expect(metaConfigMock).toHaveBeenCalledTimes(1); + }); + + test('enabled, single tenant, broken compile → DOWN', async () => { + process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true'; + + compilerApi.mockImplementation(async () => ({ + metaConfig: async () => { throw new Error('Compile errors: duplicate view'); }, + })); + + const { app } = await createApiGateway(); + const res = await request(app) + .get('/readyz') + .set('Content-type', 'application/json') + .expect(500); + + expect(res.body).toMatchObject({ health: 'DOWN' }); + }); + + test('enabled, multi-tenant, all healthy', async () => { + process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true'; + + const metaConfigMock = jest.fn().mockResolvedValue([]); + compilerApi.mockImplementation(async () => ({ metaConfig: metaConfigMock })); + + const { app } = await createApiGateway(new AdapterApiMock(), new DataSourceStorageMock(), { + scheduledRefreshContexts: async () => [ + { securityContext: { tenant: 'a' } }, + { securityContext: { tenant: 'b' } }, + { securityContext: { tenant: 'c' } }, + ], + }); + + const res = await request(app) + .get('/readyz') + .set('Content-type', 'application/json') + .expect(200); + + expect(res.body).toMatchObject({ health: 'HEALTH' }); + expect(metaConfigMock).toHaveBeenCalledTimes(3); + }); + + test('enabled, multi-tenant, one tenant broken → DOWN', async () => { + process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true'; + + const healthy = { metaConfig: async () => [] }; + const broken = { metaConfig: async () => { throw new Error('Compile errors: duplicate view'); } }; + compilerApi + .mockImplementationOnce(async () => healthy) + .mockImplementationOnce(async () => broken) + .mockImplementation(async () => healthy); + + const { app } = await createApiGateway(new AdapterApiMock(), new DataSourceStorageMock(), { + scheduledRefreshContexts: async () => [ + { securityContext: { tenant: 'a' } }, + { securityContext: { tenant: 'b' } }, + { securityContext: { tenant: 'c' } }, + ], + }); + + const res = await request(app) + .get('/readyz') + .set('Content-type', 'application/json') + .expect(500); + + expect(res.body).toMatchObject({ health: 'DOWN' }); + }); + + test('enabled, scheduledRefreshContexts itself throws → DOWN', async () => { + process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true'; + + const { app } = await createApiGateway(new AdapterApiMock(), new DataSourceStorageMock(), { + scheduledRefreshContexts: async () => { throw new Error('Tenant directory unavailable'); }, + }); + + const res = await request(app) + .get('/readyz') + .set('Content-type', 'application/json') + .expect(500); + + expect(res.body).toMatchObject({ health: 'DOWN' }); + }); + }); }); describe('/v1/cubesql', () => { diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 8b05ebbddb3b5..a9d5b51ee7a19 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -243,6 +243,9 @@ const variables: Record any> = { rollupOnlyMode: () => get('CUBEJS_ROLLUP_ONLY') .default('false') .asBoolStrict(), + readinessCheckDataModel: () => get('CUBEJS_READINESS_CHECK_DATA_MODEL') + .default('false') + .asBoolStrict(), schemaPath: () => get('CUBEJS_SCHEMA_PATH') .default('model') .asString(),