feat(api-gateway): CORE-418 readiness probe data model check

docs-mintlify/cube-core/deployment.mdx

@@ -463,6 +463,16 @@ monitoring service of choice to use the [`/readyz`][ref-api-readyz] and
 [`/livez`][ref-api-livez] API endpoints so you can check on the Cube
 deployment's health and be alerted to any issues.
 
+For rolling deploys where a broken data model should not be promoted to
+production, set the
+[`CUBEJS_READINESS_CHECK_DATA_MODEL`](/reference/configuration/environment-variables#cubejs_readiness_check_data_model)
+environment variable to `true`. The `/readyz` probe will then compile the
+data model for every tenant, fail with a `500` response if any tenant fails
+to compile, and let the previous (working) build continue serving traffic
+until the issue is resolved. Compiled schemas are cached by version, and
+probe responses are coalesced for one second, so steady-state cost is
+minimal even with many tenants.
+
 ### Appropriate cluster sizing
 
 There's no one-size-fits-all when it comes to sizing a Cube cluster and its

docs-mintlify/reference/configuration/environment-variables.mdx

@@ -1431,6 +1431,20 @@ For example, if set to `/`, a nested folder structure like the `Customer Informa
 folder with the `Personal Details` subfolder will be flattened to `Customer
 Information / Personal Details` at the root level.
 
+## `CUBEJS_READINESS_CHECK_DATA_MODEL`
+
+If `true`, the [`/readyz`](/reference/rest-api/reference#readyz) readiness
+probe additionally compiles the data model for every tenant returned by
+[`scheduled_refresh_contexts`](/reference/configuration/config#scheduled_refresh_contexts)
+(or once with an empty security context in single-tenant deployments). The
+probe returns a `500` response if compilation fails for any tenant, allowing
+container orchestrators to keep the previous build serving traffic when a
+new build ships with a broken data model.
+
+| Possible Values | Default in Development | Default in Production |
+| --------------- | ---------------------- | --------------------- |
+| `true`, `false` | `false`                | `false`               |
+
 ## `CUBEJS_TELEMETRY`
 
 If `true`, then send telemetry to Cube.

packages/cubejs-api-gateway/src/gateway.ts

@@ -2783,9 +2783,54 @@ class ApiGateway {
       }
     }
 
+    if (health === 'HEALTH' && getEnv('readinessCheckDataModel')) {
+      health = await this.checkDataModelCompiles();
+    }
+
     return this.healthResponse(res, health);
   };
 
+  // Returns 'DOWN' on the first per-tenant compile failure so the probe
+  // fails fast and the deployment platform refuses to promote a broken
+  // build. The previous (working) version keeps serving traffic.
+  protected async checkDataModelCompiles(): Promise<'HEALTH' | 'DOWN'> {
+    let backgroundContexts: UserBackgroundContext[];
+    try {
+      backgroundContexts = this.scheduledRefreshContexts
+        ? await this.scheduledRefreshContexts()
+        : [];
+    } catch (e: any) {
+      this.logProbeError(e, 'Internal Server Error on readiness probe (scheduledRefreshContexts)');
+      return 'DOWN';
+    }
+
+    if (backgroundContexts.length === 0 && this.scheduledRefreshContexts) {
+      this.log({
+        type: 'Readiness probe data model check',
+        warning: 'scheduledRefreshContexts returned no contexts; checking with empty security context',
+      });
+    }
+
+    const contexts = backgroundContexts.length > 0 ? backgroundContexts : [{ securityContext: {} }];
+
+    for (const backgroundContext of contexts) {
+      const requestContext: RequestContext = {
+        securityContext: backgroundContext?.securityContext || backgroundContext?.authInfo || {},
+        requestId: `readiness-${uuidv4()}`,
+      };
+
+      try {
+        const compilerApi = await this.getCompilerApi(requestContext);
+        await compilerApi.metaConfig(requestContext, { requestId: requestContext.requestId });
+      } catch (e: any) {
+        this.logProbeError(e, 'Internal Server Error on readiness probe (data model compilation)');
+        return 'DOWN';
+      }
+    }
+
+    return 'HEALTH';
+  }
+
   protected liveness: RequestHandler = async (req, res) => {
     let health: 'HEALTH' | 'DOWN' = 'HEALTH';
 

packages/cubejs-api-gateway/test/index.test.ts

@@ -1186,6 +1186,135 @@ describe('API Gateway', () => {
       expect(dataSourceStorage.$testConnectionsDone).toEqual(true);
       expect(dataSourceStorage.$testOrchestratorConnectionsDone).toEqual(false);
     });
+
+    describe('readyz with CUBEJS_READINESS_CHECK_DATA_MODEL', () => {
+      const originalEnv = process.env.CUBEJS_READINESS_CHECK_DATA_MODEL;
+      const originalCompilerApiImpl = compilerApi.getMockImplementation();
+
+      afterEach(() => {
+        if (originalEnv === undefined) {
+          delete process.env.CUBEJS_READINESS_CHECK_DATA_MODEL;
+        } else {
+          process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = originalEnv;
+        }
+        if (originalCompilerApiImpl) {
+          compilerApi.mockImplementation(originalCompilerApiImpl);
+        }
+        compilerApi.mockClear();
+      });
+
+      test('disabled by default — /readyz does not invoke compilerApi', async () => {
+        delete process.env.CUBEJS_READINESS_CHECK_DATA_MODEL;
+
+        const metaConfigMock = jest.fn();
+        compilerApi.mockImplementation(async () => ({ metaConfig: metaConfigMock }));
+
+        const { app } = await createApiGateway();
+        const res = await request(app)
+          .get('/readyz')
+          .set('Content-type', 'application/json')
+          .expect(200);
+
+        expect(res.body).toMatchObject({ health: 'HEALTH' });
+        expect(metaConfigMock).not.toHaveBeenCalled();
+      });
+
+      test('enabled, single tenant, healthy compile', async () => {
+        process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true';
+
+        const metaConfigMock = jest.fn().mockResolvedValue([]);
+        compilerApi.mockImplementation(async () => ({ metaConfig: metaConfigMock }));
+
+        const { app } = await createApiGateway();
+        const res = await request(app)
+          .get('/readyz')
+          .set('Content-type', 'application/json')
+          .expect(200);
+
+        expect(res.body).toMatchObject({ health: 'HEALTH' });
+        expect(metaConfigMock).toHaveBeenCalledTimes(1);
+      });
+
+      test('enabled, single tenant, broken compile → DOWN', async () => {
+        process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true';
+
+        compilerApi.mockImplementation(async () => ({
+          metaConfig: async () => { throw new Error('Compile errors: duplicate view'); },
+        }));
+
+        const { app } = await createApiGateway();
+        const res = await request(app)
+          .get('/readyz')
+          .set('Content-type', 'application/json')
+          .expect(500);
+
+        expect(res.body).toMatchObject({ health: 'DOWN' });
+      });
+
+      test('enabled, multi-tenant, all healthy', async () => {
+        process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true';
+
+        const metaConfigMock = jest.fn().mockResolvedValue([]);
+        compilerApi.mockImplementation(async () => ({ metaConfig: metaConfigMock }));
+
+        const { app } = await createApiGateway(new AdapterApiMock(), new DataSourceStorageMock(), {
+          scheduledRefreshContexts: async () => [
+            { securityContext: { tenant: 'a' } },
+            { securityContext: { tenant: 'b' } },
+            { securityContext: { tenant: 'c' } },
+          ],
+        });
+
+        const res = await request(app)
+          .get('/readyz')
+          .set('Content-type', 'application/json')
+          .expect(200);
+
+        expect(res.body).toMatchObject({ health: 'HEALTH' });
+        expect(metaConfigMock).toHaveBeenCalledTimes(3);
+      });
+
+      test('enabled, multi-tenant, one tenant broken → DOWN', async () => {
+        process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true';
+
+        const healthy = { metaConfig: async () => [] };
+        const broken = { metaConfig: async () => { throw new Error('Compile errors: duplicate view'); } };
+        compilerApi
+          .mockImplementationOnce(async () => healthy)
+          .mockImplementationOnce(async () => broken)
+          .mockImplementation(async () => healthy);
+
+        const { app } = await createApiGateway(new AdapterApiMock(), new DataSourceStorageMock(), {
+          scheduledRefreshContexts: async () => [
+            { securityContext: { tenant: 'a' } },
+            { securityContext: { tenant: 'b' } },
+            { securityContext: { tenant: 'c' } },
+          ],
+        });
+
+        const res = await request(app)
+          .get('/readyz')
+          .set('Content-type', 'application/json')
+          .expect(500);
+
+        expect(res.body).toMatchObject({ health: 'DOWN' });
+      });
+
+      test('enabled, scheduledRefreshContexts itself throws → DOWN', async () => {
+        process.env.CUBEJS_READINESS_CHECK_DATA_MODEL = 'true';
+
+        const { app } = await createApiGateway(new AdapterApiMock(), new DataSourceStorageMock(), {
+          scheduledRefreshContexts: async () => { throw new Error('Tenant directory unavailable'); },
+        });
+
+        const res = await request(app)
+          .get('/readyz')
+          .set('Content-type', 'application/json')
+          .expect(500);
+
+        expect(res.body).toMatchObject({ health: 'DOWN' });
+      });
+    });
   });
 
   describe('/v1/cubesql', () => {

packages/cubejs-backend-shared/src/env.ts

@@ -243,6 +243,9 @@ const variables: Record<string, (...args: any) => any> = {
   rollupOnlyMode: () => get('CUBEJS_ROLLUP_ONLY')
     .default('false')
     .asBoolStrict(),
+  readinessCheckDataModel: () => get('CUBEJS_READINESS_CHECK_DATA_MODEL')
+    .default('false')
+    .asBoolStrict(),
   schemaPath: () => get('CUBEJS_SCHEMA_PATH')
     .default('model')
     .asString(),