feat: support mongodb cloud customizations (#1097)

realSpok · web-flow · commit 65ce409f1d10 · 2024-04-09T11:23:33.000+02:00
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -32,7 +32,7 @@ jobs:
           path: |
             node_modules
             packages/*/node_modules
-          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('packages/*/package.json') }}
       - name: Install & Bootstrap
         run: yarn && yarn bootstrap --ci
       - name: Build
@@ -89,7 +89,7 @@ jobs:
           path: |
             node_modules
             packages/*/node_modules
-          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('packages/*/package.json') }}
           fail-on-cache-miss: true
       - name: Restore build from cache
         uses: actions/cache/restore@v3
@@ -142,7 +142,7 @@ jobs:
           path: |
             node_modules
             packages/*/node_modules
-          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('packages/*/package.json') }}
           fail-on-cache-miss: true
       - name: Restore build from cache
         uses: actions/cache/restore@v3
@@ -184,7 +184,7 @@ jobs:
           path: |
             node_modules
             packages/*/node_modules
-          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('packages/*/package.json') }}
           fail-on-cache-miss: true
       - name: Restore build from cache
         uses: actions/cache/restore@v3
diff --git a/packages/datasource-mongo/src/index.ts b/packages/datasource-mongo/src/index.ts
@@ -1,10 +1,9 @@
 import type { Introspection } from './introspection/types';
 import type { ConnectionParams, IntrospectorParams, MongoDatasourceParams } from './types';
 import type { DataSourceFactory, Logger } from '@forestadmin/datasource-toolkit';
-import type { Connection } from 'mongoose';
 
 import { MongooseDatasource } from '@forestadmin/datasource-mongoose';
-import mongoose from 'mongoose';
+import mongoose, { Connection, Mongoose } from 'mongoose';
 
 import Introspector from './introspection/introspector';
 import listCollectionsFromIntrospection from './introspection/list-collections-from-introspection';
@@ -56,6 +55,13 @@ export async function buildMongooseInstance(
   return connection;
 }
 
+export function buildDisconnectedMongooseInstance(introspection: Introspection): Mongoose {
+  const mongooseInstance = new Mongoose();
+  OdmBuilder.defineModels(mongooseInstance, introspection.models);
+
+  return mongooseInstance;
+}
+
 export function createMongoDataSource(
   params: MongoDatasourceParams,
   options?: { introspection: Introspection },
diff --git a/packages/datasource-mongo/src/odm-builder/index.ts b/packages/datasource-mongo/src/odm-builder/index.ts
@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
-import { Connection, Schema } from 'mongoose';
+import { Connection, Mongoose, Schema } from 'mongoose';
 
 import { ModelAnalysis, ModelDefinition, PrimitiveDefinition } from '../introspection/types';
 
@@ -14,7 +14,7 @@ export default class OdmBuilder {
     ObjectId: Schema.Types.ObjectId,
   };
 
-  static defineModels(connection: Connection, study: ModelDefinition[]) {
+  static defineModels(connection: Connection | Mongoose, study: ModelDefinition[]) {
     for (const collection of study) {
       const definition = this.buildDefinition(collection.analysis);
       connection.model(collection.name, new Schema(definition), collection.name);
diff --git a/packages/datasource-mongo/test/index.integration.test.ts b/packages/datasource-mongo/test/index.integration.test.ts
@@ -0,0 +1,63 @@
+import { Introspection, buildDisconnectedMongooseInstance } from '../src';
+
+describe('Datasource Mongo', () => {
+  describe('buildDisconnectedMongooseInstance', () => {
+    it('should build a mongoose instance with the given introspection', () => {
+      const introspection: Introspection = {
+        source: '@forestadmin/datasource-mongo',
+        version: 3,
+        models: [
+          {
+            name: 'comments',
+            analysis: {
+              type: 'object',
+              object: {
+                _id: { type: 'ObjectId', nullable: false },
+                date: { type: 'Date', nullable: false },
+                name: { type: 'string', nullable: false },
+                text: { type: 'string', nullable: false },
+                email: { type: 'string', nullable: false },
+                movie_id: { type: 'ObjectId', nullable: false },
+              },
+              nullable: false,
+            },
+          },
+          {
+            name: 'movies',
+            analysis: {
+              type: 'object',
+              object: {
+                _id: { type: 'ObjectId', nullable: false },
+                title: { type: 'string', nullable: false },
+                tomatoes: {
+                  type: 'object',
+                  object: {
+                    critic: {
+                      type: 'object',
+                      object: {
+                        meter: { type: 'number', nullable: false },
+                        rating: { type: 'number', nullable: false },
+                        numReviews: { type: 'number', nullable: false },
+                      },
+                      nullable: false,
+                    },
+                  },
+                  nullable: false,
+                },
+                lastupdated: { type: 'string', nullable: false },
+              },
+              nullable: false,
+            },
+          },
+        ],
+      };
+
+      const mongoose = buildDisconnectedMongooseInstance(introspection);
+
+      expect(mongoose).toBeDefined();
+
+      expect(mongoose.models.comments).toBeDefined();
+      expect(mongoose.models.movies).toBeDefined();
+    });
+  });
+});
diff --git a/packages/datasource-mongoose/src/datasource.ts b/packages/datasource-mongoose/src/datasource.ts
@@ -1,5 +1,5 @@
 import { BaseDataSource, Logger } from '@forestadmin/datasource-toolkit';
-import { Connection, Model } from 'mongoose';
+import { Connection, Model, Mongoose } from 'mongoose';
 
 import MongooseCollection from './collection';
 import MongooseSchema from './mongoose/schema';
@@ -9,7 +9,11 @@ import OptionsParser from './utils/options';
 import RelationGenerator from './utils/schema/relations';
 
 export default class MongooseDatasource extends BaseDataSource<MongooseCollection> {
-  constructor(connection: Connection, options: MongooseOptions = {}, logger: Logger = null) {
+  constructor(
+    connection: Connection | Mongoose,
+    options: MongooseOptions = {},
+    logger: Logger = null,
+  ) {
     super();
 
     if (options && !options.flattenMode) {
diff --git a/packages/datasource-mongoose/src/index.ts b/packages/datasource-mongoose/src/index.ts
@@ -1,5 +1,5 @@
 import { DataSourceFactory } from '@forestadmin/datasource-toolkit';
-import { Connection } from 'mongoose';
+import { Connection, Mongoose } from 'mongoose';
 
 import MongooseDatasource from './datasource';
 import { MongooseOptions } from './types';
@@ -9,7 +9,7 @@ export { default as MongooseDatasource } from './datasource';
 export type { MongooseOptions };
 
 export function createMongooseDataSource(
-  connection: Connection,
+  connection: Connection | Mongoose,
   options: MongooseOptions = {},
 ): DataSourceFactory {
   return async logger => new MongooseDatasource(connection, options, logger);
diff --git a/packages/datasource-sql/src/index.ts b/packages/datasource-sql/src/index.ts
@@ -6,6 +6,7 @@ import { Sequelize } from 'sequelize';
 
 import connect from './connection';
 import ConnectionOptions from './connection/connection-options';
+import SequelizeFactory from './connection/sequelize-factory';
 import SqlDatasource from './decorators/sql-datasource';
 import Introspector from './introspection/introspector';
 import listCollectionsFromIntrospection from './introspection/list-collections-from-introspection';
@@ -55,6 +56,17 @@ async function buildModelsAndRelations(
   }
 }
 
+export async function buildDisconnectedSequelizeInstance(
+  introspection: SupportedIntrospection,
+  logger: Logger,
+): Promise<Sequelize> {
+  const options = new ConnectionOptions({ dialect: 'sqlite', sslMode: 'disabled' }, logger);
+  const sequelize = SequelizeFactory.build(await options.buildSequelizeCtorOptions());
+  await buildModelsAndRelations(sequelize, logger, introspection);
+
+  return sequelize;
+}
+
 export async function buildSequelizeInstance(
   uriOrOptions: PlainConnectionOptionsOrUri,
   logger: Logger,
@@ -91,5 +103,11 @@ export async function preprocessOptions(
 }
 
 export * from './connection/errors';
-export type { PlainConnectionOptionsOrUri as ConnectionOptions, Table, SslMode, Introspection };
+export type {
+  PlainConnectionOptionsOrUri as ConnectionOptions,
+  Table,
+  SslMode,
+  Introspection,
+  SupportedIntrospection,
+};
 export { listCollectionsFromIntrospection };
diff --git a/packages/datasource-sql/test/datasource/disconnected-sequlieze-factory.unit.test.ts b/packages/datasource-sql/test/datasource/disconnected-sequlieze-factory.unit.test.ts
@@ -0,0 +1,18 @@
+import { Sequelize } from 'sequelize';
+
+import { SupportedIntrospection, buildDisconnectedSequelizeInstance } from '../../src';
+
+describe('buildDisconnectedSequelizeInstance', () => {
+  const introspection: SupportedIntrospection = {
+    source: '@forestadmin/datasource-sql',
+    version: 3,
+    tables: [{ name: 'books', columns: [], schema: 'public', unique: [] }],
+    views: [],
+  };
+
+  it('should return a Sequelize instance without a running database', async () => {
+    const sequelize = await buildDisconnectedSequelizeInstance(introspection, jest.fn());
+    expect(sequelize).toBeInstanceOf(Sequelize);
+    expect(sequelize.models.books).toBeDefined();
+  });
+});
diff --git a/packages/forest-cloud/package.json b/packages/forest-cloud/package.json
@@ -6,6 +6,9 @@
     "@forestadmin/agent": "1.38.10",
     "@forestadmin/datasource-customizer": "1.44.1",
     "@forestadmin/datasource-sql": "1.12.3",
+    "@forestadmin/datasource-mongo": "1.1.1",
+    "@forestadmin/datasource-mongoose": "^1.6.7",
+    "@forestadmin/datasource-sequelize": "^1.8.0",
     "adm-zip": "^0.5.10",
     "apollo-cache-inmemory": "^1.6.6",
     "apollo-client": "^2.6.10",
@@ -19,7 +22,6 @@
     "graphql-tag": "^2.12.6",
     "joi": "^17.12.2",
     "ora": "^3.2.0",
-    "sqlite3": "^5.1.7",
     "subscriptions-transport-ws": "^0.9.19",
     "ws": "^8.16.0"
   },
diff --git a/packages/forest-cloud/src/services/load-customization.ts b/packages/forest-cloud/src/services/load-customization.ts
@@ -0,0 +1,22 @@
+import type { Agent } from '../types';
+
+import { CustomizationError } from '../errors';
+
+export default function loadCustomization(agent: Agent, builtCodePath: string): void {
+  // eslint-disable-next-line
+  const customization = require(builtCodePath);
+  const entryPoint = customization?.default || customization;
+
+  if (typeof entryPoint !== 'function') {
+    throw new CustomizationError('Customization file must export a function');
+  }
+
+  try {
+    entryPoint(agent);
+  } catch (error) {
+    throw new CustomizationError(
+      `Issue with customizations: ${error.name}\n${error.message}`,
+      error.stack,
+    );
+  }
+}
diff --git a/packages/forest-cloud/src/services/update-typings.ts b/packages/forest-cloud/src/services/update-typings.ts
@@ -1,63 +1,62 @@
+import type { Introspection as DataSourceMongoIntrospection } from '@forestadmin/datasource-mongo';
+import type { SupportedIntrospection as DataSourceSQLIntrospection } from '@forestadmin/datasource-sql';
+
 import { AgentOptions, createAgent } from '@forestadmin/agent';
-import { Table, createSqlDataSource } from '@forestadmin/datasource-sql';
+import { buildDisconnectedMongooseInstance } from '@forestadmin/datasource-mongo';
+import { createMongooseDataSource } from '@forestadmin/datasource-mongoose';
+import { createSequelizeDataSource } from '@forestadmin/datasource-sequelize';
+import { buildDisconnectedSequelizeInstance } from '@forestadmin/datasource-sql';
 import path from 'path';
 
 import { throwIfNoBuiltCode } from './access-file';
 import BootstrapPathManager from './bootstrap-path-manager';
 import DistPathManager from './dist-path-manager';
-import { CustomizationError } from '../errors';
-import { Agent } from '../types';
-
-function loadCustomization(agent: Agent, builtCodePath: string): void {
-  // eslint-disable-next-line
-  const customization = require(builtCodePath);
-  const entryPoint = customization?.default || customization;
-
-  if (typeof entryPoint !== 'function') {
-    throw new CustomizationError('Customization file must export a function');
-  }
+import loadCustomization from './load-customization';
 
-  try {
-    entryPoint(agent);
-  } catch (error) {
-    throw new CustomizationError(
-      `Issue with customizations: ${error.name}\n${error.message}`,
-      error.stack,
-    );
-  }
+function isDatasourceMongoIntrospection(
+  introspection: DataSourceSQLIntrospection | DataSourceMongoIntrospection,
+): introspection is DataSourceMongoIntrospection {
+  return 'source' in introspection && introspection.source === '@forestadmin/datasource-mongo';
 }
 
-function buildAgent(introspection: Table[]) {
+async function buildAgent(
+  introspection: DataSourceSQLIntrospection | DataSourceMongoIntrospection,
+) {
   const agentOptions: AgentOptions = {
     authSecret: 'a'.repeat(64),
     envSecret: 'a'.repeat(64),
     loggerLevel: 'Error',
     isProduction: false,
   };
   const agent = createAgent(agentOptions);
-  agent.addDataSource(
-    createSqlDataSource({ dialect: 'sqlite', storage: ':memory:' }, { introspection }),
-  );
+
+  if (isDatasourceMongoIntrospection(introspection)) {
+    const mongoose = buildDisconnectedMongooseInstance(introspection);
+    agent.addDataSource(createMongooseDataSource(mongoose, { flattenMode: 'auto' }));
+  } else {
+    const sequelize = await buildDisconnectedSequelizeInstance(introspection, null);
+    agent.addDataSource(createSequelizeDataSource(sequelize));
+  }
 
   return agent;
 }
 
 export async function updateTypings(
-  introspection: Table[],
+  introspection: DataSourceSQLIntrospection | DataSourceMongoIntrospection,
   bootstrapPathManager: BootstrapPathManager,
 ): Promise<void> {
-  const agent = buildAgent(introspection);
+  const agent = await buildAgent(introspection);
   await agent.updateTypesOnFileSystem(bootstrapPathManager.typingsDuringBootstrap, 3);
 }
 
 export async function updateTypingsWithCustomizations(
-  introspection: Table[],
+  introspection: DataSourceSQLIntrospection | DataSourceMongoIntrospection,
   distPathManager: DistPathManager,
   bootstrapPathManager: BootstrapPathManager,
 ): Promise<void> {
   const builtCodePath = path.resolve(distPathManager.distCodeCustomizations);
   await throwIfNoBuiltCode(builtCodePath);
-  const agent = buildAgent(introspection);
+  const agent = await buildAgent(introspection);
   loadCustomization(agent, builtCodePath);
   await agent.updateTypesOnFileSystem(bootstrapPathManager.typings, 3);
 }
diff --git a/packages/forest-cloud/test/commands/update-typings.test.ts b/packages/forest-cloud/test/commands/update-typings.test.ts
diff --git a/packages/forest-cloud/test/services/access-file.unit.test.ts b/packages/forest-cloud/test/services/access-file.unit.test.ts
diff --git a/packages/forest-cloud/test/services/load-customization.unit.test.ts b/packages/forest-cloud/test/services/load-customization.unit.test.ts
diff --git a/packages/forest-cloud/test/services/update-typings.unit.test.ts b/packages/forest-cloud/test/services/update-typings.unit.test.ts
diff --git a/yarn.lock b/yarn.lock
