Running Github Actions + NestJs + Mongodb Memory Server raising (JavaScript heap out of memory)

[d4vz]

Versions

• NodeJS: 20.x
• mongodb-memory-server-*: 9.5.0
• mongodb(the binary version): 6.0.14
• mongodb(the js package): 0.0.0
• mongoose: @nestjs/mongoose 10.0.0
• system: ubuntu-24.04

package: mongo-memory-server

What is the Problem?

When running tests using Nestjs Testing Module i'm receiving out of memory from Github Actions (Working on every local machine):

Code Example

action.yaml

name: Node.js CI

on:
  push:
  pull_request:
    branches:
      - main
      - dev
permissions:
  contents: read

env:
  # Download mongodb binaries to ~/.cache/mongodb-binaries instead of local node_modules
  # Used for separate cache
  MONGOMS_PREFER_GLOBAL_PATH: true

jobs:
  tests:
    permissions:
      contents: read # for actions/checkout to fetch code
    runs-on: ${{ matrix.distro }}
    strategy:
      matrix:
        # this is a hack as there is currently no (public) way to access "runs-on" with version
        distro: [ubuntu-24.04]
        node-version: [22.x]
    steps:
      # Install libssl1.1 for libcrypto.so.1.1, which is required for binaries before 22.04 is available (4.0, 4.2, 4.4, 5.0)
      # currently 5.0 is still in LTS
      - name: Load libssl chache
        # cache the package so that we dont need to hit the debian server so often
        id: cache-libssl
        uses: actions/cache@v4
        with:
          path: ~/ssl
          key: libssl1.1
      - name: install libssl1.1
        # ubuntu seemingly does not have that package anymore, but the one from debian still works
        run: |
          mkdir -p ~/ssl && cd ~/ssl
          wget -nc https://ftp.debian.org/debian/pool/main/o/openssl/libssl1.1_1.1.1w-0+deb11u1_amd64.deb
          sudo dpkg -i libssl1.1_1.1.1w-0+deb11u1_amd64.deb
      # end libssl1.1 install
      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Load MongoDB binary cache
        id: cache-mongodb-binaries
        uses: actions/cache@v4
        with:
          path: ~/.cache/mongodb-binaries
          key: ${{ matrix.distro }}-${{ hashFiles('**/globalSetup.ts') }}
          restore-keys: |
            ${{ matrix.distro }}-
            ${{ matrix.distro }}
      - name: Install Dependencies
        run: npm ci --legacy-peer-deps
      - name: Test
        run: npm run test:ci --colors
        env:
          CI: true

File setups like typegoose repository

jest.config.ts

import type { Config } from 'jest';

const config: Config = {
  moduleFileExtensions: ['js', 'json', 'ts'],
  testTimeout: 30000,
  rootDir: 'src',
  testRegex: '.*\\.spec\\.ts$',
  transform: {
    '^.+\\.(t|j)s$': 'ts-jest',
  },
  collectCoverageFrom: ['**/*.(controller|service).(t|j)s'],
  moduleNameMapper: {
    '@common/(.*)': '<rootDir>/common/$1',
    '@modules/(.*)': '<rootDir>/modules/$1',
    '@services/(.*)': '<rootDir>/services/$1',
    '@utils/(.*)': '<rootDir>/utils/$1',
    'src/(.*)': '<rootDir>/$1',
  },
  coverageDirectory: '../coverage',
  testEnvironment: 'node',
  setupFilesAfterEnv: ['<rootDir>/../test/setup-test.ts'],
  globalSetup: '<rootDir>/../test/global-setup.ts',
  globalTeardown: '<rootDir>/../test/global-teardown.ts',
  globals: {
    'ts-jest': {
      isolatedModules: true,
    },
  },
  testPathIgnorePatterns: ['/node_modules/', '/dist/'],
  maxConcurrency: 2,
};

export default config;

package.json

{
  "name": "api",
  "version": "1.2.1",
  "description": "",
  "author": "",
  "private": true,
  "license": "UNLICENSED",
  "scripts": {
    "prebuild": "rimraf dist",
    "build": "npx @nestjs/cli build",
    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
    "dev": "nest start --watch",
    "start": "node dist/src/main",
    "start:dev": "nest start --watch",
    "start:debug": "nest start --debug --watch",
    "start:prod": "node dist/src/main",
    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
    "test": "jest --force-exit --detectOpenHandles",
    "test:ci": "jest --ci --logHeapUsage --runInBand --detectOpenHandles",
    "test:watch": "jest --watch --logHeapUsage",
    "test:cov": "jest --coverage --logHeapUsage",
    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand --logHeapUsage",
    "test:e2e": "jest --config ./test/jest-e2e.json --logHeapUsage",
    "knip": "knip"
  },
  "dependencies": {
    "@liaoliaots/nestjs-redis": "^9.0.5",
    "@nestjs/axios": "^0.1.0",
    "@nestjs/cache-manager": "^2.3.0",
    "@nestjs/cli": "^10.0.0",
    "@nestjs/common": "^10.0.0",
    "@nestjs/config": "^2.2.0",
    "@nestjs/core": "^10.0.0",
    "@nestjs/event-emitter": "^2.0.2",
    "@nestjs/jwt": "^10.0.0",
    "@nestjs/mapped-types": "*",
    "@nestjs/mongoose": "^10.0.0",
    "@nestjs/passport": "^10.0.0",
    "@nestjs/platform-express": "^10.0.0",
    "@nestjs/swagger": "^7.0.0",
    "@nestjs/testing": "^10.0.0",
    "@prisma/client": "^4.0.0",
    "@types/amqplib": "^0.10.5",
    "@types/geojson": "^7946.0.10",
    "@types/sharp": "^0.31.1",
    "@willsoto/nestjs-prometheus": "^6.0.1",
    "ajv": "^8.12.0",
    "amqp-connection-manager": "^4.1.10",
    "amqplib": "^0.10.3",
    "aws-sdk": "^2.1323.0",
    "axios": "^1.6.8",
    "cache-manager": "^5.7.6",
    "cheerio": "^1.0.0-rc.12",
    "class-transformer": "^0.5.1",
    "class-validator": "^0.13.2",
    "compression": "^1.7.4",
    "cron-parser": "^4.7.1",
    "crypto": "^1.0.1",
    "dotenv": "^16.4.1",
    "express": "^4.18.2",
    "geojson": "^0.5.0",
    "helmet": "^5.1.0",
    "ioredis": "^5.3.2",
    "joi": "^17.6.0",
    "lodash": "^4.17.21",
    "lodash.groupby": "^4.6.0",
    "moment": "^2.29.4",
    "mongodb": "^5.9.2",
    "mongodb-memory-server": "^9.5.0",
    "mongoose-delete": "^0.5.4",
    "mongoose-paginate-ts": "^1.2.7",
    "multer": "^1.4.5-lts.1",
    "multer-s3": "^3.0.1",
    "passport": "^0.6.0",
    "passport-jwt": "^4.0.0",
    "passport-local": "^1.0.0",
    "prom-client": "^15.1.3",
    "reflect-metadata": "^0.1.13",
    "rimraf": "^3.0.2",
    "rrule": "^2.8.1",
    "rxjs": "^7.2.0",
    "sharp": "^0.31.3"
  },
  "devDependencies": {
    "@faker-js/faker": "^8.4.1",
    "@nestjs/cli": "^10.0.0",
    "@nestjs/schematics": "^10.0.0",
    "@types/express": "^4.17.13",
    "@types/jest": "28.1.4",
    "@types/lodash": "^4.14.197",
    "@types/node": "^18.11.18",
    "@types/passport-local": "^1.0.34",
    "@types/supertest": "^2.0.11",
    "@typescript-eslint/eslint-plugin": "^5.0.0",
    "@typescript-eslint/parser": "^5.0.0",
    "eslint": "^8.36.0",
    "eslint-config-prettier": "^8.3.0",
    "eslint-plugin-prettier": "^4.0.0",
    "jest": "^29.7.0",
    "knip": "^5.51.0",
    "prettier": "^2.3.2",
    "source-map-support": "^0.5.20",
    "supertest": "^6.1.3",
    "ts-jest": "28.0.5",
    "ts-loader": "^9.2.3",
    "ts-node": "^10.0.0",
    "tsconfig-paths": "4.0.0",
    "typescript": "^5.1.6"
  },
  "husky": {
    "hooks": {
      "pre-commit": "lint-staged"
    }
  },
  "lint-staged": {
    "src/**/*": [
      "yarn format",
      "yarn lint",
      "yarn test --bail --findRelatedTests"
    ]
  }
}

Debug Output

Debug Output

Run npm run test:ci --colors
> api@1.2.1 test:ci
> jest --ci --logHeapUsage --runInBand --detectOpenHandles
ts-jest[versions] (WARN) Version 29.[7](https://github.com/marine-br/api/actions/runs/15281021852/job/42979812813?pr=981#step:9:8).0 of jest installed has not been tested with ts-jest. If you're experiencing issues, consider using a supported version (>=2[8](https://github.com/marine-br/api/actions/runs/15281021852/job/42979812813?pr=981#step:9:9).0.0 <29.0.0-0). Please do not report issues in ts-jest if you are using unsupported versions.
ts-jest[versions] (WARN) Version 5.8.3 of typescript installed has not been tested with ts-jest. If you're experiencing issues, consider using a supported version (>=4.3.0 <5.0.0-0). Please do not report issues in ts-jest if you are using unsupported versions.
PASS src/modules/installation/installer/usecases/send-otp-token.usecase.spec.ts (266 MB heap size)
PASS src/modules/installation/installer/usecases/update-installer.usecase.spec.ts (285 MB heap size)
PASS src/modules/installation/installer/usecases/sign-in-installer.usecase.spec.ts (26[9](https://github.com/marine-br/api/actions/runs/15281021852/job/42979812813?pr=981#step:9:10) MB heap size)
PASS src/modules/installation/installer/usecases/delete-installer.usecase.spec.ts (300 MB heap size)
<--- Last few GCs --->
[2108:0x40301000]   297700 ms: Mark-Compact 2005.9 (2081.6) -> 1990.2 (2081.9) MB, pooled: 9 MB, 271.05 / 0.00 ms  (average mu = 0.147, current mu = 0.[10](https://github.com/marine-br/api/actions/runs/15281021852/job/42979812813?pr=981#step:9:11)6) allocation failure; scavenge might not succeed
[2108:0x40301000]   297997 ms: Mark-Compact 2006.0 (2081.9) -> 1990.3 (2082.1) MB, pooled: 9 MB, 233.49 / 0.00 ms  (average mu = 0.179, current mu = 0.213) allocation failure; scavenge might not succeed
<--- JS stacktrace --->
FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
----- Native stack trace -----
 1: 0xe09a56 node::OOMErrorHandler(char const*, v8::OOMDetails const&) [node]
 2: 0x[11](https://github.com/marine-br/api/actions/runs/15281021852/job/42979812813?pr=981#step:9:12)ba250 v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, v8::OOMDetails const&) [node]
 3: 0x11ba527 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, v8::OOMDetails const&) [node]
 4: 0x[13](https://github.com/marine-br/api/actions/runs/15281021852/job/42979812813?pr=981#step:9:14)e7f55  [node]
 5: 0x13e7f83  [node]
 6: 0x140105a  [node]
 7: 0x1404228  [node]
 8: 0x1c69ef1  [node]
Aborted (core dumped)

Do you know why it happenes?

no

[hasezoey]

As i dont see how you actually start a MMS instance, i assume you use the default options for mongodb 6.x, which is storageEngine: 'ephermeralForTest', which in turn is a in-memory only engine. So unless you are clearing the database or restarting the instance each test suite, it could easily run out of memory if you store enough in there.
I would recommend to set the storageEngine to wiredTiger and/or check that you clear the database at some point. Also note that since mongodb 8 the ephermeralForTest storage engine was removed and so wiredTiger became the default there.

As a aside, i would also recommend to upgrade MMS to 10.x (currently the latest is 10.1.4) and upgrade the mongodb version so that the ssl workaround would not be required anymore.

[d4vz]

import { MongooseModule, MongooseModuleOptions } from '@nestjs/mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { config } from '../../../test/utils/config';

import { disconnect } from 'mongoose';

let mongo: MongoMemoryServer;

export const rootMongooseTestModule = (options: MongooseModuleOptions = {}) =>
  MongooseModule.forRootAsync({
    useFactory: async () => {
      const instance = (global as any).__MONGOINSTANCE;
      mongo =
        instance ||
        (await MongoMemoryServer.create({
          instance: { ip: config.IP, dbName: config.DataBase, port: config.Port },
        }));
      const uri = mongo.getUri();
      return {
        uri: uri,
        ...options,
      };
    },
  });

export const closeMongoConnection = async () => {
  await disconnect();
  if (mongo) await mongo.stop();
};

global-setup.ts

import { MongoMemoryServer } from 'mongodb-memory-server';
import { config } from './utils/config';
import { connect } from './utils/connect';

export = async function globalSetup() {
  if (config.Memory) {
    /** it's needed in global space, because we don't want to create a new instance every time */
    const instance = await MongoMemoryServer.create();
    const uri = instance.getUri();
    (global as any).__MONGOINSTANCE = instance;
    process.env.MONGO_URI = uri.slice(0, uri.lastIndexOf('/'));
  } else {
    process.env.MONGO_URI = `mongodb://${config.IP}:${config.Port}`;
  }

  const connection = await connect({ dbName: config.DataBase });
  await connection.db?.dropDatabase();
  await connection.close();
};

global-teardown.ts

import { MongoMemoryServer } from 'mongodb-memory-server';
import { config } from './utils/config';

export = async function globalTeardown() {
  if (config.Memory) {
    const instance: MongoMemoryServer = (global as any).__MONGOINSTANCE;
    await instance.stop();
  }
};

setup-test.ts

import { connect, disconnect } from './utils/connect';

beforeAll(async () => {
  await connect();
});

afterAll(async () => {
  await disconnect();
});

jest.config.ts

  setupFilesAfterEnv: ['<rootDir>/../test/setup-test.ts'],
  globalSetup: '<rootDir>/../test/global-setup.ts',
  globalTeardown: '<rootDir>/../test/global-teardown.ts',

utils/connect.ts

import mongoose from 'mongoose';
import { config } from './config';

interface ExtraConnectionConfig {
  dbName?: string;
  createNewConnection?: boolean;
  differentMongoose?: mongoose.Mongoose;
}

// to not duplicate code
const staticOptions: mongoose.ConnectOptions = {
  autoIndex: true,
};

/**
 * Make a Connection to MongoDB
 */
export async function connect(extraConfig: ExtraConnectionConfig = {}): Promise<mongoose.Connection> {
  const mongooseInstance: mongoose.Mongoose = extraConfig.differentMongoose ?? mongoose;
  let connection: mongoose.Connection;

  const options = Object.assign({}, staticOptions);

  if (config.Memory) {
    if (config?.Auth?.User?.length > 0) {
      Object.assign(options, {
        user: config.Auth.User,
        pass: config.Auth.Passwd,
        authSource: config.Auth.DB,
      });
    }
  }

  // to not duplicate code
  const connectionString = `${process.env.MONGO_URI}/${extraConfig.dbName ?? config.DataBase}`;

  if (extraConfig.createNewConnection) {
    connection = await mongooseInstance.createConnection(connectionString, options).asPromise();
  } else {
    await mongoose.connect(connectionString, options);
    connection = mongooseInstance.connection;
  }

  return connection;
}

/**
 * Disconnect from MongoDB
 * @returns when it is disconnected
 */
export async function disconnect(): Promise<void> {
  await mongoose.disconnect();

  return;
}

config.ts

import * as fs from 'fs';

interface IConfig {
  Memory: boolean;
  DataBase: string;
  Port: number;
  Auth: IAuth;
  IP: string;
}

interface IAuth {
  User: string;
  Passwd: string;
  DB: string;
}

enum EConfig {
  MONGODB_IP = 'MongoDB IP is not specified!',
  MONGODB_DB = 'MongoDB DataBase is not specified!',
  MONGODB_PORT = 'MongoDB Port is not specified!',
  MONGODB_AUTH = 'You should activate & use MongoDB Authentication!',
}

const env: NodeJS.ProcessEnv = process.env; // just to write less

let path: string = env.CONFIG ?? './test/config.json';
path = fs.existsSync(path) ? path : './test/config_default.json';

const configRAW: Readonly<IConfig> = JSON.parse(fs.readFileSync(path).toString());

// ENV || CONFIG-FILE || DEFAULT
const configFINAL: Readonly<IConfig> = {
  Memory: env.C_USE_IN_MEMORY !== undefined || (typeof configRAW.Memory === 'boolean' ? configRAW.Memory : true),
  DataBase: env.C_DATABASE ?? configRAW?.DataBase ?? 'typegooseTest',
  Port: parseInt(env.C_PORT as string, 10) || configRAW.Port || 27017,
  Auth: {
    User: env.C_AUTH_USER ?? configRAW?.Auth?.User ?? '',
    Passwd: env.C_AUTH_PASSWD || configRAW?.Auth?.Passwd || '',
    DB: env.C_AUTH_DB || configRAW?.Auth?.DB || '',
  },
  IP: env.C_IP ?? configRAW.IP ?? 'localhost',
};

/** Small callback for the tests below */
function cb(text: string): void {
  console.error(text);
  process.exit(-1);
}

if (!configFINAL.Memory) {
  if (!configFINAL.IP) {
    cb(EConfig.MONGODB_IP);
  }
  if (!configFINAL.DataBase) {
    cb(EConfig.MONGODB_DB);
  }
  if (!configFINAL.Port) {
    cb(EConfig.MONGODB_PORT);
  }
}

export { configFINAL as config };

[hasezoey]

Thanks for providing the code, but from what i can tell, the 2 places where a instance could be created, both use the default ephermeralForTest, which can be quite memory heavy if not cleared.

Also i dont quite know much about how or where rootMongooseTestModule is used, but if it is in a different context than globalSetup and globalTeardown, at least in jest (global as any).__MONGOINSTANCE will always be false-y as to my knowledge in jest only process.env is shared to the workers after setup (not global).
Could you verify it is not always creating another instance? (or worse multiple in a single suite?)