Merge branch 'bug/prevent_manifest_service_crashing' into 'main'

Improve health checks to prevent manifest service from crashing, add jest

See merge request frontend/Infrastructure/manifest-service!1

.gitignore

@@ -6,6 +6,7 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
+/output
 
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

index.js

@@ -2,78 +2,10 @@
 // Note: actual env vars supersede .env file and .env file supersedes .env.defaults file
 require('dotenv-defaults').config()
 
-// Ignore paths for logging, metrics and docs
-const ignorePaths = ['/ready', '/healthy']
-
-// Fast, minimalist web framework for node.
-const express = require('express')
-
-// Helmet helps you secure your Express app by setting various HTTP headers
-const helmet = require('helmet')
-
-// Very low overhead Node.js logger. Logs in json use pino-pretty for dev.
 const logger = require('pino')()
+const createApp = require('./src/createApp')
 
-// Fastest HTTP logger for Node.js in town
-const httpLogger = require('pino-http')({ logger, autoLogging: { ignorePaths } })
-
-// Readiness and liveness checks middleware
-const health = require('@cloudnative/health-connect')
-const healthCheck = new health.HealthChecker()
-
-// Prometheus middleware for standard api metrics
-const apiMetrics = require('prometheus-api-metrics')
-
-// Swagger UI for api-docs
-const swaggerUi = require('swagger-ui-express')
-const yaml = require('js-yaml')
-const fs = require('fs')
-const swaggerDocument = yaml.load(fs.readFileSync('./swagger.yaml', 'utf8'))
-
-// The app returned by express() is in fact a JavaScript Function, designed to be passed to Node’s HTTP servers as a callback to handle requests.
-const app = express()
-
-const fetch = require('node-fetch')
-
-// Application-level middleware
-app.use(httpLogger)
-app.use(helmet())
-app.use('/healthy', health.LivenessEndpoint(healthCheck))
-app.use('/ready', health.ReadinessEndpoint(healthCheck))
-app.use(apiMetrics({ excludeRoutes: ignorePaths }))
-app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument))
-app.use('/swagger.json', (req, res) => res.json(swaggerDocument))
-
-const urls = yaml.load(fs.readFileSync('./config/manifests/urls.yaml', 'utf8')).manifests
-
-let manifestCache = []
-let lastCached
-
-const getManifests = async () => {
-  if (+new Date() < lastCached + 30000) return
-  const results = urls.map(url => fetch(url).then(result => {
-    if (!result.ok) throw new Error(`Failed to load manifest for url ${result.url} (Status: ${result.status}: ${result.statusText})`)
-    return result.json().catch(err => { throw new Error(`Failed to load manifest for url ${result.url}: ${err}`) })
-  }))
-  manifestCache = (await Promise.all(results)).flat()
-  lastCached = +new Date()
-}
-
-getManifests()
-
-app.get('/api/manifest.json', async (req, res, next) => {
-  try {
-    getManifests()
-    res.json(manifestCache || [])
-  } catch (err) {
-    next(err)
-  }
-})
-
-app.use(function (err, req, res, next) {
-  logger.error(err)
-  res.status(500).end()
-})
+const app = createApp()
 
 // Binds and listens for connections on the specified host and port
 app.listen(process.env.PORT, () => {

jest.config.js

+/*
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+module.exports = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/private/var/folders/51/r4dxfwvd2cn70x7vhpyh88480000gn/T/jest_dx",
+
+  // Automatically clear mock calls and instances between every test
+  clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+  collectCoverageFrom: ['index.js'],
+
+  // The directory where Jest should output its coverage files
+  coverageDirectory: 'output/coverage',
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: 'v8',
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+  coverageReporters: [
+    'text',
+    'cobertura',
+    'text-summary'
+  ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "json",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: undefined,
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state between every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state between every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: ['./spec/setup.js'],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  setupFilesAfterEnv: ['jest-extended'],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  // testEnvironment: "jest-environment-node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).[tj]s?(x)"
+  // ],
+  testMatch: [
+    '**/spec/**/*_(spec|test).js'
+  ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+  testResultsProcessor: 'jest-junit'
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
+  // testURL: "http://localhost",
+
+  // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
+  // timers: "real",
+
+  // A map from regular expressions to paths to transformers
+  // transform: undefined,
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/",
+  //   "\\.pnp\\.[^\\/]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+}

package.json

@@ -7,7 +7,8 @@
     "lint": "eslint .",
     "start": "node index.js",
     "dev": "nodemon index.js | pino-pretty",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "test": "jest --no-cache"
   },
   "author": "Open-Xchange",
   "license": "CC-BY-NC-SA-2.5",
@@ -26,16 +27,31 @@
     "swagger-ui-express": "^4.1.6"
   },
   "devDependencies": {
+    "babel-cli": "^6.26.0",
+    "babel-preset-env": "^1.7.0",
     "eslint": "^7.20.0",
     "eslint-plugin-import": "^2.22.1",
+    "eslint-plugin-jest": "^24.4.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^5.1.0",
+    "fetch-mock-jest": "^1.5.1",
     "husky": ">=6",
+    "jest": "^27.1.0",
+    "jest-extended": "^0.11.5",
+    "jest-junit": "^12.2.0",
     "lint-staged": ">=10",
+    "mock-fs": "^5.0.0",
     "nodemon": "^2.0.7",
-    "pino-pretty": "^4.7.1"
+    "pino-pretty": "^4.7.1",
+    "superagent": "^6.1.0",
+    "supertest": "^6.1.6"
   },
   "lint-staged": {
     "*.js": "eslint --cache --fix"
+  },
+  "jest-junit": {
+    "classNameTemplate": "{classname}",
+    "titleTemplate": "{title}",
+    "outputDirectory": "output/"
   }
 }

spec/.eslintrc.js

+module.exports = {
+  env: {
+    'jest/globals': true
+  },
+  plugins: ['jest']
+}

spec/server_test.js

+const { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } = require('@jest/globals')
+const mock = require('mock-fs')
+const fetchMock = require('node-fetch')
+const request = require('supertest')
+const createApp = require('../src/createApp')
+
+jest.mock('node-fetch', () => require('fetch-mock-jest').sandbox())
+
+describe('Manifest service', () => {
+  let app
+
+  beforeAll(() => {
+    mock({
+      './config/manifests': {
+        'urls.yaml': `manifests:
+          - https://some-fake-url.k3s.de/api/manifest.json`
+      }
+    })
+    app = createApp()
+  })
+
+  afterAll(() => {
+    mock.restore()
+  })
+
+  beforeEach(() => {
+    fetchMock.get('https://some-fake-url.k3s.de/api/manifest.json', JSON.parse('{"some":"thing"}'))
+  })
+
+  afterEach(() => {
+    fetchMock.restore()
+    app.timeout = 30000
+  })
+
+  it('is healthy', async () => {
+    return await request(app)
+      .get('/healthy')
+      .then(response => {
+        expect(response.statusCode).toBe(200)
+      })
+  })
+
+  it('fetches manifest data', async () => {
+    return await request(app)
+      .get('/api/manifest.json')
+      .then(response => {
+        expect(response.statusCode).toBe(200)
+        expect(response.body).toEqual([JSON.parse('{"some":"thing"}')])
+      })
+  })
+
+  it('caches manifest data', async () => {
+    await request(app)
+      .get('/api/manifest.json')
+      .then(response => {
+        expect(response.statusCode).toBe(200)
+        expect(response.body).toEqual([JSON.parse('{"some":"thing"}')])
+      })
+
+    fetchMock.restore()
+    fetchMock.get('https://some-fake-url.k3s.de/api/manifest.json', JSON.parse('{"some":"different"}'))
+
+    await new Promise((resolve, reject) => setTimeout(resolve, 150))
+
+    await request(app)
+      .get('/api/manifest.json')
+      .then(response => {
+        expect(response.statusCode).toBe(200)
+        expect(response.body).toEqual([JSON.parse('{"some":"thing"}')])
+      })
+  })
+
+  it('refreshes manifest data after caching timeout', async () => {
+    app.timeout = 100
+
+    await request(app)
+      .get('/api/manifest.json')
+      .then(response => {
+        expect(response.statusCode).toBe(200)
+        expect(response.body).toEqual([JSON.parse('{"some":"thing"}')])
+      })
+
+    fetchMock.restore()
+    fetchMock.get('https://some-fake-url.k3s.de/api/manifest.json', JSON.parse('{"some":"different"}'))
+
+    await new Promise((resolve, reject) => setTimeout(resolve, 150))
+
+    await request(app)
+      .get('/api/manifest.json')
+      .then(response => {
+        expect(response.statusCode).toBe(200)
+        expect(response.body).toEqual([JSON.parse('{"some":"different"}')])
+      })
+  })
+
+  it.skip('can load multiple configurations', async () => {
+    mock({
+      './config/manifests': {
+        'urls.yaml': `manifests:
+          - https://some-fake-url.k3s.de/api/manifest.json
+          - https://some-other-fake-url.k3s.de/api/manifest.json`
+      }
+    })
+
+    fetchMock.get('https://some-other-fake-url.k3s.de/api/manifest.json', JSON.parse('{"some":"other"}'))
+
+    const app = createApp()
+
+    await request(app)
+      .get('/api/manifest.json')
+      .then(response => {
+        expect(response.statusCode).toBe(200)
+        expect(response.body).toEqual([JSON.parse('{"some":"thing"}'), JSON.parse('{"some":"other"}')])
+      })
+  })
+})

src/createApp.js

+// Ignore paths for logging, metrics and docs
+const ignorePaths = ['/ready', '/healthy']
+
+// Fast, minimalist web framework for node.
+const express = require('express')
+
+// Helmet helps you secure your Express app by setting various HTTP headers
+const helmet = require('helmet')
+
+// Very low overhead Node.js logger. Logs in json use pino-pretty for dev.
+const logger = require('pino')()
+
+// Fastest HTTP logger for Node.js in town
+const httpLogger = require('pino-http')({ logger, autoLogging: { ignorePaths } })
+
+// Readiness and liveness checks middleware
+const health = require('@cloudnative/health-connect')
+
+// Prometheus middleware for standard api metrics
+const apiMetrics = require('prometheus-api-metrics')
+
+// Swagger UI for api-docs
+const swaggerUi = require('swagger-ui-express')
+const yaml = require('js-yaml')
+const fs = require('fs')
+const fetch = require('node-fetch')
+const swaggerDocument = yaml.load(fs.readFileSync('./swagger.yaml', 'utf8'))
+
+module.exports = () => {
+  // The app returned by express() is in fact a JavaScript Function, designed to be passed to Node’s HTTP servers as a callback to handle requests.
+
+  const app = express()
+
+  const healthCheck = new health.HealthChecker()
+
+  // Application-level middleware
+  app.use(httpLogger)
+  app.use(helmet())
+  app.use('/healthy', health.LivenessEndpoint(healthCheck))
+  app.use('/ready', health.ReadinessEndpoint(healthCheck))
+  app.use(apiMetrics({ excludeRoutes: ignorePaths }))
+  app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument))
+  app.use('/swagger.json', (req, res) => res.json(swaggerDocument))
+
+  const urls = yaml.load(fs.readFileSync('./config/manifests/urls.yaml', 'utf8')).manifests
+
+  let manifestCache = []
+  let lastCached
+  app.timeout = 30000
+
+  const fetchManifest = async () => {
+    if (+new Date() < lastCached + (app.timeout || 30000)) return
+    const results = urls.map(url => fetch(url).then(result => {
+      if (!result.ok) throw new Error(`Failed to load manifest for url ${result.url} (Status: ${result.status}: ${result.statusText})`)
+      return result.json().catch(err => { throw new Error(`Failed to load manifest for url ${result.url}: ${err}`) })
+    }))
+    manifestCache = (await Promise.all(results)).flat()
+    lastCached = +new Date()
+  }
+
+  app.get('/api/manifest.json', async (req, res, next) => {
+    try {
+      await fetchManifest()
+      res.json(manifestCache || [])
+    } catch (err) {
+      next(err)
+    }
+  })
+
+  app.use(function (err, req, res, next) {
+    logger.error(err)
+    res.status(500).end()
+  })
+
+  return app
+}