Add optional flag for redis configuration

.gitlab/autodeploy/values.yaml

@@ -6,6 +6,7 @@ baseUrls:
   - http://main-core-ui.main-e2e-stack.svc.cluster.local
 
 redis:
+  enabled: true
   host: main-redis-master.main-e2e-stack.svc.cluster.local
   prefix: ${CI_COMMIT_REF_SLUG}-${OX_COMPONENT}
 

helm/core-ui-middleware/templates/deployment.yaml

@@ -27,6 +27,7 @@ spec:
               value: "{{ .Values.logLevel }}"
             - name: APP_ROOT
               value: "{{ .Values.appRoot }}"
+            {{- if .Values.redis.enabled }}
             - name: REDIS_HOST
               value: "{{ required "redis.host required" .Values.redis.host }}"
             - name: REDIS_PORT
@@ -37,6 +38,7 @@ spec:
               value: "{{ .Values.redis.password }}"
             - name: REDIS_PREFIX
               value: "{{ .Values.redis.prefix }}"
+            {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.containerPort | default 8080 }}

helm/core-ui-middleware/values.yaml

@@ -105,6 +105,7 @@ baseUrls: []
 appRoot: '/'
 
 redis:
+  enabled: false
   host: ''
   port: 6379
   db: 0

integration/.mocharc.cjs

 module.exports = {
   spec: ['integration/**/*_test.js'],
+  file: ['integration/global-setup.js']
 }
\ No newline at end of file

integration/global-setup.js

+process.env.REDIS_HOST = 'localhost'

package.json

@@ -9,7 +9,7 @@
     "start": "node src/index.js",
     "dev": "nodemon index.js",
     "prepare": "husky install",
-    "test": "LOG_LEVEL=error mocha --loader=testdouble --config spec/.mocharc.cjs",
+    "test": "LOG_LEVEL=error mocha --loader=testdouble --config spec/.mocharc.cjs --exit",
     "integration": "LOG_LEVEL=error mocha --loader=testdouble --config integration/.mocharc.cjs --exit",
     "release-chart": "cd helm/core-ui-middleware/ && npx --package=@open-xchange/release-it -- release-it",
     "release-app": "npx --package=@open-xchange/release-it@latest -- release-it-auto-keepachangelog",

spec/redis_test.js

+import request from 'supertest'
+import { generateSimpleViteManifest, mockApp, mockConfig, mockFetch } from './util.js'
+import { expect } from 'chai'
+import * as td from 'testdouble'
+import { Response } from 'node-fetch'
+import sinon from 'sinon'
+
+const sandbox = sinon.createSandbox()
+
 describe('Redis', function () {
-  it('first instance updates version', async function () {
-    // TODO
+  let app
+  let spy
+
+  beforeEach(async function () {
+    // no redis mock!!
+    await import('../src/create-queues.js').then(({ default: createQueues }) => createQueues())
+    mockConfig({ urls: ['http://ui-server/'] })
+    mockFetch({
+      'http://ui-server': {
+        '/manifest.json': generateSimpleViteManifest({}),
+        '/example.js': spy = sandbox.spy(() => {
+          return new Response('this is example', { headers: { 'content-type': 'application/javascript' } })
+        })
+      }
+    })
+    app = await mockApp()
+  })
+
+  afterEach(async function () {
+    td.reset()
+  })
+
+  it('use internal cache, when redis is disabled', async function () {
+    expect(spy.callCount).to.equal(0)
+    let response = await request(app).get('/example.js')
+    expect(response.statusCode).to.equal(200)
+    expect(spy.callCount).to.equal(1)
+    response = await request(app).get('/example.js')
+    expect(response.statusCode).to.equal(200)
+    expect(spy.callCount).to.equal(1)
   })
 })

spec/util.js

@@ -42,9 +42,10 @@ export function mockFetch (servers = {}) {
   })
 }
 
-export function mockRedis (data = {}) {
+export function mockRedis (data = {}, isEnabled = true) {
   const mock = {
     isReady () { return Promise.resolve() },
+    isEnabled () { return isEnabled },
     client: new RedisMock({ data }),
     pubClient: new RedisMock(),
     subClient: new RedisMock()

src/cache.js

+import * as redis from './redis.js'
+
+const cache = {}
+
+export function set (key, value) {
+  if (cache[key] === value) return
+  cache[key] = value
+  if (redis.isEnabled()) {
+    return redis.client.set(key, value)
+  }
+}
+
+export async function getBuffer (key) {
+  return get(key, { method: 'getBuffer' })
+}
+
+export async function get (key, { method = 'get' } = {}) {
+  if (cache[key]) return cache[key]
+
+  if (redis.isEnabled()) {
+    const result = await redis.client[method]?.(key)
+    cache[key] = result
+    return result
+  }
+}

src/create-queues.js

-import { getQueue, subClient } from './redis.js'
+import * as redis from './redis.js'
 import { updateVersionProcessor, registerLatestVersionListener } from './version.js'
 
+const { getQueue, subClient } = redis
+
 export default function createQueues () {
-  const updateVersionQueue = getQueue('update-version')
-  updateVersionQueue.process(updateVersionProcessor)
-  updateVersionQueue.add({}, {
-    jobId: 'update-version-job',
-    repeat: { every: Number(process.env.CACHE_TTL) },
-    removeOnComplete: true
-  })
+  if (redis.isEnabled()) {
+    const updateVersionQueue = getQueue('update-version')
+    updateVersionQueue.process(updateVersionProcessor)
+    updateVersionQueue.add({}, {
+      jobId: 'update-version-job',
+      repeat: { every: Number(process.env.CACHE_TTL) },
+      removeOnComplete: true
+    })
+  }
 
   // not a queue but though, used by redis
   registerLatestVersionListener(subClient)

src/files.js

@@ -3,12 +3,10 @@ import crypto from 'crypto'
 import { config } from './config.js'
 import { getRedisKey, isJSFile } from './util.js'
 import { getCSSDependenciesFor, getViteManifests } from './manifests.js'
-import { client } from './redis.js'
+import * as cache from './cache.js'
 import { logger } from './logger.js'
 import { NotFoundError } from './errors.js'
 
-const fileCache = {}
-
 export async function fetchFileWithHeadersFromBaseUrl (path, baseUrl, version) {
   const [response, dependencies] = await Promise.all([
     fetch(new URL(path, baseUrl)),
@@ -56,27 +54,17 @@ export async function fetchFileWithHeaders ({ path, version }) {
 
 export async function saveToCache ({ version, path, body, headers, ...rest }) {
   if (typeof body !== 'string' && !(body instanceof Buffer)) body = JSON.stringify(body)
-  fileCache[version] = fileCache[version] || {}
-  fileCache[version][path] = {
-    body,
-    headers,
-    ...rest
-  }
   return Promise.all([
-    client.set(getRedisKey({ version, name: `${path}:body` }), body),
-    client.set(getRedisKey({ version, name: `${path}:meta` }), JSON.stringify({ headers, ...rest }))
+    cache.set(getRedisKey({ version, name: `${path}:body` }), body),
+    cache.set(getRedisKey({ version, name: `${path}:meta` }), JSON.stringify({ headers, ...rest }))
   ])
 }
 
 export async function loadFromCache ({ version, path }) {
-  if (!fileCache[version]?.[path]) {
-    const [body, meta = '{}'] = await Promise.all([
-      client.getBuffer(getRedisKey({ version, name: `${path}:body` })),
-      client.get(getRedisKey({ version, name: `${path}:meta` }))
-    ])
-    if (!body) return
-    fileCache[version] = fileCache[version] || {}
-    fileCache[version][path] = { ...JSON.parse(meta), body }
-  }
-  return fileCache[version][path]
+  const [body, meta = '{}'] = await Promise.all([
+    cache.getBuffer(getRedisKey({ version, name: `${path}:body` })),
+    cache.get(getRedisKey({ version, name: `${path}:meta` }))
+  ])
+  if (!body) return
+  return { ...JSON.parse(meta), body }
 }

src/manifests.js

@@ -2,10 +2,10 @@ import fetch from 'node-fetch'
 import { config } from './config.js'
 import { getRedisKey, viteManifestToDeps, viteToOxManifest } from './util.js'
 import { logger } from './logger.js'
-import { client } from './redis.js'
+import * as cache from './cache.js'
 
 export async function getViteManifests ({ version }) {
-  const manifests = await client.get(getRedisKey({ version, name: 'viteManifests' }))
+  const manifests = await cache.get(getRedisKey({ version, name: 'viteManifests' }))
   if (manifests) return JSON.parse(manifests)
 
   await config.load()
@@ -30,27 +30,27 @@ export async function getViteManifests ({ version }) {
 
   // combine all manifests by keys. With duplicates, last wins
   const newManifests = viteManifests.reduce((memo, manifest) => Object.assign(memo, manifest), {})
-  await client.set(getRedisKey({ version, name: 'viteManifests' }), JSON.stringify(newManifests))
+  await cache.set(getRedisKey({ version, name: 'viteManifests' }), JSON.stringify(newManifests))
   return newManifests
 }
 
 export async function getOxManifests ({ version }) {
-  const manifests = await client.get(getRedisKey({ version, name: 'oxManifests' }))
+  const manifests = await cache.get(getRedisKey({ version, name: 'oxManifests' }))
   if (manifests) return JSON.parse(manifests)
 
   const viteManifests = await getViteManifests({ version })
   const newManifests = viteToOxManifest(viteManifests)
-  await client.set(getRedisKey({ version, name: 'oxManifests' }), JSON.stringify(newManifests))
+  await cache.set(getRedisKey({ version, name: 'oxManifests' }), JSON.stringify(newManifests))
   return newManifests
 }
 
 export async function getDependencies ({ version }) {
-  const deps = await client.get(getRedisKey({ version, name: 'dependencies' }))
+  const deps = await cache.get(getRedisKey({ version, name: 'dependencies' }))
   if (deps) return JSON.parse(deps)
 
   const viteManifests = await getViteManifests({ version })
   const newDeps = viteManifestToDeps(viteManifests)
-  await client.set(getRedisKey({ version, name: 'dependencies' }), JSON.stringify(newDeps))
+  await cache.set(getRedisKey({ version, name: 'dependencies' }), JSON.stringify(newDeps))
   return newDeps
 }
 

src/redis.js

@@ -5,6 +5,14 @@ import Queue from 'bull'
 const commonQueueOptions = { enableReadyCheck: false, maxRetriesPerRequest: null }
 
 const createClient = (type, options = {}) => {
+  if (!isEnabled()) {
+    return new Proxy({}, {
+      get () {
+        throw new Error('Redis is disabled. Do not use it.')
+      }
+    })
+  }
+
   const client = new Redis({
     host: process.env.REDIS_HOST,
     port: process.env.REDIS_PORT || 6379,
@@ -64,3 +72,7 @@ export async function closeQueue (name) {
   }
   return queue.close()
 }
+
+export function isEnabled () {
+  return !!process.env.REDIS_HOST
+}

src/routes/health.js

@@ -7,8 +7,10 @@ import { once } from '../util.js'
 const router = Router()
 const healthCheck = new health.HealthChecker()
 
-const redisReady = new health.ReadinessCheck('Redis ready', redis.isReady)
-healthCheck.registerReadinessCheck(redisReady)
+if (redis.isEnabled()) {
+  const redisReady = new health.ReadinessCheck('Redis ready', redis.isReady)
+  healthCheck.registerReadinessCheck(redisReady)
+}
 
 const startupCheck = new health.StartupCheck('check latest version', once(async function () {
   await getLatestVersion()

src/version.js

@@ -2,7 +2,7 @@ import fetch from 'node-fetch'
 import { config } from './config.js'
 import { getRedisKey, hash } from './util.js'
 import { logger } from './logger.js'
-import { client, pubClient } from './redis.js'
+import * as redis from './redis.js'
 
 let latestVersion
 
@@ -34,30 +34,40 @@ export const fetchLatestVersion = async () => {
 export async function getLatestVersion () {
   if (latestVersion) return latestVersion
 
-  const version = await client.get(getRedisKey({ name: 'latestVersion' }))
-  if (version) return (latestVersion = version)
+  if (redis.isEnabled()) {
+    const version = await redis.client.get(getRedisKey({ name: 'latestVersion' }))
+    if (version) return (latestVersion = version)
+  }
 
   const newVersion = await fetchLatestVersion()
-  pubClient.publish(getRedisKey({ name: 'updateLatestVersion' }), newVersion)
-  await client.set(getRedisKey({ name: 'latestVersion' }), newVersion)
+
+  if (redis.isEnabled()) {
+    redis.pubClient.publish(getRedisKey({ name: 'updateLatestVersion' }), newVersion)
+    await redis.client.set(getRedisKey({ name: 'latestVersion' }), newVersion)
+  }
+
   return (latestVersion = newVersion)
 }
 
 export function registerLatestVersionListener (client) {
-  const key = getRedisKey({ name: 'updateLatestVersion' })
-  client.subscribe(key, (errs, count) => logger.info(`Subscribed to ${key}.`))
-  client.on('message', (channel, message) => {
-    if (channel === key) latestVersion = message
-  })
+  if (redis.isEnabled()) {
+    const key = getRedisKey({ name: 'updateLatestVersion' })
+    client.subscribe(key, (errs, count) => logger.info(`Subscribed to ${key}.`))
+    client.on('message', (channel, message) => {
+      if (channel === key) latestVersion = message
+    })
+  } else {
+    setInterval(updateVersionProcessor, Number(process.env.CACHE_TTL))
+  }
 }
 
-export async function updateVersionProcessor (job) {
+export async function updateVersionProcessor () {
   const [storedVersion, fetchedVersion] = await Promise.all([
     getLatestVersion(),
     fetchLatestVersion()
   ])
   if (storedVersion === fetchedVersion) return fetchedVersion
-  pubClient.publish(getRedisKey({ name: 'updateLatestVersion' }), fetchedVersion)
-  await client.set(getRedisKey({ name: 'latestVersion' }), fetchedVersion)
+  redis.pubClient.publish(getRedisKey({ name: 'updateLatestVersion' }), fetchedVersion)
+  await redis.client.set(getRedisKey({ name: 'latestVersion' }), fetchedVersion)
   return fetchedVersion
 }