From 2c3a21e5b60a6a139d8ef3ed9a1407a2751a22b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Julian=20B=C3=A4ume?= <julian.baeume@open-xchange.com>
Date: Mon, 27 Sep 2021 12:57:14 +0200
Subject: [PATCH] implement local file cache
the file cache is populated with data defined in the manifest files configured for
all components and will basically fetch everything from all components. This allows
us to use this manifest service as the only source of truth for UI related sources and
simplifies ingres setup a lot.
---
spec/dependencies_test.js | 70 ++++++++++++++++++++++++++++++++---
spec/file_caching_test.js | 77 +++++++++++++++++++++++++++++++++++++++
spec/util.js | 11 ++++--
src/createApp.js | 15 ++++++++
src/files.js | 40 ++++++++++++++++++++
src/manifests.js | 16 +++++++-
6 files changed, 218 insertions(+), 11 deletions(-)
create mode 100644 spec/file_caching_test.js
create mode 100644 src/files.js
diff --git a/spec/dependencies_test.js b/spec/dependencies_test.js
index f89e4eb..e97e419 100644
--- a/spec/dependencies_test.js
+++ b/spec/dependencies_test.js
@@ -3,7 +3,7 @@ import { describe, it, expect } from '@jest/globals'
import { viteManifestToDeps } from '../src/manifests.js'
describe('Vite manifest parsing', () => {
- it('should', () => {
+ it('works for simple modules', () => {
const deps = viteManifestToDeps({
'../io.ox/guidedtours/i18n.de_DE.js': {
file: 'io.ox/guidedtours/i18n.de_DE.js',
@@ -96,7 +96,16 @@ describe('Vite manifest parsing', () => {
},
'_preload-helper-a7bbbf37.js': {
file: 'io.ox/guidedtours/preload-helper-a7bbbf37.js'
- },
+ }
+ })
+ expect(typeof deps).toBe('object')
+ expect(Object.keys(deps).length).toBe(8)
+ expect(deps['io.ox/guidedtours/main.07676e21.js']).toEqual(['io.ox/guidedtours/preload-helper-a7bbbf37.js', 'io.ox/guidedtours/i18n.3de05d46.js'])
+ expect(deps['io.ox/guidedtours/multifactor.22d3e17d.js']).toEqual(['io.ox/guidedtours/preload-helper-a7bbbf37.js', 'io.ox/guidedtours/i18n.3de05d46.js', 'io.ox/guidedtours/main.07676e21.js', 'io.ox/guidedtours/assets/multifactor.91962241.css'])
+ })
+
+ it('exports assets as entrypoints without dependencies', async () => {
+ const deps = viteManifestToDeps({
'themes/icons/alarm.svg': {
file: 'assets/alarm.6d2fbb40.js',
src: 'themes/icons/alarm.svg',
@@ -107,10 +116,59 @@ describe('Vite manifest parsing', () => {
meta: {}
}
})
- expect(typeof deps).toBe('object')
- expect(Object.keys(deps).length).toBe(8)
- expect(deps['io.ox/guidedtours/main.07676e21.js']).toEqual(['io.ox/guidedtours/preload-helper-a7bbbf37.js', 'io.ox/guidedtours/i18n.3de05d46.js'])
- expect(deps['io.ox/guidedtours/multifactor.22d3e17d.js']).toEqual(['io.ox/guidedtours/preload-helper-a7bbbf37.js', 'io.ox/guidedtours/i18n.3de05d46.js', 'io.ox/guidedtours/main.07676e21.js', 'io.ox/guidedtours/assets/multifactor.91962241.css'])
+ expect(Object.keys(deps).length).toBe(2)
expect(deps['assets/alarm.6d2fbb40.js']).toEqual(['assets/alarm.310541a0.svg'])
+ expect(deps['assets/alarm.310541a0.svg']).toEqual([])
+ })
+
+ it('exports css as entrypoints without dependencies', async () => {
+ const deps = viteManifestToDeps({
+ 'main.js': {
+ file: 'main.js',
+ src: 'main.js',
+ isEntry: true,
+ css: ['assets/main.3b761440.css'],
+ meta: {}
+ }
+ })
+ expect(deps['main.js']).toEqual(['assets/main.3b761440.css'])
+ expect(deps['assets/main.3b761440.css']).toEqual([])
+ })
+
+ it('separately exports HTML entrypoints', async () => {
+ const deps = viteManifestToDeps({
+ 'index.html': {
+ file: 'index.html.js',
+ src: 'index.html',
+ isEntry: true,
+ imports: ['main.js', '_preload-helper.a295b1c6.js', '_vendor.ae457d06.js'],
+ meta: {}
+ },
+ '_preload-helper.a295b1c6.js': { file: 'assets/preload-helper.a295b1c6.js' },
+ '_vendor.ae457d06.js': {
+ file: 'assets/vendor.ae457d06.js',
+ isDynamicEntry: true,
+ dynamicImports: ['io.ox/core/a11y.js']
+ },
+ 'io.ox/core/a11y.js': {
+ file: 'io.ox/core/a11y.js',
+ src: 'io.ox/core/a11y.js',
+ isEntry: true,
+ isDynamicEntry: true,
+ imports: ['_vendor.ae457d06.js'],
+ meta: {}
+ },
+ 'main.js': {
+ file: 'main.js',
+ src: 'main.js',
+ isEntry: true,
+ imports: ['_preload-helper.a295b1c6.js', '_vendor.ae457d06.js', 'io.ox/core/a11y.js'],
+ css: ['assets/main.3b761440.css'],
+ meta: {}
+ }
+ })
+
+ expect(Object.keys(deps).length).toBe(7)
+ expect(deps['index.html']).toEqual(['index.html.js'])
})
})
diff --git a/spec/file_caching_test.js b/spec/file_caching_test.js
new file mode 100644
index 0000000..7a30e60
--- /dev/null
+++ b/spec/file_caching_test.js
@@ -0,0 +1,77 @@
+import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from '@jest/globals'
+import mockfs from 'mock-fs'
+import request from 'supertest'
+import { createApp } from '../src/createApp'
+import { createMockServer, generateSimpleViteManifest, getRandomPort } from './util.js'
+
+describe('File caching service', () => {
+ let app
+ let mockserver
+ const port = getRandomPort()
+
+ beforeAll(() => {
+ mockfs({
+ './config/manifests': {
+ 'urls.yaml': `manifests:
+ - http://localhost:${port}/manifest.json`
+ }
+ })
+ app = createApp()
+ })
+
+ afterAll(() => {
+ mockfs.restore()
+ })
+
+ beforeEach(async () => {
+ mockserver = await createMockServer({ port })
+ mockserver.respondWith({
+ '/manifest.json': generateSimpleViteManifest({
+ 'example.js': { imports: ['test.txt'] },
+ 'test.txt': { },
+ 'main.css': {},
+ 'index.html': {
+ file: 'index.html.js',
+ isEntry: true,
+ imports: ['example.js'],
+ css: ['main.css']
+ }
+ }),
+ '/example.js': (req, res) => res.setHeader('content-type', 'application/javascript').status(200).send('this is example'),
+ '/test.txt': (req, res) => res.setHeader('content-type', 'text/plain').status(200).send('this is test'),
+ '/index.html.js': (req, res) => res.setHeader('content-type', 'application/javascript').status(200).send('this is index.html.js'),
+ '/index.html': (req, res) => res.setHeader('content-type', 'text/html').status(200).send('<html><head></head><body>it\'s me</body></html>'),
+ '/main.css': (req, res) => res.setHeader('content-type', 'text/css').status(200).send('.foo { color: #000; }')
+
+ })
+ })
+
+ afterEach(() => {
+ mockserver.close()
+ process.env.CACHE_TTL = 30000
+ })
+
+ it('serves files defined in manifest.json file', async () => {
+ const response = await request(app).get('/example.js')
+ expect(response.statusCode).toBe(200)
+ expect(response.headers['content-type']).toBe('application/javascript; charset=utf-8')
+ expect(response.text).toBe('this is example')
+ const response2 = await request(app).get('/test.txt')
+ expect(response2.statusCode).toBe(200)
+ expect(response2.headers['content-type']).toBe('text/plain; charset=utf-8')
+ expect(response2.text).toBe('this is test')
+ })
+
+ it('serves css files', async () => {
+ const response = await request(app).get('/main.css')
+ expect(response.statusCode).toBe(200)
+ expect(response.headers['content-type']).toBe('text/css; charset=utf-8')
+ })
+
+ it('serves / as index.html', async () => {
+ const response = await request(app).get('/')
+ expect(response.statusCode).toBe(200)
+ expect(response.headers['content-type']).toBe('text/html; charset=utf-8')
+ expect(response.text).toBe('<html><head></head><body>it\'s me</body></html>')
+ })
+})
diff --git a/spec/util.js b/spec/util.js
index cd78917..5453dd9 100644
--- a/spec/util.js
+++ b/spec/util.js
@@ -11,7 +11,11 @@ export async function createMockServer ({ port }) {
})
server.respondWith = function (routes) {
for (const [route, data] of Object.entries(routes)) {
- app.get(route, (req, res) => res.json(data))
+ if (typeof data === 'function') {
+ app.get(route, data)
+ } else {
+ app.get(route, (req, res) => res.json(data))
+ }
}
}
return server
@@ -19,11 +23,12 @@ export async function createMockServer ({ port }) {
export function generateSimpleViteManifest (mapping) {
const viteManifest = {}
- for (const [file, namespace] of Object.entries(mapping)) {
+ for (const [file, value] of Object.entries(mapping)) {
viteManifest[file] = {
file,
- meta: namespace ? { manifests: [{ namespace }] } : {}
+ meta: typeof value === 'string' ? { manifests: [{ namespace: value }] } : {}
}
+ if (typeof value === 'object') Object.assign(viteManifest[file], value)
}
return viteManifest
}
diff --git a/src/createApp.js b/src/createApp.js
index e4cca76..ffdaa78 100644
--- a/src/createApp.js
+++ b/src/createApp.js
@@ -19,6 +19,7 @@ import swaggerUi from 'swagger-ui-express'
import yaml from 'js-yaml'
import fs from 'fs'
import { getDependencies, getOxManifests } from './manifests.js'
+import { fileCache } from './files.js'
const ignorePaths = ['/ready', '/healthy']
const swaggerDocument = yaml.load(fs.readFileSync('./src/swagger.yaml', 'utf8'))
@@ -61,6 +62,20 @@ export function createApp () {
}
})
+ app.get('/', async (req, res, next) => {
+ await getDependencies()
+ const { 'content-type': contentType, content } = fileCache.get('/index.html')
+ if (content) return res.setHeader('content-type', contentType).status(200).send(content)
+ next()
+ })
+
+ app.use(async (req, res, next) => {
+ await getDependencies()
+ const { 'content-type': contentType, content } = fileCache.get(req.path)
+ if (content) return res.setHeader('content-type', contentType).status(200).send(content)
+ next()
+ })
+
app.use(function (err, req, res, next) {
logger.error(err)
res.status(500).end()
diff --git a/src/files.js b/src/files.js
new file mode 100644
index 0000000..9d2c45b
--- /dev/null
+++ b/src/files.js
@@ -0,0 +1,40 @@
+import fetch from 'node-fetch'
+
+class FileCache {
+ async warmUp (manifests, deps) {
+ const cache = Object.fromEntries(await (async function () {
+ const files = Object.keys(deps)
+ const chunkSize = Math.ceil(files.length / 50)
+ const result = []
+ while (files.length > 0) {
+ result.push.apply(result, (await Promise.all(files.splice(0, chunkSize).map(async file => {
+ try {
+ const manifest = manifests[file] || Object.values(manifests).find(m =>
+ m.file === file ||
+ (m?.assets?.indexOf(file) >= 0) ||
+ (m?.css?.indexOf(file) >= 0)
+ )
+ if (!manifest) {
+ console.error('could not find manifest for', file)
+ return null
+ }
+ const response = await fetch(new URL(file, manifest.meta.baseUrl))
+ if (!response.ok) return null
+ return [file, {
+ 'content-type': response.headers.get('content-type'),
+ content: await response.buffer()
+ }]
+ } catch (e) { console.error(e) }
+ }))).filter(data => Array.isArray(data) && data.length === 2))
+ }
+ return result
+ }()))
+ this._cache = cache
+ }
+
+ get (path) {
+ return this?._cache[path.slice(1)] || {}
+ }
+}
+
+export const fileCache = new FileCache()
diff --git a/src/manifests.js b/src/manifests.js
index 4b95b7f..44b9250 100644
--- a/src/manifests.js
+++ b/src/manifests.js
@@ -2,6 +2,8 @@ import fs from 'fs/promises'
import yaml from 'js-yaml'
import fetch from 'node-fetch'
import path from 'path'
+import { URL } from 'url'
+import { fileCache } from './files.js'
export const loadViteManifests = (() => {
let cache
@@ -17,11 +19,17 @@ export const loadViteManifests = (() => {
// vite manifests contains a set of objects with the vite-manifests
// from the corresponding registered services
const viteManifests = await Promise.all(urls.map(async url => {
+ const { origin } = new URL(url)
// fetch the manifests
const result = await fetch(url)
if (!result.ok) throw new Error(`Failed to load manifest for url ${result.url} (Status: ${result.status}: ${result.statusText})`)
try {
- return result.json()
+ const manifest = await result.json()
+ for (const file in manifest) {
+ manifest[file].meta = manifest[file].meta || {}
+ manifest[file].meta.baseUrl = origin
+ }
+ return manifest
} catch (err) {
throw new Error(`Failed to load manifest for url ${result.url}: ${err}`)
}
@@ -73,7 +81,10 @@ export const getOxManifests = (() => {
export function viteManifestToDeps (viteManifest) {
const deps = {}
- for (const { file, imports, css, assets } of Object.values(viteManifest)) {
+ for (const [codePoint, { isEntry, file, imports, css, assets }] of Object.entries(viteManifest)) {
+ if (isEntry && codePoint.endsWith('.html')) deps[codePoint] = [file]
+ if (Array.isArray(assets)) assets.forEach(asset => { deps[asset] = [] })
+ if (Array.isArray(css)) css.forEach(css => { deps[css] = [] })
deps[file] = []
.concat(imports?.map(path => viteManifest[path].file))
.concat(css)
@@ -90,6 +101,7 @@ export const getDependencies = (() => {
const viteManifest = await loadViteManifests()
if (viteManifest !== prevViteManifest) {
depCache = viteManifestToDeps(viteManifest)
+ await fileCache.warmUp(viteManifest, depCache)
prevViteManifest = viteManifest
}
return depCache
--
GitLab