From 1fff10d3c37cb598286e21fd14b3cdb006ecf5ee Mon Sep 17 00:00:00 2001
From: Richard Petersen <richard.petersen@open-xchange.com>
Date: Tue, 7 Sep 2021 12:21:09 +0200
Subject: [PATCH] Add dependency parsing
---
spec/dependencies_test.js | 102 ++++++++++++++++++++++++++++++++++
spec/manifest_parsing_test.js | 6 +-
src/createApp.js | 29 ++++------
src/manifests.js | 79 +++++++++++++++++++++++++-
4 files changed, 193 insertions(+), 23 deletions(-)
create mode 100644 spec/dependencies_test.js
diff --git a/spec/dependencies_test.js b/spec/dependencies_test.js
new file mode 100644
index 0000000..3978ba1
--- /dev/null
+++ b/spec/dependencies_test.js
@@ -0,0 +1,102 @@
+import { describe, it, expect } from '@jest/globals'
+
+import { viteManifestToDeps } from '../src/manifests.js'
+
+describe('Vite manifest parsing', () => {
+ it('should', () => {
+ const deps = viteManifestToDeps({
+ '../io.ox/guidedtours/i18n.de_DE.js': {
+ file: 'io.ox/guidedtours/i18n.de_DE.js',
+ src: '../io.ox/guidedtours/i18n.de_DE.js',
+ isEntry: true,
+ meta: {}
+ },
+ '../io.ox/guidedtours/i18n': {
+ file: 'io.ox/guidedtours/i18n.3de05d46.js',
+ src: '../io.ox/guidedtours/i18n',
+ isEntry: true,
+ imports: [
+ '_preload-helper-a7bbbf37.js'
+ ],
+ meta: {
+ gettext: {
+ dictionary: true
+ },
+ manifests: [
+ {
+ namespace: 'i18n'
+ }
+ ]
+ }
+ },
+ 'io.ox/guidedtours/intro.js': {
+ file: 'io.ox/guidedtours/intro.e84819a5.js',
+ src: 'io.ox/guidedtours/intro.js',
+ isEntry: true,
+ isDynamicEntry: true,
+ imports: [
+ '../io.ox/guidedtours/i18n',
+ '_preload-helper-a7bbbf37.js'
+ ],
+ meta: {}
+ },
+ 'io.ox/guidedtours/main.js': {
+ file: 'io.ox/guidedtours/main.07676e21.js',
+ src: 'io.ox/guidedtours/main.js',
+ isEntry: true,
+ imports: [
+ '_preload-helper-a7bbbf37.js',
+ '../io.ox/guidedtours/i18n'
+ ],
+ dynamicImports: [
+ 'io.ox/guidedtours/intro.js',
+ 'io.ox/guidedtours/multifactor.js'
+ ],
+ meta: {
+ manifests: [
+ {
+ namespace: 'settings'
+ },
+ {
+ namespace: 'io.ox/core/main',
+ title: 'Guided tours',
+ company: 'Open-Xchange',
+ icon: '/images/icon.png',
+ category: 'Dev',
+ settings: false,
+ index: 100,
+ package: 'open-xchange-guidedtours'
+ }
+ ]
+ }
+ },
+ 'io.ox/guidedtours/multifactor.js': {
+ file: 'io.ox/guidedtours/multifactor.22d3e17d.js',
+ src: 'io.ox/guidedtours/multifactor.js',
+ isEntry: true,
+ isDynamicEntry: true,
+ imports: [
+ '_preload-helper-a7bbbf37.js',
+ '../io.ox/guidedtours/i18n',
+ 'io.ox/guidedtours/main.js'
+ ],
+ meta: {}
+ },
+ 'io.ox/guidedtours/utils.js': {
+ file: 'io.ox/guidedtours/utils.91ad511f.js',
+ src: 'io.ox/guidedtours/utils.js',
+ isEntry: true,
+ imports: [
+ '_preload-helper-a7bbbf37.js'
+ ],
+ meta: {}
+ },
+ '_preload-helper-a7bbbf37.js': {
+ file: 'io.ox/guidedtours/preload-helper-a7bbbf37.js'
+ }
+ })
+ expect(typeof deps).toBe('object')
+ expect(Object.keys(deps).length).toBe(7)
+ expect(deps['io.ox/guidedtours/main.07676e21.js']).toEqual(['_preload-helper-a7bbbf37.js', '../io.ox/guidedtours/i18n'])
+ })
+})
diff --git a/spec/manifest_parsing_test.js b/spec/manifest_parsing_test.js
index 596b5c4..71ecd14 100644
--- a/spec/manifest_parsing_test.js
+++ b/spec/manifest_parsing_test.js
@@ -98,9 +98,9 @@ describe('Vite manifest parsing', () => {
expect(Array.isArray(manifests)).toBe(true)
expect(manifests.length).toBe(3)
expect(manifests.map(manifest => manifest.path)).toEqual([
- 'io.ox/guidedtours/i18n.3de05d46.js',
- 'io.ox/guidedtours/main.07676e21.js',
- 'io.ox/guidedtours/main.07676e21.js'
+ 'io.ox/guidedtours/i18n.3de05d46',
+ 'io.ox/guidedtours/main.07676e21',
+ 'io.ox/guidedtours/main.07676e21'
])
expect(manifests.map(manifest => manifest.namespace)).toEqual([
'i18n',
diff --git a/src/createApp.js b/src/createApp.js
index e42c9ef..c622bca 100644
--- a/src/createApp.js
+++ b/src/createApp.js
@@ -18,10 +18,9 @@ import promBundle from 'express-prom-bundle'
import swaggerUi from 'swagger-ui-express'
import yaml from 'js-yaml'
import fs from 'fs'
-import fetch from 'node-fetch'
+import { getDependencies, getOxManifests } from './manifests.js'
const ignorePaths = ['/ready', '/healthy']
-const logger = new Logger()
const httpLogger = pinoHttp({ logger, autoLogging: { ignorePaths } })
const swaggerDocument = yaml.load(fs.readFileSync('./src/swagger.yaml', 'utf8'))
const bypass = (request) => ignorePaths.includes(request.path)
@@ -42,27 +41,19 @@ export function createApp () {
app.use(metricsMiddleware)
app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument))
app.use('/swagger.json', (req, res) => res.json(swaggerDocument))
-
- const urls = yaml.load(fs.readFileSync('./config/manifests/urls.yaml', 'utf8')).manifests
-
- let manifestCache = []
- let lastCached
app.timeout = 30000
- const fetchManifest = async () => {
- if (+new Date() < lastCached + (app.timeout || 30000)) return
- const results = urls.map(url => fetch(url).then(result => {
- if (!result.ok) throw new Error(`Failed to load manifest for url ${result.url} (Status: ${result.status}: ${result.statusText})`)
- return result.json().catch(err => { throw new Error(`Failed to load manifest for url ${result.url}: ${err}`) })
- }))
- manifestCache = (await Promise.all(results)).flat()
- lastCached = +new Date()
- }
-
app.get('/api/manifest.json', async (req, res, next) => {
try {
- await fetchManifest()
- res.json(manifestCache || [])
+ res.json(await getOxManifests())
+ } catch (err) {
+ next(err)
+ }
+ })
+
+ app.get('/api/deps.json', async (req, res, next) => {
+ try {
+ res.json(await getDependencies())
} catch (err) {
next(err)
}
diff --git a/src/manifests.js b/src/manifests.js
index d949959..d84d783 100644
--- a/src/manifests.js
+++ b/src/manifests.js
@@ -1,3 +1,45 @@
+import fs from 'fs/promises'
+import yaml from 'js-yaml'
+import fetch from 'node-fetch'
+import path from 'path'
+
+const CACHE_TTL = parseInt(process.env.CACHE_TTL)
+
+export const loadViteManifests = (() => {
+ let cache
+ let lastCacheTime
+
+ return async function loadViteManifests ({ useCache = true } = {}) {
+ if (!cache || useCache === false || +new Date() > lastCacheTime + CACHE_TTL) {
+ const urlsSource = await fs.readFile('./config/manifests/urls.yaml', 'utf8')
+ const urls = yaml.load(urlsSource).manifests
+
+ // vite manifests contains a set of objects with the vite-manifests
+ // from the corresponding registered services
+ const viteManifests = await Promise.all(urls.map(async url => {
+ // fetch the manifests
+ const result = await fetch(url)
+ if (!result.ok) throw new Error(`Failed to load manifest for url ${result.url} (Status: ${result.status}: ${result.statusText})`)
+ try {
+ return result.json()
+ } catch (err) {
+ throw new Error(`Failed to load manifest for url ${result.url}: ${err}`)
+ }
+ }))
+
+ // combine all manifests by keys. With duplicates, last wins
+ const viteManifest = viteManifests.reduce((memo, manifest) => Object.assign(memo, manifest), {})
+ // only update that object if it really changed to prevent any further parsing from being triggered
+ if (!cache || JSON.stringify(viteManifest) !== JSON.stringify(cache)) {
+ cache = viteManifest
+ }
+
+ lastCacheTime = +new Date()
+ }
+ return cache
+ }
+})()
+
export function viteToOxManifest (viteManifests) {
return Object.values(viteManifests)
.filter(manifest => Array.isArray(manifest?.meta?.manifests))
@@ -5,9 +47,44 @@ export function viteToOxManifest (viteManifests) {
manifest.meta.manifests.map(oxManifest => {
return {
...oxManifest,
- path: manifest.file
+ path: manifest.file.slice(0, -path.parse(manifest.file).ext.length)
}
})
)
.flat()
}
+
+export const getOxManifests = (() => {
+ let prevViteManifest
+ let oxManifestCache
+ return async function getOxManifests () {
+ const viteManifest = await loadViteManifests()
+ if (viteManifest !== prevViteManifest) {
+ oxManifestCache = viteToOxManifest(viteManifest)
+ prevViteManifest = viteManifest
+ }
+ return oxManifestCache
+ }
+})()
+
+export function viteManifestToDeps (viteManifest) {
+ const deps = {}
+ for (const { file, imports, css } of Object.values(viteManifest)) {
+ deps[file] = [].concat(imports).concat(css).filter(Boolean)
+ }
+ return deps
+}
+
+export const getDependencies = (() => {
+ let prevViteManifest
+ let depCache
+ return async function getDependencies () {
+ const viteManifest = await loadViteManifests()
+ console.log('viteManifests', Object.keys(viteManifest).length)
+ if (viteManifest !== prevViteManifest) {
+ depCache = viteManifestToDeps(viteManifest)
+ prevViteManifest = viteManifest
+ }
+ return depCache
+ }
+})()
--
GitLab