Add dependency parsing

spec/dependencies_test.js

+import { describe, it, expect } from '@jest/globals'
+
+import { viteManifestToDeps } from '../src/manifests.js'
+
+describe('Vite manifest parsing', () => {
+  it('should', () => {
+    const deps = viteManifestToDeps({
+      '../io.ox/guidedtours/i18n.de_DE.js': {
+        file: 'io.ox/guidedtours/i18n.de_DE.js',
+        src: '../io.ox/guidedtours/i18n.de_DE.js',
+        isEntry: true,
+        meta: {}
+      },
+      '../io.ox/guidedtours/i18n': {
+        file: 'io.ox/guidedtours/i18n.3de05d46.js',
+        src: '../io.ox/guidedtours/i18n',
+        isEntry: true,
+        imports: [
+          '_preload-helper-a7bbbf37.js'
+        ],
+        meta: {
+          gettext: {
+            dictionary: true
+          },
+          manifests: [
+            {
+              namespace: 'i18n'
+            }
+          ]
+        }
+      },
+      'io.ox/guidedtours/intro.js': {
+        file: 'io.ox/guidedtours/intro.e84819a5.js',
+        src: 'io.ox/guidedtours/intro.js',
+        isEntry: true,
+        isDynamicEntry: true,
+        imports: [
+          '../io.ox/guidedtours/i18n',
+          '_preload-helper-a7bbbf37.js'
+        ],
+        meta: {}
+      },
+      'io.ox/guidedtours/main.js': {
+        file: 'io.ox/guidedtours/main.07676e21.js',
+        src: 'io.ox/guidedtours/main.js',
+        isEntry: true,
+        imports: [
+          '_preload-helper-a7bbbf37.js',
+          '../io.ox/guidedtours/i18n'
+        ],
+        dynamicImports: [
+          'io.ox/guidedtours/intro.js',
+          'io.ox/guidedtours/multifactor.js'
+        ],
+        meta: {
+          manifests: [
+            {
+              namespace: 'settings'
+            },
+            {
+              namespace: 'io.ox/core/main',
+              title: 'Guided tours',
+              company: 'Open-Xchange',
+              icon: '/images/icon.png',
+              category: 'Dev',
+              settings: false,
+              index: 100,
+              package: 'open-xchange-guidedtours'
+            }
+          ]
+        }
+      },
+      'io.ox/guidedtours/multifactor.js': {
+        file: 'io.ox/guidedtours/multifactor.22d3e17d.js',
+        src: 'io.ox/guidedtours/multifactor.js',
+        isEntry: true,
+        isDynamicEntry: true,
+        imports: [
+          '_preload-helper-a7bbbf37.js',
+          '../io.ox/guidedtours/i18n',
+          'io.ox/guidedtours/main.js'
+        ],
+        meta: {}
+      },
+      'io.ox/guidedtours/utils.js': {
+        file: 'io.ox/guidedtours/utils.91ad511f.js',
+        src: 'io.ox/guidedtours/utils.js',
+        isEntry: true,
+        imports: [
+          '_preload-helper-a7bbbf37.js'
+        ],
+        meta: {}
+      },
+      '_preload-helper-a7bbbf37.js': {
+        file: 'io.ox/guidedtours/preload-helper-a7bbbf37.js'
+      }
+    })
+    expect(typeof deps).toBe('object')
+    expect(Object.keys(deps).length).toBe(7)
+    expect(deps['io.ox/guidedtours/main.07676e21.js']).toEqual(['_preload-helper-a7bbbf37.js', '../io.ox/guidedtours/i18n'])
+  })
+})

spec/manifest_parsing_test.js

@@ -98,9 +98,9 @@ describe('Vite manifest parsing', () => {
     expect(Array.isArray(manifests)).toBe(true)
     expect(manifests.length).toBe(3)
     expect(manifests.map(manifest => manifest.path)).toEqual([
-      'io.ox/guidedtours/i18n.3de05d46.js',
-      'io.ox/guidedtours/main.07676e21.js',
-      'io.ox/guidedtours/main.07676e21.js'
+      'io.ox/guidedtours/i18n.3de05d46',
+      'io.ox/guidedtours/main.07676e21',
+      'io.ox/guidedtours/main.07676e21'
     ])
     expect(manifests.map(manifest => manifest.namespace)).toEqual([
       'i18n',

src/createApp.js

@@ -18,10 +18,9 @@ import promBundle from 'express-prom-bundle'
 import swaggerUi from 'swagger-ui-express'
 import yaml from 'js-yaml'
 import fs from 'fs'
-import fetch from 'node-fetch'
+import { getDependencies, getOxManifests } from './manifests.js'
 
 const ignorePaths = ['/ready', '/healthy']
-const logger = new Logger()
 const httpLogger = pinoHttp({ logger, autoLogging: { ignorePaths } })
 const swaggerDocument = yaml.load(fs.readFileSync('./src/swagger.yaml', 'utf8'))
 const bypass = (request) => ignorePaths.includes(request.path)
@@ -42,27 +41,19 @@ export function createApp () {
   app.use(metricsMiddleware)
   app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument))
   app.use('/swagger.json', (req, res) => res.json(swaggerDocument))
-
-  const urls = yaml.load(fs.readFileSync('./config/manifests/urls.yaml', 'utf8')).manifests
-
-  let manifestCache = []
-  let lastCached
   app.timeout = 30000
 
-  const fetchManifest = async () => {
-    if (+new Date() < lastCached + (app.timeout || 30000)) return
-    const results = urls.map(url => fetch(url).then(result => {
-      if (!result.ok) throw new Error(`Failed to load manifest for url ${result.url} (Status: ${result.status}: ${result.statusText})`)
-      return result.json().catch(err => { throw new Error(`Failed to load manifest for url ${result.url}: ${err}`) })
-    }))
-    manifestCache = (await Promise.all(results)).flat()
-    lastCached = +new Date()
-  }
-
   app.get('/api/manifest.json', async (req, res, next) => {
     try {
-      await fetchManifest()
-      res.json(manifestCache || [])
+      res.json(await getOxManifests())
+    } catch (err) {
+      next(err)
+    }
+  })
+
+  app.get('/api/deps.json', async (req, res, next) => {
+    try {
+      res.json(await getDependencies())
     } catch (err) {
       next(err)
     }

src/manifests.js

+import fs from 'fs/promises'
+import yaml from 'js-yaml'
+import fetch from 'node-fetch'
+import path from 'path'
+
+const CACHE_TTL = parseInt(process.env.CACHE_TTL)
+
+export const loadViteManifests = (() => {
+  let cache
+  let lastCacheTime
+
+  return async function loadViteManifests ({ useCache = true } = {}) {
+    if (!cache || useCache === false || +new Date() > lastCacheTime + CACHE_TTL) {
+      const urlsSource = await fs.readFile('./config/manifests/urls.yaml', 'utf8')
+      const urls = yaml.load(urlsSource).manifests
+
+      // vite manifests contains a set of objects with the vite-manifests
+      // from the corresponding registered services
+      const viteManifests = await Promise.all(urls.map(async url => {
+        // fetch the manifests
+        const result = await fetch(url)
+        if (!result.ok) throw new Error(`Failed to load manifest for url ${result.url} (Status: ${result.status}: ${result.statusText})`)
+        try {
+          return result.json()
+        } catch (err) {
+          throw new Error(`Failed to load manifest for url ${result.url}: ${err}`)
+        }
+      }))
+
+      // combine all manifests by keys. With duplicates, last wins
+      const viteManifest = viteManifests.reduce((memo, manifest) => Object.assign(memo, manifest), {})
+      // only update that object if it really changed to prevent any further parsing from being triggered
+      if (!cache || JSON.stringify(viteManifest) !== JSON.stringify(cache)) {
+        cache = viteManifest
+      }
+
+      lastCacheTime = +new Date()
+    }
+    return cache
+  }
+})()
+
 export function viteToOxManifest (viteManifests) {
   return Object.values(viteManifests)
     .filter(manifest => Array.isArray(manifest?.meta?.manifests))
@@ -5,9 +47,44 @@ export function viteToOxManifest (viteManifests) {
       manifest.meta.manifests.map(oxManifest => {
         return {
           ...oxManifest,
-          path: manifest.file
+          path: manifest.file.slice(0, -path.parse(manifest.file).ext.length)
         }
       })
     )
     .flat()
 }
+
+export const getOxManifests = (() => {
+  let prevViteManifest
+  let oxManifestCache
+  return async function getOxManifests () {
+    const viteManifest = await loadViteManifests()
+    if (viteManifest !== prevViteManifest) {
+      oxManifestCache = viteToOxManifest(viteManifest)
+      prevViteManifest = viteManifest
+    }
+    return oxManifestCache
+  }
+})()
+
+export function viteManifestToDeps (viteManifest) {
+  const deps = {}
+  for (const { file, imports, css } of Object.values(viteManifest)) {
+    deps[file] = [].concat(imports).concat(css).filter(Boolean)
+  }
+  return deps
+}
+
+export const getDependencies = (() => {
+  let prevViteManifest
+  let depCache
+  return async function getDependencies () {
+    const viteManifest = await loadViteManifests()
+    console.log('viteManifests', Object.keys(viteManifest).length)
+    if (viteManifest !== prevViteManifest) {
+      depCache = viteManifestToDeps(viteManifest)
+      prevViteManifest = viteManifest
+    }
+    return depCache
+  }
+})()