Introduce contstraints, when to use gzip

.env.defaults

@@ -4,6 +4,8 @@ PORT=8080
 LOG_LEVEL=info
 APP_ROOT=/
 EXPOSE_API_DOCS=false
+COMPRESS_FILE_SIZE=600
+COMPRESS_FILE_TYPES=application/javascript application/json application/x-javascript application/xml application/xml+rss text/css text/html text/javascript text/plain text/xml image/svg+xml
 
 REDIS_PORT=6379
 REDIS_DB=0

README.md

@@ -37,28 +37,33 @@ It is possible to horizontally scale the UI Middleware, as more clients are fetc
 
 **local, docker**
 
-| Parameter       | Description                     | Default  |
-|-----------------|---------------------------------|----------|
-| `PORT`          | Exposed port                    | `"8080"` |
-| `CACHE_TTL`     | Vite manifest caching time      | `30000`  |
-| `LOG_LEVEL`     | Pino log level                  | `"info"` |
-| `REDIS_HOST`    | Redis host (required)           |          |
-| `REDIS_PORT`    | Redis port (optional)           | `6379`   |
-| `REDIS_DB`      | Redis DB, e.g. `"1"` (optional) | null     |
-| `REDIS_PASSWORD`| Redis password (optional)       | null     |
+| Parameter             | Description                     | Default  |
+|-----------------------|---------------------------------|----------|
+| `PORT`                | Exposed port                    | `"8080"` |
+| `CACHE_TTL`           | Vite manifest caching time      | `30000`  |
+| `LOG_LEVEL`           | Pino log level                  | `"info"` |
+| `REDIS_HOST`          | Redis host (required)           |          |
+| `REDIS_PORT`          | Redis port (optional)           | `6379`   |
+| `REDIS_DB`            | Redis DB, e.g. `"1"` (optional) | null     |
+| `REDIS_PASSWORD`      | Redis password (optional)       | null     |
+| `COMPRESS_FILE_SIZE`  | Larger files will be gzipped    | `600`    |
+| `COMPRESS_FILE_TYPES` | Set of compression mime types   |see values|
+
 
 **kubernetes**
 
-| Parameter       | Description                     | Default  |
-|-----------------|---------------------------------|----------|
-| `port`          | Exposed port                    | `"8080"` |
-| `cacheTTL`      | Vite manifest caching time      | `30000`  |
-| `logLevel`      | Pino log level                  | `"info"` |
-| `redis.enabled` | Global switch Redis integration |  false   |
-| `redis.host`    | Redis host                      |          |
-| `redis.port`    | Redis port (optional)           | `6379`   |
-| `redis.db`      | Redis DB, e.g. `"1"` (optional) | null     |
-| `redis.password`| Redis password (optional)       | null     |
+| Parameter             | Description                     | Default  |
+|-----------------------|---------------------------------|----------|
+| `port`                | Exposed port                    | `"8080"` |
+| `cacheTTL`            | Vite manifest caching time      | `30000`  |
+| `logLevel`            | Pino log level                  | `"info"` |
+| `redis.enabled`       | Global switch Redis integration |  false   |
+| `redis.host`          | Redis host                      |          |
+| `redis.port`          | Redis port (optional)           | `6379`   |
+| `redis.db`            | Redis DB, e.g. `"1"` (optional) | null     |
+| `redis.password`      | Redis password (optional)       | null     |
+| `compressFileSize`    | Larger files will be gzipped    | `600`    |
+| `compressFileTypes`   | Set of compression mime types   |see values|
 
 ## Ingress
 

helm/core-ui-middleware/templates/deployment.yaml

@@ -27,6 +27,10 @@ spec:
               value: "{{ .Values.logLevel }}"
             - name: APP_ROOT
               value: "{{ .Values.appRoot }}"
+            - name: COMPRESS_FILE_SIZE
+              value: "{{ .Values.compressFileSize }}"
+            - name: COMPRESS_FILE_TYPES
+              value: "{{ .Values.compressFileTypes }}"
             {{- if .Values.redis.enabled }}
             - name: REDIS_HOST
               value: "{{ required "redis.host required" .Values.redis.host }}"

helm/core-ui-middleware/values.yaml

@@ -103,6 +103,8 @@ cacheTTL: 30000
 logLevel: info
 baseUrls: []
 appRoot: '/'
+compressFileSize: 600
+compressFileTypes: application/javascript application/json application/x-javascript application/xml application/xml+rss text/css text/html text/javascript text/plain text/xml image/svg+xml
 
 redis:
   enabled: false

integration/caching_test.js

@@ -4,7 +4,6 @@ import { generateSimpleViteManifest, mockApp, mockConfig, mockFetch } from '../s
 import { client } from '../src/redis.js'
 import * as td from 'testdouble'
 import { getRedisKey } from '../src/util.js'
-import { gunzipSync } from 'node:zlib'
 
 describe('File caching service', function () {
   let app
@@ -42,9 +41,9 @@ describe('File caching service', function () {
     const version = response.headers.version
 
     const body = (await client.getBuffer(getRedisKey({ version, name: '/index.html:body' }))) || ''
-    expect(gunzipSync(body).toString()).to.equal('<html><head></head><body>it\'s me</body></html>')
+    expect(body.toString()).to.equal('<html><head></head><body>it\'s me</body></html>')
     const meta = await client.get(getRedisKey({ version, name: '/index.html:meta' }))
-    expect(meta).to.equal('{"headers":{"content-type":"text/html","dependencies":false,"content-encoding":"gzip"}}')
+    expect(meta).to.equal('{"headers":{"content-type":"text/html","dependencies":false}}')
   })
 
   it('serves files from redis and stores them in local cache', async function () {

spec/file_caching_test.js

@@ -350,11 +350,53 @@ describe('File caching service', function () {
   })
 
   it('serves files as gzip', async function () {
-    const response = await request(app.server).get('/example.js')
+    mockFetch({
+      'http://ui-server': {
+        '/manifest.json': generateSimpleViteManifest({}),
+        '/large.js': () => new Response([...new Array(2500)].join(' '), { headers: { 'content-type': 'application/javascript' } })
+      }
+    })
+    app = await mockApp()
+
+    const response = await request(app.server).get('/large.js')
     expect(response.statusCode).to.equal(200)
     expect(response.headers['content-encoding']).to.equal('gzip')
 
     // check for files in redis
-    expect(zlib.gunzipSync(await redis.client.getBuffer('ui-middleware:554855300:/example.js:body')).toString()).to.equal(response.text)
+    expect(zlib.gunzipSync(await redis.client.getBuffer('ui-middleware:554855300:/large.js:body')).toString()).to.equal(response.text)
+  })
+
+  it('does not serve small files with gzip', async function () {
+    mockFetch({
+      'http://ui-server': {
+        '/manifest.json': generateSimpleViteManifest({}),
+        '/small.js': () => new Response('small', { headers: { 'content-type': 'application/javascript' } })
+      }
+    })
+    app = await mockApp()
+
+    const response = await request(app.server).get('/small.js')
+    expect(response.statusCode).to.equal(200)
+    expect(response.headers).to.not.have.property('content-encoding')
+
+    // check for files in redis
+    expect((await redis.client.getBuffer('ui-middleware:554855300:/small.js:body')).toString()).to.equal(response.text)
+  })
+
+  it('does not serve other mime types with gzip', async function () {
+    mockFetch({
+      'http://ui-server': {
+        '/manifest.json': generateSimpleViteManifest({}),
+        '/file.mp3': () => new Response('123', { headers: { 'content-type': 'audio/mpeg' } })
+      }
+    })
+    app = await mockApp()
+
+    const response = await request(app.server).get('/file.mp3')
+    expect(response.statusCode).to.equal(200)
+    expect(response.headers).to.not.have.property('content-encoding')
+
+    // check for files in redis
+    expect(await redis.client.getBuffer('ui-middleware:554855300:/file.mp3:body')).to.deep.equal(response.body)
   })
 })

src/files.js

@@ -9,6 +9,10 @@ import zlib from 'node:zlib'
 import { promisify } from 'node:util'
 const gzip = promisify(zlib.gzip)
 
+const compressFileSize = Number(process.env.COMPRESS_FILE_SIZE)
+const compressionMimeTypes = (process.env.COMPRESS_FILE_TYPES || '').split(' ')
+const compressionWhitelistRegex = new RegExp(`^(${compressionMimeTypes.join('|')})$`)
+
 export function createWritable (body) {
   if (typeof body !== 'string' && !(body instanceof Buffer)) return JSON.stringify(body)
   return body
@@ -24,7 +28,7 @@ async function createFileBuffer (response, dependencies) {
   buffer.fill(Buffer.from(resBuffer), 0, resBuffer.byteLength)
   if (appendix) buffer.write(appendix, resBuffer.byteLength)
 
-  return await gzip(buffer)
+  return buffer
 }
 
 export async function fetchFileWithHeadersFromBaseUrl (path, baseUrl, version) {
@@ -35,16 +39,20 @@ export async function fetchFileWithHeadersFromBaseUrl (path, baseUrl, version) {
 
   if (!response.ok) throw new NotFoundError(`Error fetching file: ${path}`)
 
-  const body = await createFileBuffer(response, dependencies)
-
-  return {
-    body,
+  const result = {
+    body: await createFileBuffer(response, dependencies),
     headers: {
       'content-type': response.headers.get('content-type'),
-      dependencies,
-      'content-encoding': 'gzip'
+      dependencies
     }
   }
+
+  if (result.body.length > compressFileSize && compressionWhitelistRegex.test(result.headers['content-type'])) {
+    result.body = await gzip(result.body)
+    result.headers['content-encoding'] = 'gzip'
+  }
+
+  return result
 }
 
 export async function fetchFileWithHeaders ({ path, version }) {