refactor(core): Port cache config (no-changelog) (#10286)

This commit is contained in:
Iván Ovejero
2024-08-02 17:10:03 +02:00
committed by GitHub
parent aa0a470dce
commit acbae928f2
20 changed files with 190 additions and 162 deletions

View File

@@ -660,43 +660,6 @@ export const schema = {
},
},
cache: {
backend: {
doc: 'Backend to use for caching',
format: ['memory', 'redis', 'auto'] as const,
default: 'auto',
env: 'N8N_CACHE_BACKEND',
},
memory: {
maxSize: {
doc: 'Maximum size of memory cache in bytes',
format: Number,
default: 3 * 1024 * 1024, // 3 MB
env: 'N8N_CACHE_MEMORY_MAX_SIZE',
},
ttl: {
doc: 'Time to live for cached items in memory (in ms)',
format: Number,
default: 3600 * 1000, // 1 hour
env: 'N8N_CACHE_MEMORY_TTL',
},
},
redis: {
prefix: {
doc: 'Prefix for all cache keys',
format: String,
default: 'cache',
env: 'N8N_CACHE_REDIS_KEY_PREFIX',
},
ttl: {
doc: 'Time to live for cached items in redis (in ms), 0 for no TTL',
format: Number,
default: 3600 * 1000, // 1 hour
env: 'N8N_CACHE_REDIS_TTL',
},
},
},
/**
* @important Do not remove until after cloud hooks are updated to stop using convict config.
*/

View File

@@ -13,6 +13,7 @@ import type {
} from '@/services/cache/cache.types';
import { TIME } from '@/constants';
import { TypedEmitter } from '@/TypedEmitter';
import { GlobalConfig } from '@n8n/config';
type CacheEvents = {
'metrics.cache.hit': never;
@@ -22,12 +23,15 @@ type CacheEvents = {
@Service()
export class CacheService extends TypedEmitter<CacheEvents> {
constructor(private readonly globalConfig: GlobalConfig) {
super();
}
private cache: TaggedRedisCache | TaggedMemoryCache;
async init() {
const backend = config.getEnv('cache.backend');
const { backend } = this.globalConfig.cache;
const mode = config.getEnv('executions.mode');
const ttl = config.getEnv('cache.redis.ttl');
const useRedis = backend === 'redis' || (backend === 'auto' && mode === 'queue');
@@ -36,8 +40,9 @@ export class CacheService extends TypedEmitter<CacheEvents> {
const redisClientService = Container.get(RedisClientService);
const prefixBase = config.getEnv('redis.prefix');
const cachePrefix = config.getEnv('cache.redis.prefix');
const prefix = redisClientService.toValidPrefix(`${prefixBase}:${cachePrefix}:`);
const prefix = redisClientService.toValidPrefix(
`${prefixBase}:${this.globalConfig.cache.redis.prefix}:`,
);
const redisClient = redisClientService.createClient({
type: 'client(cache)',
@@ -45,7 +50,9 @@ export class CacheService extends TypedEmitter<CacheEvents> {
});
const { redisStoreUsingClient } = await import('@/services/cache/redis.cache-manager');
const redisStore = redisStoreUsingClient(redisClient, { ttl });
const redisStore = redisStoreUsingClient(redisClient, {
ttl: this.globalConfig.cache.redis.ttl,
});
const redisCache = await caching(redisStore);
@@ -54,7 +61,7 @@ export class CacheService extends TypedEmitter<CacheEvents> {
return;
}
const maxSize = config.getEnv('cache.memory.maxSize');
const { maxSize, ttl } = this.globalConfig.cache.memory;
const sizeCalculation = (item: unknown) => {
const str = jsonStringify(item, { replaceCircularRefs: true });

View File

@@ -1,8 +1,13 @@
import { ActivationErrorsService } from '@/ActivationErrors.service';
import { CacheService } from '@/services/cache/cache.service';
import { GlobalConfig } from '@n8n/config';
import { mockInstance } from '@test/mocking';
describe('ActivationErrorsService', () => {
const cacheService = new CacheService();
const globalConfig = mockInstance(GlobalConfig, {
cache: { backend: 'memory', memory: { maxSize: 3 * 1024 * 1024, ttl: 3600 * 1000 } },
});
const cacheService = new CacheService(globalConfig);
const activationErrorsService = new ActivationErrorsService(cacheService);
const firstWorkflowId = 'GSG0etbfTA2CNPDX';

View File

@@ -12,8 +12,8 @@ jest.unmock('@/eventbus/MessageEventBus/MessageEventBus');
const toLines = (response: Response) => response.text.trim().split('\n');
const globalConfig = Container.get(GlobalConfig);
// @ts-expect-error `metrics` is a readonly property
globalConfig.endpoints.metrics = {
enable: true,
prefix: 'n8n_test_',
includeDefaultMetrics: true,
includeApiEndpoints: true,

View File

@@ -1,6 +1,8 @@
import { CacheService } from '@/services/cache/cache.service';
import config from '@/config';
import { sleep } from 'n8n-workflow';
import { GlobalConfig } from '@n8n/config';
import Container from 'typedi';
jest.mock('ioredis', () => {
const Redis = require('ioredis-mock');
@@ -13,10 +15,12 @@ jest.mock('ioredis', () => {
for (const backend of ['memory', 'redis'] as const) {
describe(backend, () => {
let cacheService: CacheService;
let globalConfig: GlobalConfig;
beforeAll(async () => {
config.set('cache.backend', backend);
cacheService = new CacheService();
globalConfig = Container.get(GlobalConfig);
globalConfig.cache.backend = backend;
cacheService = new CacheService(globalConfig);
await cacheService.init();
});
@@ -43,7 +47,7 @@ for (const backend of ['memory', 'redis'] as const) {
if (backend === 'memory') {
test('should honor max size when enough', async () => {
config.set('cache.memory.maxSize', 16); // enough bytes for "withoutUnicode"
globalConfig.cache.memory.maxSize = 16; // enough bytes for "withoutUnicode"
await cacheService.init();
await cacheService.set('key', 'withoutUnicode');
@@ -51,12 +55,12 @@ for (const backend of ['memory', 'redis'] as const) {
await expect(cacheService.get('key')).resolves.toBe('withoutUnicode');
// restore
config.set('cache.memory.maxSize', 3 * 1024 * 1024);
globalConfig.cache.memory.maxSize = 3 * 1024 * 1024;
await cacheService.init();
});
test('should honor max size when not enough', async () => {
config.set('cache.memory.maxSize', 16); // not enough bytes for "withUnicodeԱԲԳ"
globalConfig.cache.memory.maxSize = 16; // not enough bytes for "withUnicodeԱԲԳ"
await cacheService.init();
await cacheService.set('key', 'withUnicodeԱԲԳ');
@@ -64,7 +68,8 @@ for (const backend of ['memory', 'redis'] as const) {
await expect(cacheService.get('key')).resolves.toBeUndefined();
// restore
config.set('cache.memory.maxSize', 3 * 1024 * 1024);
globalConfig.cache.memory.maxSize = 3 * 1024 * 1024;
// restore
await cacheService.init();
});
}