INIT

.dockerignore

@@ -0,0 +1,4 @@
+.wappler
+**/.git
+**/.svn
+node_modules

.gitignore

@@ -0,0 +1,4 @@
+node_modules
+.svn
+.env
+**/.DS_Store

.npmrc

@@ -0,0 +1 @@
+scripts-prepend-node-path=true

.wappler/project.json

@@ -0,0 +1,52 @@
+{
+  "projectName": "ERTFastFiller",
+  "styleFile": "/css/style.css",
+  "assetsFolder": "/assets",
+  "designFramework": "bootstrap5",
+  "frameworks": [
+    {
+      "name": "fontawesome_5",
+      "type": "cdn"
+    },
+    {
+      "name": "bootstrap5",
+      "type": "local"
+    },
+    {
+      "name": "appConnect",
+      "type": "local"
+    }
+  ],
+  "hostingType": "docker",
+  "projectServerModel": "node",
+  "runtime": "capacitor",
+  "webRootFolder": "/public",
+  "useRouting": true,
+  "addBase": true,
+  "routingHandler": "node",
+  "projectLinksType": "site",
+  "targets": [
+    {
+      "name": "Development",
+      "remoteURL": "http://localhost:8100",
+      "webServerPort": 8100,
+      "webServerLang": "node",
+      "targetType": "docker",
+      "webServer": "node",
+      "NodeVersion": "lts",
+      "NodeOS": "alpine",
+      "NodeImageType": "slim",
+      "webLoggingMaxFiles": "5",
+      "webLoggingMaxFileSize": "10m",
+      "databaseLoggingMaxFiles": "5",
+      "databaseLoggingMaxFileSize": "10m"
+    }
+  ],
+  "activeTarget": "Development",
+  "projectType": "web",
+  "extensions": [
+    {
+      "name": "pdf-lib"
+    }
+  ]
+}

.wappler/targets/Development/docker-compose.yml

@@ -0,0 +1,22 @@
+services:
+  web:
+    volumes:
+      - '../../../app:/opt/node_app/app'
+      - '../../../lib:/opt/node_app/lib'
+      - '../../../views:/opt/node_app/views'
+      - '../../../public:/opt/node_app/public'
+      - '../../../extensions:/opt/node_app/extensions'
+      - '../../../db:/opt/node_app/db'
+      - '../../../certs:/opt/node_app/certs'
+    ports:
+      - '8100:3000'
+    restart: 'always'
+    stdin_open: true
+    tty: true
+    build:
+      context: '../../../'
+      dockerfile: '.wappler/targets/Development/web/Dockerfile'
+    logging:
+      options:
+        max-file: '5'
+        max-size: '10m'

.wappler/targets/Development/web/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:lts-alpine
+
+ARG NODE_ENV=production
+ENV NODE_ENV $NODE_ENV
+
+ARG PORT=3000
+ENV PORT $PORT
+EXPOSE $PORT
+
+ENV PATH /opt/node_app/node_modules/.bin:$PATH
+ENV NODE_ENV development
+WORKDIR /opt/node_app
+COPY index.js .
+COPY package.json .
+RUN npm install --no-optional --no-package-lock
+CMD [ "nodemon", "--polling-interval", "5000", "--legacy-watch", "./index.js" ]

app/config/routes.json

@@ -0,0 +1,10 @@
+{
+  "routes": [
+    {
+      "path": "/",
+      "page": "index",
+      "routeType": "page",
+      "layout": "main"
+    }
+  ]
+}

index.js

@@ -0,0 +1,3 @@
+const server = require('./lib/server');
+
+server.start();

lib/auth/database.js

@@ -0,0 +1,72 @@
+const AuthProvider = require('./provider');
+
+class DatabaseProvider extends AuthProvider {
+
+    constructor(app, opts, name) {
+        super(app, opts, name);
+        this.users = opts.users;
+        this.perms = opts.permissions;
+        this.db = app.getDbConnection(opts.connection);
+    }
+
+    async validate(username, password, passwordVerify) {
+        if (!username) return false;
+        if (!password) password = '';
+
+        let results = await this.db
+            .select(this.users.identity, this.users.username, this.users.password)
+            .from(this.users.table)
+            .where(this.users.username, username);
+
+        for (let result of results) {
+            if (result[this.users.username] == username) {
+                if (passwordVerify && result[this.users.password].startsWith('$')) {
+                    const argon2 = require('argon2');
+                    const valid = await argon2.verify(result[this.users.password], password);
+                    return valid ? result[this.users.identity] : false;
+                } else if (result[this.users.password] == password) {
+                    return result[this.users.identity];
+                }
+            }
+        }
+        
+        return false;
+    }
+
+    async permissions(identity, permissions) {
+        for (let permission of permissions) {
+            if (!this.perms[permission]) return false;
+
+            let perm = this.perms[permission];
+            let table = perm.table || this.users.table;
+            let ident = perm.identity || this.users.identity;
+
+            let results = await this.db
+                .select(ident)
+                .from(table)
+                .where(ident, identity)
+                .where(function() {
+                    for (let condition of perm.conditions) {
+                        if (condition.operator == 'in') {
+                            this.whereIn(condition.column, condition.value);
+                        } else if (condition.operator == 'not in') {
+                            this.whereNotIn(condition.column, condition.value);
+                        } else if (condition.operator == 'is null') {
+                            this.whereNull(condition.column);
+                        } else if (condition.operator == 'is not null') {
+                            this.whereNotNull(condition.column);
+                        } else {
+                            this.where(condition.column, condition.operator, condition.value);
+                        }
+                    }
+                });
+
+            if (!results.length) return false;
+        }
+
+        return true;
+    }
+
+}
+
+module.exports = DatabaseProvider;

lib/auth/passport.js

@@ -0,0 +1,25 @@
+const PassportStrategy = require('passport-strategy');
+
+class ServerConnectStrategy extends PassportStrategy {
+  constructor (options) {
+    const { provider } = options;
+
+    if (!provider) { throw new TypeError('ServerConnectStrategy requires a provider'); }
+
+    super();
+
+    this.name = 'server-connect';
+    this._key = provider + 'Id';
+  }
+
+  authenticate (req, options = {}) {
+    if (req.session && req.session[this._key]) {
+      const property = req._userProperty || 'user';
+      req[property] = { id: req.session[this._key] };
+    }
+
+    this.pass();
+  }
+}
+
+module.exports = ServerConnectStrategy;

lib/auth/provider.js

@@ -0,0 +1,159 @@
+const config = require('../setup/config');
+const crypto = require('crypto');
+const debug = require('debug')('server-connect:auth');
+
+class AuthProvider {
+
+    constructor(app, opts, name) {
+        this.app = app;
+        this.name = name;
+        this.identity = this.app.getSession(this.name + 'Id') || false;
+        this.secret = opts.secret || config.secret;
+        this.basicAuth = opts.basicAuth;
+        this.basicRealm = opts.basicRealm;
+        this.passwordVerify = opts.passwordVerify || false;
+
+        this.cookieOpts = {
+            domain: opts.domain || undefined,
+            httpOnly: true,
+            maxAge: (opts.expires || 30) * 24 * 60 * 60 * 1000, // from days to ms
+            path: opts.path || '/',
+            secure: !!opts.secure,
+            sameSite: opts.sameSite || 'Strict',
+            signed: true
+        };
+    }
+
+    async autoLogin() {
+        if (this.basicAuth) {
+            const auth = require('../core/basicauth')(this.app.req);
+            debug(`Basic auth credentials received: %o`, auth);
+            if (auth) await this.login(auth.username, auth.password, false, true);
+        }
+
+        const cookie = this.app.getCookie(this.name + '.auth', true);
+        if (cookie) {
+            const auth = this.decrypt(cookie);
+            debug(`Login with cookie: %o`, auth);
+            if (auth) await this.login(auth.username, auth.password, true, true);
+        } else {
+            debug(`No login cookie found`);
+        }
+    }
+
+    async impersonate(identity) {
+        this.app.setSession(this.name + 'Id', identity);
+        this.app.set('identity', identity);
+        this.identity = identity;
+        await this.app.regenerateSessionId();
+    }
+
+    async login(username, password, remember, autoLogin) {
+        const identity = await this.validate(username, password, this.passwordVerify);
+
+        if (!identity) {
+            await this.logout();
+
+            if (!autoLogin) {
+                this.unauthorized();
+                return false;
+            }
+        } else {
+            this.app.setSession(this.name + 'Id', identity);
+            this.app.set('identity', identity);
+
+            if (remember) {
+                debug('setCookie', identity, username, password);
+                this.app.setCookie(this.name + '.auth', this.encrypt({ username, password }), this.cookieOpts);
+            }
+
+            this.identity = identity;
+        }
+
+        await this.app.regenerateSessionId();
+
+        return identity;
+    }
+
+    async logout() {
+        this.app.removeSession(this.name + 'Id');
+        this.app.removeCookie(this.name + '.auth', this.cookieOpts);
+        this.app.remove('identity');
+        this.identity = false;
+        await this.app.regenerateSessionId();
+    }
+
+    async restrict(opts) {
+        if (this.identity === false) {
+            if (opts.loginUrl) {
+                if (this.app.req.fragment) {
+                    this.app.res.status(222).send(opts.loginUrl);
+                } else {
+                    this.app.res.redirect(opts.loginUrl);
+                }
+            } else {
+                this.unauthorized();
+            }
+
+            return;
+        }
+
+        if (opts.permissions) {
+            const allowed = await this.permissions(this.identity, opts.permissions);
+            if (!allowed) {
+                if (opts.forbiddenUrl) {
+                    if (this.app.req.fragment) {
+                        this.app.res.status(222).send(opts.forbiddenUrl);
+                    } else {
+                        this.app.res.redirect(opts.forbiddenUrl);
+                    }
+                } else {
+                    this.forbidden();
+                }
+            }
+        }
+    }
+
+    encrypt(data) {
+        const iv = crypto.randomBytes(16);
+        const key = crypto.scryptSync(this.secret, iv, 32);
+        const cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
+        const encrypted = cipher.update(JSON.stringify(data), 'utf8', 'base64');
+        return iv.toString('base64') + '.' + encrypted + cipher.final('base64');
+    }
+
+    decrypt(data) {
+        // try/catch to prevent errors on currupt cookies
+        try {
+            const iv = Buffer.from(data.split('.')[0], 'base64');
+            const key = crypto.scryptSync(this.secret, iv, 32);
+            const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
+            const decrypted = decipher.update(data.split('.')[1], 'base64', 'utf8');
+            return JSON.parse(decrypted + decipher.final('utf8'));
+        } catch (err) {
+            return undefined;
+        }
+    }
+
+    unauthorized() {
+        if (this.basicAuth) {
+            this.app.res.set('WWW-Authenticate', `Basic Realm="${this.basicRealm}"`);
+        }
+        this.app.res.sendStatus(401);
+    }
+
+    forbidden() {
+        this.app.res.sendStatus(403);
+    }
+
+    async validate(username, password) {
+        throw new Error('auth.validate needs to be extended.');
+    }
+
+    async permissions(identity, permissions) {
+        throw new Error('auth.permissions needs to be extended.');
+    }
+
+}
+
+module.exports = AuthProvider;

lib/auth/provider_experimental.js

@@ -0,0 +1,181 @@
+const config = require('../setup/config');
+const crypto = require('crypto');
+const debug = require('debug')('server-connect:auth');
+
+class AuthProvider {
+
+    constructor(app, opts, name) {
+        this.app = app;
+        this.name = name;
+        this.identity = options.jwt ? false : this.app.getSession(this.name + 'Id') || false;
+        this.jwt = options.jwt || false;
+        this.jwtExpires = options.jwtExpires || '1h';
+        this.jwtIssuer  = options.jwtIssuer || 'serverconnect';
+        this.secret = opts.secret || config.secret;
+        this.basicAuth = opts.basicAuth;
+        this.basicRealm = opts.basicRealm;
+        this.passwordVerify = opts.passwordVerify || false;
+
+        this.cookieOpts = {
+            domain: opts.domain || undefined,
+            httpOnly: true,
+            maxAge: (opts.expires || 30) * 24 * 60 * 60 * 1000, // from days to ms
+            path: opts.path || '/',
+            secure: !!opts.secure,
+            sameSite: opts.sameSite || 'Strict',
+            signed: true
+        };
+    }
+
+    async autoLogin() {
+        if (this.basicAuth) {
+            const auth = require('../core/basicauth')(this.app.req);
+            debug(`Basic auth credentials received: %o`, auth);
+            if (auth) await this.login(auth.username, auth.password, false, true);
+        }
+
+        if (this.jwt && this.app.req.headers['authorization']) {
+            const matches = /(\S+)\s+(\S+)/;
+
+            if (matches && matches[1].toLowerCase == 'bearer') {
+                const jwt = require('jsonwebtoken');
+                const { identity } = jwt.verify(matches[2], this.secret, {
+                    algorithm: 'HS256',
+                    issuer: this.jwtIssuer,
+                });
+
+                this.app.set('identity', identity);
+                this.identity = identity;
+            }
+        }
+
+        const cookie = this.app.getCookie(this.name + '.auth', true);
+        if (cookie) {
+            const auth = this.decrypt(cookie);
+            debug(`Login with cookie: %o`, auth);
+            if (auth) await this.login(auth.username, auth.password, true, true);
+        } else {
+            debug(`No login cookie found`);
+        }
+    }
+
+    async login(username, password, remember, autoLogin) {
+        const identity = await this.validate(username, password, this.passwordVerify);
+
+        if (!identity) {
+            await this.logout();
+
+            if (!autoLogin) {
+                this.unauthorized();
+                return false;
+            }
+        } else {
+            this.app.set('identity', identity);
+
+            if (!this.jwt) {
+                this.app.setSession(this.name + 'Id', identity);
+
+                if (remember) {
+                    debug('setCookie', identity, username, password);
+                    this.app.setCookie(this.name + '.auth', this.encrypt({ username, password }), this.cookieOpts);
+                }
+            }
+
+            this.identity = identity;
+        }
+
+        if (this.jwt) {
+            const jwt = require('jsonwebtoken');
+            return jwt.sign({ identity }, this.secret, {
+                algorithm: 'HS256',
+                expiresIn: this.jwtExpires,
+                issuer: this.jwtIssuer,
+            });
+        }
+
+        return identity;
+    }
+
+    async logout() {
+        if (!this.jwt) {
+            this.app.removeSession(this.name + 'Id');
+            this.app.removeCookie(this.name + '.auth', this.cookieOpts);
+        }
+        this.app.remove('identity');
+        this.identity = false;
+    }
+
+    async restrict(opts) {
+        if (this.identity === false) {
+            if (opts.loginUrl) {
+                if (this.app.req.fragment) {
+                    this.app.res.status(222).send(opts.loginUrl);
+                } else {
+                    this.app.res.redirect(opts.loginUrl);
+                }
+            } else {
+                this.unauthorized();
+            }
+
+            return;
+        }
+
+        if (opts.permissions) {
+            const allowed = await this.permissions(this.identity, opts.permissions);
+            if (!allowed) {
+                if (opts.forbiddenUrl) {
+                    if (this.app.req.fragment) {
+                        this.app.res.status(222).send(opts.forbiddenUrl);
+                    } else {
+                        this.app.res.redirect(opts.forbiddenUrl);
+                    }
+                } else {
+                    this.forbidden();
+                }
+            }
+        }
+    }
+
+    encrypt(data) {
+        const iv = crypto.randomBytes(16);
+        const key = crypto.scryptSync(this.secret, iv, 32);
+        const cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
+        const encrypted = cipher.update(JSON.stringify(data), 'utf8', 'base64');
+        return iv.toString('base64') + '.' + encrypted + cipher.final('base64');
+    }
+
+    decrypt(data) {
+        // try/catch to prevent errors on currupt cookies
+        try {
+            const iv = Buffer.from(data.split('.')[0], 'base64');
+            const key = crypto.scryptSync(this.secret, iv, 32);
+            const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
+            const decrypted = decipher.update(data.split('.')[1], 'base64', 'utf8');
+            return JSON.parse(decrypted + decipher.final('utf8'));
+        } catch (err) {
+            return undefined;
+        }
+    }
+
+    unauthorized() {
+        if (this.basicAuth) {
+            this.app.res.set('WWW-Authenticate', `Basic Realm="${this.basicRealm}"`);
+        }
+        this.app.res.sendStatus(401);
+    }
+
+    forbidden() {
+        this.app.res.sendStatus(403);
+    }
+
+    async validate(username, password) {
+        throw new Error('auth.validate needs to be extended.');
+    }
+
+    async permissions(identity, permissions) {
+        throw new Error('auth.permissions needs to be extended.');
+    }
+
+}
+
+module.exports = AuthProvider;

lib/auth/single.js

@@ -0,0 +1,26 @@
+const AuthProvider = require('./provider');
+const debug = require('debug')('server-connect:auth');
+
+class SingleProvider extends AuthProvider {
+
+    constructor(app, opts, name) {
+        super(app, opts, name);
+        this.username = opts.username;
+        this.password = opts.password;
+    }
+
+    validate(username, password) {
+        if (username == this.username && password == this.password) {
+            return username;
+        }
+
+        return false;
+    }
+
+    permissions() {
+        return true;
+    }
+
+}
+
+module.exports = SingleProvider;

lib/auth/static.js

@@ -0,0 +1,31 @@
+const AuthProvider = require('./provider');
+
+class StaticProvider extends AuthProvider {
+
+    constructor(app, opts, name) {
+        super(app, opts, name);
+        this.users = opts.users;
+        this.perms = opts.perms;
+    }
+
+    validate(username, password) {
+        if (this.users[username] == password) {
+            return username;
+        }
+
+        return false;
+    }
+
+    permissions(username, permissions) {
+        for (let permission of permissions) {
+            if (!this.perms[permission] || !this.perms[permission].includes(username)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+}
+
+module.exports = StaticProvider;

lib/core/app.js

@@ -0,0 +1,840 @@
+const fs = require('fs-extra');
+const Scope = require('./scope');
+const Parser = require('./parser');
+const db = require('./db');
+const validator = require('../validator');
+const config = require('../setup/config');
+const debug = require('debug')('server-connect:app');
+const debug2 = require('debug')('server-connect:output');
+const { clone, formatDate, parseDate } = require('../core/util');
+const { toSystemPath } = require('../core/path');
+const os = require('os');
+
+if (!global.fileCache) {
+  global.fileCache = new Map();
+}
+
+if (!global.db) {
+  global.db = {};
+}
+
+if (!global.rateLimiter) {
+  global.rateLimiter = {};
+}
+
+function App(req = {}, res = {}) {
+  this.error = false;
+  this.data = {};
+  this.meta = {};
+  this.settings = {};
+  this.modules = {};
+
+  this.io = global.io;
+  this.req = req;
+  this.res = res;
+  this.global = new Scope();
+  this.scope = this.global;
+
+  this.rateLimiter = {};
+  this.mail = {};
+  this.auth = {};
+  this.oauth = {};
+  this.db = {};
+  this.s3 = {};
+  this.jwt = {};
+  this.trx = {};
+
+  if (config.globals.data) {
+    this.set(config.globals.data);
+  }
+
+  this.set({
+    $_ERROR: null,
+    //$_SERVER: process.env,
+    $_ENV: process.env,
+    $_GET: req.query,
+    $_POST: req.body,
+    $_PARAM: req.params,
+    $_HEADER: req.headers,
+    $_COOKIE: req.cookies,
+    $_SESSION: req.session,
+  });
+
+  let urlParts = req.originalUrl ? req.originalUrl.split('?') : ['', ''];
+  let $server = {
+    CONTENT_TYPE: req.headers && req.headers['content-type'],
+    HTTPS: req.protocol == 'https',
+    PATH_INFO: req.path,
+    QUERY_STRING: urlParts[1],
+    REMOTE_ADDR: req.ip,
+    REQUEST_PROTOCOL: req.protocol,
+    REQUEST_METHOD: req.method,
+    SERVER_NAME: req.hostname,
+    BASE_URL: req.headers && req.protocol + '://' + req.headers['host'],
+    URL: urlParts[0],
+    HOSTNAME: os.hostname(),
+  };
+
+  if (req.headers) {
+    for (let header in req.headers) {
+      $server['HTTP_' + header.toUpperCase().replace(/-/, '_')] = req.headers[header];
+    }
+  }
+
+  // Try to keep same as ASP/PHP
+  this.set('$_SERVER', $server);
+}
+
+App.prototype = {
+  set: function (key, value) {
+    this.global.set(key, value);
+  },
+
+  get: function (key, def) {
+    let value = this.global.get(key);
+    return value !== undefined ? value : def;
+  },
+
+  remove: function (key) {
+    this.global.remove(key);
+  },
+
+  setSession: function (key, value) {
+    this.req.session[key] = value;
+  },
+
+  getSession: function (key) {
+    return this.req.session[key];
+  },
+
+  removeSession: function (key) {
+    delete this.req.session[key];
+  },
+
+  regenerateSessionId: function () {
+    return new Promise((resolve, reject) => {
+      const oldSession = this.req.session;
+      this.req.session.regenerate((err) => {
+        if (err) return reject(err);
+
+        for (let key in oldSession) {
+          this.req.session[key] = oldSession[key];
+        }
+
+        resolve();
+      });
+    });
+  },
+
+  setCookie: function (name, value, opts) {
+    if (!this.res.headersSent) {
+      if (this.res.cookie) {
+        this.res.cookie(name, value, opts);
+      }
+    } else {
+      debug(`Trying to set ${name} cookie while headers were already sent.`);
+    }
+  },
+
+  getCookie: function (name, signed) {
+    return signed ? this.req.signedCookies[name] : this.req.cookies[name];
+  },
+
+  removeCookie: function (name, opts) {
+    if (this.res.clearCookie) {
+      // copy all options except expires and maxAge
+      const clearOpts = Object.assign({}, opts);
+      delete clearOpts.expires;
+      delete clearOpts.maxAge;
+      this.res.clearCookie(name, clearOpts);
+    }
+  },
+
+  setRateLimiter: function (name, options) {
+    const { RateLimiterMemory, RateLimiterRedis } = require('rate-limiter-flexible');
+
+    if (global.redisClient) {
+      global.rateLimiter[name] = new RateLimiterRedis({
+        ...options,
+        storeClient: global.redisClient,
+        keyPrefix: 'ac:' + name + ':',
+      });
+    } else {
+      global.rateLimiter[name] = new RateLimiterMemory({
+        ...options,
+        keyPrefix: 'ac:' + name + ':',
+      });
+    }
+
+    return global.rateLimiter[name];
+  },
+
+  getRateLimiter: function (name) {
+    if (global.rateLimiter[name]) {
+      return global.rateLimiter[name];
+    }
+
+    if (config.rateLimiter[name]) {
+      return this.setRateLimiter(name, config.rateLimiter[name]);
+    }
+
+    if (fs.existsSync(`app/modules/RateLimiter/${name}.json`)) {
+      let action = fs.readJSONSync(`app/modules/RateLimiter/${name}.json`);
+      return this.setRateLimiter(name, action.options);
+    }
+
+    throw new Error(`Couldn't find rate limiter "${name}".`);
+  },
+
+  setMailer: function (name, options) {
+    let setup = {};
+
+    options = this.parse(options);
+
+    switch (options.server) {
+      case 'mail':
+        setup.sendmail = true;
+        break;
+
+      case 'ses':
+        const aws = require('@aws-sdk/client-ses');
+        const ses = new AWS.SES({
+          endpoint: options.endpoint,
+          credentials: {
+            accessKeyId: options.accessKeyId,
+            secretAccessKey: options.secretAccessKey,
+          },
+        });
+        setup.SES = { ses, aws };
+        break;
+
+      default:
+        // https://nodemailer.com/smtp/
+        setup.host = options.host || 'localhost';
+        setup.port = options.port || 25;
+        setup.secure = options.useSSL || false;
+        setup.auth = {
+          user: options.username,
+          pass: options.password,
+        };
+        setup.tls = {
+          rejectUnauthorized: false,
+        };
+    }
+
+    return (this.mail[name] = setup);
+  },
+
+  getMailer: function (name) {
+    if (this.mail[name]) {
+      return this.mail[name];
+    }
+
+    if (config.mail[name]) {
+      return this.setMailer(name, config.mail[name]);
+    }
+
+    if (fs.existsSync(`app/modules/Mailer/${name}.json`)) {
+      let action = fs.readJSONSync(`app/modules/Mailer/${name}.json`);
+      return this.setMailer(name, action.options);
+    }
+
+    throw new Error(`Couldn't find mailer "${name}".`);
+  },
+
+  setAuthProvider: async function (name, options) {
+    const Provider = require('../auth/' + options.provider.toLowerCase());
+    const provider = new Provider(this, options, name);
+    if (!provider.identity) await provider.autoLogin();
+    this.set('identity', provider.identity);
+    return (this.auth[name] = provider);
+  },
+
+  getAuthProvider: async function (name) {
+    if (this.auth[name]) {
+      return this.auth[name];
+    }
+
+    if (config.auth[name]) {
+      return await this.setAuthProvider(name, config.auth[name]);
+    }
+
+    if (fs.existsSync(`app/modules/SecurityProviders/${name}.json`)) {
+      let action = fs.readJSONSync(`app/modules/SecurityProviders/${name}.json`);
+      return await this.setAuthProvider(name, action.options);
+    }
+
+    throw new Error(`Couldn't find security provider "${name}".`);
+  },
+
+  setOAuthProvider: async function (name, options) {
+    const OAuth2 = require('../oauth');
+    const services = require('../oauth/services');
+
+    options = this.parse(options);
+
+    let service = this.parseOptional(options.service, 'string', null);
+    let opts = service ? services[service] : {};
+    opts.client_id = this.parseOptional(options.client_id, 'string', null);
+    opts.client_secret = this.parseOptional(options.client_secret, 'string', null);
+    opts.token_endpoint =
+      opts.token_endpoint ||
+      this.parseRequired(options.token_endpoint, 'string', 'oauth.provider: token_endpoint is required.');
+    opts.auth_endpoint = opts.auth_endpoint || this.parseOptional(options.auth_endpoint, 'string', '');
+    opts.scope_separator = opts.scope_separator || this.parseOptional(options.scope_separator, 'string', ' ');
+    opts.access_token = this.parseOptional(options.access_token, 'string', null);
+    opts.refresh_token = this.parseOptional(options.refresh_token, 'string', null);
+    opts.jwt_bearer = this.parseOptional(options.jwt_bearer, 'string', false);
+    opts.client_credentials = this.parseOptional(options.client_credentials, 'boolean', false);
+    opts.params = Object.assign({}, opts.params, this.parseOptional(options.params, 'object', {}));
+
+    this.oauth[name] = new OAuth2(this, this.parse(options), name);
+    await this.oauth[name].init();
+
+    return this.oauth[name];
+  },
+
+  getOAuthProvider: async function (name) {
+    if (this.oauth[name]) {
+      return this.oauth[name];
+    }
+
+    if (config.oauth[name]) {
+      return await this.setOAuthProvider(name, config.oauth[name]);
+    }
+
+    if (fs.existsSync(`app/modules/oauth/${name}.json`)) {
+      let action = fs.readJSONSync(`app/modules/oauth/${name}.json`);
+      return await this.setOAuthProvider(name, action.options);
+    }
+
+    throw new Error(`Couldn't find oauth provider "${name}".`);
+  },
+
+  setDbConnection: function (name, options) {
+    if (global.db[name]) {
+      return global.db[name];
+    }
+
+    options = this.parse(options);
+
+    switch (options.client) {
+      case 'couchdb':
+        const { host, port, user, password, database } = options.connection;
+        const nano = require('nano');
+        global.db[name] = nano(
+          `http://${user ? user + (password ? ':' + password : '') + '@' : ''}${host}${
+            port ? ':' + port : ''
+          }/${database}`
+        );
+        global.db[name].client = options.client;
+        return global.db[name];
+
+      case 'sqlite3':
+        if (options.connection.filename) {
+          options.connection.filename = toSystemPath(options.connection.filename);
+        }
+        break;
+
+      case 'mysql':
+      case 'mysql2':
+        options.connection = {
+          supportBigNumbers: true,
+          dateStrings: !!options.tz,
+          decimalNumbers: true,
+          ...options.connection,
+        };
+        /*
+                options.connection.typeCast = function(field, next) {
+                    if (field.type == 'NEWDECIMAL') {
+                        return field.string() + 'm';
+                    }
+
+                    return next();
+                };
+                */
+        break;
+
+      case 'mssql':
+        if (options.tz) {
+          // use local timezone to have same behavior as the other drivers
+          // to prevent problems node should have same timezone as database
+          options.connection.options = { useUTC: false, ...options.connection.options };
+        }
+        break;
+
+      case 'postgres':
+        if (options.tz) {
+          const types = require('pg').types;
+          const parseFn = (val) => val;
+          types.setTypeParser(types.builtins.TIME, parseFn);
+          types.setTypeParser(types.builtins.TIMETZ, parseFn);
+          types.setTypeParser(types.builtins.TIMESTAMP, parseFn);
+          types.setTypeParser(types.builtins.TIMESTAMPTZ, parseFn);
+        }
+        break;
+    }
+
+    if (options.connection && options.connection.ssl) {
+      if (options.connection.ssl.key) {
+        options.connection.ssl.key = fs.readFileSync(toSystemPath(options.connection.ssl.key));
+      }
+
+      if (options.connection.ssl.ca) {
+        options.connection.ssl.ca = fs.readFileSync(toSystemPath(options.connection.ssl.ca));
+      }
+
+      if (options.connection.ssl.cert) {
+        options.connection.ssl.cert = fs.readFileSync(toSystemPath(options.connection.ssl.cert));
+      }
+    }
+
+    options.useNullAsDefault = true;
+
+    const formatRecord = (record, meta) => {
+      for (column in record) {
+        if (record[column] != null) {
+          if (meta.has(column)) {
+            const info = meta.get(column);
+
+            if (['json', 'object', 'array'].includes(info.type)) {
+              if (typeof record[column] == 'string') {
+                try {
+                  // column of type json returned as string, need parse
+                  record[column] = JSON.parse(record[column]);
+                } catch (err) {
+                  console.warn(err);
+                }
+              }
+            }
+
+            if (info.type == 'date') {
+              if (typeof record[column] == 'string' && record[column].startsWith('0000-00-00')) {
+                record[column] = undefined;
+              } else {
+                record[column] = formatDate(record[column], 'yyyy-MM-dd');
+              }
+            }
+
+            if (info.type == 'time') {
+              if (options.tz == 'local') {
+                record[column] = formatDate(record[column], 'HH:mm:ss.v');
+              } else if (options.tz == 'utc') {
+                record[column] = parseDate(
+                  parseDate(formatDate('now', 'yyyy-MM-dd ') + formatDate(record[column], 'HH:mm:ss.v'))
+                )
+                  .toISOString()
+                  .slice(11);
+              }
+            }
+
+            if (
+              ['datetime', 'timestamp'].includes(info.type) ||
+              Object.prototype.toString.call(record[column]) == '[object Date]'
+            ) {
+              if (typeof record[column] == 'string' && record[column].startsWith('0000-00-00')) {
+                record[column] = undefined;
+              } else if (options.tz == 'local') {
+                record[column] = formatDate(record[column], 'yyyy-MM-dd HH:mm:ss.v');
+              } else if (options.tz == 'utc') {
+                record[column] = parseDate(record[column]).toISOString();
+              }
+            }
+          } else {
+            // try to detect datetime
+            if (Object.prototype.toString.call(record[column]) == '[object Date]') {
+              if (options.tz == 'local') {
+                record[column] = formatDate(record[column], 'yyyy-MM-dd HH:mm:ss.v');
+              } else if (options.tz == 'utc') {
+                record[column] = parseDate(record[column]).toISOString();
+              }
+            } else if (
+              typeof record[column] == 'string' &&
+              /^\d{4}-\d{2}-\d{2}([T\s]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}(:\d{2})?)?)?$/.test(record[column])
+            ) {
+              if (record[column].startsWith('0000-00-00')) {
+                record[column] = undefined;
+              } else if (options.tz == 'local') {
+                record[column] = formatDate(record[column], 'yyyy-MM-dd HH:mm:ss.v');
+              } else if (options.tz == 'utc') {
+                record[column] = parseDate(record[column]).toISOString();
+              }
+            }
+          }
+
+          if (record[column] == undefined) {
+            record[column] = null;
+          } else if (record[column].toJSON) {
+            record[column] = record[column].toJSON();
+          }
+        }
+      }
+
+      return record;
+    };
+
+    options.postProcessResponse = function (result, queryContext) {
+      const meta = new Map();
+
+      if (Array.isArray(queryContext)) {
+        for (let item of queryContext) {
+          if (item.name && item.type) {
+            meta.set(item.name, item);
+          }
+        }
+      }
+
+      if (Array.isArray(result)) {
+        return result.map((record) => formatRecord(record, meta));
+      } else {
+        return formatRecord(result, meta);
+      }
+    };
+
+    global.db[name] = db(options);
+
+    return global.db[name];
+  },
+
+  getDbConnection: function (name) {
+    if (config.db[name]) {
+      return this.setDbConnection(name, config.db[name]);
+    }
+
+    const path = `app/modules/connections/${name}.json`;
+    const action = global.fileCache.get(path) || fs.readJSONSync(`app/modules/connections/${name}.json`);
+    const isDynamic = this.isDynamic(action);
+    const options = clone(action.options); // need to clone because this.parse will update object by ref
+
+    if (!global.fileCache.has(path)) {
+      // if action was not already cached do it now
+      global.fileCache.set(path, action);
+    }
+
+    if (isDynamic) {
+      // since it is only used internal json stringify is good enough for creating a unique name
+      name = JSON.stringify(this.parse(options));
+    }
+
+    if (this.trx[name]) return this.trx[name];
+
+    return this.setDbConnection(name, options);
+  },
+
+  setS3Provider: function (name, options) {
+    options = this.parse(options);
+
+    const { S3 } = require('@aws-sdk/client-s3');
+    const endpoint = this.parseRequired(options.endpoint, 'string', 's3.provider: endpoint is required.');
+    const accessKeyId = this.parseRequired(options.accessKeyId, 'string', 's3.provider: accessKeyId is required.');
+    const secretAccessKey = this.parseRequired(
+      options.secretAccessKey,
+      'string',
+      's3.provider: secretAccessKey is required.'
+    );
+
+    let region = options.region || 'us-east-1';
+    let pos = endpoint.indexOf('.amazonaws');
+    if (pos > 3) region = endpoint.substr(3, pos - 3);
+    let forcePathStyle = options.forcePathStyle || false;
+
+    this.s3[name] = new S3({
+      endpoint: 'https://' + endpoint,
+      credentials: { accessKeyId, secretAccessKey },
+      region,
+      signatureVersion: 'v4',
+      forcePathStyle,
+    });
+
+    return this.s3[name];
+  },
+
+  getS3Provider: function (name) {
+    if (this.s3[name]) {
+      return this.s3[name];
+    }
+
+    if (config.s3[name]) {
+      return this.setS3Provider(name, config.s3[name]);
+    }
+
+    if (fs.existsSync(`app/modules/s3/${name}.json`)) {
+      let action = fs.readJSONSync(`app/modules/s3/${name}.json`);
+      return this.setS3Provider(name, action.options);
+    }
+
+    throw new Error(`Couldn't find S3 provider "${name}".`);
+  },
+
+  setJSONWebToken: function (name, options) {
+    const jwt = require('jsonwebtoken');
+
+    options = this.parse(options);
+
+    let opts = {};
+
+    if (options.alg) opts.algorithm = options.alg;
+    if (options.iss) opts.issuer = options.iss;
+    if (options.sub) opts.subject = options.sub;
+    if (options.aud) opts.audience = options.aud;
+    if (options.jti) opts.jwtid = options.jti;
+    if (options.expiresIn) opts.expiresIn = options.expiresIn;
+
+    return (this.jwt[name] = jwt.sign(
+      {
+        ...options.claims,
+      },
+      options.key,
+      {
+        expiresIn: 3600, // use as default (required by google)
+        ...opts,
+      }
+    ));
+  },
+
+  getJSONWebToken: function (name) {
+    if (config.jwt[name]) {
+      return this.setJSONWebToken(name, config.jwt[name]);
+    }
+
+    if (fs.existsSync(`app/modules/jwt/${name}.json`)) {
+      let action = fs.readJSONSync(`app/modules/jwt/${name}.json`);
+      return this.setJSONWebToken(name, action.options);
+    }
+
+    throw new Error(`Couldn't find JSON Web Token "${name}".`);
+  },
+
+  define: async function (cfg, internal) {
+    if (cfg.settings) {
+      this.settings = clone(cfg.settings);
+    }
+
+    if (cfg.vars) {
+      this.set(clone(cfg.vars));
+    }
+
+    if (cfg.meta) {
+      this.meta = clone(cfg.meta);
+      await validator.init(this, this.meta);
+    }
+
+    if (fs.existsSync('app/modules/global.json')) {
+      await this.exec(await fs.readJSON('app/modules/global.json'), true);
+    }
+
+    /*
+        debug('body: %o', this.req.body);
+        debug('query: %o', this.req.query);
+        debug('params: %o', this.req.params);
+        debug('headers: %o', this.req.headers);
+        debug('cookies: %o', this.req.cookies);
+        debug('session: %o', this.req.session);
+        */
+
+    await this.exec(cfg.exec || cfg, internal);
+  },
+
+  sub: async function (actions, scope) {
+    const subApp = new App(this.req, this.res);
+    subApp.global = this.global;
+    subApp.scope = this.scope.create(scope);
+    await subApp.exec(actions, true);
+    return subApp.data;
+  },
+
+  exec: async function (actions, internal) {
+    if (actions.exec) {
+      return this.exec(actions.exec, internal);
+    }
+
+    actions = clone(actions);
+
+    await this._exec(actions.steps || actions);
+
+    if (this.error !== false) {
+      if (actions.catch) {
+        this.scope.set('$_ERROR', this.error.message);
+        this.error = false;
+        await this._exec(actions.catch, true);
+      } else {
+        throw this.error;
+      }
+    }
+
+    if (!internal && !this.res.headersSent && !this.noOutput) {
+      this.res.json(this.data);
+    }
+  },
+
+  _exec: async function (steps, ignoreAbort) {
+    if (this.res.headersSent) return;
+
+    if (typeof steps == 'string') {
+      return this.exec(await fs.readJSON(`app/modules/${steps}.json`), true);
+    }
+
+    if (this.res.headersSent) {
+      // do not execute other steps after headers has been sent
+      return;
+    }
+
+    if (Array.isArray(steps)) {
+      for (let step of steps) {
+        await this._exec(step);
+        if (this.error) return;
+      }
+      return;
+    }
+
+    if (steps.disabled) {
+      return;
+    }
+
+    if (steps.action) {
+      try {
+        let module;
+
+        if (fs.existsSync(`extensions/server_connect/modules/${steps.module}.js`)) {
+          module = require(`../../extensions/server_connect/modules/${steps.module}`);
+        } else if (fs.existsSync(`lib/modules/${steps.module}.js`)) {
+          module = require(`../modules/${steps.module}`);
+        } else {
+          throw new Error(`Module ${steps.module} doesn't exist`);
+        }
+
+        if (typeof module[steps.action] != 'function') {
+          throw new Error(`Action ${steps.action} doesn't exist in ${steps.module || 'core'}`);
+        }
+
+        debug(`Executing action step ${steps.action}`);
+        debug(`options: %O`, steps.options);
+
+        if (!ignoreAbort && config.abortOnDisconnect && this.req.isDisconnected) {
+          throw new Error('Aborted');
+        }
+
+        const data = await module[steps.action].call(this, clone(steps.options), steps.name, steps.meta);
+
+        if (data instanceof Error) {
+          throw data;
+        }
+        
+        debug2(`${steps.name || steps.action}: %O`, data);
+
+        if (steps.name) {
+          this.scope.set(steps.name, data);
+
+          if (steps.output) {
+            this.data[steps.name] = data;
+          }
+        }
+      } catch (e) {
+        debug2(`Error at ${steps.name || steps.action}: %O`, e);
+
+        this.error = e;
+        return;
+      }
+    }
+  },
+
+  parse: function (value, scope) {
+    return Parser.parseValue(value, scope || this.scope);
+  },
+
+  parseRequired: function (value, type, err) {
+    if (value === undefined) {
+      throw new Error(err);
+    }
+
+    let val = Parser.parseValue(value, this.scope);
+
+    if (type == '*') {
+      if (val === undefined) {
+        throw new Error(err);
+      }
+    } else if (type == 'boolean') {
+      val = !!val;
+    } else if (typeof val != type) {
+      throw new Error(err);
+    }
+
+    return val;
+  },
+
+  parseOptional: function (value, type, def) {
+    if (value === undefined) return def;
+
+    let val = Parser.parseValue(value, this.scope);
+
+    if (type == '*') {
+      if (val === undefined) val = def;
+    } else if (type == 'boolean') {
+      if (val === undefined) {
+        val = def;
+      } else {
+        val = !!val;
+      }
+    } else if (typeof val != type) {
+      val = def;
+    }
+
+    return val;
+  },
+
+  parseSQL: function (sql) {
+    if (!sql) return null;
+
+    ['values', 'orders'].forEach((prop) => {
+      if (Array.isArray(sql[prop])) {
+        sql[prop] = sql[prop].filter((value) => {
+          if (!value.condition) return true;
+          return !!Parser.parseValue(value.condition, this.scope);
+        });
+      }
+    });
+
+    if (sql.wheres && sql.wheres.rules) {
+      if (sql.wheres.conditional && !Parser.parseValue(sql.wheres.conditional, this.scope)) {
+        delete sql.wheres;
+      } else {
+        sql.wheres.rules = sql.wheres.rules.filter(function filterConditional(rule) {
+          if (!rule.rules) return true;
+          if (rule.conditional && !Parser.parseValue(rule.conditional, this.scope)) return false;
+          rule.rules = rule.rules.filter(filterConditional, this);
+          return rule.rules.length;
+        }, this);
+
+        if (!sql.wheres.rules.length) {
+          delete sql.wheres;
+        }
+      }
+    }
+
+    if (sql.sub) {
+      for (const name in sql.sub) {
+        sql.sub[name] = this.parseSQL(sql.sub[name]);
+      }
+    }
+
+    return Parser.parseValue(sql, this.scope);
+  },
+
+  isDynamic: function (value) {
+    if (typeof value == 'string') {
+      return value.includes('{{');
+    }
+
+    if (typeof value == 'object') {
+      for (const prop in value) {
+        if (this.isDynamic(value[prop])) {
+          return true;
+        }
+      }
+    }
+
+    return false;
+  },
+};
+
+module.exports = App;

lib/core/async.js

@@ -0,0 +1,97 @@
+function defered() {
+    let resolve, reject;
+    let promise = new Promise((_resolve, _reject) => {
+        resolve = _resolve;
+        reject = _reject;
+    });
+
+    return {
+        promise,
+        resolve,
+        reject
+    };
+}
+
+function limit(concurrency) {
+    let running = 0;
+    let queue = [];
+
+    function init() {
+        if (running < concurrency) {
+            running++;
+            return Promise.resolve();
+        }
+
+        let defer = defered();
+        queue.push(defer);
+        return defer.promise;
+    }
+
+    function complete(a) {
+        let next = queue.shift();
+
+        if (next) {
+            next.resolve();
+        } else {
+            running--;
+        }
+
+        return a;
+    }
+
+    return function(fn) {
+        return function() {
+            let args = Array.prototype.slice.apply(arguments);
+
+            return init().then(() => {
+                return fn.apply(null, args);
+            }).finally(complete);
+        }
+    }
+}
+
+module.exports = {
+
+    map: function(arr, fn, concurrency) {
+        arr = Array.isArray(arr) ? arr : [arr];
+
+        if (concurrency) {
+            const limiter = limit(concurrency)
+            return Promise.all(arr.map(limiter(fn)));
+        }
+
+        return Promise.all(arr.map(fn));
+    },
+
+    mapSeries: function(arr, fn) {
+        arr = Array.isArray(arr) ? arr : [arr];
+
+        return arr.reduce((promise, curr, index, arr) => {
+            return promise.then(prev => {
+                return fn(curr, index, arr).then(val => {
+                    prev.push(val);
+                    return prev;
+                });
+            })
+        }, Promise.resolve([]));
+    },
+
+    reduce: function(arr, fn, start) {
+        arr = Array.isArray(arr) ? arr : [arr];
+
+        if (!arr.length) {
+            return Promise.resolve(start);
+        }
+
+        return arr.reduce((promise, curr, index, arr) => {
+            return promise.then(prev => {
+                if (prev === undefined && arr.length === 1) {
+                    return curr;
+                }
+
+                return fn(prev, curr, index, arr);
+            });
+        }, Promise.resolve(start));
+    },
+
+};

lib/core/base32.js

@@ -0,0 +1,84 @@
+const Variants = {
+  'RFC4648': {
+    alphabet: 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567',
+    padding: true,
+  },
+  'RFC4648-HEX': {
+    alphabet: '0123456789ABCDEFGHIJKLMNOPQRSTUV',
+    padding: true,
+  },
+  'CROCKFORD': {
+    alphabet: '0123456789ABCDEFGHJKMNPQRSTVWXYZ',
+    padding: false,
+  },
+};
+
+exports.encode = (input, opts = {}) => {
+  const Variant = Variants[opts.variant] || Variants['RFC4648'];
+  const alphabet = Variant.alphabet;
+  const padding = opts.padding != null ? opts.padding : Variant.padding;
+  const data = Buffer.from(input);
+
+  let bits = 0;
+  let value = 0;
+  let output = '';
+
+  for (let i = 0; i < data.length; i++) {
+    value = (value << 8) | data.readUInt8(i);
+    bits += 8;
+
+    while (bits >= 5) {
+      output += alphabet[(value >>> (bits - 5)) & 31];
+      bits -= 5;
+    }
+  }
+
+  if (bits > 0) {
+    output += alphabet[(value << (5 - bits)) & 31];
+  }
+
+  if (padding) {
+    while ((output.length % 8) !== 0) {
+      output += '=';
+    }
+  }
+
+  return output;
+};
+
+exports.decode = (input, opts = {}) => {
+  const Variant = Variants[opts.variant] || Variants['RFC4648'];
+  const alphabet = Variant.alphabet;
+
+  let bits = 0;
+  let value = 0;
+  
+  if (opts.variant === 'CROCKFORD') {
+    input = input.toUpperCase().replace(/O/g, '0').replace(/[IL]/g, '1');
+  } else {
+    input = input.replace(/=/g, '');
+  }
+  
+  let index = 0;
+  let length = input.length;
+  let output = Buffer.alloc(Math.ceil(length * 5 / 8));
+  
+  for (let i = 0; i < length; i++) {
+    let char = input[i];
+    let char_index = alphabet.indexOf(char);
+
+    if (char_index === -1) {
+      throw new Error('Invalid character: ' + char);
+    }
+
+    value = (value << 5) | char_index;
+    bits += 5;
+
+    if (bits >= 8) {
+      output.writeUInt8((value >>> (bits - 8)) & 255, index++);
+      bits -= 8;
+    }
+  }
+
+  return output;
+};

lib/core/basicauth.js

@@ -0,0 +1,43 @@
+// Code taken from https://github.com/jshttp/basic-auth/blob/master/index.js
+// Copyright(c) 2013 TJ Holowaychuk
+// Copyright(c) 2014 Jonathan Ong
+// Copyright(c) 2015-2016 Douglas Christopher Wilson
+
+module.exports = auth;
+module.exports.parse = parse;
+
+const CREDENTIALS_REGEXP = /^ *(?:[Bb][Aa][Ss][Ii][Cc]) +([A-Za-z0-9._~+/-]+=*) *$/;
+const USER_PASS_REGEXP = /^([^:]*):(.*)$/;
+
+function auth(req) {
+    if (!req) throw new Error('argument req is required.');
+    if (typeof req != 'object') throw new Error('argument req needs to be an object.');
+
+    return parse(getAuthorization(req));
+}
+
+function decodeBase64(str) {
+    return Buffer.from(str, 'base64').toString();
+}
+
+function getAuthorization(req) {
+    if (!req.headers || typeof req.headers != 'object') {
+        throw new Error('argument req needs to have headers.');
+    }
+
+    return req.headers.authorization;
+}
+
+function parse(str) {
+    if (typeof str != 'string') {
+        return undefined;
+    }
+
+    const match = CREDENTIALS_REGEXP.exec(str);
+    if (!match) return undefined;
+
+    const userPass = USER_PASS_REGEXP.exec(decodeBase64(match[1]));
+    if (!userPass) return undefined;
+
+    return { username: userPass[1], password: userPass[2] };
+}

lib/core/db.js

@@ -0,0 +1,217 @@
+const knex = require('knex');
+
+knex.QueryBuilder.extend('whereGroup', function(condition, rules) {
+    condition = condition.toLowerCase();
+
+    for (let rule of rules) {
+        if (!rule.condition) {
+            let where = condition == 'or' ? 'orWhere' : 'where';
+            let column = rule.data ? rule.data.column : rule.column;
+
+            if (rule.data && rule.data.table) {
+                if (rule.data.schema) {
+                    column = rule.data.schema + '.' + rule.data.table + '.' + column;
+                } else {
+                    column = rule.data.table + '.' + column;
+                }
+            } else if (rule.table) {
+                if (rule.schema) {
+                    column = rule.schema + '.' + rule.table + '.' + column;
+                } else {
+                    column = rule.table + '.' + column;
+                }
+            }
+
+            if (typeof rule.value == 'undefined') {
+                rule.value = null;
+            }
+
+            if (rule.operator == 'between') {
+                this[where + 'Between'](column, rule.value);
+            } else if (rule.operator == 'not_between') {
+                this[where + 'NotBetween'](column, rule.value);
+            } else if (rule.operator == 'is_null') {
+                this[where + 'Null'](column);
+            } else if (rule.operator == 'is_not_null') {
+                this[where + 'NotNull'](column);
+            } else if (rule.operator == 'in') {
+                this[where + 'In'](column, rule.value);
+            } else if (rule.operator == 'not_in') {
+                this[where + 'NotIn'](column, rule.value);
+            } else if (rule.operator == 'begins_with') {
+                // simple escaping for likes, knex doesn't do this (\ is default in most databases)
+                // perhaps improve by using raw query like
+                // this[where](knex.raw("?? like ? escape '\'"), [column, escapedValue]);
+                let escapedValue = rule.value.replace(/[\\%_^]/g, '\\$&');
+                this[where](column, 'like', escapedValue + '%');
+            } else if (rule.operator == 'not_begins_with') {
+                let escapedValue = rule.value.replace(/[\\%_^]/g, '\\$&');
+                this[where + 'Not'](column, 'like', escapedValue + '%');
+            } else if (rule.operator == 'ends_with') {
+                let escapedValue = rule.value.replace(/[\\%_^]/g, '\\$&');
+                this[where](column, 'like', '%' + escapedValue);
+            } else if (rule.operator == 'not_ends_with') {
+                let escapedValue = rule.value.replace(/[\\%_^]/g, '\\$&');
+                this[where + 'Not'](column, 'like', '%' + escapedValue);
+            } else if (rule.operator == 'contains') {
+                let escapedValue = rule.value.replace(/[\\%_^]/g, '\\$&');
+                this[where](column, 'like', '%' + escapedValue + '%');
+            } else if (rule.operator == 'not_contains') {
+                let escapedValue = rule.value.replace(/[\\%_^]/g, '\\$&');
+                this[where + 'Not'](column, 'like', '%' + escapedValue + '%');
+            } else {
+                this[where](column, rule.operation, rule.value);
+            }
+        } else {
+            this[condition + 'Where'](function() {
+                this.whereGroup(rule.condition, rule.rules);
+            });
+        }
+    }
+
+    return this;
+});
+
+knex.QueryBuilder.extend('fromJSON', function(ast, meta) {
+    if (ast.type == 'count') {
+        return this.count('* as Total').from(function() {
+            this.fromJSON(Object.assign({}, ast, {
+                type: 'select',
+                offset: null,
+                limit: null,
+                orders: null
+            })).as('t1');
+        }).first();
+    } else if (ast.type == 'insert' || ast.type == 'update') {
+        let values = {};
+
+        for (let val of ast.values) {
+            if (val.type == 'json') {
+                // json support
+                values[val.column] = JSON.stringify(val.value);
+            } else {
+                values[val.column] = val.value;
+            }
+        }
+
+        this[ast.type](values);
+    } else {
+        this[ast.type]();
+    }
+
+    if (ast.returning && !(this.client?.config?.client?.includes('mysql'))) {
+        // Adding the option includeTriggerModifications allows you
+        // to run statements on tables that contain triggers. Only affects MSSQL.
+        this.returning(ast.returning, { includeTriggerModifications: true });
+    }
+
+    if (ast.table) {
+        let table = ast.table.name || ast.table;
+
+        if (ast.table.schema) {
+            table = ast.table.schema + '.' + table;
+        }
+
+        if (ast.table.alias) {
+            table += ' as ' + ast.table.alias;
+        }
+
+        this.from(table);
+    }
+
+    if ((ast.type == 'select' || ast.type == 'first') && ast.joins && ast.joins.length) {
+        for (let join of ast.joins) {
+            let table = join.table;
+
+            if (join.schema) {
+                table = join.schema + '.' + table;
+            }
+
+            if (join.alias) {
+                table += ' as ' + join.alias;
+            }
+
+            this[(join.type || 'inner').toLowerCase() + 'Join'](table, function() {
+                for (let clause of join.clauses.rules) {
+                    this.on(
+                        (clause.schema ? clause.schema + '.' : '') + clause.table + '.' + clause.column,
+                        clause.operation,
+                        (clause.value.schema ? clause.value.schema + '.' : '') + clause.value.table + '.' + clause.value.column
+                    );
+                }
+            });
+        }
+    }
+
+    if ((ast.type == 'select' || ast.type == 'first') && ast.columns) {
+        for (let col of ast.columns) {
+            let column = col.column || col;
+
+            if (ast.joins && ast.joins.length && col.table) {
+                column = col.table + '.' + column;
+    
+                if (col.schema) {
+                    column = col.schema + '.' + column;
+                }
+            }
+
+            if (col.alias) {
+                column += ' as ' + col.alias;
+            }
+
+            this[col.aggregate ? col.aggregate.toLowerCase() : 'column'](column);
+        }
+    }
+
+    if ((ast.type == 'select' || ast.type == 'first') && ast.groupBy && ast.groupBy.length) {
+        for (let col of ast.groupBy) {
+            if (ast.joins && ast.joins.length && col.table) {
+                if (col.schema) {
+                    this.groupBy(col.schema + '.' + col.table + '.' + col.column);
+                } else {
+                    this.groupBy(col.table + '.' + col.column);
+                }
+            } else {
+                this.groupBy(col.column || col);
+            }
+        }
+    }
+
+    if (ast.wheres && ast.wheres.condition) {
+        this.whereGroup(ast.wheres.condition, ast.wheres.rules);
+    }
+
+    if ((ast.type == 'select' || ast.type == 'first') && ast.orders && ast.orders.length) {
+        for (let order of ast.orders) {
+            if (ast.joins && ast.joins.length && order.table) {
+                if (order.schema) {
+                    this.orderBy(order.schema + '.' + order.table + '.' + order.column, order.direction);
+                } else {
+                    this.orderBy(order.table + '.' + order.column, order.direction);
+                }
+            } else {
+                this.orderBy(order.column, order.direction);
+            }
+        }
+    }
+
+    if (ast.distinct) {
+        this.distinct();
+    }
+
+    if (ast.type == 'select' && ast.limit) {
+        this.limit(ast.limit);
+    }
+
+    if (ast.type == 'select' && ast.offset) {
+        this.offset(ast.offset);
+    }
+
+    if (meta) {
+        this.queryContext(meta);
+    }
+    
+	return this;
+});
+
+module.exports = knex;

lib/core/diacritics.js

@@ -0,0 +1,96 @@
+const diacriticsMap = [
+    {'base':'A', 'letters':/[\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F]/g},
+    {'base':'AA','letters':/[\uA732]/g},
+    {'base':'AE','letters':/[\u00C6\u01FC\u01E2]/g},
+    {'base':'AO','letters':/[\uA734]/g},
+    {'base':'AU','letters':/[\uA736]/g},
+    {'base':'AV','letters':/[\uA738\uA73A]/g},
+    {'base':'AY','letters':/[\uA73C]/g},
+    {'base':'B', 'letters':/[\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181]/g},
+    {'base':'C', 'letters':/[\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E]/g},
+    {'base':'D', 'letters':/[\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779]/g},
+    {'base':'DZ','letters':/[\u01F1\u01C4]/g},
+    {'base':'Dz','letters':/[\u01F2\u01C5]/g},
+    {'base':'E', 'letters':/[\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E]/g},
+    {'base':'F', 'letters':/[\u0046\u24BB\uFF26\u1E1E\u0191\uA77B]/g},
+    {'base':'G', 'letters':/[\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E]/g},
+    {'base':'H', 'letters':/[\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D]/g},
+    {'base':'I', 'letters':/[\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197]/g},
+    {'base':'J', 'letters':/[\u004A\u24BF\uFF2A\u0134\u0248]/g},
+    {'base':'K', 'letters':/[\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2]/g},
+    {'base':'L', 'letters':/[\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780]/g},
+    {'base':'LJ','letters':/[\u01C7]/g},
+    {'base':'Lj','letters':/[\u01C8]/g},
+    {'base':'M', 'letters':/[\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C]/g},
+    {'base':'N', 'letters':/[\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4]/g},
+    {'base':'NJ','letters':/[\u01CA]/g},
+    {'base':'Nj','letters':/[\u01CB]/g},
+    {'base':'O', 'letters':/[\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C]/g},
+    {'base':'OI','letters':/[\u01A2]/g},
+    {'base':'OO','letters':/[\uA74E]/g},
+    {'base':'OU','letters':/[\u0222]/g},
+    {'base':'P', 'letters':/[\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754]/g},
+    {'base':'Q', 'letters':/[\u0051\u24C6\uFF31\uA756\uA758\u024A]/g},
+    {'base':'R', 'letters':/[\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782]/g},
+    {'base':'S', 'letters':/[\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784]/g},
+    {'base':'T', 'letters':/[\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786]/g},
+    {'base':'TZ','letters':/[\uA728]/g},
+    {'base':'U', 'letters':/[\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244]/g},
+    {'base':'V', 'letters':/[\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245]/g},
+    {'base':'VY','letters':/[\uA760]/g},
+    {'base':'W', 'letters':/[\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72]/g},
+    {'base':'X', 'letters':/[\u0058\u24CD\uFF38\u1E8A\u1E8C]/g},
+    {'base':'Y', 'letters':/[\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE]/g},
+    {'base':'Z', 'letters':/[\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762]/g},
+    {'base':'a', 'letters':/[\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250]/g},
+    {'base':'aa','letters':/[\uA733]/g},
+    {'base':'ae','letters':/[\u00E6\u01FD\u01E3]/g},
+    {'base':'ao','letters':/[\uA735]/g},
+    {'base':'au','letters':/[\uA737]/g},
+    {'base':'av','letters':/[\uA739\uA73B]/g},
+    {'base':'ay','letters':/[\uA73D]/g},
+    {'base':'b', 'letters':/[\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253]/g},
+    {'base':'c', 'letters':/[\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184]/g},
+    {'base':'d', 'letters':/[\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A]/g},
+    {'base':'dz','letters':/[\u01F3\u01C6]/g},
+    {'base':'e', 'letters':/[\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD]/g},
+    {'base':'f', 'letters':/[\u0066\u24D5\uFF46\u1E1F\u0192\uA77C]/g},
+    {'base':'g', 'letters':/[\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F]/g},
+    {'base':'h', 'letters':/[\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265]/g},
+    {'base':'hv','letters':/[\u0195]/g},
+    {'base':'i', 'letters':/[\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131]/g},
+    {'base':'j', 'letters':/[\u006A\u24D9\uFF4A\u0135\u01F0\u0249]/g},
+    {'base':'k', 'letters':/[\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3]/g},
+    {'base':'l', 'letters':/[\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747]/g},
+    {'base':'lj','letters':/[\u01C9]/g},
+    {'base':'m', 'letters':/[\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F]/g},
+    {'base':'n', 'letters':/[\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5]/g},
+    {'base':'nj','letters':/[\u01CC]/g},
+    {'base':'o', 'letters':/[\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275]/g},
+    {'base':'oi','letters':/[\u01A3]/g},
+    {'base':'ou','letters':/[\u0223]/g},
+    {'base':'oo','letters':/[\uA74F]/g},
+    {'base':'p','letters':/[\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755]/g},
+    {'base':'q','letters':/[\u0071\u24E0\uFF51\u024B\uA757\uA759]/g},
+    {'base':'r','letters':/[\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783]/g},
+    {'base':'s','letters':/[\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B]/g},
+    {'base':'t','letters':/[\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787]/g},
+    {'base':'tz','letters':/[\uA729]/g},
+    {'base':'u','letters':/[\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289]/g},
+    {'base':'v','letters':/[\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C]/g},
+    {'base':'vy','letters':/[\uA761]/g},
+    {'base':'w','letters':/[\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73]/g},
+    {'base':'x','letters':/[\u0078\u24E7\uFF58\u1E8B\u1E8D]/g},
+    {'base':'y','letters':/[\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF]/g},
+    {'base':'z','letters':/[\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763]/g}
+];
+
+module.exports = {
+    replace: function(str) {
+        for (let change of diacriticsMap) {
+            str = str.replace(change.letters, change.base);
+        }
+
+        return str;
+    }
+};

lib/core/memoryStore.js

@@ -0,0 +1,92 @@
+module.exports = function(session) {
+  const Store = session.Store
+
+  const defer = (cb, ...args) => {
+    if (typeof cb != 'function') return
+    setImmediate(cb, ...args)
+  }
+
+  class ExtendedMap extends Map {
+    set(key, value, maxAge) {
+      const cached = super.get(key)
+      if (cached && cached.timer) clearTimeout(cached.timer)
+      const timer = maxAge ? setTimeout(super.delete.bind(this, key), maxAge) : null
+      return super.set(key, {timer, value})
+    }
+  
+    get(key) {
+      const cached = super.get(key)
+      return cached && cached.value
+    }
+  
+    delete(key) {
+      const cached = super.get(key)
+      if (cached && cached.timer) clearTimeout(cached.timer)
+      return super.delete(key)
+    }
+  }
+
+  class MemoryStore extends Store {
+    constructor(options = {}) {
+      super(options)
+      this.sessions = new ExtendedMap()
+      this.ttl = options.ttl
+    }
+
+    get(sid, callback) {
+      let session = this.sessions.get(sid)
+      defer(callback, null, session)
+    }
+
+    set(sid, session, callback) {
+      let ttl = this._getTTL(session)
+      if (ttl > 0) {
+        this.sessions.set(sid, session, ttl)
+      } else {
+        this.sessions.delete(sid)
+      }
+      defer(callback, null)
+    }
+
+    touch(sid, session, callback) {
+      let ttl = this._getTTL(session)
+      let stored = this.sessions.get(sid)
+      stored.cookie = session.cookie
+      this.sessions.set(sid, stored, ttl)
+      defer(callback, null)
+    }
+
+    destroy(sid, callback) {
+      this.sessions.delete(sid)
+      defer(callback, null)
+    }
+
+    clear(callback) {
+      this.sessions.clear()
+      defer(callback, null)
+    }
+
+    length(callback) {
+      let len = this.sessions.size
+      defer(callback, null, len)
+    }
+
+    ids(callback) {
+      let keys = Array.from(this.sessions.keys())
+      defer(callback, null, keys)
+    }
+
+    all(callback) {
+      let sessions = Array.from(this.sessions.values())
+      defer(callback, null, sessions)
+    }
+
+    _getTTL(session) {
+      if (typeof this.ttl == 'number') return this.ttl
+      let maxAge = (session && session.cookie) ? session.cookie.maxAge : null
+      return (typeof maxAge == 'number') ? maxAge : 86400000
+    }
+  }
+
+  return MemoryStore
+}

lib/core/middleware.js

@@ -0,0 +1,163 @@
+const config = require('../setup/config');
+const debug = require('debug')('server-connect:router');
+const { clone } = require('./util');
+const App = require('./app');
+
+module.exports = {
+    cache: function(options) {
+        return async function(req, res, next) {
+            if (!options.ttl || !global.redisClient) {
+                // no caching
+                return next();
+            }
+
+            let key = 'erc:' + (req.originalUrl || req.url);
+            let nocache = false;
+            let nostore = false;
+
+            if (req.fragment) {
+                key += '-fragment';
+            }
+
+            if (req.query.nocache) {
+                nocache = true;
+            }
+
+            try {
+                const cacheControl = req.get('Cache-Control');
+                if (cacheControl) {
+                    if (cacheControl.includes('no-cache')) nocache = true;
+                    if (cacheControl.includes('no-store')) nostore = true;
+                }
+            } catch (err) {
+                // Ignore but log errors
+                console.error(err);
+            }
+
+            if (!nocache) {
+                try {
+                    const cache = await global.redisClient.hgetall(key);
+                    if (cache.body) {
+                        res.type(cache.type || 'text/html');
+                        res.send(cache.body);
+                        return;
+                    }
+                } catch (err) {
+                    // Ignore but log errors
+                    console.error(err);
+                }
+            }
+
+            if (!nostore) {
+                // wrap res.send
+                const send = res.send.bind(res);
+                res.send = function (body) {
+                    const ret = send(body);
+
+                    if (this.statusCode !== 200 || typeof body !== 'string') {
+                        // do not cache when not status 200 or body is not a string
+                        return ret;
+                    }
+
+                    global.redisClient.hset(key, {
+                        body: body,
+                        type: this.get('Content-Type') || 'text/html'
+                    }).then(() => {
+                        return global.redisClient.expire(key, +options.ttl);
+                    }).catch(err => {
+                        // Ignore but log errors
+                        console.error(err);
+                    });
+
+                    return ret;
+                };
+            }
+
+            return next();
+        };
+    },
+
+    serverConnect: function(json) {
+        return async function(req, res, next) {
+            const app = new App(req, res);
+
+            debug(`Serving serverConnect ${req.path}`);
+
+            return Promise.resolve(app.define(json)).catch(next);
+        };
+    },
+
+    templateView: function(layout, page, data, exec) {
+        return async function(req, res, next) {
+            const app = new App(req, res);
+            
+            debug(`Serving templateView ${req.path}`);
+
+            const routeData = clone(data);
+            const methods = {
+                _: (expr, data = {}) => {
+                    return app.parse(`{{${expr}}}`, app.scope.create(data));
+                },
+
+                _exec: async (name, steps, data = {}) => {
+                    let context = {};
+                    app.scope = app.scope.create(data, context);
+                    await app.exec(steps, true);
+                    app.scope = app.scope.parent;
+                    app.set(name, context);
+                    return context;
+                },
+
+                _repeat: (expr, cb) => {
+                    let data = app.parse(`{{${expr}}}`);
+                    if (Array.isArray(data)) return data.forEach(cb);
+                    return '';
+                },
+
+                _csrfToken: (overwrite) => {
+                    return req.csrfToken(overwrite);
+                },
+
+                _csrfMeta: () => {
+                    return `<meta name="csrf-token" content="${req.csrfToken()}">`;
+                },
+
+                _csrfInput: () => {
+                    return `<input type="hidden" name="CSRFToken" value="${req.csrfToken()}">`;
+                },
+
+                _route: req.route.path
+            };
+
+            let template = page;
+            if (layout && !req.fragment) {
+                routeData.content = '/' + page;
+                template = 'layouts/' + layout;
+            }
+
+            if (exec) {
+                return Promise.resolve(app.define(exec, true)).then(() => {
+                    if (!res.headersSent) {
+                        app.set(app.parse(routeData));
+                        debug(`Render template ${template}`);
+                        debug(`Template data: %O`, Object.assign({}, app.global.data, app.data));
+                        res.render(template, Object.assign({}, app.global.data, app.data, methods), async (err, html) => {
+                            // callback is needed when using ejs with aync, html is a promise
+                            if (err) return next(err);
+                            html.then(html => res.send(html)).catch(err => next(err));
+                        });
+                    }
+                }).catch(next)
+            } else {
+                app.set(app.parse(routeData));
+                debug(`Render template ${template}`);
+                debug(`Template data: %O`, app.global.data);
+                res.render(template, Object.assign({}, app.global.data, methods), async (err, html) => {
+                    // callback is needed when using ejs with aync, html is a promise
+                    if (err) return next(err);
+                    html.then(html => res.send(html)).catch(err => next(err));
+                });
+            }
+        };
+    }
+};

lib/core/parser.js

@@ -0,0 +1,752 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { randomUUID } = require('crypto');
+const { v4: uuidv4 } = require('uuid');
+
+const NOOP = function() {};
+
+const OPERATORS = {
+    '{'  : 'L_CURLY',
+    '}'  : 'R_CURLY',
+    '('  : 'L_PAREN',
+    ')'  : 'R_PAREN',
+    '['  : 'L_BRACKET',
+    ']'  : 'R_BRACKET',
+    '.'  : 'PERIOD',
+    ','  : 'COMMA',
+    ':'  : 'COLON',
+    '?'  : 'QUESTION',
+    // Arithmetic operators
+    '+'  : 'ADDICTIVE',
+    '-'  : 'ADDICTIVE',
+    '*'  : 'MULTIPLICATIVE',
+    '/'  : 'MULTIPLICATIVE',
+    '%'  : 'MULTIPLICATIVE',
+    // Comparison operators
+    '===': 'EQUALITY',
+    '!==': 'EQUALITY',
+    '==' : 'EQUALITY',
+    '!=' : 'EQUALITY',
+    '<'  : 'RELATIONAL',
+    '>'  : 'RELATIONAL',
+    '<=' : 'RELATIONAL',
+    '>=' : 'RELATIONAL',
+    'in' : 'RELATIONAL',
+    // Logical operators
+    '&&' : 'LOGICAL_AND',
+    '||' : 'LOGICAL_OR',
+    '!'  : 'LOGICAL_NOT',
+    // Bitwise operators
+    '&'  : 'BITWISE_AND',
+    '|'  : 'BITWISE_OR',
+    '^'  : 'BITWISE_XOR',
+    '~'  : 'BITWISE_NOT',
+    '<<' : 'BITWISE_SHIFT',
+    '>>' : 'BITWISE_SHIFT',
+    '>>>': 'BITWISE_SHIFT'
+};
+
+const EXPRESSIONS = {
+    'in' : function(a, b)    { return a() in b(); },
+    '?'  : function(a, b, c) { return a() ? b() : c(); },
+    '+'  : function(a, b)    { a = a(); b = b(); return a == null ? b : b == null ? a : a + b; },
+    '-'  : function(a, b)    { return a() - b(); },
+    '*'  : function(a, b)    { return a() * b(); },
+    '/'  : function(a, b)    { return a() / b(); },
+    '%'  : function(a, b)    { return a() % b(); },
+    '===': function(a, b)    { return a() === b(); },
+    '!==': function(a, b)    { return a() !== b(); },
+    '==' : function(a, b)    { return a() == b(); },
+    '!=' : function(a, b)    { return a() != b(); },
+    '<'  : function(a, b)    { return a() < b(); },
+    '>'  : function(a, b)    { return a() > b(); },
+    '<=' : function(a, b)    { return a() <= b(); },
+    '>=' : function(a, b)    { return a() >= b(); },
+    '&&' : function(a, b)    { return a() && b(); },
+    '||' : function(a, b)    { return a() || b(); },
+    '&'  : function(a, b)    { return a() & b(); },
+    '|'  : function(a, b)    { return a() | b(); },
+    '^'  : function(a, b)    { return a() ^ b(); },
+    '<<' : function(a, b)    { return a() << b(); },
+    '>>' : function(a, b)    { return a() >> b(); },
+    '>>>': function(a, b)    { return a() >>> b(); },
+    '~'  : function(a)       { return ~a(); },
+    '!'  : function(a)       { return !a(); }
+};
+
+const ESCAPE = {
+    'n': '\n',
+    'f': '\f',
+    'r': '\r',
+    't': '\t',
+    'v': '\v',
+    "'": "'",
+    '"': '"',
+    '`': '`'
+};
+
+const formatters = require('../formatters');
+
+// User formatters
+if (fs.existsSync('extensions/server_connect/formatters')) {
+    const files = fs.readdirSync('extensions/server_connect/formatters');
+    for (let file of files) {
+        if (path.extname(file) == '.js') {
+            Object.assign(formatters, require(`../../extensions/server_connect/formatters/${file}`));
+        }
+    }
+}
+
+function lexer(expr) {
+    let tokens = [],
+        token,
+        name,
+        start,
+        index = 0,
+        op = true,
+        ch,
+        ch2,
+        ch3;
+    
+    while (index < expr.length) {
+        start = index;
+
+        ch = read();
+
+        if (isQuote(ch) && op) {
+            name = 'STRING';
+            token = readString(ch);
+            op = false;
+        } else if ((isDigid(ch) || (is('.') && peek() && isDigid(peek()))) && op) {
+            name = 'NUMBER';
+            token = readNumber();
+            op = false;
+        } else if (isAlpha(ch) && op) {
+            name = 'IDENT';
+            token = readIdent();
+            if (is('(')) {
+                name = 'METHOD';
+            }
+            op = false;
+        } else if (is('/') && op && (token == '(' || token == ',' || token == '?' || token == ':') && testRegexp()) {
+            name = 'REGEXP';
+            token = readRegexp();
+            op = false;
+        } else if (isWhitespace(ch)) {
+            index++;
+            continue;
+        } else if ((ch3 = read(3)) && OPERATORS[ch3]) {
+            name = OPERATORS[ch3];
+            token = ch3;
+            op = true;
+            index += 3;
+        } else if ((ch2 = read(2)) && OPERATORS[ch2]) {
+            name = OPERATORS[ch2];
+            token = ch2;
+            op = true;
+            index += 2;
+        } else if (OPERATORS[ch]) {
+            name = OPERATORS[ch];
+            token = ch;
+            op = true;
+            index++;
+        } else {
+            throw new Error('Lexer Error: Unexpected token "' + ch + '" at column ' + index + ' in expression {{' + expr + '}}');
+        }
+
+        tokens.push({
+            name: name,
+            index: start,
+            value: token
+        });
+    }
+
+    return tokens;
+
+    function read(n) {
+        if (!n) n = 1;
+        return expr.substr(index, n);
+    }
+
+    function peek(n) {
+        n = n || 1;
+        return index + n < expr.length ? expr[index + n] : false;
+    }
+
+    function is(chars) {
+        return chars.indexOf(ch) != -1;
+    }
+
+    function isQuote(ch) {
+        return ch == '"' || ch == "'";
+    }
+
+    function isDigid(ch) {
+        return ch >= '0' && ch <= '9';
+    }
+
+    function isAlpha(ch) {
+        return (ch >= 'a' && ch <= 'z') ||
+               (ch >= 'A' && ch <= 'Z') ||
+                ch == '_' || ch == '$';
+    }
+
+    function isAlphaNum(ch) {
+        return isAlpha(ch) || isDigid(ch);
+    }
+
+    function isWhitespace(ch) {
+        return ch == ' ' || ch == '\r' || ch == '\t' || ch == '\n' || ch == '\v' || ch == '\u00A0';
+    }
+
+    function isExpOperator(ch) {
+        return ch == '-' || ch == '+' || isDigid(ch);
+    }
+
+    function readString(quote) {
+        let str = '', esc = false;
+
+        index++;
+        
+        while (index < expr.length) {
+            ch = read();
+
+            if (esc) {
+                if (ch == 'u') {
+                    // unicode escape
+                    index++;
+                    let hex = read(4);
+                    if (!hex.match(/[\da-f]{4}/i)) {
+                        throw new Error('Lexer Error: Invalid unicode escape at column ' + index + ' in expression {{' + expr + '}}');
+                    }
+                    str += String.fromCharCode(parseInt(hex, 16));
+                    index += 3;
+                } else {
+                    str += ESCAPE[ch] ? ESCAPE[ch] : ch;
+                }
+
+                esc = false;
+            } else if (ch == '\\') {
+                // escape character
+                esc = true;
+            } else if (ch == quote) {
+                // end of string
+                index ++;
+                return str;
+            } else {
+                str += ch;
+            }
+
+            index++;
+        }
+
+        throw new Error('Lexer Error: Unterminated string at column ' + index + ' in expression {{' + expr + '}}');
+    }
+
+    function readNumber() {
+        let num = '', exp = false;
+
+        while (index < expr.length) {
+            ch = read();
+
+            if (isDigid(ch) || (is('.') && peek() && isDigid(peek()))) {
+                num += ch;
+            } else {
+                let next = peek();
+
+                if (is('eE') && isExpOperator(next)) {
+                    num += 'e';
+                    exp = true;
+                } else if (isExpOperator(ch) && next && isDigid(next) && exp) {
+                    num += ch;
+                    exp = false;
+                } else if (isExpOperator(ch) && (!next || !isDigid(next)) && exp) {
+                    throw new Error('Lexer Error: Invalid exponent at column ' + index + ' in expression {{' + expr + '}}');
+                } else {
+                    break;
+                }
+            }
+
+            index++;
+        }
+
+        return +num;
+    }
+
+    function readIdent() {
+        let ident = '';
+
+        while (index < expr.length) {
+            ch = read();
+
+            if (isAlphaNum(ch)) {
+                ident += ch;
+            } else {
+                break;
+            }
+
+            index++;
+        }
+
+        return ident;
+    }
+
+    function readRegexp() {
+        let re = '', mod = '', esc = false;
+
+        index ++;
+
+        while (index < expr.length) {
+            ch = read();
+
+            if (esc) {
+                esc = false;
+            } else if (ch == '\\') {
+                esc = true;
+            } else if (ch == '/') {
+                index++;
+
+                while ('ign'.indexOf(ch = read()) != -1) {
+                    mod += ch;
+                    index++;
+                }
+
+                return re + '%%%' + mod;
+            }
+
+            re += ch;
+            index++;
+        }
+
+        throw new Error('Lexer Error: Unterminated regexp at column ' + index + ' in expression {{' + expr + '}}');
+    }
+
+    function testRegexp() {
+        var idx = index, ok = true;
+
+        try {
+            readRegexp();
+        } catch (e) {
+            ok = false;
+        }
+
+        // reset our index and ch
+        index = idx;
+        ch = '/';
+
+        return ok;
+    }
+}
+
+function parser(expr, scope) {
+    let tokens = lexer(expr),
+        context = undefined,
+        RESERVED = {
+            'PI'       : function() { return Math.PI; },
+            'UUID'     : function() { return randomUUID ? randomUUID() : uuidv4(); },
+            'NOW'      : function() { return date(); },
+            'NOW_UTC'  : function() { return utc_date(); },
+            'TIMESTAMP': function() { return timestamp(); },
+            '$this'    : function() { return scope.data; },
+            '$global'  : function() { return globalScope.data; },
+            '$parent'  : function() { return scope.parent && scope.parent.data; },
+            'null'     : function() { return null; },
+            'true'     : function() { return true; },
+            'false'    : function() { return false; },
+            '_'        : function() { return { __dmxScope__: true } }
+        };
+
+    return start()();
+
+    function pad(s, n) {
+        return ('000' + s).substr(-n);
+    }
+
+    function date(dt) {
+        dt = dt || new Date();
+        return pad(dt.getFullYear(), 4) + '-' + pad(dt.getMonth() + 1, 2) + '-' + pad(dt.getDate(), 2) + ' ' +
+               pad(dt.getHours(), 2) + ':' + pad(dt.getMinutes(), 2) + ':' + pad(dt.getSeconds(), 2);
+    }
+
+    function utc_date(dt) {
+        dt = dt || new Date();
+        return pad(dt.getUTCFullYear(), 4) + '-' + pad(dt.getUTCMonth() + 1, 2) + '-' + pad(dt.getUTCDate(), 2) + 'T' +
+               pad(dt.getUTCHours(), 2) + ':' + pad(dt.getUTCMinutes(), 2) + ':' + pad(dt.getUTCSeconds(), 2) + 'Z';
+    }
+
+    function timestamp(dt) {
+        dt = dt || new Date();
+        return ~~(dt / 1000);
+    }
+
+    function read() {
+        if (tokens.length === 0) {
+            throw new Error('Parser Error: Unexpected end of expression {{' + expr + '}}');
+        }
+
+        return tokens[0];
+    }
+
+    function peek(e) {
+        if (tokens.length > 0) {
+            let token = tokens[0];
+
+            if (!e || token.name == e) {
+                return token;
+            }
+        }
+
+        return false;
+    }
+
+    function expect(e) {
+        let token = peek(e);
+
+        if (token) {
+            tokens.shift();
+            return token;
+        }
+
+        return false;
+    }
+
+    function consume(e) {
+        if (!expect(e)) {
+            throw new Error('Parser Error: Unexpected token, expecting ' + e + ' in expression {{' + expr + '}}');
+        }
+    }
+
+    function fn(expr) {
+        let args = [].slice.call(arguments, 1);
+
+        return function() {
+            if (EXPRESSIONS[expr]) {
+                return EXPRESSIONS[expr].apply(context, args);
+            } else {
+                return expr;
+            }
+        }
+    }
+
+    function start() {
+        return conditional();
+    }
+
+    function conditional() {
+        let left = logicalOr(), middle, token;
+
+        if ((token = expect('QUESTION'))) {
+            middle = conditional();
+
+            if ((token = expect('COLON'))) {
+                return fn('?', left, middle, conditional());
+            } else {
+                throw new Error('Parse Error: Expecting : in expression {{' + expr + '}}');
+            }
+        } else {
+            return left;
+        }
+    }
+
+    function logicalOr() {
+        let left = logicalAnd(), token;
+
+        while (true) {
+            if ((token = expect('LOGICAL_OR'))) {
+                left = fn(token.value, left, logicalAnd());
+            } else {
+                return left;
+            }
+        }
+    }
+
+    function logicalAnd() {
+        let left = bitwiseOr(), token;
+
+        if ((token = expect('LOGICAL_AND'))) {
+            left = fn(token.value, left, logicalAnd());
+        }
+
+        return left;
+    }
+
+    function bitwiseOr() {
+        let left = bitwiseXor(), token;
+
+        if ((token = expect('BITWISE_OR'))) {
+            left = fn(token.value, left, bitwiseXor());
+        }
+
+        return left;
+    }
+
+    function bitwiseXor() {
+        let left = bitwiseAnd(), token;
+
+        if ((token = expect('BITWISE_XOR'))) {
+            left = fn(token.value, left, bitwiseAnd());
+        }
+
+        return left;
+    }
+
+    function bitwiseAnd() {
+        let left = equality(), token;
+
+        if ((token = expect('BITWISE_AND'))) {
+            left = fn(token.value, left, bitwiseAnd());
+        }
+
+        return left;
+    }
+
+    function equality() {
+        let left = relational(), token;
+
+        if ((token = expect('EQUALITY'))) {
+            left = fn(token.value, left, equality());
+        }
+
+        return left;
+    }
+
+    function relational() {
+        let left = bitwiseShift(), token;
+
+        if ((token = expect('RELATIONAL'))) {
+            left = fn(token.value, left, relational());
+        }
+
+        return left;
+    }
+
+    function bitwiseShift() {
+        let left = addictive(), token;
+
+        if ((token = expect('BITWISE_SHIFT'))) {
+            left = fn(token.value, left, addictive());
+        }
+
+        return left;
+    }
+
+    function addictive() {
+        let left = multiplicative(), token;
+
+        while ((token = expect('ADDICTIVE'))) {
+            left = fn(token.value, left, multiplicative());
+        }
+
+        return left;
+    }
+
+    function multiplicative() {
+        let left = unary(), token;
+
+        while ((token = expect('MULTIPLICATIVE'))) {
+            left = fn(token.value, left, unary());
+        }
+
+        return left;
+    }
+
+    function unary() {
+        let token;
+
+        if ((token = expect('ADDICTIVE'))) {
+            if (token.value == '+') {
+                return primary();
+            } else {
+                return fn(token.value, function() { return 0; }, unary());
+            }
+        } else if ((token = expect('LOGICAL_NOT'))) {
+            return fn(token.value, unary());
+        }
+
+        return primary();
+    }
+
+    function primary() {
+        let value, next;
+
+        if (expect('L_PAREN')) {
+            value = start();
+            consume('R_PAREN');
+        } else if (expect('L_CURLY')) {
+            let obj = {};
+
+            if (read().name != 'R_CURLY') {
+                do {
+                    let key = expect().value;
+                    consume('COLON');
+                    obj[key] = start()();
+                } while (expect('COMMA'));
+            }
+
+            value = fn(obj);
+
+            consume('R_CURLY');
+        } else if (expect('L_BRACKET')) {
+            let arr = [];
+
+            if (read().name != 'R_BRACKET') {
+                do {
+                    arr.push(start()());
+                } while (expect('COMMA'));
+            }
+
+            value = fn(arr);
+
+            consume('R_BRACKET');
+        } else if (expect('PERIOD')) {
+            value = peek() ? objectMember(fn(scope.data)) : fn(scope.data);
+        } else {
+            let token = expect();
+
+            if (token === false) {
+                throw new Error('Parser Error: Not a primary expression {{' + expr + '}}');
+            }
+
+            if (token.name == 'IDENT') {
+                value = RESERVED.hasOwnProperty(token.value)
+                      ? RESERVED[token.value]
+                      : function() { return scope.get(token.value) };
+            } else if (token.name == 'METHOD') {
+                if (!formatters[token.value]) {
+                    throw new Error('Parser Error: Formatter "' + token.value + '" does not exist, expression {{' + expression + '}}');
+                }
+
+                value = fn(formatters[token.value]);
+            } else if (token.name == 'REGEXP') {
+                value = function() {
+                    let re = token.value.split('%%%');
+                    return new RegExp(re[0], re[1]);
+                };
+            } else {
+                value = function() { return token.value };
+            }
+        }
+
+        while ((next = expect('L_PAREN') || expect('L_BRACKET') || expect('PERIOD'))) {
+            if (next.value == '(') {
+                value = functionCall(value, context);
+            } else if (next.value == '[') {
+                value = objectIndex(value);
+            } else if (next.value == '.') {
+                context = value;
+                value = objectMember(value);
+            } else {
+                throw new Error('Parser Error: Parse error in expression {{' + expr + '}}');
+            }
+        }
+
+        context = undefined;
+
+        return value;
+    }
+
+    function functionCall(func, ctx) {
+        let argsFn = [];
+
+        if (read().name != 'R_PAREN') {
+            do {
+                argsFn.push(start());
+            } while (expect('COMMA'));
+        }
+
+        consume('R_PAREN');
+
+        return function() {
+            let args = [];
+
+            if (ctx) args.push(ctx());
+
+            for (let argFn of argsFn) {
+                args.push(argFn());
+            }
+
+            let fnPtr = func() || NOOP;
+
+            return fnPtr.apply(null, args);
+        }
+    }
+
+    function objectIndex(obj) {
+        let indexFn = start();
+
+        consume('R_BRACKET');
+
+        return function() {
+            let o = obj(),
+                i = indexFn();
+            
+            if (typeof o != 'object') return undefined;
+
+            if (o.__dmxScope__) {
+                return scope.get(i);
+            }
+
+            return o[i];
+        }
+    }
+
+    function objectMember(obj) {
+        let token = expect();
+
+        return function() {
+            let o = obj();
+
+            if (token.name == 'METHOD') {
+                if (!formatters[token.value]) {
+                    throw new Error('Parser Error: Formatter "' + token.value + '" does not exist, expression {{' + expr + '}}');
+                }
+
+                return formatters[token.value];
+            }
+
+            if (o && o.__dmxScope) {
+                return scope.get(token.value);
+            }
+
+            return o && o[token.value];
+        }
+    }
+}
+
+function parseValue(value, scope) {
+    if (value == null) return value;
+
+    value = value.valueOf();
+
+    if (typeof value == 'object') {
+        for (let key in value) {
+            value[key] = parseValue(value[key], scope);
+        }
+    }
+
+    if (typeof value == 'string') {
+        if (value.substr(0, 2) == '{{' && value.substr(-2) == '}}') {
+            let expr = value.replace(/^\{\{|\}\}$/g, '');
+
+            if (expr.indexOf('{{') == -1) {
+                return parser(expr, scope);
+            }
+        }
+
+        return parseTemplate(value, scope);
+    }
+
+    return value;
+}
+
+function parseTemplate(template, scope) {
+    return template.replace(/\{\{(.*?)\}\}/g, function(a, m) {
+        var value = parser(m, scope);
+        return value != null ? String(value) : '';
+    });
+}
+
+exports.lexer = lexer;
+exports.parse = parser;
+exports.parseValue = parseValue;
+exports.parseTemplate = parseTemplate;

lib/core/path.js

@@ -0,0 +1,97 @@
+const { existsSync: exists } = require('fs');
+const { dirname, basename, extname, join, resolve, relative, posix } = require('path');
+const { v4: uuidv4 } = require('uuid');
+const debug = require('debug')('server-connect:path');
+
+module.exports = {
+
+    getFilesArray: function(paths) {
+        let files = [];
+
+        if (!Array.isArray(paths)) {
+            paths = [paths];
+        }
+
+        for (let path of paths) {
+            if (Array.isArray(path)) {
+                files = files.concat(module.exports.getFilesArray(path));
+            } else if (path && path.path) {
+                files.push(module.exports.toSystemPath(path.path));
+            } else if (path) {
+                files.push(module.exports.toSystemPath(path));
+            }
+        }
+
+        return files;
+    },
+
+    toSystemPath: function(path) {
+        if (path[0] != '/' || path.includes('../')) {
+            throw new Error(`path.toSystemPath: Invalid path "${path}".`);
+        }
+
+        return resolve('.' + path);
+    },
+
+    toAppPath: function(path) {
+        let root = resolve('.');
+        let rel = relative(root, path).replace(/\\/g, '/');
+
+        debug('toAppPath: %O', { root, path, rel });
+
+        if (rel.includes('../')) {
+            throw new Error(`path.toAppPath: Invalid path "${rel}".`);
+        }
+
+        return '/' + rel;
+    },
+
+    toSiteUrl: function(path) {
+        let root = resolve('public');
+        let rel = relative(root, path).replace(/\\/g, '/');
+
+        debug('toSiteUrl: %O', { root, path, rel });
+
+        if (rel.includes('../')) {
+            return '';
+        }
+
+        return '/' + rel;
+    },
+
+    getUniqFile: function(path) {
+        let n = 1;
+
+        while (exists(path)) {
+            path = path.replace(/(_(\d+))?(\.\w+)$/, (a, b, c, d) => '_' + (n++) + (d || a));
+            if (n > 999) throw new Error(`path.getUniqFile: Couldn't create a unique filename for ${path}`);
+        }
+
+        return path;
+    },
+
+    parseTemplate: function(path, template) {
+        let n = 1, dir = dirname(path), file = template.replace(/\{([^\}]+)\}/g, (a, b) => {
+            switch (b) {
+                case 'name': return basename(path, extname(path));
+                case 'ext' : return extname(path);
+                case 'guid': return uuidv4();
+            }
+
+            return a;
+        });
+
+        if (file.includes('{_n}')) {
+            template = file;
+            file = template.replace('{_n}', '');
+
+            while (exists(join(dir, file))) {
+                file = template.replace('{_n}', n++);
+                if (n > 999) throw new Error(`path.parseTemplate: Couldn't create a unique filename for ${path}`);
+            }
+        }
+
+        return join(dir, file);
+    }
+
+};

lib/core/scope.js

@@ -0,0 +1,63 @@
+function Scope(data, parent, context) {
+    if (typeof data != 'object') {
+        data = { $value: data };
+    }
+
+    this.data = data;
+    this.parent = parent;
+    this.context = context;
+};
+
+Scope.prototype = {
+    create: function(data, context) {
+        return new Scope(data, this, context);
+    },
+
+    get: function(name) {
+        if (this.data[name] !== undefined) {
+            return this.data[name];
+        }
+
+        if (this.parent) {
+            return this.parent.get(name);
+        }
+
+        return undefined;
+    },
+
+    set: function(name, value) {
+        if (typeof name == 'object') {
+            for (let prop in name) {
+                this.set(prop, name[prop]);
+            }
+        } else {
+            this.data[name] = value;
+
+            if (this.context) {
+                this.context[name] = value;
+            }
+        }
+    },
+
+    has: function(name) {
+        if (this.data[name] !== undefined) {
+            return true;
+        }
+
+        if (this.parent) {
+            return this.parent.has(name);
+        }
+
+        return false;
+    },
+
+    remove: function(name) {
+        delete this.data[name];
+
+        if (this.context) {
+            delete this.context[name];
+        }
+    },
+}
+
+module.exports = Scope;

lib/core/util.js

@@ -0,0 +1,186 @@
+const reGrouping = /(\d+)(\d{3})/;
+
+function clone(o) {
+    if (o == null) {
+        return o;
+    }
+    
+    if (Array.isArray(o)) {
+        return o.map(clone);
+    }
+
+    if (o instanceof Date) {
+        // we do not clone date objects but convert them to an iso string
+        return o.toISOString();
+    }
+
+    if (typeof o == 'object') {
+        let oo = {};
+        for (let key in o) {
+            oo[key] = clone(o[key]);
+        }
+        return oo;
+    }
+
+    return o;
+}
+
+function mergeDeep(target, source) {
+    if (isObject(target) && isObject(source)) {
+        for (const key in source) {
+            if (isObject(source[key])) {
+                if (!target[key]) {
+                    Object.assign(target, { [key]: {} });
+                }
+
+                mergeDeep(target[key], source[key]);
+            } else {
+                Object.assign(target, { [key]: source[key] });
+            }
+        }
+    }
+
+    return target;
+}
+
+function isObject(item) {
+    return (item && typeof item === 'object' && !Array.isArray(item) && item !== null);
+  }
+
+module.exports = {
+
+    clone,
+
+    mergeDeep,
+
+    escapeRegExp: function(val) {
+        // https://github.com/benjamingr/RegExp.escape
+        return val.replace(/[\\^$*+?.()|[\]{}]/g, '\\$&');
+    },
+
+    keysToLowerCase: function(o) {
+        return Object.keys(o).reduce((c, k) => (c[k.toLowerCase()] = o[k], c), {});
+    },
+
+    padNumber: function(num, digids, trim) {
+        let sign = '';
+
+        if (num < 0) {
+            sign = '-';
+            num = -num;
+        }
+
+        num = String(num);
+
+        while (num.length < digids) {
+            num = '0' + num;
+        }
+
+        if (trim) {
+            num = num.substr(num.length - digids);
+        }
+
+        return sign + num;
+    },
+
+    formatNumber: function(num, decimals, decimalSeparator, groupingSeparator) {
+        num = Number(num);
+
+        if (isNaN(num) || !isFinite(num)) return '';
+
+        decimalSeparator = typeof decimalSeparator == 'string' ? decimalSeparator : '.';
+        groupingSeparator = typeof groupingSeparator == 'string' ? groupingSeparator : '';
+        precision = typeof precision == 'number' ? Math.abs(precision) : null;
+
+        let minus = num < 0;
+        let parts = (decimals == null ? String(Math.abs(num)) : Math.abs(num).toFixed(decimals)).split('.');
+        let wholePart = parts[0];
+        let decimalPart = parts.length > 1 ? decimalSeparator + parts[1] : '';
+
+        if (groupingSeparator) {
+            while (reGrouping.test(wholePart)) {
+                wholePart = wholePart.replace(reGrouping, '$1' + groupingSeparator + '$2');
+            }
+        }
+
+        return (minus ? '-' : '') + wholePart + decimalPart;
+    },
+
+    parseDate: function(obj) {
+        if (Object.prototype.toString.call(obj) == '[object Date]') return new Date(obj);
+
+        let date = new Date(obj);
+
+        if (typeof obj == 'number') {
+            date = new Date(obj * 1000);
+        }
+
+        if (typeof obj == 'string') {
+            if (obj == 'now') {
+                date = new Date();
+            } else if (/^\d{4}-\d{2}-\d{2}$/.test(obj)) {
+                let parts = obj.split('-');
+                date = new Date(parts[0], parts[1] - 1, parts[2]);
+            } else if (/^\d{2}:\d{2}:\d{2}$/.test(obj)) {
+                let parts = obj.split(':');
+                date = new Date();
+                date.setHours(parts[0]);
+                date.setMinutes(parts[1]);
+                date.setSeconds(parts[2]);
+                date.setMilliseconds(0);
+            }
+        }
+
+        if (date.toString() == 'Invalid Date') {
+            return undefined;
+        }
+
+        return date;
+    },
+
+    formatDate: function(date, format, utc, locale) {
+        date = module.exports.parseDate(date);
+        if (date == null) return date;
+        locale = typeof locale == 'string' ? locale : 'en-US';
+        const lang = require('../locale/' + locale);
+        const pad = (v, n) => `0000${v}`.substr(-n);
+
+        let y = utc ? date.getUTCFullYear() : date.getFullYear(),
+            n = utc ? date.getUTCMonth() : date.getMonth(),
+            d = utc ? date.getUTCDate() : date.getDate(),
+            w = utc ? date.getUTCDay() : date.getDay(),
+            h = utc ? date.getUTCHours() : date.getHours(),
+            m = utc ? date.getUTCMinutes() : date.getMinutes(),
+            s = utc ? date.getUTCSeconds() : date.getSeconds(),
+            v = utc ? date.getUTCMilliseconds() : date.getMilliseconds();
+        
+        return format.replace(/([yMdHhmsaAvw])(\1+)?/g, part => {
+            switch (part) {
+                case 'yyyy': return pad(y, 4);
+                case 'yy': return pad(y, 2);
+                case 'y': return y;
+                case 'MMMM': return lang.months[n];
+                case 'MMM': return lang.monthsShort[n];
+                case 'MM': return pad(n + 1, 2);
+                case 'M': return n + 1;
+                case 'dddd': return lang.days[w];
+                case 'ddd': return lang.daysShort[w];
+                case 'dd': return pad(d, 2);
+                case 'd': return d;
+                case 'HH': return pad(h, 2);
+                case 'H': return h;
+                case 'hh': return pad((h % 12) || 12, 2);
+                case 'h': return (h % 12) || 12;
+                case 'mm': return pad(m, 2);
+                case 'm': return m;
+                case 'ss': return pad(s, 2);
+                case 's': return s;
+                case 'a': return h < 12 ? 'am' : 'pm';
+                case 'A': return h < 12 ? 'AM' : 'PM';
+                case 'v': return pad(v, 3);
+                case 'w': return w;
+            }
+        });
+    }
+
+};

lib/core/webhook.js

@@ -0,0 +1,24 @@
+// Helper methods for webhooks
+
+const fs = require('fs-extra');
+const App = require('./app');
+
+exports.createHandler = function(name, fn = () => 'handler') {
+    return async (req, res, next) => {
+        const action = await fn(req, res, next);
+
+        if (typeof action == 'string') {
+            const path = `app/webhooks/${name}/${action}.json`;
+
+            if (fs.existsSync(path)) {
+                const app = new App(req, res);
+                let json = await fs.readJSON(path);
+                return Promise.resolve(app.define(json)).catch(next);
+            } else {
+                res.json({error: `No action found for ${action}.`});
+                // do not return 404 else stripe will retry
+                //next();
+            }
+        }
+    }
+};

lib/formatters/collections.js

@@ -0,0 +1,169 @@
+module.exports = {
+
+    join: function(arr, separator, prop) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.map(item => prop ? item[prop] : item).join(separator);
+    },
+
+    first: function(arr) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.length ? arr[0] : null;
+    },
+
+    top: function(arr, count) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.slice(0, count);
+    },
+
+    last: function(arr, count) {
+        if (!Array.isArray(arr)) return arr;
+        if (count) return arr.slice(-count);
+        return arr[arr.length - 1];
+    },
+
+    where: function(arr, prop, operator, value) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.filter(item => {
+            let val = item[prop];
+
+            switch (operator) {
+                case 'startsWith': return String(val).startsWith(value);
+                case 'endsWith': return String(val).endsWith(value);
+                case 'contains': return String(val).includes(value);
+                case '===': return val === value;
+                case '!==': return val !== value;
+                case '==': return val == value;
+                case '!=': return val != value;
+                case '<=': return val <= value;
+                case '>=': return val >= value;
+                case '<': return val < value;
+                case '>': return val > value;
+            };
+
+            return true;
+        });
+    },
+
+    unique: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        if (prop) arr = arr.map(item => item[prop]);
+
+        let lookup = [];
+        return arr.filter(item => {
+            if (lookup.includes(item)) return false;
+            lookup.push(item);
+            return true;
+        });
+    },
+
+    groupBy: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.reduce((groups, item) => {
+            let group = String(item[prop]);
+            if (!groups[group]) groups[group] = [];
+            groups[group].push(item);
+            return groups;
+        }, {});
+    },
+
+    sort: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.slice(0).sort((a, b) => {
+            a = prop ? a && a[prop] : a;
+            b = prop ? b && b[prop] : b;
+
+            if (typeof a == 'string' && typeof b == 'string') {
+                return a.localeCompare(b);
+            }
+
+            return a < b ? -1 : a > b ? 1 : 0;
+        });
+    },
+
+    randomize: function(arr) {
+        if (!Array.isArray(arr)) return arr;
+        arr = arr.slice(0);
+        let len = arr.length;
+        if (len) {
+            while (--len) {
+                let rnd = Math.floor(Math.random() * (len + 1));
+                let val = arr[len];
+                arr[len] = arr[rnd];
+                arr[rnd] = val;
+            }
+        }
+        return arr;
+    },
+
+    reverse: function(arr) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.slice(0).reverse();
+    },
+
+    count: function(arr) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.length;
+    },
+
+    min: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.reduce((min, num) => {
+            num = prop ? num[prop] : num;
+            if (num == null) return min;
+            num = Number(num);
+            if (isNaN(num) || !isFinite(num)) return min;
+            return min == null || num < min ? num : min;
+        }, null);
+    },
+
+    max: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.reduce((max, num) => {
+            num = prop ? num[prop] : num;
+            if (num == null) return max;
+            num = Number(num);
+            if (isNaN(num) || !isFinite(num)) return max;
+            return max == null || num > max ? num : max;
+        }, null);
+    },
+
+    sum: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.reduce((sum, num) => {
+            num = prop ? num[prop] : num;
+            if (num == null) return sum;
+            num = Number(num);
+            if (isNaN(num) || !isFinite(num)) return sum;
+            return sum + num;
+        }, 0);
+    },
+
+    avg: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        let cnt = 0;
+        return arr.reduce((avg, num) => {
+            num = prop ? num[prop] : num;
+            if (num == null) return avg;
+            num = Number(num);
+            if (isNaN(num) || !isFinite(num)) return avg;
+            cnt++;
+            return avg + num;
+        }, 0) / cnt;
+    },
+
+    flatten: function(arr, prop) {
+        if (!Array.isArray(arr)) return arr;
+        return arr.map(item => item[prop]);
+    },
+
+    keys: function(obj) {
+        if (typeof obj != 'object') return obj;
+        return Object.keys(obj);
+    },
+
+    values: function(obj) {
+        if (typeof obj != 'object') return obj;
+        return Object.values(obj);
+    },
+
+};

lib/formatters/conditional.js

@@ -0,0 +1,30 @@
+module.exports = {
+
+    startsWith: function(val, str) {
+        if (val == null) return false;
+        return String(val).startsWith(str);
+    },
+
+    endsWith: function(val, str) {
+        if (val == null) return false;
+        return String(val).endsWith(str);
+    },
+
+    contains: function(val, str) {
+        if (val == null) return false;
+        return String(val).includes(str);
+    },
+
+    between: function(val, min, max) {
+        return val >= min && val <= max;
+    },
+
+    inRange: function(val, min, max) {
+        val = Number(val);
+        min = Number(min);
+        max = Number(max);
+
+        return val >= min && val <= max;
+    },
+
+};

lib/formatters/core.js

@@ -0,0 +1,27 @@
+module.exports = {
+
+    default: function(val, def) {
+        return val ? val : def;
+    },
+
+    then: function(val, trueVal, falseVal) {
+        return val ? trueVal : falseVal;
+    },
+
+    toNumber: function(val) {
+        return Number(val);
+    },
+
+    toString: function(val) {
+        return String(val);
+    },
+
+    toJSON: function(val) {
+        return JSON.stringify(val);
+    },
+
+    parseJSON: function(val) {
+        return JSON.parse(val);
+    },
+
+};

lib/formatters/crypto.js

@@ -0,0 +1,99 @@
+const { createHash, createHmac, createCipheriv, createDecipheriv, randomBytes, scryptSync, randomUUID } = require('crypto');
+const { v4: uuidv4 } = require('uuid');
+
+module.exports = {
+
+    randomUUID: function() {
+        return randomUUID ? randomUUID() : uuidv4();
+    },
+
+    md5: function(data, salt, enc) {
+        return hash('md5', data, salt, enc)
+    },
+
+    sha1: function(data, salt, enc) {
+        return hash('sha1', data, salt, enc)
+    },
+
+    sha256: function(data, salt, enc) {
+        return hash('sha256', data, salt, enc)
+    },
+
+    sha512: function(data, salt, enc) {
+        return hash('sha512', data, salt, enc)
+    },
+
+    hash: function(data, alg, enc) {
+        return hash(alg, data, '', enc);
+    },
+
+    hmac: function(data, alg, secret, enc) {
+        return hmac(alg, data, secret, enc);
+    },
+
+    transform: function(data, from, to) {
+        return transform(data, from, to);
+    },
+
+    encodeBase64: function(data, enc) {
+        enc = typeof enc == 'string' ? enc : 'utf8';
+        return transform(data, enc, 'base64');
+    },
+
+    decodeBase64: function(data, enc) {
+        enc = typeof enc == 'string' ? enc : 'utf8';
+        return transform(data, 'base64', enc);
+    },
+
+    encodeBase64Url: function(data, enc) {
+        enc = typeof enc == 'string' ? enc : 'utf8';
+        return transform(data, enc, 'base64').replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_');
+    },
+
+    decodeBase64Url: function(data, enc) {
+        enc = typeof enc == 'string' ? enc : 'utf8';
+        return transform(data.replace(/-/g, '+').replace(/_/g, '/'), 'base64', enc);
+    },
+
+    encrypt: function(data, password, enc) {
+        const config = require('../setup/config');
+        password = typeof password == 'string' ? password : config.secret;
+        enc = typeof enc == 'string' ? enc : 'base64';
+        const iv = randomBytes(16);
+        const key = scryptSync(password, iv, 32);
+        const cipher = createCipheriv('aes-256-cbc', key, iv);
+        const encrypted = cipher.update(data, 'utf8', enc);
+        return iv.toString(enc) + '.' + encrypted + cipher.final(enc);
+    },
+
+    decrypt: function(data, password, enc) {
+        const config = require('../setup/config');
+        password = typeof password == 'string' ? password : config.secret;
+        enc = typeof enc == 'string' ? enc : 'base64';
+        const iv = Buffer.from(data.split('.')[0], enc);
+        const key = scryptSync(password, iv, 32);
+        const decipher = createDecipheriv('aes-256-cbc', key, iv);
+        const decrypted = decipher.update(data.split('.')[1], enc, 'utf8');
+        return decrypted + decipher.final('utf8');
+    },
+
+};
+
+function hash(alg, data, salt, enc) {
+    if (data == null) return data;
+    salt = typeof salt == 'string' ? salt : '';
+    enc = typeof enc == 'string' ? enc : 'hex';
+    return createHash(alg).update(data + salt).digest(enc);
+}
+
+function hmac(alg, data, secret, enc) {
+    if (data == null) return data;
+    secret = typeof secret == 'string' ? secret : '';
+    enc = typeof enc == 'string' ? enc : 'hex';
+    return createHmac(alg, secret).update(data).digest(enc);
+}
+
+function transform(data, from, to) {
+    if (data == null) return data;
+    return Buffer.from(String(data), from).toString(to);
+}

lib/formatters/date.js

@@ -0,0 +1,83 @@
+const { padNumber, parseDate, formatDate } = require('../core/util');
+
+module.exports = {
+
+    formatDate: function(date, format, utc) {
+        date = parseDate(date);
+        if (date == null) return date;
+        
+        return formatDate(date, format, utc);
+    },
+
+    dateAdd: function(date, interval, num) {
+        date = parseDate(date);
+        if (date == null) return date;
+
+        switch (interval) {
+            case 'years': date.setFullYear(date.getFullYear() + num); break;
+            case 'months': date.setMonth(date.getMonth() + num); break;
+            case 'weeks': date.setDate(date.getDate() + (num * 7)); break;
+            case 'days': date.setDate(date.getDate() + num); break;
+            case 'hours': date.setHours(date.getHours() + num); break;
+            case 'minutes': date.setMinutes(date.getMinutes() + num); break;
+            case 'seconds': date.setSeconds(date.getSeconds() + num); break;
+        }
+
+        return date.toISOString();
+    },
+
+    dateDiff: function(date, interval, date2) {
+        date = parseDate(date);
+        date2 = parseDate(date2);
+
+        let diff = Math.abs(date2 - date);
+
+        if (isNaN(diff) || date == null || date2 == null) return undefined;
+
+        let s = 1000, m = s * 60, h = m * 60, d = h * 24, w = d * 7;
+
+        switch (interval) {
+            case 'years': return Math.abs(date2.getFullYear() - date.getFullYear());
+            case 'months': return Math.abs((date2.getFullYear() * 12 + date2.getMonth()) - (date.getFullYear() * 12 + date.getMonth()));
+            case 'weeks': return Math.floor(diff / w);
+            case 'days': return Math.floor(diff / d);
+            case 'hours': return Math.floor(diff/ h);
+            case 'minutes': return Math.floor(diff / m);
+            case 'seconds': return Math.floor(diff / s);
+            case 'hours:minutes':
+                h = Math.floor(diff / h);
+                m = Math.floor(diff / m);
+                return padNumber(h, 2) + ':' + padNumber(m - (h * 60), 2);
+            case 'minutes:seconds':
+                m = Math.floor(diff / m);
+                s = Math.floor(diff / s);
+                return padNumber(m, 2) + ':' + padNumber(s - (m * 60), 2);
+            case 'hours:minutes:seconds':
+                h = Math.floor(diff / h);
+                m = Math.floor(diff / m);
+                s = Math.floor(diff / s);
+                return padNumber(h, 2) + ':' + padNumber(m - (h * 60), 2) + ':' + padNumber(s - m * 60, 2);
+        }
+
+        return undefined;
+    },
+
+    toTimestamp: function(date) {
+        date = parseDate(date);
+        if (date == null) return date;
+        return Math.floor(date.getTime() / 1000);
+    },
+
+    toLocalTime: function(date) {
+        date = parseDate(date);
+        if (date == null) return date;
+        return formatDate(date, 'yyyy-MM-dd HH:mm:ss.v');
+    },
+
+    toUTCTime: function(date) {
+        date = parseDate(date);
+        if (date == null) return date;
+        return date.toISOString();
+    },
+
+};

lib/formatters/index.js

@@ -0,0 +1,17 @@
+const collections = require('./collections');
+const conditional = require('./conditional');
+const core = require('./core');
+const crypto = require('./crypto');
+const date = require('./date');
+const number = require('./number');
+const string = require('./string');
+
+module.exports = {
+    ...collections,
+    ...conditional,
+    ...core,
+    ...crypto,
+    ...date,
+    ...number,
+    ...string
+};

lib/formatters/number.js

@@ -0,0 +1,73 @@
+const { padNumber, formatNumber } = require('../core/util');
+
+module.exports = {
+
+    floor: function(num) {
+        return Math.floor(Number(num));
+    },
+
+    ceil: function(num) {
+        return Math.ceil(Number(num));
+    },
+
+    round: function(num) {
+        return Math.round(Number(num));
+    },
+
+    abs: function(num) {
+        return Math.abs(Number(num));
+    },
+
+    pow: function(num, exp) {
+        return Math.pow(num, exp);
+    },
+
+    padNumber: function(num, digids) {
+        num = Number(num);
+        
+        if (isNaN(num) || !isFinite(num)) return 'NaN';
+        
+        return padNumber(num, digids);
+    },
+
+    formatNumber: function(num, decimals, decimalSeparator, groupingSeparator) {
+        return formatNumber(num, decimals, decimalSeparator, groupingSeparator);
+    },
+
+    hex: function(num) {
+        return parseInt(String(num), 16) || NaN;
+    },
+
+    currency: function(num, unit, decimalSeparator, groupingSeparator, decimals) {
+        unit = typeof unit == 'string' ? unit : '$';
+        decimalSeparator = typeof decimalSeparator == 'string' ? decimalSeparator : '.';
+        groupingSeparator = typeof groupingSeparator == 'string' ? groupingSeparator : ',';
+        decimals = typeof decimals == 'number' ? Math.abs(decimals) : 2;
+        
+        let formatted = formatNumber(num, decimals, decimalSeparator, groupingSeparator);
+        let minus = formatted[0] == '-';
+
+        return (minus ? '-' : '') + unit + formatted.replace(/^\-/, '');
+    },
+
+    formatSize: function(num, decimals, binary) {
+        num = Number(num);
+
+        if (isNaN(num) || !isFinite(num)) return 'NaN';
+
+        decimals = typeof decimals == 'number' ? Math.abs(decimals) : 2;
+        
+        let base = binary ? 1024 : 1000;
+        let suffix = binary ? ['KiB', 'MiB', 'GiB', 'TiB'] : ['kB', 'MB', 'GB', 'TB'];
+
+        for (let i = 3; i >= 0; i--) {
+            let n = Math.pow(base, i + 1);
+            if (num >= n) {
+                return formatNumber(num / n, decimals) + suffix[i];
+            }
+        }
+
+        return num + 'B';
+    },
+
+};

lib/locale/en-US.js

@@ -0,0 +1,59 @@
+module.exports = {
+    months: ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'],
+    monthsShort: ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'],
+    days: ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'],
+    daysShort: ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'],
+
+    validator: {
+        core: {
+            required: 'This field is required.',
+            email: 'Please enter a valid email address.',
+            url: 'Please enter a valid URL.',
+            date: 'Please enter a valid date.',
+            time: 'Please enter a valid time.',
+            month: 'Please enter a valid month.',
+            week: 'Please enter a valid week.',
+            color: 'Please enter a color in the format #xxxxxx.',
+            pattern: 'Invalid format.',
+            number: 'Please enter a valid number.',
+            digits: 'Please enter only digits.',
+            alphanumeric: 'Letters, numbers and underscores only please.',
+            creditcard: 'Please enter a valid credit card number.',
+            bic: 'Please specify a valid BIC code.',
+            iban: 'Please specify a valid IBAN.',
+            vat: 'Please specify a valid VAT number.',
+            integer: 'A positive or negative non-decimal number please.',
+            ipv4: 'Please enter a valid IP v4 address.',
+            ipv6: 'Please enter a valid IP v6 address.',
+            lettersonly: 'Letters only please.',
+            unicodelettersonly: 'Letters only please.',
+            letterswithbasicpunc: 'Letters or punctuation only please.',
+            nowhitespace: 'No whitespace please.',
+            minlength: 'Please enter at least {0} characters.',
+            maxlength: 'Please enter no more than {0} characters.',
+            minitems: 'Please select at least {0} items.',
+            maxitems: 'Please select no more than {0} items.',
+            min: 'Please enter a value greater than or equal to {0}.',
+            max: 'Please enter a value less than or equal to {0}.',
+            equalTo: 'Please enter the same value again.',
+            notEqualTo: 'Please enter a different value, values must not be the same.',
+        },
+        db: {
+            exists: 'Value does not exist in database.',
+            notexists: 'Value already exists in database.',
+        },
+        unicode: {
+            unicodelettersonly: 'Entered characters are not allowed.',
+            unicodescripts: 'Entered characters are not allowed.',
+        },
+        upload: {
+            accept: 'Please select a file with a valid file type.',
+            minsize: 'Please select a file of at least {0} bytes.',
+            maxsize: 'Please select a file of no more than {0} bytes.',
+            mintotalsize: 'Total size of selected files should be at least {0} bytes.',
+            maxtotalsize: 'Total size of selected files should be no more than {0} bytes.',
+            minfiles: 'Please select at least {0} files.',
+            maxfiles: 'Please select no more than {0} files.',
+        },
+    },
+};

lib/modules/api.js

@@ -0,0 +1,122 @@
+const { http, https } = require('follow-redirects');
+const querystring = require('querystring');
+const zlib = require('zlib');
+const pkg = require('../../package.json');
+
+module.exports = {
+
+    send: async function(options) {
+        let url = this.parseRequired(options.url, 'string', 'api.send: url is required.');
+        let method = this.parseOptional(options.method, 'string', 'GET');
+        let data = this.parseOptional(options.data, '*', '');
+        let dataType = this.parseOptional(options.dataType, 'string', 'auto');
+        let verifySSL = this.parseOptional(options.verifySSL, 'boolean', false);
+        let params = this.parseOptional(options.params, 'object', null);
+        let headers = this.parseOptional(options.headers, 'object', {});
+        let username = this.parseOptional(options.username, 'string', '');
+        let password = this.parseOptional(options.password, 'string', '');
+        let oauth = this.parseOptional(options.oauth, 'string', '');
+        let throwErrors = this.parseOptional(options.throwErrors, 'boolean', false);
+        let passErrors = this.parseOptional(options.passErrors, 'boolean', true);
+        let timeout = this.parseOptional(options.timeout, 'number', 0);
+
+        if (params) {
+            url += '?' + querystring.stringify(params);
+        }
+
+        if (dataType == 'auto' && method == 'POST') {
+            dataType = 'x-www-form-urlencoded';
+        }
+
+        if (dataType != 'auto' && !headers['Content-Type']) {
+            headers['Content-Type'] = `application/${dataType}`;
+        }
+
+        if (dataType == 'x-www-form-urlencoded') {
+            data = querystring.stringify(data);
+        } else if (typeof data != 'string') {
+            data = JSON.stringify(data);
+        }
+
+        if (data) {
+            headers['Content-Length'] = Buffer.byteLength(data);
+        }
+
+        const Url = new URL(url);
+        const opts = { method, headers, rejectUnauthorized: !!verifySSL, maxBodyLength: 1_000_000_000 };
+
+        if (timeout > 0) {
+            opts.timeout = timeout;
+        }
+
+        if (username || password) {
+            opts.auth = `${username}:${password}`;
+        }
+
+        if (oauth) {
+            //const provider = this.oauth[oauth];
+            const provider = await this.getOAuthProvider(oauth);
+            if (provider && provider.access_token) {
+                headers['Authorization'] = 'Bearer ' + provider.access_token;
+            }
+        }
+
+        if (!headers['User-Agent']) headers['User-Agent'] = `${pkg.name}/${pkg.version}`;
+        if (!headers['Accept']) headers['Accept'] = 'application/json';
+
+        return new Promise((resolve, reject) => {
+            const req = (Url.protocol == 'https:' ? https : http).request(Url, opts, res => {
+                let body = '';
+
+                let output = res;
+                if (res.headers['content-encoding'] == 'br') {
+                    output = res.pipe(zlib.createBrotliDecompress());
+                }
+                if (res.headers['content-encoding'] == 'gzip') {
+                    output = res.pipe(zlib.createGunzip());
+                }
+                if (res.headers['content-encoding'] == 'deflate') {
+                    output = res.pipe(zlib.createInflate());
+                }
+
+                output.setEncoding('utf8');
+                output.on('data', chunk => body += chunk);
+                output.on('end', () => {
+                    if (res.statusCode >= 400) {
+                        if (throwErrors) {
+                            return reject(res.statusCode + ' ' + body);
+                        }
+
+                        if (passErrors) {
+                            this.res.status(res.statusCode).send(body);
+                            return resolve();
+                        }
+                    }
+
+                    if (body.charCodeAt(0) === 0xFEFF) {
+                        body = body.slice(1);
+                    }
+
+                    if (res.headers['content-type'] && res.headers['content-type'].includes('json')) {
+                        try {
+                            body = JSON.parse(body);
+                        } catch(e) {
+                            console.error(e);
+                        }
+                    }
+
+                    resolve({
+                        status: res.statusCode,
+                        headers: res.headers,
+                        data: body
+                    });
+                });
+            });
+
+            req.on('error', reject);
+            req.write(data);
+            req.end();
+        });
+    },
+
+};

lib/modules/arraylist.js

@@ -0,0 +1,173 @@
+// ArrayList
+
+const { clone } = require('../core/util');
+
+// Create a new ArrayList
+exports.create = function(options, name) {
+  const value = this.parseOptional(options.value, 'object', []);
+  
+  if (!name) throw Error('arraylist.create: name is required.');
+
+  if (!this.req.arrays) {
+    this.req.arrays = {};
+  }
+
+  this.req.arrays[name] = Array.isArray(value) ? clone(value) : [];
+};
+
+// Return the ArrayList as array value
+exports.value = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.value: ref is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.value: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.value: ArrayList ${ref} not found.`);
+
+  return clone(this.req.arrays[ref]);
+};
+
+// Return the size of the ArrayList
+exports.size = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.size: ref is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.size: No arraylists are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.size: ArrayList ${ref} not found.`);
+
+  return this.req.arrays[ref].length;
+};
+
+// Return the value at a specific index
+exports.get = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.get: ref is required.');
+  const index = this.parseRequired(options.index, 'number', 'arraylist.get: index is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.get: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.get: ArrayList ${ref} not found.`);
+
+  return clone(this.req.arrays[ref][index]);
+};
+
+// Add a value to the ArrayList
+exports.add = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.add: ref is required.');
+  const value = this.parseRequired(options.value, '*', 'arraylist.add: value is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.add: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.add: ArrayList ${ref} not found.`);
+
+  this.req.arrays[ref].push(clone(value));
+};
+
+// Add an array of values to the ArrayList
+exports.addAll = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.addAll: ref is required.');
+  const value = this.parseRequired(options.value, 'object', 'arraylist.addAll: value is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.addAll: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.addAll: ArrayList ${ref} not found.`);
+  if (!Array.isArray(value)) throw Error('arraylist.addAll: Value must be an array.')
+
+  for (let val of value) {
+    this.req.arrays[ref].push(clone(val));
+  }
+};
+
+// Update a value at a specific index
+exports.set = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.set: ref is required.');
+  const index = this.parseRequired(options.index, 'number', 'arraylist.set: index is required.');
+  const value = this.parseRequired(options.value, '*', 'arraylist.set: value is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.set: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.set: ArrayList ${ref} not found.`);
+
+  this.req.arrays[ref][index] = clone(value);
+};
+
+// Remove the first occurrence of a specific value
+exports.remove = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.remove: ref is required.');
+  const value = this.parseRequired(options.value, '*', 'arraylist.remove: value is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.remove: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.remove: ArrayList ${ref} not found.`);
+
+  const index = this.req.arrays[ref].indexOf(value);
+  
+  if (index == -1) throw Error('arraylist.remove: Value does not exist in the ArrayList.');
+  
+  this.req.arrays[ref].splice(index, 1);
+};
+
+// Remove a value from the ArrayList at a specific index
+exports.removeAt = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.removeAt: ref is required.');
+  const index = this.parseRequired(options.index, 'number', 'arraylist.removeAt: index is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.removeAt: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.removeAt: ArrayList ${ref} not found.`);
+
+  this.req.arrays[ref].splice(index, 1);
+};
+
+// Clear all values from the ArrayList
+exports.clear = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.clear: ref is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.clear: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.clear: ArrayList ${ref} not found.`);
+
+  this.req.arrays[ref] = [];
+};
+
+// Sort the ArrayList
+exports.sort = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.sort: ref is required.');
+  const prop = this.parseOptional(options.prop, 'string', null);
+  const desc = this.parseOptional(options.desc, 'boolean', false);
+
+  if (!this.req.arrays) throw Error('arraylist.sort: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.sort: ArrayList ${ref} not found.`);
+
+  this.req.arrays[ref].sort((a, b) => {
+    if (prop) {
+      a = a[prop];
+      b = b[prop];
+    }
+
+    if (a == b) return 0;
+    if (a < b) return desc ? 1 : -1;
+    return desc ? -1 : 1;
+  });
+};
+
+// Return the index of the first occurrence of a specific value
+exports.indexOf = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.indexOf: ref is required.');
+  const value = this.parseRequired(options.value, '*', 'arraylist.indexOf: value is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.indexOf: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.indexOf: ArrayList ${ref} not found.`);
+
+  return this.req.arrays[ref].indexOf(value);
+};
+
+// Return true if the ArrayList contains a specific value
+exports.contains = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.contains: ref is required.');
+  const value = this.parseRequired(options.value, '*', 'arraylist.contains: value is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.contains: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.contains: ArrayList ${ref} not found.`);
+
+  return this.req.arrays[ref].includes(value);
+};
+
+// Return true when ArrayList is empty
+exports.isEmpty = function(options) {
+  const ref = this.parseRequired(options.ref, 'string', 'arraylist.isEmpty: ref is required.');
+
+  if (!this.req.arrays) throw Error('arraylist.isEmpty: No ArrayList are created.');
+  if (!this.req.arrays[ref]) throw Error(`arraylist.isEmpty: ArrayList ${ref} not found.`);
+
+  return !this.req.arrays[ref].length;
+};

lib/modules/auth.js

@@ -0,0 +1,72 @@
+module.exports = {
+
+    provider: async function(options, name) {
+        const provider = await this.setAuthProvider(name, options);
+
+        return { identity: provider.identity };
+    },
+
+    identify: async function(options) {
+        const provider = await this.getAuthProvider(this.parseRequired(options.provider, 'string', 'auth.validate: provider is required.'));
+
+        return provider.identity;
+    },
+
+    validate: async function(options) {
+        const provider = await this.getAuthProvider(this.parseRequired(options.provider, 'string', 'auth.validate: provider is required.'));
+        const { action, username, password, remember } = this.req.body;
+
+        if (action == 'validate') {
+            return provider.validate(username, password);
+        }
+
+        if (action == 'login') {
+            return provider.login(username, password, remember);
+        }
+
+        if (action == 'logout') {
+            return provider.logout();
+        }
+
+        if (!provider.identity) {
+            return provider.unauthorized();
+        }
+    },
+
+    login: async function(options) {
+        const provider = await this.getAuthProvider(this.parseRequired(options.provider, 'string', 'auth.login: provider is required.'));
+        const username = this.parseOptional(options.username, 'string', this.parse('{{$_POST.username}}'));
+        const password = this.parseOptional(options.password, 'string', this.parse('{{$_POST.password}}'));
+        const remember = this.parseOptional(options.remember, '*', this.parse('{{$_POST.remember}}'));
+
+        return provider.login(username, password, remember);
+    },
+
+    logout: async function(options) {
+        const provider = await this.getAuthProvider(this.parseRequired(options.provider, 'string', 'auth.logout: provider is required.'));
+
+        return provider.logout();
+    },
+
+    restrict: async function(options) {
+        const provider = await this.getAuthProvider(this.parseRequired(options.provider, 'string', 'auth.restrict: provider is required.'));
+
+        return provider.restrict(this.parse(options));
+    },
+
+    impersonate: async function(options) {
+        const provider = await this.getAuthProvider(this.parseRequired(options.provider, 'string', 'auth.impersonate: provider is required.'));
+        const identity = this.parseRequired(options.identity, 'string', 'auth.impersonate: identity is required.');
+
+        return provider.impersonate(identity);
+    },
+
+    verify: async function(options) {
+        const provider = await this.getAuthProvider(this.parseRequired(options.provider, 'string', 'auth.verify: provider is required.'));
+        const username = this.parseRequired(options.username, 'string', 'auth.verify: username is required.');
+        const password = this.parseRequired(options.password, 'string', 'auth.verify: password is required.');
+
+        return provider.validate(username, password);
+    },
+
+};

lib/modules/collections.js

@@ -0,0 +1,185 @@
+const { clone } = require('../core/util');
+
+module.exports = {
+
+    addColumns: function(options) {
+        let collection = this.parseRequired(options.collection, 'object'/*array[object]*/, 'collections.addColumns: collection is required.');
+        let add = this.parseRequired(options.add, 'object', 'collections.addColumns: add is required.');
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let output = [];
+
+        for (let row of collection) {
+            let newRow = clone(row);
+
+            for (let column in add) {
+                if (overwrite || newRow[column] == null) {
+                    newRow[column] = add[column];
+                }
+            }
+
+            output.push(newRow);
+        }
+
+        return output;
+    },
+    
+    filterColumns: function(options) {
+        let collection = this.parseRequired(options.collection, 'object'/*array[object]*/, 'collections.filterColumns: collection is required.');
+        let columns = this.parseRequired(options.columns, 'object'/*array[string]*/, 'collections.filterColumns: columns is required.');
+        let keep = this.parseOptional(options.keep, 'boolean', false);
+        let output = [];
+
+        for (let row of collection) {
+            let newRow = {};
+
+            for (let column in row) {
+                if (columns.includes(column)) {
+                    if (keep) {
+                        newRow[column] = clone(row[column]);
+                    }
+                } else if (!keep) {
+                    newRow[column] = clone(row[column]);
+                }
+            }
+
+            output.push(newRow);
+        }
+
+        return output;
+    },
+    
+    renameColumns: function(options) {
+        let collection = this.parseRequired(options.collection, 'object'/*array[object]*/, 'collections.renamecolumns: collection is required.');
+        let rename = this.parseRequired(options.rename, 'object', 'collections.renamecolumns: rename is required.');
+        let output = [];
+
+        for (let row of collection) {
+            let newRow = {};
+
+            for (let column in row) {
+                newRow[rename[column] || column] = clone(row[column]);
+            }
+
+            output.push(newRow);
+        }
+
+        return output;
+    },
+    
+    fillDown: function(options) {
+        let collection = this.parseRequired(options.collection, 'object'/*array[object]*/, 'collections.fillDown: collection is required.');
+        let columns = this.parseRequired(options.columns, 'object'/*array[string]*/, 'collections.fillDown: columns is required.');
+        let output = [];
+        let values = {};
+
+        for (let column of columns) {
+            values[column] = null;
+        }
+
+        for (let row of collection) {
+            let newRow = clone(row);
+
+            for (let column in values) {
+                if (newRow[column] == null) {
+                    newRow[column] = values[column];
+                } else {
+                    values[column] = newRow[column];
+                }
+            }
+
+            output.push(newRow);
+        }
+
+        return output;
+    },
+    
+    addRows: function(options) {
+        let collection = this.parseRequired(options.collection, 'object'/*array[object]*/, 'collections.addRows: collection is required.');
+        let rows = this.parseRequired(options.rows, 'object'/*array[object]*/, 'collections.addRows: rows is required.');
+
+        return clone(collection).concat(clone(rows));
+    },
+    
+    addRownumbers: function(options) {
+        let collection = this.parseRequired(options.collection, 'object'/*array[object]*/, 'collections.addRownumbers: collection is required.');
+        let column = this.parseOptional(options.column, 'string', 'nr');
+        let startAt = this.parseOptional(options.startAt, 'number', 1);
+        let desc = this.parseOptional(options.desc, 'boolean', false);
+        let output = [];
+        let nr = desc ? collection.length + startAt - 1 : startAt;
+
+        for (let row of collection) {
+            let newRow = clone(row);
+            
+            newRow[column] = desc ? nr-- : nr++;
+            
+            output.push(newRow);
+        }
+
+        return output;
+    },
+    
+    join: function(options) {
+        let collection1 = this.parseRequired(options.collection1, 'object'/*array[object]*/, 'collections.join: collection1 is required.');
+        let collection2 = this.parseRequired(options.collection2, 'object'/*array[object]*/, 'collections.join: collection2 is required.');
+        let matches = this.parseRequired(options.matches, 'object', 'collections.join: matches is required.');
+        let matchAll = this.parseOptional(options.matchAll, 'boolean', false);
+        let output = [];
+
+        for (let row1 of collection1) {
+            let newRow = clone(row1);
+
+            for (let row2 of collection2) {
+                let join = false;
+
+                for (let match in matches) {
+                    if (row1[match] == row2[matches[match]]) {
+                        join = true;
+                        if (!matchAll) break;
+                    } else if (matchAll) {
+                        join = false;
+                        break;
+                    }
+                }
+
+                if (join) {
+                    for (let column in row2) {
+                        newRow[column] = clone(row2[column]);
+                    }
+                    break;
+                }
+            }
+
+            output.push(newRow);
+        }
+
+        return output;
+    },
+    
+    normalize: function(options) {
+        let collection = this.parseRequired(options.collection, 'object'/*array[object]*/, 'collections.normalize: collection is required.');
+        let columns = [];
+        let output = [];
+
+        for (let row of collection) {
+            for (let column in row) {
+                if (!columns.includes(column)) {
+                    columns.push(column);
+                }
+            }
+        }
+
+        for (let row of collection) {
+            let newRow = {};
+
+            for (let column of columns) {
+                newRow[column] = row[column] == null ? null : clone(row[column]);
+            }
+
+            output.push(newRow);
+        }
+
+        return output;
+    },
+    
+};

lib/modules/core.js

@@ -0,0 +1,273 @@
+const fs = require('fs-extra');
+const { clone } = require('../core/util');
+
+module.exports = {
+
+    wait: function(options) {
+        let delay = this.parseOptional(options.delay, 'number', 1000);
+        return new Promise(resolve => setTimeout(resolve, delay))
+    },
+
+    log: function(options) {
+        let message = this.parse(options.message);
+        console.log(message);
+        return message;
+    },
+
+    repeat: async function(options) {
+        let repeater = this.parseRequired(options.repeat, '*', 'core.repeater: repeat is required.');
+        let outputFilter = this.parseOptional(options.outputFilter, 'string', 'include'); // include/exclude
+        let outputFields = this.parseOptional(options.outputFields, 'object'/*array[string]*/, []);
+        let index = 0, data = [], parentData = this.data;
+
+        switch (typeof repeater) {
+            case 'boolean':
+                repeater = repeater ? [0] : [];
+                break;
+            
+            case 'string':
+                repeater = repeater.split(',');
+                break;
+
+            case 'number':
+                repeater = (n => {
+                    let a = [], i = 0;
+                    while (i < n) a.push(i++);
+                    return a;
+                })(repeater);
+                break;
+        }
+
+        if (!(Array.isArray(repeater) || repeater instanceof Object)) {
+            throw new Error('Repeater data is not an array or object');
+        }
+
+        for (let key in repeater) {
+            if (Object.hasOwn(repeater, key)) {
+                let scope = {};
+                this.data = {};
+
+                if (repeater[key] instanceof Object) {
+                    for (var prop in repeater[key]) {
+                        if (Object.hasOwn(repeater[key], prop)) {
+                            scope[prop] = repeater[key][prop];
+
+                            if (outputFilter == 'exclude') {
+                                if (!outputFields.includes(prop)) {
+                                    this.data[prop] = repeater[key][prop];
+                                }
+                            } else {
+                                if (outputFields.includes(prop)) {
+                                    this.data[prop] = repeater[key][prop];
+                                }
+                            }
+                        }
+                    }
+                }
+
+                scope.$key = key;
+                scope.$name = key;
+                scope.$value = clone(repeater[key]);
+                scope.$index = index;
+                scope.$number = index + 1;
+                scope.$oddeven = index % 2;
+
+                if (repeater[key] == null) {
+                    repeater[key] = {};
+                }
+
+                this.scope = this.scope.create(scope, clone(repeater[key]));
+                await this.exec(options.exec, true);
+                this.scope = this.scope.parent;
+
+                data.push({ ...this.data });
+
+                index++;
+            }
+        }
+
+        this.data = parentData;
+
+        return data;
+    },
+
+    while: async function(options) {
+        let max = this.parseOptional(options.max, 'number', Number.MAX_SAFE_INTEGER);
+        let i = 0;
+
+        while (this.parse(options.while)) {
+            await this.exec(options.exec, true);
+            if (++i == max) break;
+        }
+    },
+    
+    condition: async function(options) {
+        let condition = this.parse(options.if);
+        
+        if (!!condition) {
+            if (options.then) {
+                await this.exec(options.then, true);
+            }
+        } else if (options.else) {
+            await this.exec(options.else, true);
+        }
+    },
+
+    conditions: async function(options) {
+        if (Array.isArray(options.conditions)) {
+            for (let condition of options.conditions) {
+                let when = this.parse(condition.when);
+                
+                if (!!when) {
+                    return this.exec(condition.then, true);
+                }
+            }
+        }
+    },
+
+    select: async function(options) {
+        let expression = this.parse(options.expression);
+
+        if (Array.isArray(options.cases)) {
+            for (let item of options.cases) {
+                let value = this.parse(item.value);
+                
+                if (expression === value) {
+                    return this.exec(item.exec, true);
+                }
+            }
+        }
+    },
+
+    setvalue: function(options) {
+        let key = this.parseOptional(options.key, 'string', '');
+        let value = this.parse(options.value);
+        if (key) this.set(key, value);
+        return value;
+    },
+
+    setsession: function(options, name) {
+        let value = this.parse(options.value);
+        this.req.session[name] = value;
+        return value;
+    },
+
+    removesession: function(options, name) {
+        delete this.req.session[name];
+    },
+
+    setcookie: function(options, name) {
+        options = this.parse(options);
+
+        let cookieOptions = {
+            domain: options.domain || undefined,
+            httpOnly: !!options.httpOnly,
+            maxAge: options.expires === 0 ? undefined : (options.expires || 30) * 24 * 60 * 60 * 1000, // from days to ms
+            path: options.path || '/',
+            secure: !!options.secure,
+            sameSite: options.sameSite || false
+        };
+
+        this.setCookie(name, options.value, cookieOptions);
+    },
+
+    removecookie: function(options, name) {
+        options = this.parse(options);
+
+        let cookieOptions = {
+            domain: options.domain || undefined,
+            httpOnly: !!options.httpOnly,
+            maxAge: options.expires === 0 ? undefined : (options.expires || 30) * 24 * 60 * 60 * 1000, // from days to ms
+            path: options.path || '/',
+            secure: !!options.secure,
+            sameSite: !!options.sameSite
+        };
+
+        this.removeCookie(name, cookieOptions);
+    },
+
+    response: function(options) {
+        let data = this.parseOptional(options.data, '*', null);
+        let status = this.parseOptional(options.status, 'number', 200);
+        let contentType = this.parseOptional(options.contentType, 'string', 'application/json');
+        if (contentType != 'application/json') {
+            this.res.set('Content-Type', contentType);
+            this.res.status(status).send(data);
+        } else {
+            this.res.status(status).json(data);
+        }
+    },
+
+    end: function(options) {
+        this.res.json(this.data);
+    },
+
+    error: function(options) {
+        let message = this.parseRequired(options.message, 'string', 'core.error: message is required.');
+        throw new Error(message);
+    },
+
+    redirect: function(options) {
+        let url = this.parseRequired(options.url, 'string', 'core.redirect: url is required.');
+        let status = this.parseOptional(options.status, 'number', 302);
+
+        this.res.redirect(status, url);
+    },
+
+    trycatch: async function(options) {
+        try {
+            await this.exec(options.try, true);
+        } catch (error) {
+            this.scope.set('$_ERROR', error.message);
+            this.error = false;
+            if (options.catch) {
+                await this.exec(options.catch, true);
+            }
+        }
+    },
+
+    exec: async function(options) {
+        var data = {};
+
+        if (options.exec && fs.existsSync(`app/modules/lib/${options.exec}.json`)) {
+            let parentData = this.data;
+            this.data = {};
+            this.scope = this.scope.create({ $_PARAM: this.parse(options.params) });
+            await this.exec(await fs.readJSON(`app/modules/lib/${options.exec}.json`), true);
+            data = this.data;
+            this.scope = this.scope.parent;
+            this.data = parentData;
+        } else {
+            throw new Error(`There is no action called '${options.exec}' found in the library.`);
+        }
+
+        return data;
+    },
+
+    group: async function(options, name) {
+        if (name) {
+            return this.sub(options.exec);
+        } else {
+            return this.exec(options.exec, true);
+        }
+    },
+
+    parallel: async function(options, name) {
+        let actions = options.exec.steps || options.exec;
+        if (!Array.isArray(actions)) actions = [actions];
+        return await Promise.all(actions.map(exec => {
+            if (name) {
+                return this.sub(exec);
+            } else {
+                return this.exec(exec, true);
+            }
+        }));
+    },
+
+    randomUUID: function(options) {
+        const { randomUUID } = require('crypto');
+        const { v4: uuidv4 } = require('uuid');
+        return randomUUID ? randomUUID() : uuidv4();
+    },
+
+};

lib/modules/crypto.js

@@ -0,0 +1,31 @@
+// Perhaps use hashy as reference to implement bcrypt and needRehash action
+// https://github.com/JsCommunity/hashy
+
+// supports argon2i, argon2d and argon2id
+exports.passwordHash = async function(options) {
+  const password = this.parseRequired(options.password, 'string', 'crypto.passwordHash: password is required.')
+  const algo = this.parseOptional(options.algo, 'string', 'argon2i')
+  const argon2 = require('argon2')
+  const type = argon2[algo]
+  return argon2.hash(password, { type })
+}
+
+exports.passwordVerify = async function(options) {
+  const password = this.parseRequired(options.password, 'string', 'crypto.passwordVerify: password is required.')
+  const hash = this.parseRequired(options.hash, 'string', 'crypto.passwordVerify: hash is required.')
+  const argon2 = require('argon2')
+  return argon2.verify(hash, password)
+}
+
+exports.passwordNeedsRehash = async function(options) {
+  const hash = this.parseRequired(options.hash, 'string', 'crypto.passwordNeedsRehash: hash is required.')
+  const algo = this.parseOptional(options.algo, 'string', 'argon2i')
+  const argon2 = require('argon2')
+  return argon2.needsRehash(hash, {})
+}
+
+exports.uuid = async function(options) {
+  const { randomUUID } = require('crypto')
+  const { v4: uuidv4 } = require('uuid')
+  return randomUUID ? randomUUID() : uuidv4()
+}

lib/modules/csrf.js

@@ -0,0 +1,7 @@
+module.exports = {
+
+  generateToken: async function (options) {
+    return this.req.csrfToken(options.overwrite);
+  },
+
+};

lib/modules/dataset.js

@@ -0,0 +1,24 @@
+const { readFileSync } = require('fs');
+const { resolve } = require('path');
+
+module.exports = {
+
+    json: function(options) {
+        return JSON.parse(this.parse(options.data));
+    },
+
+    local: function(options) {
+        let path = this.parse(options.path);
+
+        if (typeof path !== 'string') throw new Error('dataset.local: path is required.');
+
+        let data = readFileSync(resolve('public', path));
+
+        return JSON.parse(data);
+    },
+
+    remote: function(options) {
+        throw new Error('dataset.remote: not implemented, use api instead.');
+    },
+
+};

lib/modules/dbconnector.js

@@ -0,0 +1,765 @@
+const db = require('../core/db');
+const { where } = require('../formatters');
+const debug = require('debug')('server-connect:db');
+
+module.exports = {
+
+    connect: function(options, name) {
+        if (!name) throw new Error('dbconnector.connect has no name.');
+        this.setDbConnection(name, options);
+    },
+
+    select: async function(options, name, meta) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbconnector.select: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.select: sql is required.');
+        if (!sql.table) throw new Error('dbconnector.select: sql.table is required.');
+        if (typeof sql.sort != 'string') sql.sort = this.parseOptional('{{ $_GET.sort }}', 'string', null);
+        if (typeof sql.dir != 'string') sql.dir = this.parseOptional('{{ $_GET.dir }}', 'string', 'asc');
+
+        if (sql.sort && sql.columns) {
+            if (!sql.orders) sql.orders = [];
+
+            for (let column of sql.columns) {
+                if (column.column == sql.sort || column.alias == sql.sort) {
+                    let order = {
+                        column: column.alias || column.column,
+                        direction: sql.dir.toLowerCase() == 'desc' ? 'desc' : 'asc'
+                    };
+
+                    if (column.table && !column.alias) order.table = column.table;
+
+                    sql.orders.unshift(order);
+
+                    break;
+                }
+            }
+        }
+
+        sql.type = 'select';
+
+        if (db.client == 'couchdb') {
+            let table = sql.table.name || sql.table;
+            let { rows } = await db.list({ include_docs: true, startkey: table + '/', endkey: table + '0' });
+
+            rows = rows.map(row => row.doc);
+
+            if (sql.wheres) {
+                const validate = (row, rule) => {
+                    if (rule.operator) {
+                        let a = row[rule.data.column];
+                        let b = rule.value;
+
+                        switch (rule.operator) {
+                            case 'equal': return a == b;
+                            case 'not_equal': return a != b;
+                            case 'in': return b.includes(a);
+                            case 'not_in': return !b.includes(a);
+                            case 'less': return a < b;
+                            case 'less_or_equal': return a <= b;
+                            case 'greater': return a > b;
+                            case 'greater_or_equal': return a >= b;
+                            case 'between': return b[0] <= a <= b[1];
+                            case 'not_between': return !(b[0] <= a <= b[1]);
+                            case 'begins_with': return String(a).startsWith(String(b));
+                            case 'not_begins_with': return !String(a).startsWith(String(b));
+                            case 'contains': return String(a).includes(String(b));
+                            case 'not_contains': return !String(a).includes(String(b));
+                            case 'ends_with': return String(a).endsWith(String(b));
+                            case 'not_ends_with': return !String(a).endsWith(String(b));
+                            case 'is_empty': return a == null || a == '';
+                            case 'is_not_empty': return a != null && a != '';
+                            case 'is_null': return a == null;
+                            case 'is_not_null': return a != null;
+                        }
+                    }
+
+                    if (rule.condition && rule.rules.length) {
+                        for (const _rule of rule.rules) {
+                            const valid = validate(row, _rule);
+                            if (!valid && rule.condition == 'AND') return false;
+                            if (valid && rule.condition == 'OR') return true;
+                        }
+
+                        return rule.condition == 'OR' ? false : true;
+                    }
+
+                    return true;
+                };
+
+                rows = rows.filter(row => {
+                    return validate(row, sql.wheres);
+                });
+            }
+
+            if (sql.orders && sql.orders.length) {
+                rows.sort((a, b) => {
+                    for (let order of sql.orders) {
+                        if (a[order.column] == b[order.column]) continue;
+                        let desc = order.direction && order.direction.toLowerCase() == 'desc';
+                        if (a[order.column] < b[order.column]) {
+                            return desc ? 1 : -1;
+                        } else {
+                            return desc ? -1 : 1;
+                        }
+                    }
+
+                    return 0;
+                });
+            }
+
+            if (sql.columns && sql.columns.length) {
+                // we can also skip if user want just all columns
+                if (!(sql.columns.length == 1 && sql.columns[0].column == '*')) {
+                    rows = rows.map(doc => {
+                        const row = {};
+
+                        for (let column of sql.columns) {
+                            if (column.column == '*') {
+                                Object.assign(row, doc);
+                            } else {
+                                // only support single level for now
+                                row[column.alias || column.column || column] = doc[column.column || column];
+                            }
+                        }
+
+                        return row;
+                    });
+                }
+            }
+
+            if (sql.distinct) {
+                rows = [...new Set(rows)];
+            }
+
+            let offset = Number(sql.offset || 0);
+            let limit = Number(sql.limit || 0);
+            return rows.slice(offset, limit ? offset + limit : undefined);
+        }
+
+        if (options.test) {
+            return {
+                options: options,
+                query: db.fromJSON(sql, meta).toSQL().toNative()
+            };
+        }
+
+        if (hasSubs(sql)) {
+            prepareColumns(sql);
+
+            const results = await db.fromJSON(sql, meta);
+
+            if (results.length) {
+                if (sql.sub) {
+                    await _processSubQueries.call(this, db, results, sql.sub, meta);
+                }
+
+                if (sql.joins && sql.joins.length) {
+                    for (const join of sql.joins) {
+                        if (join.sub) {
+                            await _processSubQueries.call(this, db, results, join.sub, meta, '_' + (join.alias || join.table));
+                        }
+                    }
+                }
+
+                cleanupResults(results);
+            }
+
+            return results;
+        }
+
+        return db.fromJSON(sql, meta);
+    },
+
+    count: async function(options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbconnector.count: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.count: sql is required.');
+        if (!sql.table) throw new Error('dbconnector.count: sql.table is required.');
+
+        sql.type = 'count';
+
+        if (db.client == 'couchdb') {
+            let table = sql.table.name || sql.table;
+            let { rows } = await db.list({ include_docs: true, startkey: table + '/', endkey: table + '0' });
+
+            rows = rows.map(row => row.doc);
+
+            if (sql.wheres) {
+                const validate = (row, rule) => {
+                    if (rule.operator) {
+                        let a = row[rule.data.column];
+                        let b = rule.value;
+
+                        switch (rule.operator) {
+                            case 'equal': return a == b;
+                            case 'not_equal': return a != b;
+                            case 'in': return b.includes(a);
+                            case 'not_in': return !b.includes(a);
+                            case 'less': return a < b;
+                            case 'less_or_equal': return a <= b;
+                            case 'greater': return a > b;
+                            case 'greater_or_equal': return a >= b;
+                            case 'between': return b[0] <= a <= b[1];
+                            case 'not_between': return !(b[0] <= a <= b[1]);
+                            case 'begins_with': return String(a).startsWith(String(b));
+                            case 'not_begins_with': return !String(a).startsWith(String(b));
+                            case 'contains': return String(a).includes(String(b));
+                            case 'not_contains': return !String(a).includes(String(b));
+                            case 'ends_with': return String(a).endsWith(String(b));
+                            case 'not_ends_with': return !String(a).endsWith(String(b));
+                            case 'is_empty': return a == null || a == '';
+                            case 'is_not_empty': return a != null && a != '';
+                            case 'is_null': return a == null;
+                            case 'is_not_null': return a != null;
+                        }
+                    }
+
+                    if (rule.condition && rule.rules.length) {
+                        for (const _rule of rule.rules) {
+                            const valid = validate(row, _rule);
+                            if (!valid && rule.condition == 'AND') return false;
+                            if (valid && rule.condition == 'OR') return true;
+                        }
+
+                        return rule.condition == 'OR' ? false : true;
+                    }
+
+                    return true;
+                };
+
+                rows = rows.filter(row => {
+                    return validate(row, sql.wheres);
+                });
+            }
+
+            if (sql.distinct) {
+                rows = [...new Set(rows)];
+            }
+
+            return rows.length;
+        }
+
+        if (options.test) {
+            return {
+                options: options,
+                query: db.fromJSON(sql, meta).toSQL().toNative()
+            };
+        }
+
+        return (await db.fromJSON(sql)).Total;
+    },
+
+    single: async function(options, name, meta) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbconnector.single: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.single: sql is required.');
+        if (!sql.table) throw new Error('dbconnector.single: sql.table is required.');
+        if (typeof sql.sort != 'string') sql.sort = this.parseOptional('{{ $_GET.sort }}', 'string', null);
+        if (typeof sql.dir != 'string') sql.dir = this.parseOptional('{{ $_GET.dir }}', 'string', 'asc');
+
+        sql.type = 'first';
+
+        if (db.client == 'couchdb') {
+            let table = sql.table.name || sql.table;
+            let { rows } = await db.list({ include_docs: true, startkey: table + '/', endkey: table + '0' });
+
+            rows = rows.map(row => row.doc);
+
+            if (sql.wheres) {
+                const validate = (row, rule) => {
+                    if (rule.operator) {
+                        let a = row[rule.data.column];
+                        let b = rule.value;
+
+                        switch (rule.operator) {
+                            case 'equal': return a == b;
+                            case 'not_equal': return a != b;
+                            case 'in': return b.includes(a);
+                            case 'not_in': return !b.includes(a);
+                            case 'less': return a < b;
+                            case 'less_or_equal': return a <= b;
+                            case 'greater': return a > b;
+                            case 'greater_or_equal': return a >= b;
+                            case 'between': return b[0] <= a <= b[1];
+                            case 'not_between': return !(b[0] <= a <= b[1]);
+                            case 'begins_with': return String(a).startsWith(String(b));
+                            case 'not_begins_with': return !String(a).startsWith(String(b));
+                            case 'contains': return String(a).includes(String(b));
+                            case 'not_contains': return !String(a).includes(String(b));
+                            case 'ends_with': return String(a).endsWith(String(b));
+                            case 'not_ends_with': return !String(a).endsWith(String(b));
+                            case 'is_empty': return a == null || a == '';
+                            case 'is_not_empty': return a != null && a != '';
+                            case 'is_null': return a == null;
+                            case 'is_not_null': return a != null;
+                        }
+                    }
+
+                    if (rule.condition && rule.rules.length) {
+                        for (const _rule of rule.rules) {
+                            const valid = validate(row, _rule);
+                            if (!valid && rule.condition == 'AND') return false;
+                            if (valid && rule.condition == 'OR') return true;
+                        }
+
+                        return rule.condition == 'OR' ? false : true;
+                    }
+
+                    return true;
+                };
+
+                rows = rows.filter(row => {
+                    return validate(row, sql.wheres);
+                });
+            }
+
+            if (sql.orders && sql.orders.length) {
+                rows.sort((a, b) => {
+                    for (let order of sql.orders) {
+                        if (a[order.column] == b[order.column]) continue;
+                        let desc = order.direction && order.direction.toLowerCase() == 'desc';
+                        if (a[order.column] < b[order.column]) {
+                            return desc ? 1 : -1;
+                        } else {
+                            return desc ? -1 : 1;
+                        }
+                    }
+
+                    return 0;
+                });
+            }
+
+            if (sql.columns && sql.columns.length) {
+                // we can also skip if user want just all columns
+                if (!(sql.columns.length == 1 && sql.columns[0].column == '*')) {
+                    rows = rows.map(doc => {
+                        const row = {};
+
+                        for (let column of sql.columns) {
+                            if (column.column == '*') {
+                                Object.assign(row, doc);
+                            } else {
+                                // only support single level for now
+                                row[column.alias || column.column || column] = doc[column.column || column];
+                            }
+                        }
+
+                        return row;
+                    });
+                }
+            }
+
+            if (sql.distinct) {
+                rows = [...new Set(rows)];
+            }
+
+            return rows.length ? rows[0] : null;
+        }
+
+        if (options.test) {
+            return {
+                options: options,
+                query: db.fromJSON(sql, meta).toSQL().toNative()
+            };
+        }
+
+        if (hasSubs(sql)) {
+            prepareColumns(sql);
+
+            const result = await db.fromJSON(sql, meta);
+
+            if (!result) return null;
+
+            if (sql.sub) {
+                await _processSubQueries.call(this, db, [result], sql.sub, meta);
+            }
+
+            if (sql.joins && sql.joins.length) {
+                for (const join of sql.joins) {
+                    if (join.sub) {
+                        await _processSubQueries.call(this, db, [result], join.sub, meta, '_' + (join.alias || join.table));
+                    }
+                }
+            }
+
+            cleanupResults([result]);
+
+            return result;
+        }
+
+        return db.fromJSON(sql, meta) || null;
+    },
+
+    paged: async function(options, name, meta) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbconnector.paged: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.paged: sql is required.');
+        if (!sql.table) throw new Error('dbconnector.paged: sql.table is required.');
+        if (typeof sql.offset != 'number') sql.offset = Number(this.parseOptional('{{ $_GET.offset }}', '*', 0));
+        if (typeof sql.limit != 'number') sql.limit = Number(this.parseOptional('{{ $_GET.limit }}', '*', 25));
+        if (typeof sql.sort != 'string') sql.sort = this.parseOptional('{{ $_GET.sort }}', 'string', null);
+        if (typeof sql.dir != 'string') sql.dir = this.parseOptional('{{ $_GET.dir }}', 'string', 'asc');
+
+        if (sql.sort && sql.columns) {
+            if (!sql.orders) sql.orders = [];
+
+            for (let column of sql.columns) {
+                if (column.column == sql.sort || column.alias == sql.sort) {
+                    let order = {
+                        column: column.alias || column.column,
+                        direction: sql.dir.toLowerCase() == 'desc' ? 'desc' : 'asc'
+                    };
+
+                    if (column.table && !column.alias) order.table = column.table;
+
+                    sql.orders.unshift(order);
+                    
+                    break;
+                }
+            }
+        }
+
+        if (db.client == 'couchdb') {
+            let table = sql.table.name || sql.table;
+            let { rows } = await db.list({ include_docs: true, startkey: table + '/', endkey: table + '0' });
+
+            rows = rows.map(row => row.doc);
+
+            if (sql.wheres) {
+                const validate = (row, rule) => {
+                    if (rule.operator) {
+                        let a = row[rule.data.column];
+                        let b = rule.value;
+
+                        switch (rule.operator) {
+                            case 'equal': return a == b;
+                            case 'not_equal': return a != b;
+                            case 'in': return b.includes(a);
+                            case 'not_in': return !b.includes(a);
+                            case 'less': return a < b;
+                            case 'less_or_equal': return a <= b;
+                            case 'greater': return a > b;
+                            case 'greater_or_equal': return a >= b;
+                            case 'between': return b[0] <= a <= b[1];
+                            case 'not_between': return !(b[0] <= a <= b[1]);
+                            case 'begins_with': return String(a).startsWith(String(b));
+                            case 'not_begins_with': return !String(a).startsWith(String(b));
+                            case 'contains': return String(a).includes(String(b));
+                            case 'not_contains': return !String(a).includes(String(b));
+                            case 'ends_with': return String(a).endsWith(String(b));
+                            case 'not_ends_with': return !String(a).endsWith(String(b));
+                            case 'is_empty': return a == null || a == '';
+                            case 'is_not_empty': return a != null && a != '';
+                            case 'is_null': return a == null;
+                            case 'is_not_null': return a != null;
+                        }
+                    }
+
+                    if (rule.condition && rule.rules.length) {
+                        for (const _rule of rule.rules) {
+                            const valid = validate(row, _rule);
+                            if (!valid && rule.condition == 'AND') return false;
+                            if (valid && rule.condition == 'OR') return true;
+                        }
+
+                        return rule.condition == 'OR' ? false : true;
+                    }
+
+                    return true;
+                };
+
+                rows = rows.filter(row => {
+                    return validate(row, sql.wheres);
+                });
+            }
+
+            if (sql.orders && sql.orders.length) {
+                rows.sort((a, b) => {
+                    for (let order of sql.orders) {
+                        if (a[order.column] == b[order.column]) continue;
+                        let desc = order.direction && order.direction.toLowerCase() == 'desc';
+                        if (a[order.column] < b[order.column]) {
+                            return desc ? 1 : -1;
+                        } else {
+                            return desc ? -1 : 1;
+                        }
+                    }
+
+                    return 0;
+                });
+            }
+
+            if (sql.columns && sql.columns.length) {
+                // we can also skip if user want just all columns
+                if (!(sql.columns.length == 1 && sql.columns[0].column == '*')) {
+                    rows = rows.map(doc => {
+                        const row = {};
+
+                        for (let column of sql.columns) {
+                            if (column.column == '*') {
+                                Object.assign(row, doc);
+                            } else {
+                                // only support single level for now
+                                row[column.alias || column.column || column] = doc[column.column || column];
+                            }
+                        }
+
+                        return row;
+                    });
+                }
+            }
+
+            if (sql.distinct) {
+                rows = [...new Set(rows)];
+            }
+
+            let offset = Number(sql.offset || 0);
+            let limit = Number(sql.limit || 0);
+            let total = rows.length;
+
+            return {
+                offset,
+                limit,
+                total,
+                page: {
+                    offset: {
+                        first: 0,
+                        prev: offset - limit > 0 ? offset - limit : 0,
+                        next: offset + limit < total ? offset + limit : offset,
+                        last: (Math.ceil(total / limit) - 1) * limit
+                    },
+                    current: Math.floor(offset / limit) + 1,
+                    total: Math.ceil(total / limit)
+                },
+                data: rows.slice(offset, limit ? offset + limit : undefined)
+            };
+        }
+
+        sql.type = 'count';
+        let total = +(await db.fromJSON(sql, meta))['Total'];
+
+        sql.type = 'select';
+        let data = [];
+
+        if (options.test) {
+            return {
+                options: options,
+                query: db.fromJSON(sql, meta).toSQL().toNative()
+            };
+        }
+
+        if (hasSubs(sql)) {
+            prepareColumns(sql);
+
+            const results = await db.fromJSON(sql, meta);
+            
+            if (results.length) {
+                if (sql.sub) {
+                    await _processSubQueries.call(this, db, results, sql.sub, meta);
+                }
+
+                if (sql.joins && sql.joins.length) {
+                    for (const join of sql.joins) {
+                        if (join.sub) {
+                            await _processSubQueries.call(this, db, results, join.sub, meta, '_' + (join.alias || join.table));
+                        }
+                    }
+                }
+
+                cleanupResults(results);
+            }
+
+            data = results;
+        } else {
+            data = await db.fromJSON(sql, meta);
+        }
+
+        return {
+            offset: sql.offset,
+            limit: sql.limit,
+            total,
+            page: {
+                offset: {
+                    first: 0,
+                    prev: sql.offset - sql.limit > 0 ? sql.offset - sql.limit : 0,
+                    next: sql.offset + sql.limit < total ? sql.offset + sql.limit : sql.offset,
+                    last: (Math.ceil(total / sql.limit) - 1) * sql.limit
+                },
+                current: Math.floor(sql.offset / sql.limit) + 1,
+                total: Math.ceil(total / sql.limit)
+            },
+            data
+        }
+    },
+
+};
+
+async function _processSubQueries(db, results, sub, meta, prefix = '') {
+    const lookup = new Map();
+    const keys = new Set();
+
+    // get keys from results and create lookup table
+    // add initial sub field to results (empty array)
+    for (const result of results) {
+        const key = String(result['__dmxPrimary' + prefix]);
+        
+        if (lookup.has(key)) {
+            lookup.get(key).push(result);
+        } else {
+            lookup.set(key, [result]);
+        }
+
+        keys.add(key);
+
+        for (const field in sub) {
+            result[field] = [];
+        }
+    }
+
+    for (const field in sub) {
+        const sql = this.parseSQL(sub[field]);
+ 
+        sql.type = 'select';
+
+        prepareColumns(sql);
+
+        let submeta = meta && meta.find(data => data.name == field);
+        if (submeta && submeta.sub) submeta = submeta.sub;
+
+        // get all subresults with a single query
+        const subResults = await db.fromJSON(sql, submeta).whereIn(sql.key, Array.from(keys));
+
+        if (subResults.length) {
+            if (sql.sub) {
+                await _processSubQueries.call(this, db, subResults, sql.sub, submeta);
+            }
+
+            if (sql.joins && sql.joins.length) {
+                for (const join of sql.joins) {
+                    if (join.sub) {
+                        await _processSubQueries.call(this, db, subResults, join.sub, submeta, '_' + (join.alias || join.table));
+                    }
+                }
+            }
+
+            // map the sub results to the parent recordset
+            for (const subResult of subResults) {
+                const results = lookup.get(String(subResult['__dmxForeign']));
+
+                if (results) {
+                    for (const result of results) {
+                        result[field].push(subResult);
+                    }
+                }
+            }
+        }
+    }
+
+    // we don't need to return anything since all is updated by reference
+}
+
+function hasSubs(sql) {
+    if (sql.sub) return true;
+
+    if (sql.joins && sql.joins.length) {
+        for (const join of sql.joins) {
+            if (join.sub) return true;
+        }
+    }
+
+    return false;
+}
+
+function prepareColumns(sql) {
+    const table = sql.table.alias || sql.table.name || sql.table;
+
+    if (!Array.isArray(sql.columns) || !sql.columns.length) {
+        sql.columns = [{
+            table: table,
+            column: '*'
+        }];
+
+        if (Array.isArray(sql.joins) && sql.joins.length) {
+            for (join of sql.joins) {
+                sql.columns.push({
+                    table: join.alias || join.table,
+                    column: '*'
+                });
+            }
+        }
+    }
+
+    if (sql.sub && sql.primary) {
+        sql.columns.push({
+            table: table,
+            column: sql.primary,
+            alias: '__dmxPrimary'
+        });
+
+        if (sql.groupBy && sql.groupBy.length) {
+            sql.groupBy.push({
+                table: table,
+                column: sql.primary
+            });
+        }
+    }
+
+    if (sql.key) {
+        sql.columns.push({
+            table: table,
+            column: sql.key,
+            alias: '__dmxForeign'
+        });
+
+        if (sql.groupBy && sql.groupBy.length) {
+            sql.groupBy.push({
+                table: table,
+                column: sql.key
+            });
+        }
+    }
+
+    if (sql.joins && sql.joins.length) {
+        for (const join of sql.joins) {
+            if (join.sub && join.primary) {
+                sql.columns.push({
+                    table: join.alias || join.table,
+                    column: join.primary,
+                    alias: '__dmxPrimary_' + (join.alias || join.table)
+                });
+
+                if (sql.groupBy && sql.groupBy.length) {
+                    sql.groupBy.push({
+                        table: join.alias || join.table,
+                        column: join.primary
+                    });
+                }
+            }
+        }
+    }
+}
+
+function cleanupResults(results) {
+    for (const result of results) {
+        for (const field of Object.keys(result)) {
+            if (field.startsWith('__dmx')) {
+                delete result[field];
+            } else if (Array.isArray(result[field])) {
+                cleanupResults(result[field]);
+            }
+        }
+    }
+}

lib/modules/dbupdater.js

@@ -0,0 +1,453 @@
+module.exports = {
+
+    insert: async function (options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbconnector.insert: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.insert: sql is required.');
+        if (!sql.table) throw new Error('dbconnector.insert: sql.table is required.');
+
+        sql.type = 'insert';
+
+        if (db.client == 'couchdb') {
+            const doc = {};
+
+            for (const value of sql.values) {
+                doc[value.column] = value.value;
+            }
+
+            const result = await db.insert(doc, sql.table + '/' + Date.now());
+
+            if (result.ok) {
+                return { affected: 1, identity: result.id };
+            } else {
+                //throw new Error('dbconnector.insert: error inserting document into couchdb.');
+                return { affected: 0 };
+            }
+        }
+
+        if (options.test) {
+            return {
+                options: options,
+                query: sql.toString()
+            };
+        }
+
+        if (sql.sub) {
+            return db.transaction(async trx => {
+                // TODO: test how identity is returned for each database
+                // main insert, returns inserted id
+                const [identity] = (await trx.fromJSON(sql)).map(value => value[sql.returning] || value);
+
+                // loop sub (relation table)
+                for (let { table, key, value, values } of Object.values(sql.sub)) {
+                    if (!Array.isArray(value)) break;
+
+                    for (const current of value) {
+                        if (typeof current == 'object') {
+                            current[key] = identity;
+                            await trx(table).insert(current);
+                        } else {
+                            if (values.length != 1) throw new Error('Invalid value mapping');
+                            await trx(table).insert({
+                                [key]: identity,
+                                [values[0].column]: current
+                            });
+                        }
+                    }
+                }
+
+                return { affected: 1, identity };
+            });
+        }
+
+        let identity = await db.fromJSON(sql);
+
+        if (identity) {
+            if (Array.isArray(identity)) {
+                identity = identity[0];
+            }
+
+            if (typeof identity == 'object') {
+                identity = identity[Object.keys(identity)[0]];
+            }
+        }
+
+        return { affected: 1, identity };
+    },
+
+    update: async function (options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbconnector.update: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.update: sql is required.');
+        if (!sql.table) throw new Error('dbconnector.update: sql.table is required.');
+
+        sql.type = 'update';
+
+        if (db.client == 'couchdb') {
+            let { rows } = await db.list({ include_docs: true, startkey: sql.table + '/', endkey: sql.table + '0' });
+
+            rows = rows.map(row => row.doc);
+
+            if (sql.wheres) {
+                const validate = (row, rule) => {
+                    if (rule.operator) {
+                        let a = row[rule.data.column];
+                        let b = rule.value;
+
+                        switch (rule.operator) {
+                            case 'equal': return a == b;
+                            case 'not_equal': return a != b;
+                            case 'in': return b.includes(a);
+                            case 'not_in': return !b.includes(a);
+                            case 'less': return a < b;
+                            case 'less_or_equal': return a <= b;
+                            case 'greater': return a > b;
+                            case 'greater_or_equal': return a >= b;
+                            case 'between': return b[0] <= a <= b[1];
+                            case 'not_between': return !(b[0] <= a <= b[1]);
+                            case 'begins_with': return String(a).startsWith(String(b));
+                            case 'not_begins_with': return !String(a).startsWith(String(b));
+                            case 'contains': return String(a).includes(String(b));
+                            case 'not_contains': return !String(a).includes(String(b));
+                            case 'ends_with': return String(a).endsWith(String(b));
+                            case 'not_ends_with': return !String(a).endsWith(String(b));
+                            case 'is_empty': return a == null || a == '';
+                            case 'is_not_empty': return a != null && a != '';
+                            case 'is_null': return a == null;
+                            case 'is_not_null': return a != null;
+                        }
+                    }
+
+                    if (rule.condition && rule.rules.length) {
+                        for (const _rule of rule.rules) {
+                            const valid = validate(row, _rule);
+                            if (!valid && rule.condition == 'AND') return false;
+                            if (valid && rule.condition == 'OR') return true;
+                        }
+
+                        return rule.condition == 'OR' ? false : true;
+                    }
+
+                    return true;
+                };
+
+                rows = rows.filter(row => {
+                    return validate(row, sql.wheres);
+                });
+            }
+
+            let result = []
+            if (rows.length) {
+                result = await db.bulk({
+                    docs: rows.map(doc => {
+                        for (const value of sql.values) {
+                            doc[value.column] = value.value;
+                        }
+                        return doc;
+                    })
+                });
+            }
+
+            return { affected: Array.isArray(result) ? result.filter(result => result.ok).length : rows.length };
+        }
+
+        if (options.test) {
+            return {
+                options: options,
+                query: sql.toString()
+            };
+        }
+
+        if (sql.sub) {
+            return db.transaction(async trx => {
+                let updated = await trx.fromJSON(sql);
+
+                if (!Array.isArray(updated)) {
+                    // check if is single update
+                    const single = (
+                        sql.wheres &&
+                        sql.wheres.rules &&
+                        sql.wheres.rules.length == 1 &&
+                        sql.wheres.rules[0].field == sql.returning &&
+                        sql.wheres.rules[0].operation == '='
+                    );
+
+                    if (single) {
+                        // get id from where condition
+                        updated = [sql.wheres.rules[0].value];
+                    } else {
+                        // create a select with same where conditions
+                        updated = await trx.fromJSON({
+                            ...sql,
+                            type: 'select',
+                            columns: [sql.returning]
+                        });
+
+                        updated = updated.map(value => value[sql.returning]);
+                    }
+                } else {
+                    updated = updated.map(value => value[sql.returning]);
+                }
+
+                // loop sub
+                for (let { table, key, value, values } of Object.values(sql.sub)) {
+                    if (!Array.isArray(value)) continue;
+
+                    // delete old related data first
+                    await trx(table).whereIn(key, updated).del();
+
+                    // for each updated item
+                    for (const identity of updated) {
+                        // insert value
+                        for (const current of value) {
+                            if (typeof current == 'object') {
+                                current[key] = identity;
+                                await trx(table).insert(current);
+                            } else {
+                                if (values.length != 1) throw new Error('Invalid value mapping');
+                                await trx(table).insert({
+                                    [key]: identity,
+                                    [values[0].column]: current
+                                });
+                            }
+                        }
+                    }
+                }
+
+                return { affected: updated.length };
+            });
+        }
+
+        let affected = await db.fromJSON(sql);
+
+        return { affected };
+    },
+
+    delete: async function (options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbconnector.delete: connection is required.');
+        const sql = this.parseSQL(options.sql);
+
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.delete: sql is required.');
+        if (!sql.table) throw new Error('dbconnector.delete: sql.table is required.');
+
+        sql.type = 'del';
+
+        if (db.client == 'couchdb') {
+
+            let { rows } = await db.list({ include_docs: true, startkey: sql.table + '/', endkey: sql.table + '0' });
+
+            rows = rows.map(row => row.doc);
+
+            if (sql.wheres) {
+                const validate = (row, rule) => {
+                    if (rule.operator) {
+                        let a = row[rule.data.column];
+                        let b = rule.value;
+
+                        switch (rule.operator) {
+                            case 'equal': return a == b;
+                            case 'not_equal': return a != b;
+                            case 'in': return b.includes(a);
+                            case 'not_in': return !b.includes(a);
+                            case 'less': return a < b;
+                            case 'less_or_equal': return a <= b;
+                            case 'greater': return a > b;
+                            case 'greater_or_equal': return a >= b;
+                            case 'between': return b[0] <= a <= b[1];
+                            case 'not_between': return !(b[0] <= a <= b[1]);
+                            case 'begins_with': return String(a).startsWith(String(b));
+                            case 'not_begins_with': return !String(a).startsWith(String(b));
+                            case 'contains': return String(a).includes(String(b));
+                            case 'not_contains': return !String(a).includes(String(b));
+                            case 'ends_with': return String(a).endsWith(String(b));
+                            case 'not_ends_with': return !String(a).endsWith(String(b));
+                            case 'is_empty': return a == null || a == '';
+                            case 'is_not_empty': return a != null && a != '';
+                            case 'is_null': return a == null;
+                            case 'is_not_null': return a != null;
+                        }
+                    }
+
+                    if (rule.condition && rule.rules.length) {
+                        for (const _rule of rule.rules) {
+                            const valid = validate(row, _rule);
+                            if (!valid && rule.condition == 'AND') return false;
+                            if (valid && rule.condition == 'OR') return true;
+                        }
+
+                        return rule.condition == 'OR' ? false : true;
+                    }
+
+                    return true;
+                };
+
+                rows = rows.filter(row => {
+                    return validate(row, sql.wheres);
+                });
+            }
+
+            let result = []
+            if (rows.length) {
+                result = await db.bulk({
+                    docs: rows.map(doc => {
+                        doc._deleted = true;
+                        return doc;
+                    })
+                });
+            }
+
+            return { affected: Array.isArray(result) ? result.filter(result => result.ok).length : rows.length };
+        }
+
+        if (options.test) {
+            return {
+                options: options,
+                query: sql.toString()
+            };
+        }
+
+        if (sql.sub) {
+            return db.transaction(async trx => {
+                const deleted = (await trx.fromJSON(sql)).map(value => value[sql.returning] || value);
+
+                // loop sub
+                for (let { table, key } of Object.values(sql.sub)) {
+                    // delete related data
+                    await trx(table).whereIn(key, deleted).del();
+                }
+
+                return { affected: deleted.length };
+            });
+        }
+
+        let affected = await db.fromJSON(sql);
+
+        return { affected };
+    },
+
+    custom: async function (options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbupdater.custom: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+        if (!sql) throw new Error('dbconnector.custom: sql is required.');
+        if (typeof sql.query != 'string') throw new Error('dbupdater.custom: sql.query is required.');
+        if (!Array.isArray(sql.params)) throw new Error('dbupdater.custom: sql.params is required.');
+
+        if (db.client == 'couchdb') {
+            throw new Error('dbupdater.custom: couchdb is not supported.');
+        }
+
+        const params = [];
+        const query = sql.query.replace(/([:@][a-zA-Z_]\w*|\?)/g, param => {
+            if (param == '?') {
+                params.push(sql.params[params.length].value);
+                return '?';
+            }
+
+            let p = sql.params.find(p => p.name == param);
+            if (p) {
+                params.push(p.value);
+                return '?';
+            }
+
+            return param;
+        });
+
+        let results = await db.raw(query, params);
+
+        if (db.client.config.client == 'mysql' || db.client.config.client == 'mysql2') {
+            results = results[0];
+        } else if (db.client.config.client == 'postgres' || db.client.config.client == 'redshift') {
+            results = results.rows;
+        }
+
+        return results;
+    },
+
+    execute: async function (options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbupdater.execute: connection is required.');
+        const query = this.parseRequired(options.query, 'string', 'dbupdater.execute: query is required.');
+        const params = this.parseOptional(options.params, 'object', []);
+        const db = this.getDbConnection(connection);
+
+        if (!db) throw new Error(`Connection "${connection}" doesn't exist.`);
+
+        if (db.client == 'couchdb') {
+            throw new Error('dbupdater.execute: couchdb is not supported.');
+        }
+
+        let results = await db.raw(query, params);
+
+        if (db.client.config.client == 'mysql' || db.client.config.client == 'mysql2') {
+            results = results[0];
+        } else if (db.client.config.client == 'postgres' || db.client.config.client == 'redshift') {
+            results = results.rows;
+        }
+
+        return results;
+    },
+
+    // bulk insert
+    bulkinsert: async function (options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbupdater.bulkinsert: connection is required.');
+        const sql = this.parseSQL(options.sql);
+        const source = this.parseRequired(options.source, 'object', 'dbupdater.bulkinsert: source is required.');
+        const batchsize = this.parseOptional(options.batchsize, 'number', 100);
+        const db = this.getDbConnection(connection);
+
+        if (db.client == 'couchdb') {
+            throw new Error('dbupdater.bulkinsert: couchdb is not supported.');
+        }
+
+        return await db.transaction(async transaction => {
+            for (let i = 0; i < source.length; i += batchsize) {
+                let batch = source.slice(i, i + batchsize).map(data => {
+                    let values = {};
+
+                    for (let value of sql.values) {
+                        if (value.type == 'json') {
+                            values[value.column] = JSON.stringify(data[value.value]);
+                        } else {
+                            values[value.column] = data[value.value];
+                        }
+                    }
+
+                    return values;
+                });
+
+                await transaction(sql.table.name || sql.table).insert(batch);
+            }
+
+            return { affected: source.length };
+        });
+    },
+
+    transaction: async function (options) {
+        const connection = this.parseRequired(options.connection, 'string', 'dbupdater.transaction: connection is required.');
+        //const exec = this.parseRequired(options.exec, 'object', 'dbupdater.transaction: exec is required.');
+        const db = this.getDbConnection(connection);
+
+        if (db.client == 'couchdb') {
+            throw new Error('dbupdater.transaction: couchdb is not supported.');
+        }
+
+        return await db.transaction(async trx => {
+            this.trx[connection] = trx;
+            return await this.exec(options.exec, true).finally(() => {
+                this.trx[connection] = null;
+            });
+        });
+    },
+
+};

lib/modules/export.js

@@ -0,0 +1,109 @@
+const fs = require('fs-extra');
+const { toSystemPath } = require('../core/path');
+
+module.exports = {
+
+    csv: async function(options) {
+        let path = this.parse(options.path);
+        let data = this.parse(options.data);
+        let header = this.parse(options.header);
+        let delimiter = this.parse(options.delimiter);
+        let overwrite = this.parse(options.overwrite);
+        
+        if (typeof path != 'string') throw new Error('export.csv: path is required.');
+        if (!Array.isArray(data) || !data.length) throw new Error ('export.csv: data is required.');
+        
+        delimiter = typeof delimiter == 'string' ? delimiter : ',';
+
+        if (delimiter == '\\t') delimiter = '\t';
+
+        const fd = await fs.open(toSystemPath(path), overwrite ? 'w' : 'wx');
+
+        if (header) {
+            await putcsv(fd, Object.keys(data[0]), delimiter);
+        }
+
+        for (let row of data) {
+            await putcsv(fd, row, delimiter);
+        }
+
+        await fs.close(fd);
+
+        return path;
+    },
+
+    xml: async function(options) {
+        let path = this.parse(options.path);
+        let data = this.parse(options.data);
+        let root = this.parse(options.root);
+        let item = this.parse(options.item);
+        let overwrite = this.parse(options.overwrite);
+
+        if (typeof path != 'string') throw new Error('export.xml: path is required.');
+        if (!Array.isArray(data) || !data.length) throw new Error('export.xml: data is required.');
+
+        root = typeof root == 'string' ? root : 'export';
+        item = typeof item == 'string' ? item : 'item';
+        
+        const fd = await fs.open(toSystemPath(path), overwrite ? 'w' : 'wx');
+
+        await fs.write(fd, `<?xml version="1.0" encoding="UTF-8" ?><${root}>`);
+        for (let row of data) {
+            await fs.write(fd, `<${item}>`);
+            for (let prop in row) {
+                await fs.write(fd, `<${prop}><![CDATA[${row[prop]}]]></${prop}>`);
+            }
+            await fs.write(fd, `</${item}>`);
+        }
+        await fs.write(fd, `</${root}>`);
+        
+        await fs.close(fd);
+
+        return path;
+    },
+
+};
+
+async function putcsv(fd, data, delimiter) {
+    let str = '';
+
+    if (typeof data != 'object') {
+        throw new Error('putcsv: Invalid data.');
+    }
+
+    for (let prop in data) {
+        if (Object.hasOwn(data, prop)) {
+            let value = String(data[prop]);
+
+            if (/["\n\r\t\s]/.test(value) || value.includes(delimiter)) {
+                let escaped = false;
+                
+                str += '"';
+
+                for (let i = 0; i < value.length; i++) {
+                    if (value.charAt(i) == '\\') {
+                        escaped = true;
+                    } else if (!escaped && value.charAt(i) == '"') {
+                        str += '"';
+                    } else {
+                        escaped = false;
+                    }
+
+                    str += value.charAt(i);
+                }
+
+                str += '"';
+            } else {
+                str += value;
+            }
+
+            str += delimiter;
+        }
+    }
+
+    if (!str) {
+        throw new Error('putcsv: No data.');
+    }
+
+    return fs.write(fd, str.substr(0, str.length - delimiter.length) + '\r\n');
+}

lib/modules/fs.js

@@ -0,0 +1,258 @@
+const fs = require('fs-extra');
+const { join, dirname, basename, extname } = require('path');
+const { toSystemPath, toAppPath, toSiteUrl, getUniqFile, parseTemplate } = require('../core/path');
+const { map } = require('../core/async');
+
+module.exports = {
+
+    download: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.download: path is required.'));
+        let filename = this.parseOptional(options.filename, 'string', basename(path));
+
+        if (fs.existsSync(path)) {
+            this.res.download(path, filename);
+            this.noOutput = true;
+        } else {
+            this.res.sendStatus(404);
+        }
+    },
+
+    exists: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.exists: path is required.'));
+
+        if (await isFile(path)) {
+            if (options.then) {
+                await this.exec(options.then, true);
+            }
+            return true;
+        } else {
+            if (options.else) {
+                await this.exec(options.else, true);
+            }
+            return false;
+        }
+    },
+
+    direxists: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.direxists: path is required.'));
+
+        if (await isDirectory(path)) {
+            if (options.then) {
+                await this.exec(options.then, true);
+            }
+            return true;
+        } else {
+            if (options.else) {
+                await this.exec(options.else, true);
+            }
+            return false;
+        }
+    },
+
+    createdir: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.createdir: path is required.'));
+
+        await fs.ensureDir(path);
+
+        return toAppPath(path);
+    },
+
+    removedir: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.removedir: path is required.'));
+
+        await fs.remove(path);
+
+        return toAppPath(path);
+    },
+
+    emptydir: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.emptydir: path is required.'));
+
+        await fs.emptyDir(path)
+
+        return toAppPath(path);
+    },
+
+    move: async function(options) {
+        let from = toSystemPath(this.parseRequired(options.from, 'string', 'fs.move: from is required.'));
+        let to = toSystemPath(this.parseRequired(options.to, 'string', 'fs.move: to is required.'));
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let createdir = this.parseOptional(options.createdir, 'boolean', true);
+
+        if (!fs.existsSync(to)) {
+            if (createdir) {
+                await fs.ensureDir(to);
+            } else {
+                throw new Error(`Destination path doesn't exists.`);
+            }
+        }
+
+        to = join(to, basename(from));
+
+        if (!overwrite) {
+            to = getUniqFile(to);
+        }
+
+        await fs.move(from, to, { overwrite: true });
+
+        return toAppPath(to);
+    },
+
+    rename: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.rename: path is required.'));
+        let template = this.parseRequired(options.template, 'string', 'fs.rename: template is required.');
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let to = parseTemplate(path, template);
+
+        if (!overwrite && fs.existsSync(to)) {
+            throw new Error(`fs.rename: file "${to}" already exists.`);
+        }
+
+        await fs.rename(path, to);
+
+        return toAppPath(to);
+    },
+
+    copy: async function(options) {
+        let from = toSystemPath(this.parseRequired(options.from, 'string', 'fs.copy: from is required.'));
+        let to = toSystemPath(this.parseRequired(options.to, 'string', 'fs.copy: to is required.'));
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let createdir = this.parseOptional(options.createdir, 'boolean', true);
+
+        if (!fs.existsSync(to)) {
+            if (createdir) {
+                await fs.ensureDir(to);
+            } else {
+                throw new Error(`Destination path doesn't exist.`);
+            }
+        }
+
+        to = join(to, basename(from));
+
+        if (!overwrite && fs.existsSync(to)) {
+            to = getUniqFile(to);
+        }
+
+        await fs.copy(from, to);
+
+        return toAppPath(to);
+    },
+
+    remove: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.remove: path is required.'));
+
+        await fs.unlink(path);
+
+        return true;
+    },
+
+    dir: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.dir: path is required.'));
+        let allowedExtensions = this.parseOptional(options.allowedExtensions, 'string', '');
+        let showHidden = this.parseOptional(options.showHidden, 'boolean', false);
+        let includeFolders = this.parseOptional(options.includeFolders, 'boolean', false);
+        let folderSize = this.parseOptional(options.folderSize, 'string', 'none');
+        let concurrency = this.parseOptional(options.concurrency, 'number', 4);
+
+        folderSize = ['none', 'files', 'recursive'].includes(folderSize) ? folderSize : 'none';
+        allowedExtensions = allowedExtensions ? allowedExtensions.split(/\s*,\s*/).map(ext => lowercase(ext[0] == '.' ? ext : '.' + ext)) : [];
+
+        let files = await fs.readdir(path, { withFileTypes: true });
+
+        files = files.filter(entry => {
+            if (!includeFolders && entry.isDirectory()) return false;
+            if (!showHidden && entry.name[0] == '.') return false;
+            if (allowedExtensions.length && entry.isFile() && !allowedExtensions.includes(lowercase(extname(entry.name)))) return false;
+            return entry.isFile() || entry.isDirectory();
+        });
+
+        // Fast parallel map
+        return map(files, async (entry) => {
+            let curr = join(path, entry.name);
+            let stat = await fs.stat(curr);
+
+            if (folderSize != 'none' && entry.isDirectory()) {
+                stat.size = await calcSize(curr, folderSize == 'recursive', concurrency);
+            }
+
+            return {
+                type: entry.isFile() ? 'file' : 'dir',
+                name: entry.name,
+                folder: toAppPath(dirname(curr)),
+                basename: basename(curr, extname(curr)),
+                extension: extname(curr),
+                path: toAppPath(curr),
+                url: toSiteUrl(curr),
+                size: stat.size,
+                created: stat.ctime,
+                accessed: stat.atime,
+                modified: stat.mtime
+            };
+        }, concurrency);
+    },
+
+    stat: async function(options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'fs.stat: path is required.'));
+        let folderSize = this.parseOptional(options.folderSize, 'string', 'none');
+        let concurrency = this.parseOptional(options.concurrency, 'number', 4);
+
+        folderSize = ['none', 'files', 'recursive'].includes(folderSize) ? folderSize : 'none';
+
+        let stat = await fs.stat(path);
+
+        if (folderSize != 'none' && stat.isDirectory()) {
+            stat.size = await calcSize(path, folderSize == 'recursive', concurrency);
+        }
+
+        return {
+            type: stat.isFile() ? 'file' : 'dir',
+            name: basename(path),
+            folder: toAppPath(dirname(path)),
+            basename: basename(path, extname(path)),
+            extension: extname(path),
+            path: toAppPath(path),
+            url: toSiteUrl(path),
+            size: stat.size,
+            created: stat.ctime,
+            accessed: stat.atime,
+            modified: stat.mtime
+        };
+    },
+
+};
+
+function lowercase(str) {
+    return str.toLowerCase();
+}
+
+async function calcSize(folder, recursive, concurrency) {
+    let entries = await fs.readdir(folder);
+
+    return map(entries, async (entry) => {
+        let stat = await fs.stat(join(folder, entry));
+
+        if (stat.isDirectory() && recursive) {
+            return calcSize(join(folder, entry), recursive, concurrency);
+        }
+
+        return stat.size;
+    }, concurrency).then(arr => arr.reduce((size, curr) => size + curr, 0));
+};
+
+async function isFile(path) {
+    try {
+        let stats = await fs.stat(path);
+        return stats.isFile();
+    } catch (err) {
+        return false;
+    }
+}
+
+async function isDirectory(path) {
+    try {
+        let stats = await fs.stat(path);
+        return stats.isDirectory();
+    } catch (err) {
+        return false;
+    }
+}

lib/modules/image.js

@@ -0,0 +1,505 @@
+const fs = require('fs-extra');
+const Sharp = require('sharp');
+const debug = require('debug')('server-connect:image');
+const { basename, extname, join } = require('path');
+const { toAppPath, toSystemPath, parseTemplate, getUniqFile } = require('../core/path');
+
+const positions = {
+    'center': 0,
+    'centre': 0,
+    'top': 1,
+    'north': 1,
+    'right': 2,
+    'east': 2,
+    'bottom': 3,
+    'south': 3,
+    'left': 4,
+    'west': 4,
+    'top right': 5,
+    'right top': 5,
+    'northeast': 5,
+    'bottom right': 6,
+    'right bottom': 6,
+    'southeast': 6,
+    'bottom left': 7,
+    'left bottom': 7,
+    'southwest': 7,
+    'top left': 8,
+    'left top': 8,
+    'northwest': 8,
+    'entropy': 16,
+    'attention': 17
+};
+
+function cw(w, meta) {
+    if (typeof w == 'string') {
+        if (/%$/.test(w)) {
+            w = meta.width * parseFloat(w) / 100;
+        }
+    }
+
+    if (w < 0) {
+        w = meta.width + w;
+    }
+
+    return parseInt(w);
+}
+
+function ch(h, meta) {
+    if (typeof h == 'string') {
+        if (/%$/.test(h)) {
+            h = meta.height * parseFloat(h) / 100;
+        }
+    }
+
+    if (h < 0) {
+        h = meta.height + h;
+    }
+
+    return parseInt(h);
+}
+
+function cx(x, w, meta) {
+    if (typeof x == 'string') {
+        switch (x) {
+            case 'left':
+                x = 0;
+                break;
+            case 'center':
+                x = (meta.width - w) / 2;
+                break;
+            case 'right':
+                x = meta.width - w;
+                break;
+            default:
+                if (/%$/.test(x)) {
+                    x = (meta.width - w) * parseFloat(x) / 100;
+                }
+        }
+    }
+
+    if (x < 0) {
+        x = meta.width - w + x;
+    }
+
+    return parseInt(x);
+}
+
+function cy(y, h, meta) {
+    if (typeof y == 'string') {
+        switch (y) {
+            case 'top':
+                y = 0;
+                break;
+            case 'middle':
+                y = (meta.height - h) / 2;
+                break;
+            case 'bottom':
+                y = meta.height - h;
+                break;
+            default:
+                if (/%$/.test(y)) {
+                    y = (meta.height - h) * parseFloat(y) / 100;
+                }
+        }
+    }
+
+    if (y < 0) {
+        y = meta.height - h + y;
+    }
+
+    return parseInt(y);
+}
+
+async function updateImage(sharp) {
+    sharp.image = Sharp(await sharp.image.toBuffer());
+    sharp.metadata = await sharp.image.metadata();
+}
+
+module.exports = {
+
+    getImageSize: async function (options) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'image.getImageSize: path is required.'));
+
+        const image = Sharp(path);
+        const metadata = await image.metadata();
+
+        return {
+            width: metadata.width,
+            height: metadata.height
+        };
+    },
+
+    load: async function (options, name) {
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'image.load: path is required.'));
+        let orient = this.parseOptional(options.autoOrient, 'boolean', false);
+
+        this.req.image = this.req.image || {};
+        this.req.image[name] = { name: basename(path), image: Sharp(path), metadata: null };
+
+        const sharp = this.req.image[name];
+        if (orient) sharp.image.rotate();
+
+        await updateImage(sharp);
+
+        return {
+            name: basename(path),
+            width: sharp.metadata.width,
+            height: sharp.metadata.height
+        };
+    },
+
+    save: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.save: instance "${options.instance} doesn't exist.`);
+
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'image.save: path is required.'));
+        let format = this.parseOptional(options.format, 'string', 'jpeg').toLowerCase();
+        let template = this.parseOptional(options.template, 'string', '{name}{ext}');
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let createPath = this.parseOptional(options.createPath, 'boolean', true);
+        let background = this.parseOptional(options.background, 'string', '#FFFFFF');
+        let quality = this.parseOptional(options.quality, 'number', 75);
+
+        if (!fs.existsSync(path)) {
+            if (createPath) {
+                await fs.ensureDir(path);
+            } else {
+                throw new Error(`image.save: path "${path}" doesn't exist.`);
+            }
+        }
+
+        let file = join(path, sharp.name);
+
+        if (template) {
+            file = parseTemplate(file, template);
+        }
+
+        if (format == 'auto') {
+            switch (extname(file).toLowerCase()) {
+                case '.png': format = 'png'; break;
+                case '.gif': format = 'gif'; break;
+                case '.webp': format = 'webp'; break;
+                default: format = 'jpeg';
+            }
+        }
+        
+        if (format == 'jpeg') {
+            sharp.image.flatten({ background });
+            sharp.image.toFormat(format, { quality });
+        } else if (format == 'webp') {
+            sharp.image.toFormat(format, { quality });
+        } else {
+            sharp.image.toFormat(format);
+        }
+
+        const data = await sharp.image.toBuffer();
+        
+        file = file.replace(extname(file), '.' + format.replace('jpeg', 'jpg'));
+        
+        if (fs.existsSync(file)) {
+            if (overwrite) {
+                await fs.unlink(file);
+            } else {
+                file = getUniqFile(file);
+            }
+        }
+
+        await fs.writeFile(file, data) //.toFile(file);
+
+        return toAppPath(file);
+    },
+
+    resize: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.resize: instance "${options.instance} doesn't exist.`);
+
+        let width = this.parseOptional(cw(this.parse(options.width), sharp.metadata), 'number', null);
+        let height = this.parseOptional(ch(this.parse(options.height), sharp.metadata), 'number', null);
+        let upscale = this.parseOptional(options.upscale, 'boolean', false);
+
+        if (isNaN(width)) width = null;
+        if (isNaN(height)) height = null;
+
+        sharp.image.resize(width, height, { fit: width && height ? 'fill' : 'cover', withoutEnlargement: !upscale });
+
+        await updateImage(sharp);
+    },
+
+    crop: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.crop: instance "${options.instance} doesn't exist.`);
+
+        let width = this.parseRequired(cw(this.parse(options.width)), 'number', 'image.crop: width is required.');
+        let height = this.parseRequired(ch(this.parse(options.height)), 'number', 'image.crop: height is required.');
+        if (width > sharp.metadata.width) width = sharp.metadata.width;
+        if (height > sharp.metadata.height) height = sharp.metadata.height;
+        let left = this.parseRequired(cx(this.parse(options.x), width, sharp.metadata), 'number', 'image.crop: x is required.');
+        let top = this.parseRequired(cy(this.parse(options.y), height, sharp.metadata), 'number', 'image.crop: y is required.');
+
+        sharp.image.extract({ left, top, width, height });
+        
+        await updateImage(sharp);
+    },
+
+    cover: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.cover: instance "${options.instance}" doesn't exist.`);
+
+        let width = this.parseRequired(options.width, 'number', 'image.cover: width is required.');
+        let height = this.parseRequired(options.height, 'number', 'image.cover: height is required.');
+        // position: see positions object for options
+        let position = this.parseOptional(options.position, 'string', 'center');
+        // kernel: 'nearest', 'cubic', 'mitchell', 'lanczos2', 'lanczos3'
+        let kernel = this.parseOptional(options.kernel, 'string', 'lanczos3');
+
+        position = positions[position] || 0;
+
+        sharp.image.resize({ width, height, position, kernel });
+        
+        await updateImage(sharp);
+    },
+
+    watermark: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.watermark: instance "${options.instance} doesn't exist.`);
+
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'image.watermark: path is required.'));
+        let image = Sharp(path);
+        let metadata = await image.metadata();
+        let input = await image.toBuffer();
+        let left = this.parseRequired(cx(this.parse(options.x), metadata.width, sharp.metadata), 'number', 'image.watermark: x is required.');
+        let top = this.parseRequired(cy(this.parse(options.y), metadata.height, sharp.metadata), 'number', 'image.watermark: y is required.');
+
+        sharp.image.composite([{ input, left, top }]);
+    },
+
+    text: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.text: instance "${options.instance} doesn't exist.`);
+
+        let x = this.parse(options.x);
+        let y = this.parse(options.y);
+        let text = this.parseRequired(options.text, 'string', 'image.text: text is required.');
+        let font = this.parseOptional(options.font, 'string', 'Verdana');
+        let size = this.parseOptional(options.size, 'number', 24);
+        let color = this.parseOptional(options.color, 'string', '#ffffff');
+
+        let width = sharp.metadata.width;
+        let height = sharp.metadata.height;
+        let anchor = 'start';
+
+        switch (x) {
+            case 'left':
+                x = '0%';
+                anchor = 'start';
+                break;
+            case 'center':
+                x = '50%';
+                anchor = 'middle';
+                break;
+            case 'right':
+                x = '100%';
+                anchor = 'end';
+                break;
+            default:
+                if (x < 0) {
+                    x = width - x;
+                    anchor = 'end';
+                }
+        }
+
+        switch (y) {
+            case 'top':
+                y = size;
+                break;
+            case 'middle':
+                y = (height / 2) - (size / 2);
+                break;
+            case 'bottom':
+                y = height;
+                break;
+            default:
+                if (y < 0) {
+                    y = height - size - y;
+                }
+        }
+
+        let svg = `
+            <svg width="${width}" height="${height}">
+                <style>
+                    .text {
+                        fill: ${color};
+                        font-family: "${font}";
+                        font-size: ${size}px;
+                        line-height: 1;
+                    }
+                </style>
+                <text x="${x}" y="${y}" text-anchor="${anchor}" class="text">${text}</text>
+            </svg>
+        `;
+
+        const input = await Sharp(Buffer.from(svg)).toBuffer();
+
+        sharp.image.composite([{ input, left: 0, top: 0 }]);
+    },
+
+    tiled: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.tiled: instance "${options.instance} doesn't exist.`);
+
+        let input = toSystemPath(this.parseRequired(options.path, 'string', 'image.tiled: path is required.'));
+        let padding = this.parseOptional(options.padding, 'number', 0);
+
+        if (padding) {
+            input = await Sharp(input).extend({
+                top: padding, left: padding, bottom: 0, right: 0, background: { r: 0, g: 0, b: 0, alpha: 0 }
+            }).toBuffer();
+        }
+
+        sharp.image.composite([{ input, left: 0, top: 0, tile: true }]);
+    },
+
+    flip: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.flip: instance "${options.instance} doesn't exist.`);
+
+        let horizontal = this.parseOptional(options.horizontal, 'boolean', false);
+        let vertical = this.parseOptional(options.vertical, 'boolean', false);
+
+        if (horizontal) sharp.image.flop();
+        if (vertical) sharp.image.flip();
+    },
+
+    rotateLeft: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.rotateLeft: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.rotate(-90);
+
+        await updateImage(sharp);
+    },
+
+    rotateRight: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.rotateRight: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.rotate(90);
+
+        await updateImage(sharp);
+    },
+
+    smooth: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.smooth: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.convolve({
+            width: 3,
+            height: 3,
+            kernel: [
+                1, 1, 1,
+                1, 1, 1,
+                1, 1, 1
+            ]
+        });
+    },
+
+    blur: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.blur: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.convolve({
+            width: 3,
+            height: 3,
+            kernel: [
+                1, 2, 1,
+                2, 4, 2,
+                1, 2, 1
+            ]
+        });
+    },
+
+    sharpen: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.sharpen: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.convolve({
+            width: 3,
+            height: 3,
+            kernel: [
+                0, -2, 0,
+                -2, 15, -2,
+                0, -2, 0
+            ]
+        });
+    },
+
+    meanRemoval: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.meanRemoval: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.convolve({
+            width: 3,
+            height: 3,
+            kernel: [
+                -1, -1, -1,
+                -1, 9, -1,
+                -1, -1, -1
+            ]
+        });
+    },
+
+    emboss: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.emboss: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.convolve({
+            width: 3,
+            height: 3,
+            kernel: [
+                -1, 0, -1,
+                0, 4, 0,
+                -1, 0, -1
+            ],
+            offset: 127
+        });
+    },
+
+    edgeDetect: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.edgeDetect: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.convolve({
+            width: 3,
+            height: 3,
+            kernel: [
+                -1, -1, -1,
+                0, 0, 0,
+                1, 1, 1
+            ],
+            offset: 127
+        });
+    },
+
+    grayscale: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.grayscale: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.grayscale();
+    },
+
+    sepia: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.sepia: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.tint({ r: 112, g: 66, b: 20 });
+    },
+
+    invert: async function (options) {
+        const sharp = this.req.image[options.instance];
+        if (!sharp) throw new Error(`image.invert: instance "${options.instance} doesn't exist.`);
+
+        sharp.image.negate();
+    },
+
+};

lib/modules/import.js

@@ -0,0 +1,129 @@
+const fs = require('fs-extra');
+const { toSystemPath } = require('../core/path');
+const { keysToLowerCase } = require('../core/util');
+
+// simple inital implementation
+// better implementation at:
+// https://github.com/adaltas/node-csv-parse
+// https://github.com/mafintosh/csv-parser
+function parseCSV(csv, options) {
+    if (!csv) return [];
+
+    if (csv.charCodeAt(0) === 0xFEFF) {
+        csv = csv.slice(1);
+    }
+
+    let delimiter = options.delimiter.replace('\\t', '\t');
+    let keys = options.fields;
+    let line = 1;
+    let data = [];
+
+    if (options.header) {
+        keys = getcsv();
+
+        options.fields.forEach(field => {
+            if (keys.indexOf(field) == -1) {
+                throw new Error('parseCSV: ' + field + ' is missing in ' + options.path);
+            }
+        });
+
+        line++;
+    }
+
+    let size = keys.length;
+
+    while (csv.length) {
+        let values = getcsv();
+        let o = {};
+
+        if (values.length != size) {
+            throw new Error('parseCSV: columns do not match. keys: ' + size + ', values: ' + values.length + ' at line ' + line);
+        }
+
+        for (let i = 0; i < size; i++) {
+            o[keys[i]] = values[i];
+        }
+
+        data.push(o);
+
+        line++;
+    }
+
+    return data;
+
+    function getcsv() {
+        let data = [''], l = csv.length,
+            esc = false, escesc = false,
+            n = 0, i = 0;
+
+        while (i < l) {
+            let s = csv.charAt(i);
+
+            if (s == '\n') {
+                if (esc) {
+                    data[n] += s;
+                } else {
+                    i++;
+                    break;
+                }
+            } else if (s == '\r') {
+                if (esc) {
+                    data[n] += s;
+                }
+            } else if (s == delimiter) {
+                if (esc) {
+                    data[n] += s;
+                } else {
+                    data[++n] = '';
+                    esc = false;
+                    escesc = false;
+                }
+            } else if (s == '"') {
+                if (escesc) {
+                    data[n] += s;
+                    escesc = false;
+                }
+
+                if (esc) {
+                    esc = false;
+                    escesc = true;
+                } else {
+                    esc = true;
+                    escesc = false;
+                }
+            } else {
+                if (escesc) {
+                    data[n] += '"';
+                    escesc = false;
+                }
+
+                data[n] += s;
+            }
+
+            i++;
+        }
+
+        csv = csv.substr(i);
+
+        return data;
+    }
+}
+
+module.exports = {
+
+    csv: async function(options) {
+        let path = this.parseRequired(options.path, 'string', 'export.csv: path is required.');
+        let fields = this.parseOptional(options.fields, 'object', []);
+        let header = this.parseOptional(options.header, 'boolean', false);
+        let delimiter = this.parseOptional(options.delimiter, 'string', ',');
+        let csv = await fs.readFile(toSystemPath(path), 'utf8');
+
+        return parseCSV(csv, { fields, header, delimiter });
+    },
+
+    xml: async function(options) {
+        // TODO: import.xml
+        throw new Error('import.xml: not implemented.');
+    },
+
+};

lib/modules/jwt.js

@@ -0,0 +1,22 @@
+exports.sign = function(options, name) {
+    return this.setJSONWebToken(name, options)
+};
+
+exports.decode = function(options) {
+    const jwt = require('jsonwebtoken');
+    return jwt.decode(this.parse(options.token), { complete: true });
+};
+
+exports.verify = function(options) {
+    const jwt = require('jsonwebtoken');
+    options = this.parse(options);
+
+    try {
+        return jwt.verify(options.token, options.key, options);
+    } catch (error) {
+        const debug = require('debug')('server-connect:jwt');
+        debug('jwt verify failed: %o', error);
+        if (options.throw) throw error;
+        return { error };
+    }
+};

lib/modules/mail.js

@@ -0,0 +1,92 @@
+const fs = require('fs-extra');
+const { getFilesArray, toSystemPath } = require('../core/path');
+const { basename, posix } = require('path');
+const { v4: uuidv4 } = require('uuid');
+const IMPORTANCE = { 0: 'low', 1: 'normal', 2: 'high' };
+
+module.exports = {
+
+    setup: function(options, name) {
+        if (!name) throw new Error('mail.setup has no name.');
+        this.setMailer(name, options);
+    },
+
+    send: async function(options) {
+        let setup = this.getMailer(this.parseOptional(options.instance, 'string', 'system'));
+        let subject = this.parseRequired(options.subject, 'string', 'mail.send: subject is required.');
+        let fromEmail = this.parseRequired(options.fromEmail, 'string', 'mail.send: fromEmail is required.');
+        let fromName = this.parseOptional(options.fromName, 'string', '');
+        let toEmail = this.parseRequired(options.toEmail, 'string', 'mail.send: toEmail is required.');
+        let toName = this.parseOptional(options.toName, 'string', '');
+        let replyTo = this.parseOptional(options.replyTo, 'string', '');
+        let cc = this.parseOptional(options.cc, 'string', '');
+        let bcc = this.parseOptional(options.bcc, 'string', '');
+        let source = this.parseOptional(options.source, 'string', 'static'); // static, file
+        let contentType = this.parseOptional(options.contentType, 'string', 'text'); // text / html
+        let body = this.parseOptional(options.body, 'string', '');
+        let bodyFile = this.parseOptional(options.bodyFile, 'string', '');
+        let embedImages = this.parseOptional(options.embedImages, 'boolean', false);
+        let priority = IMPORTANCE[this.parseOptional(options.importance, 'number', 1)];
+        let attachments = this.parseOptional(options.attachments, '*', []); // "/file.ext" / ["/file.ext"] / {path:"/file.ext"} / [{path:"/file.ext"}]
+
+        let from = fromName ? `"${fromName}" <${fromEmail}>` : fromEmail;
+        let to = toName ? `"${toName}" <${toEmail}>` : toEmail;
+        let text = body;
+        let html = null;
+
+        if (source == 'file') {
+            body = this.parse(await fs.readFile(toSystemPath(bodyFile), 'utf8'));
+        }
+
+        if (attachments) {
+            attachments = getFilesArray(attachments).map((path) => ({ filename: basename(path), path }));
+        }
+
+        if (contentType == 'html') {
+            html = body;
+
+            if (embedImages) {
+                let cid = {};
+
+                html = html.replace(/(?:"|')([^"']+\.(jpg|png|gif))(?:"|')/gi, (m, url) => {
+                    let path = toSystemPath(url);
+
+                    if (fs.existsSync(path)) {
+                        if (!cid[path]) {
+                            cid[path] = uuidv4();
+                            attachments.push({
+                                filename: basename(path),
+                                path: path,
+                                cid: cid[path]
+                            });
+                        }
+
+                        return `"cid:${cid[path]}"`;
+                    } else {
+                        console.warn(`${path} not found`);
+                    }
+
+                    return `"${url}"`;
+                });
+            }
+
+            if (this.req.get) { // we can only do this if we have a request to get our hostname
+                const hasProxy = !!this.req.get('x-forwarded-host');
+                const host = hasProxy ? `${this.req.protocol}://${this.req.hostname}` : this.req.get('host');
+
+                html = html.replace(/(href|src)(?:\s*=\s*)(?:"|')([^"']+)(?:"|')/gi, (m, attr, url) => {
+                    if (!url.includes(':')) {
+                        url = posix.join(host, url);
+                    }
+
+                    return `${attr}="${url}"`;
+                });
+            }
+        }
+
+        const nodemailer = require('nodemailer');
+        let transport = nodemailer.createTransport(setup);
+        return transport.sendMail({ from, to, cc, bcc, replyTo, subject, html, text, priority, attachments });
+    },
+
+};

lib/modules/metadata.js

@@ -0,0 +1,484 @@
+const fs = require('fs-extra');
+
+const imageTypes = ['PNG', 'GIF', 'BMP', 'JPEG', 'TIFF'];
+const videoTypes = ['AVI', 'MP4', 'MOV', 'MKV', 'WEBM', 'OGV'];
+const soundTypes = ['OGG', 'WAV', 'MP3', 'FLAC'];
+
+const read = async (path, offset, length) => {
+    const fp = await fs.open(path);
+    const buff = Buffer.alloc(length);
+    
+    await fs.read(fd, buff, 0, length, offset);
+    await fs.close();
+
+    return buff;
+};
+
+
+const parser = {
+
+    PNG: async (path, result) => {
+        const buff = await read(path, 18, 6);
+        result.width = buff.readUInt16BE(0);
+        result.height = buff.readUInt16BE(4);
+    },
+
+    GIF: async (path, result) => {
+        const buff = await read(path, 6, 4);
+        result.width = buff.readUInt16LE(0);
+        result.height = buff.readUInt16LE(2);
+    },
+
+    BMP: async (path, result) => {
+        const buff = await read(path, 18, 8);
+        result.width = buff.readUInt32LE(0);
+        result.height = buff.readUInt32LE(4);
+    },
+
+    JPEG: async (path, result) => {
+        const sof = [0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf, 0xde];
+        const buff = await read(path, 2, 64000);
+        
+        let pos = 0;
+
+        while (buff[pos++] == 0xff) {
+            let marker = buff[pos++];
+            let size = buff.readUInt16BE(pos);
+
+            if (marker == 0xda) break;
+
+            if (sof.includes(marker)) {
+                result.height = buff.readUInt16BE(pos + 3);
+                result.width = buff.readUInt16BE(pos + 5);
+                break;
+            }
+
+            pos += size;
+        }
+    },
+
+    TIFF: async (path, result) => {
+        const buff = await read(path, 0, 64000);
+        const le = buff.toString('ascii', 0, 2) == 'II';
+        let pos = 0;
+        const readUInt16 = () => { pos += 2; return buff[le ? 'readUInt16LE' : 'readUInt16BE'](pos - 2); }
+        const readUInt32 = () => { pos += 4; return buff[le ? 'readUInt32LE' : 'readUInt32BE'](pos - 4); }
+
+        let offset = readUInt32();
+        
+        while (pos < buff.length && offset > 0) {
+            let entries = readUInt16(offset);
+            let start = pos;
+
+            for (let i = 0; i < entries; i++) {
+                let tag = readUInt16();
+                let type = readUInt16();
+                let length = readUInt32();
+                let data = (type == 3) ? readUInt16() : readUInt32();
+                if (type == 3) pos += 2;
+
+                if (tag == 256) {
+                    result.width = data;
+                } else if (tag == 257) {
+                    result.height = data;
+                }
+
+                if (result.width > 0 && result.height > 0) {
+                    return;
+                }
+            }
+
+            offset = readUInt32();
+            pos += offset;
+        }
+    },
+
+    AVI: async (path, result) => {
+        const buff = await read(path, 0, 144);
+        result.width = buff.readUInt32LE(64);
+        result.height = buff.readUInt32LE(68);
+        result.duration = ~~(buff.readUInt32LE(128) / buff.readUInt32LE(132) * buff.readUInt32LE(140));
+    },
+
+    MP4: async (path, result) => {
+        return parser.MOV(path, result);
+    },
+
+    MOV: async (path, result, pos = 0) => {
+        const buff = await read(path, 0, 64000);
+
+        while (pos < buff.length) {
+            let size = buff.readUInt32BE(pos);
+            let name = buff.toString('ascii', pos + 4, 4);
+
+            if (name == 'mvhd') {
+                let scale = buff.readUInt32BE(pos + 20);
+                let duration = buff.readUInt32BE(pos + 24);
+                result.duration = ~~(duration / scale);
+            }
+
+            if (name == 'tkhd') {
+                let m0 = buff.readUInt32BE(pos + 48);
+                let m4 = buff.readUInt32BE(pos + 64);
+                let w = buff.readUInt32BE(pos + 84);
+                let h = buff.readUInt32BE(pos + 88);
+                if (w > 0 && h > 0) {
+                    result.width = w / m0;
+                    result.height = h / m4;
+                    return;
+                }
+            }
+
+            if (name == 'moov' || name == 'trak') {
+                await parser.MOV(path, pos + 8);
+            }
+
+            pos += size;
+        }
+    },
+
+    WEBM: async (path, result) => {
+        return parser.EBML(path, result);
+    },
+
+    MKV: async (path, result) => {
+        return parser.EBML(path, result);
+    },
+
+    EBML: async (path, result) => {
+        const containers = ['\x1a\x45\xdf\xa3', '\x18\x53\x80\x67', '\x15\x49\xa9\x66', '\x16\x54\xae\x6b', '\xae', '\xe0'];
+        const buff = await read(path, 0, 64000);
+
+        // TODO parse EBML
+    },
+
+    OGV: async (path, result) => {
+        return parser.OGG(path, result);
+    },
+
+    OGG: async (path, result) => {
+        const buff = await read(apth, 0, 64000);
+        let pos = 0, vorbis;
+
+        while (buff.toString('ascii', pos, pos + 4) == 'OggS') {
+            let version = buff[pos + 4];
+            let b = buff[pos + 5];
+            let continuation = !!(b & 0x01);
+            let bos = !!(b & 0x02);
+            let eos = !!(b & 0x04);
+            let position = Number(buff.readBigUInt64LE(pos + 6));
+            let serial = buff.readUInt32LE(pos + 14);
+            let pageNumber = buff.readUInt32LE(pos + 18);
+            let checksum = buff.readUInt32LE(pos + 22);
+            let pageSegments = buff[path + 26];
+            let lacing = buff.slice(pos + 27, pos + 27 + pageSegments);
+            let pageSize = lacing.reduce((p, v) => p + v, 0);
+            let start = pos + 27 + pageSegments;
+            let pageHeader = buff.slice(start, start + 7);
+
+            if (pageHeader.compare(Buffer.from([0x01, 'v', 'o', 'r', 'b', 'i', 's']))) {
+                vorbis = { serial, sampleRate: buff.readUInt32LE(start + 12) };
+            }
+
+            if (pageHeader.compare(Buffer.from([0x80, 't', 'h', 'e', 'o', 'r', 'a']))) {
+                let version = buff.slice(start + 7, start + 10);
+                result.width = buff.readUInt16BE(start + 10) << 4;
+                result.height = buff.readUInt16BE(start + 12) << 4;
+
+                if (version >= 0x030200) {
+                    let width = buff.slice(start + 14, start + 17);
+                    let height = buff.slice(start + 17, start + 20);
+
+                    if (width <= result.width && width > result.width - 16 && height <= result.height && height > result.height - 16) {
+                        result.width = width;
+                        result.height = height;
+                    }
+                }
+            }
+
+            if (eos && vorbis && serail == vorbis.serial) {
+                result.duration = ~~(position / vorbis.sampleRate);
+            }
+
+            pos = start + pageSize;
+        }
+    },
+
+    WAV: async (path, result) => {
+        const buff = await read(path, 0, 32);
+        let size = buff.readUInt32LE(4);
+        let rate = buff.readUInt32LE(28);
+        result.duration = ~~(size / rate);
+    },
+
+    MP3: async (path, result) => {
+        const versions = [2.5, 0, 2, 1];
+        const layers = [0, 3, 2, 1];
+        const bitrates = [
+            [ // version 2.5
+                [0, 0, 0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], // reserved
+                [0, 8,16,24, 32, 40, 48, 56, 64, 80, 96,112,128,144,160], // layer 3
+                [0, 8,16,24, 32, 40, 48, 56, 64, 80, 96,112,128,144,160], // layer 2
+                [0,32,48,56, 64, 80, 96,112,128,144,160,176,192,224,256]  // layer 1
+            ],
+            [ // reserved
+                [0, 0, 0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], // reserved
+                [0, 0, 0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], // reserved
+                [0, 0, 0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], // reserved
+                [0, 0, 0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]  // reserved
+            ],
+            [ // version 2
+                [0, 0, 0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], // reserved
+                [0, 8,16,24, 32, 40, 48, 56, 64, 80, 96,112,128,144,160], // layer 3
+                [0, 8,16,24, 32, 40, 48, 56, 64, 80, 96,112,128,144,160], // layer 2
+                [0,32,48,56, 64, 80, 96,112,128,144,160,176,192,224,256]  // layer 1
+            ],
+            [ // version 1
+                [0, 0, 0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], // reserved
+                [0,32,40,48, 56, 64, 80, 96,112,128,160,192,224,256,320], // layer 3
+                [0,32,48,56, 64, 80, 96,112,128,160,192,224,256,320,384], // layer 2
+                [0,32,64,96,128,160,192,224,256,288,320,352,384,416,448]  // layer 1
+            ]
+        ]
+        const srates = [
+            [11025, 12000,  8000, 0], // mpeg 2.5
+            [    0,     0,     0, 0], // reserved
+            [22050, 24000, 16000, 0], // mpeg 2
+            [44100, 48000, 32000, 0]  // mpeg 1
+        ]
+        const tsamples = [
+            [0,  576, 1152, 384], // mpeg 2.5
+            [0,    0,    0,   0], // reserved
+            [0,  576, 1152, 384], // mpeg 2
+            [0, 1152, 1152, 384]  // mpeg 1
+        ];
+        const slotSizes = [0, 1, 1, 4];
+        const modes = ['stereo', 'joint_stereo', 'dual_channel', 'mono'];
+        const buff = await read(path, 0, 64000);
+
+        let duration = 0;
+        let count = 0;
+        let skip = 0;
+        let pos = 0;
+
+        while (pos < buff.length) {
+            let start = pos;
+
+            if (buff.toString('ascii', pos, 4) == 'TAG+') {
+                skip += 227;
+                pos += 227;
+            } else if (buff.toString('ascii', pos, 3) == 'TAG') {
+                skip += 128;
+                pos += 128;
+            } else if (buff.toString('ascii', pos, 3) == 'ID3') {
+                let bytes = buff.readUInt32BE(pos + 6);
+                let size = 10 + (bytes[0] << 21 | bytes[1] << 14 | bytes[2] << 7 | bytes[3]);
+                skip += size;
+                pos += size;
+            } else {
+                let hdr = buff.slice(pos, pos + 4);
+
+                while (pos < buff.length && !(hdr[0] == 0xff && (hdr[1] & 0xe0) == 0xe0)) {
+                    pos++;
+                    hdr = buff.slice(pos, pos + 4);
+                }
+
+                let ver = (hdr[1] & 0x18) >> 3;
+                let lyr = (hdr[1] & 0x06) >> 1;
+                let pad = (hdr[2] & 0x02) >> 1;
+                let brx = (hdr[2] & 0xf0) >> 4;
+                let srx = (hdr[2] & 0x0c) >> 2;
+                let mdx = (hdr[3] & 0xc0) >> 6;
+
+                let version = versions[ver];
+                let layer = layers[lyr];
+                let bitrate = bitrates[ver][lyr][brx] * 1000;
+                let samprate = srates[ver][srx];
+                let samples = tsamples[ver][lyr];
+                let slotSize = slotSizes[lyr];
+                let mode = modes[mdx];
+                let fsize = ~~(((samples / 8 * bitrate) / samprate) + (pad ? slotSize : 0));
+
+                count++;
+
+                if (count == 1) {
+                    if (layer != 3) {
+                        pos += 2;
+                    } else {
+                        if (mode != 'mono') {
+                            if (version == 1) {
+                                pos += 32;
+                            } else {
+                                pos += 17;
+                            }
+                        } else {
+                            if (version == 1) {
+                                pos += 17;
+                            } else {
+                                pos += 9;
+                            }
+                        }
+                    }
+
+                    if (buff.toString('ascii', pos, pos + 4) == 'Xing' && (buff.readUInt32BE(pos + 4) & 0x0001) == 0x0001) {
+                        let totalFrames = buff.readUInt32BE(pos + 8);
+                        duration = totalFrames * samples / samprate;
+                        break;
+                    }
+                }
+
+                if (fsize < 1) break;
+
+                pos = start + fsize;
+
+                duration += (samples / samprate);
+            }
+        }
+        
+        result.duration = ~~duration;
+    },
+
+    FLAC: async (path, result) => {
+        const buff = await read(path, 18, 8);
+        let rate = (buff[0] << 12) | (buff[1] << 4) | ((buff[2] & 0xf0) >> 4);
+        let size = ((buff[3] & 0x0f) << 32) | (buff[4] << 24) | (buff[5] << 16) | (buff[6] << 8) | buff[7];
+        result.duration = ~~(size / rate);
+    },
+
+};
+
+async function detect(path) {
+    const buff = await read(path, 0, 12);
+
+    if (buff.slice(0, 8).compare(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
+        return 'PNG';
+    }
+
+    if (buff.toString('ascii', 0, 3) == 'GIF') {
+        return 'GIF';
+    }
+
+    if (buff.toString('ascii', 0, 2) == 'BM') {
+        return 'BMP';
+    }
+
+    if (buff.slice(0, 2).compare(Buffer.from([0xff, 0xd8]))) {
+        return 'JPEG';
+    }
+
+    if (buff.toString('ascii', 0, 2) == 'II' && buff.readUInt16LE(2) == 42) {
+        return 'TIFF';
+    }
+
+    if (buff.toString('ascii', 0, 2) == 'MM' && buff.readUInt16BE(2) == 42) {
+        return 'TIFF';
+    }
+
+    if (buff.toString('ascii', 0, 4) == 'RIFF' && buff.toString('ascii', 8, 4) == 'AVI ') {
+        return 'AVI';
+    }
+
+    if (buff.toString('ascii', 4, 4) == 'ftyp') {
+        return 'MP4';
+    }
+
+    if (buff.toString('ascii', 4, 4) == 'moov') {
+        return 'MOV';
+    }
+
+    if (buff.slice(0, 4).compare(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) {
+        // TODO detect MKV
+
+        return 'EBML';
+    }
+
+    if (buff.toString('ascii', 0, 4) == 'OggS') {
+        // TODO detect OGV
+        
+        return 'OGG'
+    }
+
+    if (buff.toString('ascii', 0, 4) == 'RIFF', buff.toString('ascii', 8, 4) == 'WAVE') {
+        return 'WAV';
+    }
+
+    if (buff.toString('ascii', 0, 3) == 'ID3' || (buf[0] == 0xff && (buff[1] & 0xe0))) {
+        return 'MP3';
+    }
+
+    if (buff.toString('ascii', 0, 4) == 'fLaC') {
+        return 'FLAC';
+    }
+
+    return null
+}
+
+module.exports = {
+
+    detect: async function(options) {
+        let path = this.parseRequired(options.path, 'string', 'metadata.detect: path is required.');
+
+        return detect(path);
+    },
+
+    isImage: async function(options) {
+        let path = this.parseRequired(options.path, 'string', 'metadata.isImage: path is required.');
+        let type = await detect(path);
+        let cond = imageTypes.includes(type);
+        
+        if (cond) {
+            if (options.then) {
+                await this.exec(options.then, true);
+            }
+        } else if (options.else) {
+            await this.exec(options.else, true);
+        }
+
+        return cond;
+    },
+
+    isVideo: async function(options) {
+        let path = this.parseRequired(options.path, 'string', 'metadata.isVideo: path is required.');
+        let type = await detect(path);
+        let cond = videoTypes.includes(type);
+        
+        if (cond) {
+            if (options.then) {
+                await this.exec(options.then, true);
+            }
+        } else if (options.else) {
+            await this.exec(options.else, true);
+        }
+
+        return cond;
+    },
+
+    isSound: async function(options) {
+        let path = this.parseRequired(options.path, 'string', 'metadata.isSound: path is required.');
+        let type = await detect(path);
+        let cond = soundTypes.includes(type);
+        
+        if (cond) {
+            if (options.then) {
+                await this.exec(options.then, true);
+            }
+        } else if (options.else) {
+            await this.exec(options.else, true);
+        }
+
+        return cond;
+    },
+
+    fileinfo: async function(options) {
+        let path = this.parseRequired(options.path, 'string', 'metadata.fileinfo: path is required.');
+        let type = await detect(path);
+        let result = { type, width: null, height: null, duration: null };
+
+        if (parser[type]) {
+            await parser[type](path, result);
+        }
+
+        return result;
+    },
+
+};

lib/modules/oauth.js

@@ -0,0 +1,24 @@
+module.exports = {
+
+    provider: async function(options, name) {
+        const oauth = await this.setOAuthProvider(name, options)
+       
+        return {
+            access_token: oauth.access_token,
+            refresh_token: oauth.refresh_token
+        };
+    },
+
+    authorize: async function(options) {
+        const oauth = await this.getOAuthProvider(options.provider);
+
+        return oauth.authorize(this.parse(options.scopes), this.parse(options.params));
+    },
+
+    refresh: async function(options) {
+        const oauth = await this.getOAuthProvider(options.provider);
+
+        return oauth.refreshToken(this.parse(options.refresh_token));
+    },
+
+};

lib/modules/otp.js

@@ -0,0 +1,174 @@
+const crypto = require('crypto');
+const base32 = require('../core/base32');
+
+const secretLength = {
+  'sha1': 20,
+  'sha224': 28,
+  'sha256': 32,
+  'sha384': 48,
+  'sha512': 64,
+  'sha3-224': 28,
+  'sha3-256': 32,
+  'sha3-384': 48,
+  'sha3-512': 64,
+};
+
+const hotp = {}; // HOTP: An HMAC-Based One-Time Password Algorithm
+const totp = {}; // TOTP: Time-Based One-Time Password Algorithm
+
+hotp.counter = 0n;
+hotp.defaults = {
+  algorithm: 'sha1',
+  digits: 6,
+};
+
+hotp.generate = function (secret, opts) {
+  opts = { ...hotp.defaults, ...opts };
+  const counter = opts.counter == null ? ++hotp.counter : opts.counter;
+  const buffer = Buffer.alloc(8);
+  buffer.writeBigUInt64BE(BigInt(counter));
+  const hash = crypto.createHmac(opts.algorithm, secret).update(buffer.slice(0, secretLength[opts.algorithm])).digest();
+  return truncate(hash, opts.digits);
+};
+
+hotp.validate = function (code, secret, opts) {
+  opts = { ...hotp.defaults, ...opts };
+  if (opts.counter == null) opts.counter = hotp.counter;
+  if (code === hotp.generate(secret, opts)) return true;
+  if (typeof opts.window == 'number' && opts.window > 0) {
+    for (let n = 1; n < opts.window + 1; n++) {
+      if (code === hotp.generate(secret, { ...opts, counter: opts.counter - n })) {
+        return -n;
+      }
+    }
+  }
+  return false;
+};
+
+totp.defaults = {
+  ...hotp.defaults,
+  time: null,
+  period: 30,
+};
+
+totp.generate = function(secret, opts) {
+  opts = { ...totp.defaults, ...opts };
+  const counter = ~~((opts.time || Date.now()) / 1000 / opts.period);
+  return hotp.generate(secret, { ...opts, counter });
+};
+
+totp.validate = function(code, secret, opts) {
+  opts = { ...totp.defaults, ...opts };
+  const counter = ~~((opts.time || Date.now()) / 1000 / opts.period);
+  return code === hotp.generate(secret, { ...opts, counter });
+};
+
+function truncate(hash, digits) {
+  const offset = hash[hash.length-1] & 0xf;
+  const binary = (hash[offset] & 0x7f) << 24
+    | (hash[offset+1] & 0xff) << 16
+    | (hash[offset+2] & 0xff) << 8
+    | (hash[offset+3] & 0xff);
+  const otp = binary % Math.pow(10, digits);
+
+  return String(otp).padStart(digits, '0');
+};
+
+module.exports = {
+
+  hotpGenerage (options) {
+    const secret = this.parseRequired(options.secret, 'string', 'otp.hotpGenerate: secret is required.');
+    const counter = this.parseOptional(options.counter, 'number', null);
+    const digits = this.parseOptional(options.digits, 'number', 6); // 6 | 8
+    const algorithm = this.parseOptional(options.algorithm, 'string', 'sha1'); // 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512' | 'sha3-224' | 'sha3-256' | 'sha3-384' | 'sha3-512'
+
+    return hotp.generate(secret, { counter, digits, algorithm });
+  },
+
+  hotpValidate (options) {
+    const code = this.parseRequired(options.code, 'string', 'otp.hotpValidate: code is required.');
+    const secret = this.parseRequired(options.secret, 'string', 'otp.hotpValidate: secret is required.');
+    const counter = this.parseOptional(options.counter, 'number', null);
+    const digits = this.parseOptional(options.digits, 'number', 6); // 6 | 8
+    const algorithm = this.parseOptional(options.algorithm, 'string', 'sha1'); // 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512' | 'sha3-224' | 'sha3-256' | 'sha3-384' | 'sha3-512'
+
+    return hotp.validate(code, secret, { counter, digits, algorithm });
+  },
+
+  totpGenerate (options) {
+    const secret = this.parseRequired(options.secret, 'string', 'otp.totpGenerate: secret is required.');
+    const time = this.parseOptional(options.time, 'string', null);
+    const period = this.parseOptional(options.period, 'number', 30); // 30 | 60
+    const digits = this.parseOptional(options.digits, 'number', 6); // 6 | 8
+    const algorithm = this.parseOptional(options.algorithm, 'string', 'sha1'); // 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512' | 'sha3-224' | 'sha3-256' | 'sha3-384' | 'sha3-512'
+
+    if (typeof time == 'string') time = +(new Date(time));
+
+    return totp.generate(secret, { time, period, digits, algorithm });
+  },
+
+  totpValidate (options) {
+    const code = this.parseRequired(options.code, 'string', 'otp.totpValidate: code is required.');
+    const secret = this.parseRequired(options.secret, 'string', 'otp.totpValidate: secret is required.');
+    const time = this.parseOptional(options.time, 'string', null);
+    const period = this.parseOptional(options.period, 'number', 30); // 30 | 60
+    const digits = this.parseOptional(options.digits, 'number', 6); // 6 | 8
+    const algorithm = this.parseOptional(options.algorithm, 'string', 'sha1'); // 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512' | 'sha3-224' | 'sha3-256' | 'sha3-384' | 'sha3-512'
+
+    if (typeof time == 'string') time = +(new Date(time));
+
+    return totp.validate(code, secret, { time, period, digits, algorithm });
+  },
+
+  // https://git.coolaj86.com/coolaj86/browser-authenticator.js
+  // 20 cryptographically random binary bytes (160-bit key)
+  generateSecret (options) {
+    const size = this.parseOptional(options.size, 'number', 20);
+    return base32.encode(crypto.randomBytes(size), { padding: false }).toString();
+  },
+
+  // generates a 6-digit (20-bit) decimal time-based token
+  generateToken (options) {
+    const secret = this.parseRequired(options.secret, 'string', 'otp.generateToken: secret is required.');
+    const period = this.parseOptional(options.period, 'number', 30); // 30 | 60
+    const digits = this.parseOptional(options.digits, 'number', 6); // 6 | 8
+    const algorithm = this.parseOptional(options.algorithm, 'string', 'sha1'); // 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512' | 'sha3-224' | 'sha3-256' | 'sha3-384' | 'sha3-512'
+
+    return totp.generate(base32.decode(secret), { period, digits, algorithm });
+  },
+
+  // validates a time-based token within a +/- 30 second (90 seconds) window
+  //returns null on failure or an object such as { delta: 0 } on success
+  verifyToken (options) {
+    const code = this.parseRequired(options.code, 'string', 'otp.verifyToken: code is required.')
+    const secret = this.parseRequired(options.secret, 'string', 'otp.verifyToken: secret is required.');
+    const period = this.parseOptional(options.period, 'number', 30); // 30 | 60
+    const digits = this.parseOptional(options.digits, 'number', 6); // 6 | 8
+    const algorithm = this.parseOptional(options.algorithm, 'string', 'sha1'); // 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512' | 'sha3-224' | 'sha3-256' | 'sha3-384' | 'sha3-512'
+
+    return code === totp.generate(base32.decode(secret), { period, digits, algorithm, window: 1 });
+  },
+
+  // generates an OTPAUTH:// scheme URI for QR Code generation.
+  // https://github.com/google/google-authenticator/wiki/Key-Uri-Format
+  generateOtpUri (options) {
+    const secret = this.parseRequired(options.secret, 'string', 'otp.generateTotpUri: secret is required.');
+    const issuer = this.parseRequired(options.issuer, 'string', 'otp.generateTotpUri: issuer is required.');
+    const account = this.parseRequired(options.account, 'string', 'otp.generateTotpUri: account is required.');
+    const digits = this.parseOptional(options.digits, 'number', 6); // 6 | 8
+    const period = this.parseOptional(options.period, 'number', 30); // 30 | 60
+    const algorithm = this.parseOptional(options.algorithm, 'string', 'sha1'); // 'sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512' | 'sha3-224' | 'sha3-256' | 'sha3-384' | 'sha3-512'
+
+    return `otpauth://totp/${encodeURI(issuer)}:${encodeURI(account)}?secret=${secret}&issuer=${encodeURIComponent(issuer)}&algorithm=${encodeURIComponent(algorithm.toUpperCase())}&digits=${digits}&period=${period}`;
+  },
+
+  generateQrCodeUrl (options) {
+    const data = this.parseRequired(options.data, 'string', 'otp.generateQrCodeUrl: data is required.');
+    const size = this.parseOptional(options.size, 'number', 200);
+    const ecl = this.parseOptional(options.ecl, 'string', 'L'); // error_correction_level: L | M | Q | H
+    const margin = this.parseOptional(options.margin, 'number', 4);
+
+    return `https://www.google.com/chart?cht=qr&chs=${size}x${size}&chl=${encodeURIComponent(data)}&chld=${ecl}|${margin}`;
+  },
+
+};

lib/modules/passkeys.js

@@ -0,0 +1,86 @@
+module.exports = {
+
+  // rpName: User-visible, "friendly" website/service name
+  // rpID: Valid domain name (after `https://`)
+  // userID: User's website-specific unique ID (must be a string, do not use integer, user GUID instead)
+  // userName: User's website-specific username (email, etc...)
+  // userDisplayName: User's actual name
+  // timeout: How long (in ms) the user can take to complete attestation
+  // attestationType: Specific attestation statement
+  // excludeCredentials: Authenticators registered by the user so the user can't register the same credential multiple times
+  // supportedAlgorithmIDs Array of numeric COSE algorithm identifiers supported for attestation by this RP. See https://www.iana.org/assignments/cose/cose.xhtml#algorithms
+  async generateRegistrationOptions (options) {
+    const { generateRegistrationOptions } = require('@simplewebauthn/server');
+    const rpName = this.parseRequired(options.rpName, 'string', 'passkeys.generateRegistrationOptions: rpName is required.');
+    const rpID = this.parseRequired(options.rpID, 'string', 'passkeys.generateRegistrationOptions: rpID is required.');
+    const userID = this.parseRequired(options.userID, 'string', 'passkeys.generateRegistrationOptions: userID is required.');
+    const userName = this.parseRequired(options.userName, 'string', 'passkeys.generateRegistrationOptions: userName is required.');
+    const userDisplayName = this.parseOptional(options.userDisplayName, 'string', '');
+    const timeout = this.parseOptional(options.timeout, 'number', 60000);
+    const attestationType = this.parseOptional(options.attestationType, 'string', 'none'); // "direct" | "enterprise" | "indirect" | "none"
+    const excludeCredentials = this.parseOptional(options.excludeCredentials, 'object', []); // array[{id, transports}]
+    const supportedAlgorithmIDs = this.parseOptional(options.supportedAlgorithmIDs, 'object', [-8, -7, -257]) // array[number] (-8 requires Node 18 LTS)
+    // authenticator selection criteria (https://simplewebauthn.dev/docs/packages/server#1-generate-registration-options)
+    const residentKey = 'preferred'; // "discouraged" | "preferred" | "required"
+    const userVerification = 'preferred'; // "discouraged" | "preferred" | "required"
+
+    return await generateRegistrationOptions({
+      rpName, rpID, userID, userName, userDisplayName,
+      timeout, attestationType, excludeCredentials,
+      supportedAlgorithmIDs, authenticatorSelection: {
+        residentKey, userVerification,
+      },
+    });
+  },
+
+  // response: Response returned by **@simplewebauthn/browser**'s `startAuthentication()`
+  // expectedChallenge: The base64url-encoded `options.challenge` returned by `generateRegistrationOptions()`
+  // expectedOrigin: Website URL (or array of URLs) that the registration should have occurred on
+  // expectedRPID: RP ID (or array of IDs) that was specified in the registration options
+  async verifyRegistrationResponse (options) {
+    const { verifyRegistrationResponse } = require('@simplewebauthn/server');
+    const response = this.parseRequired(options.response, 'object', 'passkeys.verifyRegistrationResponse: response is required.');
+    const expectedChallenge = this.parseRequired(options.expectedChallenge, 'string', 'passkeys.verifyRegistrationResponse: expectedChallenge is required.');
+    const expectedOrigin = this.parseRequired(options.expectedOrigin, 'string', 'passkeys.verifyRegistrationResponse: expectedOrigin is required.');
+    const expectedRPID = this.parseRequired(options.expectedRPID, 'string', 'passkeys.verifyRegistrationResponse: expectedRPID is required.');
+
+    return await verifyRegistrationResponse({
+      response, expectedChallenge, expectedOrigin, expectedRPID,
+    });
+  },
+
+  // allowCredentials: Authenticators previously registered by the user, if any. If undefined the client will ask the user which credential they want to use
+  // timeout: How long (in ms) the user can take to complete authentication
+  // userVerification: Set to `'discouraged'` when asserting as part of a 2FA flow, otherwise set to `'preferred'` or `'required'` as desired.
+  // rpID: Valid domain name (after `https://`)
+  async generateAuthenticationOptions (options) {
+    const { generateAuthenticationOptions } = require('@simplewebauthn/server');
+    const allowCredentials = this.parseOptional(options.allowCredentials, 'object', undefined);
+    const timeout = this.parseOptional(options.timeout, 'number', 60000);
+    const userVerification = this.parseOptional(options.userVerification, 'string', 'preferred'); // "discouraged" | "preferred" | "required"
+    const rpID = this.parseOptional(options.rpID, 'string', undefined);
+
+    return await generateAuthenticationOptions({
+      allowCredentials, timeout, userVerification, rpID,
+    });
+  },
+
+  // response: Response returned by **@simplewebauthn/browser**'s `startAssertion()`
+  // expectedChallenge: The base64url-encoded `options.challenge` returned by `generateAuthenticationOptions()`
+  // expectedOrigin: Website URL (or array of URLs) that the registration should have occurred on
+  // expectedRPID: RP ID (or array of IDs) that was specified in the registration options
+  // authenticator: An internal {@link AuthenticatorDevice} matching the credential's ID
+  async verifyAuthenticationResponse (options) {
+    const { verifyAuthenticationResponse } = require('@simplewebauthn/server');
+    const response = this.parseRequired(options.response, 'object', 'passkeys.verifyAuthenticationResponse: response is required.');
+    const expectedChallenge = this.parseRequired(options.expectedChallenge, 'string', 'passkeys.verifyAuthenticationResponse: expectedChallenge is required.');
+    const expectedOrigin = this.parseRequired(options.expectedOrigin, 'string', 'passkeys.verifyAuthenticationResponse: expectedOrigin is required.');
+    const expectedRPID = this.parseRequired(options.expectedRPID, 'string', 'passkeys.verifyAuthenticationResponse: expectedRPID is required.');
+    const authenticator = this.parseRequired(options.authenticator, 'object', 'passkeys.verifyAuthenticationResponse: authenticator is required.')
+
+    return await verifyAuthenticationResponse({
+      response, expectedChallenge, expectedOrigin, expectedRPID, authenticator,
+    });
+  },
+
+};

lib/modules/ratelimit.js

@@ -0,0 +1,84 @@
+module.exports = {
+  /**
+   * Setup a rate limiter instance
+   *
+   * @param {Object} options
+   * @param {number} options.points - Maximum number of points
+   * @param {number} options.duration - Duration in seconds
+   * @param {number} options.blockDuration - Duration in seconds to block the user
+   * @param {string} name - The name of the rate limiter instance
+   */
+  setup: function (options, name) {
+    if (!name) throw new Error('rateLimit.setup has no name.');
+    this.setRateLimiter(name, options);
+  },
+
+  /**
+   * Consume points from a rate limiter instance
+   *
+   * @typedef {Object} options
+   * @param {string} options.instance - The rate limiter instance name
+   * @param {number} [options.points=1] - The number of points to consume
+   * @param {number} [options.status=429] - The status code to return
+   * @param {string} [options.message=Too Many Requests] - The message to return
+   * @param {boolean} [options.throw=false] - Throw an error instead of sending a response
+   *
+   * @return {Promise} - Resolves if the points were consumed, rejects if the rate limit was exceeded and options.throw is true
+   */
+  consume: function (options) {
+    options = options || {};
+    options.points = options.points || 1;
+    options.status = options.status || 429;
+    options.message = options.message || 'Too Many Requests';
+
+    if (options.instance) {
+      const rateLimiter = this.getRateLimiter(options.instance);
+
+      return rateLimiter.consume(this.req.ip, options.points).catch(() => {
+        if (options.throw) {
+          throw new Error(options.message);
+        } else {
+          if (this.req.is('json')) {
+            this.res.status(options.status).json({ error: options.message });
+          } else {
+            this.res.status(options.status).send(options.message);
+          }
+        }
+      });
+    } else if (this.req.app.rateLimiter) {
+      const config = require('../setup/config');
+      let isPrivate = false;
+      let key = this.req.ip;
+
+      if (this.req.app.privateRateLimiter) {
+        if (this.req.session && this.req.session[config.rateLimit.private.provider + 'Id']) {
+          isPrivate = true;
+          key = this.req.session[config.rateLimit.private.provider + 'Id'];
+        }
+      }
+
+      const limit = isPrivate ? config.rateLimit.private.points : config.rateLimit.points;
+      const duration = isPrivate ? config.rateLimit.private.duration : config.rateLimit.duration;
+      this.req.app[isPrivate ? 'privateRateLimiter' : 'rateLimiter'].consume(key, points).then((rateLimiterRes) => {
+        const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
+        this.res.set('RateLimit-Policy', `${limit};w=${duration}`);
+        this.res.set('RateLimit', `limit=${limit}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+        next();
+      }).catch((rateLimiterRes) => {
+        if (options.throw) {
+          throw new Error(options.message);
+        } else {
+          const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
+          this.res.set('RateLimit-Policy', `${limit};w=${duration}`);
+          this.res.set('RateLimit', `limit=${limit}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+          this.res.set('Retry-After', reset);
+          if (this.req.is('json')) {
+            this.res.status(options.status).json({ error: options.message });
+          } else {
+            this.res.status(options.status).send(options.message);
+          }
+        }
+      });
+    }
+  },
+};

lib/modules/recaptcha.js

@@ -0,0 +1,47 @@
+const querystring = require('querystring');
+const https = require('https');
+
+exports.validate = function(options) {
+    const secret = this.parseRequired(options.secret, 'string', 'recaptcha.validate: secret is required.');
+    const msg = this.parseOptional(options.msg, 'string', 'Recaptcha check failed.');
+
+    const response = this.req.body['g-recaptcha-response'];
+    const remoteip = this.req.ip;
+    const data = querystring.stringify({ secret, response, remoteip });
+
+    return new Promise((resolve, reject) => {
+        const req = https.request('https://www.google.com/recaptcha/api/siteverify', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Content-Length': data.length
+            }
+        }, (res) => {
+            let body = '';
+
+            res.setEncoding('utf8');
+            res.on('data', (chunk) => body += chunk);
+            res.on('end', () => {
+                if (res.statusCode >= 400) return reject(body);
+                
+                if (body.charCodeAt(0) === 0xFEFF) {
+                    body = body.slice(1);
+                }
+
+                body = JSON.parse(body);
+
+                if (!body.success) {
+                    this.res.status(400).json({
+                        form: { 'g-recaptcha-response': msg }
+                    });
+                }
+
+                resolve(body);
+            });
+        });
+
+        req.on('error', reject);
+        req.write(data);
+        req.end();
+    });
+};

lib/modules/redis.js

@@ -0,0 +1,237 @@
+const client = global.redisClient;
+
+if (client) {
+    const { promisify } = require('util');
+    const commands = {};
+
+    [
+        'append', 'decr', 'decrby', 'del', 'exists', 'get', 'getset', 'incr', 'incrby',
+        'mget', 'set', 'setex', 'setnx', 'strlen', 'lindex', 'linsert', 'llen', 'lpop',
+        'lpos', 'lpush', 'lpushx', 'lrange', 'lrem', 'lset', 'ltrim', 'rpop', 'rpush',
+        'rpushx', 'copy', 'del', 'expire', 'expireat', 'keys', 'persist', 'pexpire',
+        'pexpireat', 'pttl', 'randomkey', 'rename', 'renamenx', 'touch', 'ttl', 'type',
+        'unlink'
+    ].forEach(command => {
+        commands[command] = promisify(client[command]).bind(client);
+    });
+
+    exports.append = function(options) {
+        options = this.parse(options);
+        return commands.append(options.key, options.value);
+    };
+
+    exports.decr = function(options) {
+        options = this.parse(options);
+        return commands.decr(options.key);
+    };
+
+    exports.decrby = function(options) {
+        options = this.parse(options);
+        return commands.decrby(options.key, options.decrement);
+    };
+
+    exports.del = function(options) {
+        options = this.parse(options);
+        return commands.del(options.key);
+    };
+
+    exports.exists = function(options) {
+        options = this.parse(options);
+        return commands.exists(options.key);
+    };
+
+    exports.get = function(options) {
+        options = this.parse(options);
+        return commands.get(options.key);
+    };
+
+    exports.getset = function(options) {
+        options = this.parse(options);
+        return commands.getset(options.key, options.value);
+    };
+
+    exports.incr = function(options) {
+        options = this.parse(options);
+        return commands.incr(options.key);
+    };
+
+    exports.incrby = function(options) {
+        options = this.parse(options);
+        return commands.incrby(options.key, options.increment);
+    };
+
+    exports.mget = function(options) {
+        options = this.parse(options);
+        return commands.mget(options.keys);
+    };
+
+    exports.set = function(options) {
+        options = this.parse(options);
+        return commands.set(options.key, options.value);
+    };
+
+    exports.setex = function(options) {
+        options = this.parse(options);
+        return commands.setex(options.key, options.seconds, options.value);
+    };
+
+    exports.setnx = function(options) {
+        options = this.parse(options);
+        return commands.setnx(options.key, options.value);
+    };
+
+    exports.strlen = function(options) {
+        options = this.parse(options);
+        return commands.strlen(options.key);
+    };
+
+    exports.lindex = function(options) {
+        options = this.parse(options);
+        return commands.lindex(options.key, options.index);
+    };
+
+    exports.linsert = function(options) {
+        options = this.parse(options); // position: BEFORE|AFTER
+        return commands.linsert(options.key, options.position, options.pivot, options.element);
+    };
+
+    exports.llen = function(options) {
+        options = this.parse(options);
+        return commands.llen(options.key);
+    };
+
+    exports.lpop = function(options) {
+        options = this.parse(options);
+        return commands.lpop(options.key, options.count);
+    };
+
+    exports.lpos = function(options) {
+        options = this.parse(options);
+        return commands.lpos(options.key, options.element);
+    };
+
+    exports.lpush = function(options) {
+        options = this.parse(options);
+        return commands.lpush(options.key, options.element);
+    };
+
+    exports.lpushx = function(options) {
+        options = this.parse(options);
+        return commands.lpushx(options.key, options.element);
+    };
+
+    exports.lrange = function(options) {
+        options = this.parse(options);
+        return commands.lrange(options.key, options.start, options.stop);
+    };
+
+    exports.lrem = function(options) {
+        options = this.parse(options);
+        return commands.lrem(options.key, options.count, options.element);
+    };
+
+    exports.lset = function(options) {
+        options = this.parse(options);
+        return commands.lset(options.key, options.index, options.element);
+    };
+
+    exports.ltrim = function(options) {
+        options = this.parse(options);
+        return commands.ltrim(options.key, options.start, options.stop);
+    };
+
+    exports.rpop = function(options) {
+        options = this.parse(options);
+        return commands.rpop(options.key, options.count);
+    };
+
+    exports.rpush = function(options) {
+        options = this.parse(options);
+        return commands.rpush(options.key, options.element);
+    };
+
+    exports.rpushx = function(options) {
+        options = this.parse(options);
+        return commands.rpushx(options.key, options.element);
+    };
+
+    exports.copy = function(options) {
+        options = this.parse(options);
+        return commands.copy(options.source, options.destination);
+    };
+
+    exports.del = function(options) {
+        options = this.parse(options);
+        return commands.del(options.key);
+    };
+
+    exports.expire = function(options) {
+        options = this.parse(options);
+        return commands.expire(options.key, options.seconds);
+    };
+
+    exports.expireat = function(options) {
+        options = this.parse(options);
+        return commands.expireat(options.key, options.timestamp);
+    };
+
+    exports.keys = function(options) {
+        options = this.parse(options);
+        return commands.keys(options.pattern);
+    };
+
+    exports.persist = function(options) {
+        options = this.parse(options);
+        return commands.persist(options.key);
+    };
+
+    exports.pexpire = function(options) {
+        options = this.parse(options);
+        return commands.pexpire(options.key, options.milliseconds);
+    };
+
+    exports.pexpireat = function(options) {
+        options = this.parse(options);
+        return commands.pexpireat(options.key, options.mstimestamp)
+    };
+
+    exports.pttl = function(options) {
+        options = this.parse(options);
+        return commands.pttl(options.key);
+    };
+
+    exports.randomkey = function(options) {
+        options = this.parse(options);
+        return commands.randomkey();
+    };
+
+    exports.rename = function(options) {
+        options = this.parse(options);
+        return commands.rename(options.key, options.newkey);
+    };
+
+    exports.renamenx = function(options) {
+        options = this.parse(options);
+        return commands.renamenx(options.key, options.newkey);
+    };
+
+    exports.touch = function(options) {
+        options = this.parse(options);
+        return commands.touch(options.key);
+    };
+
+    exports.ttl = function(options) {
+        options = this.parse(options);
+        return commands.ttl(options.key);
+    };
+
+    exports.type = function(options) {
+        options = this.parse(options);
+        return commands.type(options.key);
+    };
+
+    exports.unlink = function(options) {
+        options = this.parse(options);
+        return commands.unlink(options.key);
+    };
+}

lib/modules/s3.js

@@ -0,0 +1,174 @@
+const fs = require('fs-extra');
+const mime = require('mime-types');
+const { join, basename, dirname } = require('path');
+const { toSystemPath } = require('../core/path');
+
+module.exports = {
+
+    provider: function (options, name) {
+        this.setS3Provider(name, options);
+    },
+
+    createBucket: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.createBucket: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.createBucket: bucket is required.');
+        const ACL = this.parseOptional(options.acl, 'string', undefined);
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        return s3.createBucket({ Bucket, ACL });
+    },
+
+    listBuckets: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.listBuckets: provider is required.');
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        return s3.listBuckets({});
+    },
+
+    deleteBucket: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.deleteBucket: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.deleteBucket: bucket is required.');
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        return s3.deleteBucket({ Bucket });
+    },
+
+    listFiles: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.listFiles: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.listFiles: bucket is required.');
+        const MaxKeys = this.parseOptional(options.maxKeys, 'number', undefined);
+        const Prefix = this.parseOptional(options.prefix, 'string', undefined);
+        const ContinuationToken = this.parseOptional(options.continuationToken, 'string', undefined);
+        const StartAfter = this.parseOptional(options.startAfter, 'string', undefined);
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        return s3.listObjectsV2({ Bucket, MaxKeys, Prefix, ContinuationToken, StartAfter });
+    },
+
+    putFile: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.uploadFile: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.uploadFile: bucket is required.');
+        const path = toSystemPath(this.parseRequired(options.path, 'string', 's3.uploadFile: path is required.'));
+        const Key = this.parseRequired(options.key, 'string', 's3.uploadFile: key is required.');
+        const ContentType = this.parseOptional(options.contentType, 'string', mime.lookup(Key) || 'application/octet-stream');
+        const ContentDisposition = this.parseOptional(options.contentDisposition, 'string', undefined);
+        const ACL = this.parseOptional(options.acl, 'string', undefined);
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        let Body = fs.createReadStream(path);
+
+        const result = await s3.putObject({ Bucket, ACL, Key, ContentType, ContentDisposition, Body });
+
+        try {
+            const endpoint = await s3.config.endpoint();
+            result.Location = `https://${Bucket}.${endpoint.hostname}/${Key}`;
+        } catch (e) {}
+
+
+        return result;
+    },
+
+    getFile: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.getFile: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.getFile: bucket is required.');
+        const Key = this.parseRequired(options.key, 'string', 's3.getFile: key is required.');
+        const path = toSystemPath(this.parseRequired(options.path, 'string', 's3.getFile: path is required.'));
+        const stripKeyPath = this.parseOptional(options.stripKeyPath, 'boolean', false);
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        const file = Key;
+        if (stripKeyPath) file = basename(file);
+        const destination = join(path, file);
+
+        await fs.ensureDir(dirname(destination));
+
+        const writer = fs.createWriteStream(destination);
+
+        const { Body } = await s3.getObject({ Bucket, Key });
+
+        Body.pipe(writer);
+
+        return new Promise((resolve, reject) => {
+            writer.on('finish', resolve);
+            writer.on('error', reject);
+        });
+    },
+
+    deleteFile: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.deleteFile: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.deleteFile: bucket is required.');
+        const Key = this.parseRequired(options.key, 'string', 's3.deleteFile: key is required.');
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        return s3.deleteObject({ Bucket, Key });
+    },
+
+    downloadFile: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.downloadFile: profider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.downloadFile: bucket is required.');
+        const Key = this.parseRequired(options.key, 'string', 's3.downloadFile: key is required.');
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        const data = await s3.headObject({ Bucket, Key });
+
+        this.res.set('Content-Length', data.ContentLength);
+        this.res.attachment(basename(Key));
+
+        const { Body } = await s3.getObject({ Bucket, Key });
+
+        Body.pipe(this.res);
+
+        this.noOutput = true;
+    },
+
+    signDownloadUrl: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.signDownloadUrl: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.signDownloadUrl: bucket is required.');
+        const Key = this.parseRequired(options.key, 'string', 's3.signDownloadUrl: key is required.');
+        const expiresIn = this.parseOptional(options.expires, 'number', 300);
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
+        const { GetObjectCommand } = require('@aws-sdk/client-s3');
+        const command = new GetObjectCommand({ Bucket, Key });
+
+        return getSignedUrl(s3, command, { expiresIn });
+    },
+
+    signUploadUrl: async function (options) {
+        const provider = this.parseRequired(options.provider, 'string', 's3.signUploadUrl: provider is required.');
+        const Bucket = this.parseRequired(options.bucket, 'string', 's3.signUploadUrl: bucket is required.');
+        const Key = this.parseRequired(options.key, 'string', 's3.signUploadUrl: key is required.');
+        const ContentType = this.parseOptional(options.contentType, 'string', mime.lookup(Key) || 'application/octet-stream');
+        const expiresIn = this.parseOptional(options.expires, 'number', 300);
+        const ACL = this.parseOptional(options.acl, 'string', undefined);
+        const s3 = this.getS3Provider(provider);
+
+        if (!s3) throw new Error(`S3 provider "${provider}" doesn't exist.`);
+
+        const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
+        const { PutObjectCommand } = require('@aws-sdk/client-s3');
+        const command = new PutObjectCommand({ Bucket, Key, ContentType, ACL });
+
+        return getSignedUrl(s3, command, { expiresIn });
+    }
+
+};

lib/modules/sockets.js

@@ -0,0 +1,172 @@
+/**
+ * emits an event to all clients filtered by the given options
+ * @param {string} namespace - only to the specified namespace
+ * @param {string} room - emit only to the specified room (can be combined with namespace)
+ * @param {string} eventName - the name of the event
+ * @param {object} params - any parameters/data to send with the event
+ */
+exports.emit = function(options) {
+    if (this.io) {
+        options = this.parse(options);
+
+        if (options.namespace) {
+            if (options.room) {
+                this.io.of(options.namespace).to(options.room).emit(options.eventName, options.params);
+            } else {
+                this.io.of(options.namespace).emit(options.eventName, options.params);
+            }
+        } else {
+            if (options.room) {
+                this.io.in(options.room).emit(options.eventName, options.params);
+            } else {
+                this.io.emit(options.eventName, options.params);
+            }
+        }
+    }
+};
+
+/**
+ * emit an event to all clients except the sender
+ * @param {string} room - broadcast in a specific room
+ * @param {string} eventName - the name of the event
+ * @param {object} params - the parameters/data to send with the event
+ */
+exports.broadcast = function(options) {
+    if (this.socket) {
+        options = this.parse(options);
+
+        if (options.room) {
+            this.socket.to(options.room).emit(options.eventName, options.params);
+        } else {
+            this.socket.broadcast.emit(options.eventName, options.params);
+        }
+    }
+};
+
+/**
+ * emit an event to client and wait for an answer
+ * @param {string} room - broadcast in a specific room
+ * @param {string} eventName - the name of the event
+ * @param {object} params - the parameters/data to send with the event
+ */
+exports.request = function(options) {
+    if (this.socket) {
+        options = this.parse(options);
+
+        return new Promise((resolve) => {
+            this.socket.emit(options.eventName, options.params, resolve);
+        });
+    }
+
+    return null;
+};
+
+/**
+ * send a private meessage to a socket
+ * @param {string} socketId - the socket id to send the message to
+ * @param {string} eventName - the name of the event
+ * @param {object} params - the parameters/data to send with the event
+ */
+exports.message = function(options) {
+    if (this.io) {
+        options = this.parse(options);
+
+        this.io.to(options.socketId).emit(options.eventName, options.params);
+    }
+};
+
+/**
+ * special serverconnect refresh broadcast
+ * @param {string} action - the action that require refresh
+ */
+exports.refresh = async function(options) {
+    if (this.io) {
+        options = this.parse(options);
+
+        // Do we have a global redis client?
+        if (global.redisClient) {
+            try { // ignore any errors here
+                let wsKeys = await global.redisClient.keys('ws:' + options.action + ':*');
+                if (wsKeys.length) await global.redisClient.del(wsKeys);
+                let scKeys = await global.redisClient.keys('erc:' + '/api/' + options.action + '*');
+                if (scKeys.length) await global.redisClient.del(scKeys);
+            } catch (e) {
+                console.error(e);
+            }
+        }
+
+        this.io.of('/api').emit(options.action, options.params);
+    }
+};
+
+/**
+ * let current client join a room
+ * @param {string} room - the room to join
+ */
+exports.join = function(options) {
+    if (this.socket) {
+        this.socket.join(this.parse(options.room));
+    }
+};
+
+/**
+ * let current client leave a room
+ * @param {string} room - the room to leave
+ */
+exports.leave = function(options) {
+    if (this.socket) {
+        this.socket.leave(this.parse(options.room));
+    }
+};
+
+/**
+ * get the socket id of the current client
+ */
+exports.identify = function(options) {
+    return this.socket ? this.socket.id : null;
+};
+
+/**
+ * get the rooms the current client has joined
+ */
+exports.rooms = function(options) {
+    return this.socket ? Array.from(this.socket.rooms) : [];
+};
+
+/**
+ * get all the rooms
+ * @param {string} namespace - the namespace
+ */
+exports.allRooms = async function(options) {
+    if (this.io) {
+        let adapter = io.of(options.namespace || '/').adapter;
+
+        if (typeof adapter.allRooms == 'function') {
+            return Array.from(await adapter.allRooms());
+        } else if (adapter.rooms) {
+            return Array.from(adapter.rooms.keys());
+        }
+    }
+
+    return [];
+};
+
+/**
+ * get all the connected sockets
+ * @param {string} namespace - the namespace
+ * @param {string} room - return only clients in a specific room
+ */
+exports.allSockets = async function(options) {
+    if (this.io) {
+        options = this.parse(options);
+
+        if (options.room) {
+            return Array.from(await this.io.of(options.namespace || '/').in(options.room).allSockets());
+        } else {
+            return Array.from(await this.io.of(options.namespace || '/').allSockets());
+        }
+    }
+
+    return [];
+};
+

lib/modules/upload.js

@@ -0,0 +1,118 @@
+const fs = require('fs-extra');
+const debug = require('debug')('server-connect:upload');
+const { join, basename, extname } = require('path');
+const { toAppPath, toSystemPath, toSiteUrl, parseTemplate, getUniqFile } = require('../core/path');
+const diacritics = require('../core/diacritics');
+
+module.exports = {
+
+    upload: async function(options) {
+        let self = this;
+        let fields = this.parse(options.fields || this.parse('{{$_POST}}'));
+        let path = this.parseOptional(options.path, 'string', '/uploads');
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let createPath = this.parseOptional(options.createPath, 'boolean', true);
+        let throwErrors = this.parseOptional(options.throwErrors, 'boolean', false);
+        let template = typeof options.template == 'string' ? options.template : false; //this.parseOptional(options.template, 'string', '');
+        let replaceSpace = this.parseOptional(options.replaceSpace, 'boolean', false);
+        let asciiOnly = this.parseOptional(options.asciiOnly, 'boolean', false);
+        let replaceDiacritics = this.parseOptional(options.replaceDiacritics, 'boolean', false);
+
+        if (throwErrors) {
+            for (let field in this.req.files) {
+                if (Array.isArray(this.req.files[field])) {
+                    for (let file of this.req.files[field]) {
+                        if (file.truncated) {
+                            throw new Error('Some files failed to upload.');
+                        }
+                    }
+                } else {
+                    let file = this.req.files[field];
+                    if (file.truncated) {
+                        throw new Error('Some files failed to upload.');
+                    }
+                }
+            }
+        }
+
+        path = toSystemPath(path);
+
+        if (!fs.existsSync(path)) {
+            if (createPath) {
+                await fs.ensureDir(path);
+            } else {
+                throw new Error(`Upload path doesn't exist.`);
+            }
+        }
+
+        let files = this.req.files;
+        let uploaded = [];
+        
+        if (files) {
+            await processFields(fields);
+        }
+
+        return typeof fields == 'string' ? (uploaded.length ? uploaded[0] : null) : uploaded;
+
+        async function processFields(fields) {
+            debug('Process fields: %O', fields);
+        
+            if (typeof fields == 'object') {
+                for (let i in fields) {
+                    await processFields(fields[i]);
+                }
+            } else if (typeof fields == 'string' && files[fields]) {
+                let processing = files[fields];
+
+                if (!Array.isArray(processing)) processing = [processing];
+
+                for (let file of processing) {
+                    debug('Processing file: %O', file);
+                    
+                    if (!file.processed) {
+                        let name = file.name.replace(/[\x00-\x1f\x7f!%&#@$*()?:,;"'<>^`|+={}\[\]\\\/]/g, '');
+                
+                        if (replaceSpace) name = name.replace(/\s+/g, '_');
+                        if (replaceDiacritics) name = diacritics.replace(name);
+                        if (asciiOnly) name = name.replace(/[^\x00-\x7e]/g, '');
+
+                        let filepath = join(path, name);
+
+                        if (template) {
+                            let _template = self.parse(template, self.scope.create({
+                                file: basename(filepath),
+                                name: basename(filepath, extname(filepath)),
+                                ext: extname(filepath)
+                            }));
+                            filepath = parseTemplate(filepath, _template);
+                        }
+
+                        if (fs.existsSync(filepath)) {
+                            if (overwrite) {
+                                await fs.unlink(filepath);
+                            } else {
+                                filepath = getUniqFile(filepath);
+                            }
+                        }
+
+                        await file.mv(filepath);
+
+                        uploaded.push({
+                            name: basename(filepath),
+                            path: toAppPath(filepath),
+                            url: toSiteUrl(filepath),
+                            type: file.mimetype,
+                            size: file.size
+                        });
+                        
+                        file.processed = true;
+                    }
+                }
+            }
+        
+            return uploaded;
+        }
+        
+    }
+
+};

lib/modules/validator.js

@@ -0,0 +1,18 @@
+const validator = require('../validator');
+
+module.exports = {
+
+    validate: function(options) {
+        const data = this.parse(options.data);
+        const noError = this.parseOptional(options.noError, 'boolean', false);
+
+        return validator.validateData(this, data, noError);
+    },
+
+    error: function(options, name) {
+        const message = this.parseRequired(options.message, 'string', 'validator.error: message is required.');
+        const error = { [options.fieldName ? 'form' : 'data']: {[options.fieldName || name]: message  }};
+        this.res.status(400).json(error);
+    },
+
+};

lib/modules/zip.js

@@ -0,0 +1,114 @@
+const fs = require('fs-extra');
+const debug = require('debug')('server-connect:zip');
+const { basename } = require('path');
+const { toSystemPath, toAppPath, getFilesArray, getUniqFile } = require('../core/path');
+const openZip = (zipfile) => require('unzipper').Open.file(zipfile);
+
+const Zip = function(zipfile, options) {
+    this.output = fs.createWriteStream(zipfile);
+    this.archive = require('archiver')('zip', options);
+    this.archive.on('warning', (err) => {
+        if (err.code == 'ENOENT') {
+            debug('error: %O', err);
+        } else {
+            throw err;
+        }
+    });
+    this.archive.on('error', (err) => {
+        throw err;
+    });
+    this.archive.pipe(this.output);
+};
+
+Zip.prototype.addFile = function(file) {
+    this.archive.file(file, { name: basename(file) });
+};
+
+Zip.prototype.addDir = function(dir, recursive) {
+    if (recursive) {
+        this.archive.directory(dir, false);
+    } else {
+        this.archive.glob('*', { cwd: dir });
+    }
+};
+
+Zip.prototype.save = function() {
+    return new Promise((resolve, reject) => {
+        this.output.on('close', resolve);
+        this.archive.on('error', reject);
+        this.archive.finalize();
+    });
+};
+
+module.exports = {
+
+    zip: async function(options) {
+        let zipfile = toSystemPath(this.parseRequired(options.zipfile, 'string', 'zip.zip: zipfile is required.'));
+        let files = getFilesArray(this.parseRequired(options.files, 'object', 'zip.zip: files is requires.'));
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let comment = this.parseOptional(options.comment, 'string', '');
+        
+        if (!overwrite) zipfile = getUniqFile(zipfile);
+
+        const zip = new Zip(zipfile, { comment });
+        
+        for (let file of files) {
+            zip.addFile(file);
+        }
+
+        await zip.save();
+
+        return toAppPath(zipfile);
+    },
+
+    zipdir: async function(options) {
+        let zipfile = toSystemPath(this.parseRequired(options.zipfile, 'string', 'zip.zipdir: zipfile is required.'));
+        let path = toSystemPath(this.parseRequired(options.path, 'string', 'zip.zipdir: path is required.'));
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
+        let recursive = this.parseOptional(options.recursive, 'boolean', false);
+        let comment = this.parseOptional(options.comment, 'string', '');
+        
+        if (!overwrite) zipfile = getUniqFile(zipfile);
+
+        const zip = new Zip(zipfile, { comment });
+        
+        zip.addDir(path, recursive);
+
+        await zip.save();
+
+        return toAppPath(zipfile);
+    },
+
+    unzip: async function(options) {
+        let zipfile = toSystemPath(this.parseRequired(options.zipfile, 'string', 'zip.unzip: zipfile is required.'));
+        let dest = toSystemPath(this.parseRequired(options.destination, 'string', 'zip.unzip: destination is required.'));
+        let overwrite = this.parseOptional(options.overwrite, 'boolean', true);
+
+        // TODO: overwrite option
+        await openZip(zipfile).then(d => d.extract({ path: dest, concurrency: 4 }));
+
+        return true;
+    },
+
+    dir: async function(options) {
+        let zipfile = toSystemPath(this.parseRequired(options.zipfile, 'string', 'zip.dir: zipfile is required.'));
+        
+        return openZip(zipfile).then(d => {
+            return d.files.map(file => ({
+                type: file.type == 'Directory' ? 'dir': 'file',
+                path: file.path,
+                size: file.uncompressedSize,
+                compressedSize: file.compressedSize,
+                compressionMethod: file.compressionMethod == 8 ? 'Deflate' : 'None',
+                lastModified: file.lastModifiedDateTime
+            }));
+        });
+    },
+
+    comment: async function(options) {
+        let zipfile = toSystemPath(this.parseRequired(options.zipfile, 'string', 'zip.comment: zipfile is required.'));
+        
+        return openZip(zipfile).then(d => d.comment);
+    },
+
+};

lib/oauth/index.js

@@ -0,0 +1,170 @@
+const { http, https } = require('follow-redirects');
+const querystring = require('querystring');
+const services = require('./services');
+const crypto = require('crypto');
+
+class OAuth2 {
+
+    constructor(app, opts, name) {
+        this.app = app;
+        this.name = name;
+        this.opts = opts;
+
+        if (opts.service) {
+            Object.assign(this.opts, services[opts.service]);
+        }
+
+        this.access_token = opts.access_token || app.getSession(`${name}_access_token`);
+        this.refresh_token = opts.refresh_token || app.getSession(`${name}_refresh_token`);
+
+        if (this.access_token) {
+            let expires = app.getSession(`${name}_expires`);
+
+            if (expires && expires < expires_in(-10)) {
+                this.access_token = null;
+
+                app.removeSession(`${name}_access_token`);
+                app.removeSession(`${name}_expires`);
+
+                if (this.refresh_token) {
+                    this.refreshToken(this.refresh_token);
+                }
+            }
+        }
+    }
+
+    async init() {
+        if (this.opts.jwt_bearer && !this.access_token) {
+            const assertion = this.app.getJSONWebToken(this.opts.jwt_bearer);
+            let response = await this.grant('urn:ietf:params:oauth:grant-type:jwt-bearer', { assertion });
+            this.access_token = response.access_token;
+        }
+
+        if (this.opts.client_credentials && !this.access_token) {
+            let response = await this.grant('client_credentials');
+            this.access_token = response.access_token;
+        }
+    }
+
+    async authorize(scopes = [], params = {}) {
+        let query = this.app.req.query;
+
+        if (query.state && query.state == this.app.getSession(`${this.name}_state`)) {
+            this.app.removeSession(`${this.name}_state`);
+
+            if (query.error) {
+                throw new Error(query.error_message || query.error);
+            }
+
+            if (query.code) {
+                let params = {
+                    redirect_uri: redirect_uri(this.app.req),
+                    code: query.code
+                };
+
+                if (this.app.getSession(`${this.name}_code_verifier`)) {
+                    params.code_verifier = this.app.getSession(`${this.name}_code_verifier`);
+                }
+
+                return this.grant('authorization_code', params);
+            }
+        }
+
+        if (this.opts.pkce || params.code_verifier) {
+            const code_verifier = params.code_verifier || crypto.randomBytes(40).toString('hex');
+            this.app.setSession(`${this.name}_code_verifier`, code_verifier);
+            params.code_challenge_method = 'S256';
+            params.code_challenge = crypto.createHash('sha256').update(code_verifier).digest('base64').replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_');
+        }
+
+        params = Object.assign({}, {
+            response_type: 'code',
+            client_id: this.opts.client_id,
+            scope: scopes.join(this.opts.scope_separator),
+            redirect_uri: redirect_uri(this.app.req),
+            state: generate_state()
+        }, this.opts.params, params);
+
+        this.app.setSession(`${this.name}_state`, params.state);
+
+        this.app.res.redirect(build_url(this.opts.auth_endpoint, params));
+    }
+
+    async refreshToken(refresh_token) {
+        return this.grant('refresh_token', { refresh_token });
+    }
+
+    async grant(type, params) {
+        const endpoint = new URL(this.opts.token_endpoint);
+
+        return new Promise((resolve, reject) => {
+            const req = (endpoint.protocol == 'https:' ? https : http).request(endpoint, {
+                method: 'POST'
+            }, res => {
+                let body = '';
+                res.setEncoding('utf8');
+                res.on('data', chunk => body += chunk);
+                res.on('end', () => {
+                    if (res.statusCode >= 400) {
+                        return reject(new Error(`Http status code ${res.statusCode}. ${body}`));
+                    }
+
+                    try {
+                        body = JSON.parse(body);
+                    } catch (e) { }
+
+                    if (!body.access_token) {
+                        return reject(new Error(`Http response has no access_token. ${JSON.stringify(body)}`))
+                    }
+
+                    this.app.setSession(`${this.name}_access_token`, body.access_token);
+                    this.access_token = body.access_token;
+
+                    if (body.expires_in) {
+                        this.app.setSession(`${this.name}_expires`, expires_in(body.expires_in));
+                    }
+
+                    if (body.refresh_token) {
+                        this.app.setSession(`${this.name}_refresh_token`, body.refresh_token);
+                        this.refresh_token = body.refresh_token;
+                    }
+
+                    resolve(body);
+                });
+            });
+
+            const body = querystring.stringify(Object.assign({
+                grant_type: type,
+                client_id: this.opts.client_id,
+                client_secret: this.opts.client_secret
+            }, params));
+
+            req.on('error', reject);
+            req.setHeader('Content-Type', 'application/x-www-form-urlencoded');
+            req.setHeader('Content-Length', body.length); // required by azure endpoint
+            req.write(body);
+            req.end();
+        });
+    }
+
+}
+
+function build_url(url, params) {
+    return url + (url.indexOf('?') != -1 ? '&' : '?') + querystring.stringify(params);
+}
+
+function redirect_uri(req) {
+    let hasProxy = !!req.get('x-forwarded-host');
+    return `${req.protocol}://${hasProxy ? req.hostname : req.get('host')}${req.path}`;
+}
+
+function expires_in(s) {
+    return ~~(Date.now() / 1000) + s;
+}
+
+function generate_state() {
+    const { v4: uuidv4 } = require('uuid');
+    return uuidv4();
+}
+
+module.exports = OAuth2;

lib/oauth/services.js

@@ -0,0 +1,98 @@
+module.exports = {
+
+    'google': {
+        auth_endpoint: 'https://accounts.google.com/o/oauth2/v2/auth',
+        token_endpoint: 'https://www.googleapis.com/oauth2/v4/token',
+        params: { access_type: 'offline' }
+    },
+
+    'facebook': {
+        auth_endpoint: 'https://www.facebook.com/v3.2/dialog/oauth',
+        token_endpoint: 'https://graph.facebook.com/v3.2/oauth/access_token'
+    },
+
+    'linkedin': {
+        auth_endpoint: 'https://www.linkedin.com/oauth/v2/authorization',
+        token_endpoint: 'https://www.linkedin.com/oauth/v2/accessToken'
+    },
+
+    'github': {
+        auth_endpoint: 'https://github.com/login/oauth/authorize',
+        token_endpoint: 'https://github.com/login/oauth/access_token'
+    },
+
+    'instagram': {
+        auth_endpoint: 'https://api.instagram.com/oauth/authorize/',
+        token_endpoint: 'https://api.instagram.com/oauth/access_token'
+    },
+
+    'amazon': {
+        auth_endpoint: 'https://www.amazon.com/ap/oa',
+        token_endpoint: 'https://api.amazon.com/auth/o2/token'
+    },
+
+    'dropbox': {
+        auth_endpoint: 'https://www.dropbox.com/oauth2/authorize',
+        token_endpoint: 'https://api.dropbox.com/oauth2/token',
+        scope_separator: ','
+    },
+
+    'foursquare': {
+        auth_endpoint: 'https://foursquare.com/oauth2/authenticate',
+        token_endpoint: 'https://foursquare.com/oauth2/access_token'
+    },
+
+    'imgur': {
+        auth_endpoint: 'https://api.imgur.com/oauth2/authorize',
+        token_endpoint: 'https://api.imgur.com/oauth2/token'
+    },
+
+    'wordpress': {
+        auth_endpoint: 'https://public-api.wordpress.com/oauth2/authorize',
+        token_endpoint: 'https://public-api.wordpress.com/oauth2/token'
+    },
+
+    'spotify': {
+        auth_endpoint: 'https://accounts.spotify.com/authorize',
+        token_endpoint: 'https://accounts.spotify.com/api/token'
+    },
+
+    'slack': {
+        auth_endpoint: 'https://slack.com/oauth/authorize',
+        token_endpoint: 'https://slack.com/api/oauth.access'
+    },
+
+    'reddit': {
+        auth_endpoint: 'https://ssl.reddit.com/api/v1/authorize',
+        token_endpoint: 'https://ssl.reddit.com/api/v1/access_token',
+        scope_separator: ','
+    },
+
+    'twitch': {
+        auth_endpoint: 'https://api.twitch.tv/kraken/oauth2/authorize',
+        token_endpoint: 'https://api.twitch.tv/kraken/oauth2/token'
+    },
+
+    'paypal': {
+        auth_endpoint: 'https://identity.x.com/xidentity/resources/authorize',
+        token_endpoint: 'https://identity.x.com/xidentity/oauthtokenservice'
+    },
+
+    'pinterest': {
+        auth_endpoint: 'https://api.pinterest.com/oauth/',
+        token_endpoint: 'https://api.pinterest.com/v1/oauth/token',
+        scope_separator: ','
+    },
+
+    'stripe': {
+        auth_endpoint: 'https://connect.stripe.com/oauth/authorize',
+        token_endpoint: 'https://connect.stripe.com/oauth/token',
+        scope_separator: ','
+    },
+
+    'coinbase': {
+        auth_endpoint: 'https://www.coinbase.com/oauth/authorize',
+        token_endpoint: 'https://www.coinbase.com/oauth/token'
+    }
+
+};

lib/server.js

@@ -0,0 +1,110 @@
+if (process.env.NODE_ENV !== 'production') {
+    require('dotenv').config();
+}
+
+process.on('uncaughtException', (e) => {
+    // prevent errors from killing the server and just log them
+    console.error(e);
+});
+
+const config = require('./setup/config');
+const debug = require('debug')('server-connect:server');
+const secure = require('./setup/secure');
+const routes = require('./setup/routes');
+const sockets = require('./setup/sockets');
+const upload = require('./setup/upload');
+const cron = require('./setup/cron');
+const http = require('http');
+const express = require('express');
+const endmw = require('express-end');
+const cookieParser = require('cookie-parser');
+const session = require('./setup/session'); //require('express-session')(Object.assign({ secret: config.secret }, config.session));
+const cors = require('cors');
+const app = express();
+
+app.set('trust proxy', true);
+app.set('view engine', 'ejs');
+app.set('view options', { root: 'views', async: true });
+
+app.disable('x-powered-by')
+
+if (config.compression) {
+    const compression = require('compression');
+    app.use(compression());
+}
+
+if (config.abortOnDisconnect) {
+    app.use((req, res, next) => {
+        req.isDisconnected = false;
+        req.on('close', () => {
+            req.isDisconnected = true;
+        });
+
+        next();
+    });
+}
+
+app.use(cors(config.cors));
+app.use(express.static('public', config.static));
+app.use(express.urlencoded({ extended: true }));
+app.use(express.json({
+    verify: (req, res, buf) => {
+        req.rawBody = buf.toString()
+    }
+}));
+app.use(cookieParser(config.secret));
+app.use(session);
+app.use(endmw);
+
+upload(app);
+secure(app);
+routes(app);
+
+const server = http.createServer(app);
+const io = sockets(server, session);
+
+// Make sockets global available
+global.io = io;
+
+module.exports = {
+    server, app, io,
+    start: function(port) {
+        // We add the 404 and 500 routes as last
+        app.use((req, res) => {
+            // if user has a custom 404 page, redirect to it
+            if (req.accepts('html') && req.url != '/404' && app.get('has404')) {
+                //res.redirect(303, '/404');
+                req.url = '/404';
+                app.handle(req, res);
+            } else {
+                res.status(404).json({
+                    status: '404',
+                    message: `${req.url} not found.`
+                });
+            }
+        });
+        
+        app.use((err, req, res, next) => {
+            debug(`Got error? %O`, err);
+            // if user has a custom 500 page, redirect to it
+            if (req.accepts('html') && req.url != '/500' && app.get('has500')) {
+                //res.redirect(303, '/500');
+                req.url = '/500';
+                app.handle(req, res);
+            } else {
+                res.status(500).json({
+                    status: '500',
+                    code: config.debug ? err.code : undefined,
+                    message: config.debug ? err.message || err : 'A server error occured, to see the error enable the DEBUG flag.',
+                    stack: config.debug ? err.stack : undefined,
+                });
+            }
+        });
+        
+        cron.start();
+
+        server.listen(port || config.port, () => {
+            console.log(`App listening at http://localhost:${config.port}`);
+        });
+    }
+};

lib/setup/config.js

@@ -0,0 +1,105 @@
+const package = require('../../package.json');
+const fs = require('fs-extra');
+const debug = require('debug')('server-connect:setup:config');
+const { toSystemPath } = require('../core/path');
+const { mergeDeep } = require('../core/util');
+const Parser = require('../core/parser');
+const Scope = require('../core/scope');
+
+const config = {
+    port: process.env.PORT || 3000,
+    debug: false,
+    secret: 'Need to be set',
+    tmpFolder: '/tmp',
+    abortOnDisconnect: false,
+    createApiRoutes: true,
+    compression: true,
+    redis: false,
+    cron: true,
+    static: {
+        index: false,
+    },
+    session: {
+        name: package.name + '.sid',
+        resave: false,
+        saveUninitialized: false,
+        store: { $type: 'memory', ttl: 86400 },
+    },
+    cors: { // see https://github.com/expressjs/cors
+        origin: false,
+        methods: 'GET,POST',
+        credentials: true,
+    },
+    csrf: {
+        enabled: false,
+        exclude: 'GET,HEAD,OPTIONS',
+    },
+    rateLimit: {
+        enabled: false,
+        duration: 60, // duration of 60 second (1 minute)
+        points: 100, // limit to 100 requests per minute
+        private: {
+            provider: '', // security provider name
+            duration: 60, // duration of 60 second (1 minute)
+            points: 1000, // limit to 1000 requests per minute
+        },
+    },
+    globals: {},
+    rateLimiter: {},
+    mail: {},
+    auth: {},
+    oauth: {},
+    db: {},
+    s3: {},
+    jwt: {},
+    stripe: {},
+    env: {},
+};
+
+if (fs.existsSync('app/config/config.json')) {
+    mergeDeep(config, fs.readJSONSync('app/config/config.json'))
+}
+
+if (fs.existsSync('app/config/user_config.json')) {
+    mergeDeep(config, fs.readJSONSync('app/config/user_config.json'));
+}
+
+// folders are site relative
+config.tmpFolder = toSystemPath(config.tmpFolder);
+
+if (config.env) {
+    for (let key in config.env) {
+        if (!Object.hasOwn(process.env, key)) {
+            process.env[key] = config.env[key];
+        } else if (config.debug) {
+            debug(`"${key}" is already defined in \`process.env\` and will not be overwritten`);
+        }
+    }
+}
+
+Parser.parseValue(config, new Scope({
+    $_ENV: process.env
+}));
+
+// we change the cors config a bit, * will become true
+// and we split string on comma for multiple origins
+if (typeof config.cors?.origin == 'string') {
+    if (config.cors.origin === '*') {
+        config.cors.origin = true;
+    } else if (config.cors.origin.includes(',')) {
+        config.cors.origin = config.cors.origin.split(/\s*,\s*/);
+    }
+}
+
+if (config.debug) {
+    require('debug').enable(typeof config.debug == 'string' ? config.debug : 'server-connect:*');
+}
+
+if (config.redis) {
+    const Redis = require('ioredis');
+    global.redisClient = new Redis(config.redis === true ? 'redis://redis' : config.redis);
+}
+
+debug(config);
+
+module.exports = config;

lib/setup/cron.js

@@ -0,0 +1,60 @@
+const fs = require('fs-extra');
+const { isEmpty } = require('./util');
+const config = require('./config');
+const debug = require('debug')('server-connect:cron');
+
+exports.start = () => {
+    if (!config.cron || isEmpty('app/schedule')) return;
+
+    debug('Start schedule');
+
+    processEntries('app/schedule');
+};
+
+function processEntries(path) {
+    const schedule = require('node-schedule');
+    const entries = fs.readdirSync(path, { withFileTypes: true });
+
+    for (let entry of entries) {
+        if (entry.isFile() && entry.name.endsWith('.json')) {
+            try {
+                const job = fs.readJSONSync(`${path}/${entry.name}`);
+                const rule = job.settings.options.rule;
+
+                debug(`Adding schedule ${entry.name}`);
+
+                if (rule == '@reboot') {
+                    setImmediate(exec(job.exec));
+                } else {
+                    schedule.scheduleJob(rule, exec(job.exec))
+                }
+            } catch (e) {
+                console.error(e);
+            }
+        } else if (entry.isDirectory()) {
+            processEntries(`${path}/${entry.name}`);
+        }
+    }
+}
+
+function exec(action) {
+    return async () => {
+        const App = require('../core/app');
+        const app = new App({
+            params: {},
+            session: {},
+            cookies: {},
+            signedCookies: {},
+            query: {},
+            headers: {}
+        }, {
+            headersSent: false,
+            set() {},
+            status() { return this; },
+            send() { this.headersSent = true; },
+            json() { this.headersSent = true; },
+            redirect() { this.headersSent = true; }
+        });
+        return app.define(action, true);
+    }
+}

lib/setup/database.js

@@ -0,0 +1,36 @@
+// allow Wappler to run queries on the server
+
+const crypto = require('crypto');
+const App = require("../core/app");
+const config = require('./config');
+
+module.exports = function (app) {
+    app.post('/_db_', async (req, res) => {
+        const time = req.get('auth-time');
+        const hash = req.get('auth-hash');
+        if (!time || !hash || isNaN(time)) return res.status(400).json({ error: 'Auth headers missing' });
+
+        const diff = Math.abs(Date.now() - time) / 1000;
+        if (diff > 60) return res.status(400).json({ error: 'Auth time diff to high' });
+        if (hash !== crypto.createHmac('sha256', config.secret).update(req.rawBody).digest('hex')) return res.status(400).json({ error: 'Auth hash invalid' });
+
+        const sc = new App(req, res);
+        const db = sc.getDbConnection(req.body.name);
+
+        let results = [];
+
+        try {
+            results = await db.raw(req.body.query);
+        } catch (error) {
+            return res.json({ error: error.sqlMessage });
+        }
+
+        if (db.client.config.client == 'mysql' || db.client.config.client == 'mysql2') {
+            results = results[0];
+        } else if (db.client.config.client == 'postgres' || db.client.config.client == 'redshift') {
+            results = results.rows;
+        }
+
+        res.json({ results });
+    });
+}

lib/setup/redis.js

@@ -0,0 +1,9 @@
+const config = require('./config');
+
+if (config.redis) {
+    const Redis = require('ioredis');
+    //global.redisClient = redis.createClient(config.redis === true ? 'redis://redis' : config.redis);
+    global.redisClient = new Redis(config.redis === true ? 'redis://redis' : config.redis);
+}
+
+module.exports = global.redisClient;

lib/setup/routes.js

@@ -0,0 +1,298 @@
+const fs = require('fs-extra');
+const debug = require('debug')('server-connect:setup:routes');
+const config = require('./config');
+const { map } = require('../core/async');
+const { posix, extname } = require('path');
+const { cache, serverConnect, templateView } = require('../core/middleware');
+const database = require('./database');
+const webhooks = require('./webhooks');
+
+module.exports = async function (app) {
+    app.use((req, res, next) => {
+        req.fragment = (req.headers['accept'] || '*/*').includes('fragment');
+        next();
+    });
+
+    if (fs.existsSync('extensions/server_connect/routes')) {
+        const entries = fs.readdirSync('extensions/server_connect/routes', { withFileTypes: true });
+
+        for (let entry of entries) {
+            if (entry.isFile() && extname(entry.name) == '.js') {
+                let hook = require(`../../extensions/server_connect/routes/${entry.name}`);
+                if (hook.before) hook.before(app);
+                if (hook.handler) hook.handler(app);
+                debug(`Custom router ${entry.name} loaded`);
+            }
+        }
+    }
+
+    database(app);
+    webhooks(app);
+
+    if (config.createApiRoutes) {
+        fs.ensureDirSync('app/api');
+        createApiRoutes('app/api');
+    }
+
+    if (fs.existsSync('app/config/routes.json')) {
+        const { routes, layouts } = fs.readJSONSync('app/config/routes.json');
+
+        parseRoutes(routes, null);
+
+        function parseRoutes(routes, parent) {
+            for (let route of routes) {
+                if (!route.path) continue;
+
+                createRoute(route, parent);
+
+                if (Array.isArray(route.routes)) {
+                    parseRoutes(route.routes, route);
+                }
+            }
+        }
+
+        function createRoute({ auth, path, method, redirect, url, page, layout, exec, data, ttl, status, proxy }, parent) {
+            method = method || 'all';
+            data = data || {};
+            if (page) page = page.replace(/^\//, '');
+            if (layout) layout = layout.replace(/^\//, '');
+            if (parent && parent.path) path = parent.path + path;
+
+            if (auth) {
+                app.use(path, (req, res, next) => {
+                    if (typeof auth == 'string' && req.session && req.session[auth + 'Id']) {
+                        next();
+                    } else if (typeof auth == 'object' && auth.user) {
+                        const b64auth = (req.headers.authorization || '').split(' ')[1] || '';
+                        const [user, password] = Buffer.from(b64auth, 'base64').toString().split(':');
+                        if (user && password && user === auth.user && password === auth.password) {
+                            next();
+                        } else {
+                            res.set('WWW-Authenticate', 'Basic realm="401"');
+                            res.status(401).json({ error: 'Unauthorized' });
+                        }
+                    } else {
+                        res.status(401).json({ error: 'Unauthorized' });
+                    }
+                });
+            }
+
+            if (proxy) {
+                const httpProxy = require('http-proxy');
+                const proxyServer = httpProxy.createProxyServer(proxy);
+                app.use(path, (req, res) => {
+                    proxyServer.web(req, res);
+                });
+            } else if (redirect) {
+                app.get(path, (req, res) => res.redirect(status == 302 ? 302 : 301, redirect));
+            } else if (url) {
+                app[method](path, (req, res, next) => {
+                    next(parent && !req.fragment ? 'route' : null);
+                }, (req, res) => {
+                    res.sendFile(url, { root: 'public' })
+                });
+
+                if (parent) {
+                    createRoute({
+                        path,
+                        method: parent.method,
+                        redirect: parent.redirect,
+                        url: parent.url,
+                        page: parent.page,
+                        layout: parent.layout,
+                        exec: parent.exec,
+                        data: parent.data
+                    });
+                }
+            } else if (page) {
+                if (path == '/404') {
+                    app.set('has404', true);
+                }
+
+                if (path == '/500') {
+                    app.set('has500', true);
+                }
+
+                if (exec) {
+                    if (fs.existsSync(`app/${exec}.json`)) {
+                        let json = fs.readJSONSync(`app/${exec}.json`);
+
+                        if (json.exec && json.exec.steps) {
+                            json = json.exec.steps;
+                        } else if (json.steps) {
+                            json = json.steps;
+                        }
+                        
+                        if (!Array.isArray(json)) {
+                            json = [json];
+                        }
+
+
+                        if (layout && layouts && layouts[layout]) {
+                            if (layouts[layout].data) {
+                                data = Object.assign({}, layouts[layout].data, data);
+                            }
+
+                            if (layouts[layout].exec) {
+                                if (fs.existsSync(`app/${layouts[layout].exec}.json`)) {
+                                    let _json = fs.readJSONSync(`app/${layouts[layout].exec}.json`);
+
+                                    if (_json.exec && _json.exec.steps) {
+                                        _json = _json.exec.steps;
+                                    } else if (_json.steps) {
+                                        _json = _json.steps;
+                                    }
+                                    
+                                    if (!Array.isArray(_json)) {
+                                        _json = [_json];
+                                    }
+
+                                    json = _json.concat(json);
+                                } else {
+                                    debug(`Route ${path} skipped, "app/${exec}.json" not found`);
+                                    return;
+                                }
+                            }
+                        }
+
+                        app[method](path, (req, res, next) => {
+                            next(parent && !req.fragment ? 'route' : null);
+                        }, cache({ttl}), templateView(layout, page, data, json));
+                    } else {
+                        debug(`Route ${path} skipped, "app/${exec}.json" not found`);
+                        return;
+                    }
+                } else {
+                    let json = [];
+
+                    if (layout && layouts && layouts[layout]) {
+                        if (layouts[layout].data) {
+                            data = Object.assign({}, layouts[layout].data, data);
+                        }
+
+                        if (layouts[layout].exec) {
+                            if (fs.existsSync(`app/${layouts[layout].exec}.json`)) {
+                                let _json = fs.readJSONSync(`app/${layouts[layout].exec}.json`);
+
+                                if (_json.exec && _json.exec.steps) {
+                                    _json = _json.exec.steps;
+                                } else if (_json.steps) {
+                                    _json = _json.steps;
+                                }
+                                
+                                if (!Array.isArray(_json)) {
+                                    _json = [_json];
+                                }
+
+                                json = _json.concat(json);
+                            } else {
+                                debug(`Route ${path} skipped, "app/${exec}.json" not found`);
+                                return;
+                            }
+                        }
+                    }
+
+                    app[method](path, (req, res, next) => {
+                        next(parent && !req.fragment ? 'route' : null);
+                    }, cache({ttl}), templateView(layout, page, data, json));
+                }
+
+                if (parent) {
+                    createRoute({
+                        path,
+                        method: parent.method,
+                        redirect: parent.redirect,
+                        url: parent.url,
+                        page: parent.page,
+                        layout: parent.layout,
+                        exec: parent.exec,
+                        data: parent.data
+                    });
+                }
+            } else if (exec) {
+                if (fs.existsSync(`app/${exec}.json`)) {
+                    let json = fs.readJSONSync(`app/${exec}.json`);
+
+                    app[method](path, cache({ttl}), serverConnect(json));
+
+                    return;
+                }
+            }
+        }
+    }
+
+    if (fs.existsSync('extensions/server_connect/routes')) {
+        const entries = fs.readdirSync('extensions/server_connect/routes', { withFileTypes: true });
+
+        for (let entry of entries) {
+            if (entry.isFile() && extname(entry.name) == '.js') {
+                let hook = require(`../../extensions/server_connect/routes/${entry.name}`);
+                if (hook.after) hook.after(app);
+                debug(`Custom router ${entry.name} loaded`);
+            }
+        }
+    }
+
+    function createApiRoutes(dir) {
+        const entries = fs.readdirSync(dir, { withFileTypes: true });
+    
+        return map(entries, async (entry) => {
+            if (entry.name.startsWith('_')) return;
+
+            let path = posix.join(dir, entry.name);
+    
+            if (entry.isFile() && extname(path) == '.json') {
+                let json = fs.readJSONSync(path);
+                let routePath = (json.settings?.options?.path) ? json.settings.options.path : path.replace(/^app/i, '').replace(/.json$/, '(.json)?');
+                let routeMethod = (json.settings?.options?.method) ? json.settings.options.method : 'all';
+                let ttl = (json.settings?.options?.ttl) ? json.settings.options.ttl : 0;
+                let csrf = (json.settings?.options?.nocsrf) ? false : config.csrf?.enabled;
+                let points = (json.settings?.options?.points) ? json.settings.options.points : 1;
+                
+                app[routeMethod](routePath.replace(/\/\(.*?\)\//gi, '/'), (req, res, next) => {
+                    if (app.rateLimiter && points > 0) {
+                        let isPrivate = false;
+                        let key = req.ip;
+                        
+                        if (app.privateRateLimiter) {
+                            if (req.session && req.session[config.rateLimit.private.provider + 'Id']) {
+                                isPrivate = true;
+                                key = req.session[config.rateLimit.private.provider + 'Id'];
+                            }
+                        }
+
+                        app[isPrivate ? 'privateRateLimiter' : 'rateLimiter'].consume(key, points).then(rateLimiterRes => {
+                            const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
+                            res.set('RateLimit-Policy', `${config.rateLimit.points};w=${config.rateLimit.duration}`);
+                            res.set('RateLimit', `limit=${config.rateLimit.points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+                            next();
+                        }).catch(rateLimiterRes => {
+                            const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
+                            res.set('RateLimit-Policy', `${config.rateLimit.points};w=${config.rateLimit.duration}`);
+                            res.set('RateLimit', `limit=${config.rateLimit.points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+                            res.set('Retry-After', reset);
+                            if (req.is('json')) {
+                                res.status(429).json({ error: 'Too Many Requests' });
+                            } else {
+                                res.status(429).send('Too Many Requests');
+                            }
+                        });
+                    } else {
+                        next();
+                    }
+                }, (req, res, next) => {
+                    if (!csrf) return next();
+                    if (config.csrf.exclude.split(',').includes(req.method)) return next();
+                    if (!req.validateCSRF()) return res.status(403).send('Invalid CSRF token');
+                    next();
+                }, cache({ttl}), serverConnect(json));
+
+                debug(`Api route ${routePath} created`);
+            }
+    
+            if (entry.isDirectory()) {
+                return createApiRoutes(path);
+            }
+        });
+    }
+};

lib/setup/secure.js

@@ -0,0 +1,112 @@
+const debug = require('debug')('server-connect:secure');
+const config = require('./config');
+
+module.exports = function (app) {
+  const { randomBytes } = require('crypto');
+  const generateToken = (req, overwrite) => {
+    const sessionToken = req.session.csrfToken;
+
+    if (!overwrite && typeof sessionToken === 'string') {
+      return sessionToken;
+    }
+
+    const token = randomBytes(32).toString('hex');
+    req.session.csrfToken = token;
+
+    return token;
+  }
+  const getTokenFromRequest = (req) => req.headers['x-csrf-token'] || req.body.CSRFToken;
+  const isValidToken = (req) => {
+    const token = getTokenFromRequest(req);
+    const sessionToken = req.session.csrfToken;
+    return typeof token === 'string' && typeof sessionToken === 'string' && token === sessionToken;
+  }
+
+  app.get('/__csrf', (req, res) => {
+    res.json({ csrfToken: generateToken(req, true) });
+  });
+
+  app.use((req, res, next) => {
+    req.csrfToken = (overwrite) => generateToken(req, overwrite);
+    req.validateCSRF = () => isValidToken(req);
+    next();
+  });
+
+  debug('CSRF Protection initialized');
+
+  if (config.rateLimit?.enabled) {
+    const options = config.rateLimit;
+    const { RateLimiterMemory, RateLimiterRedis } = require('rate-limiter-flexible');
+
+    if (global.redisClient) {
+      app.rateLimiter = new RateLimiterRedis({
+        duration: options.duration,
+        points: options.points,
+        storeClient: global.redisClient,
+      });
+      if (options.private?.provider) {
+        app.privateRateLimiter = new RateLimiterRedis({
+          duration: options.private.duration,
+          points: options.private.points,
+          storeClient: global.redisClient,
+        });
+      }
+    } else {
+      app.rateLimiter = new RateLimiterMemory({
+        duration: config.rateLimit.duration,
+        points: config.rateLimit.points,
+      });
+      if (options.private?.provider) {
+        app.privateRateLimiter = new RateLimiterMemory({
+          duration: options.private.duration,
+          points: options.private.points,
+        });
+      }
+    }
+
+    debug('Ratelimit initialized', options);
+  }
+
+  if (config.passport) {
+    const passport = require('passport');
+    const ServerConnectStrategy = require('../auth/passport');
+
+    passport.use(new ServerConnectStrategy({ provider: 'security' }));
+
+    app.use(passport.initialize());
+    app.use(passport.session());
+    app.use(passport.authenticate('server-connect'));
+
+    debug('Passport initialized', passport.strategies);
+    
+    app.use((req, res, next) => {
+      debug('auth', req.isAuthenticated());
+      debug('Session', req.session);
+      if (req.user) {
+        debug('User', req.user);
+      }
+      next();
+    });
+
+    app.use('/api/secure', restrict());
+  }
+};
+
+// restrict middleware
+function restrict (options = {}) {
+  return async function (req, res, next) {
+    if (req.isAuthenticated()) {
+      return next();
+    }
+
+    if (req.is('json')) {
+      return res.status(401).json({ error: 'Unauthorized' });
+    }
+
+    if (options.redirect) {
+      return res.redirect(options.redirect);
+    }
+
+    res.status(401).send('Unauthorized');
+  };
+}

lib/setup/session.js

@@ -0,0 +1,94 @@
+const session = require('express-session'); //(Object.assign({ secret: config.secret }, config.session));
+const debug = require('debug')('server-connect:setup:session');
+const Parser = require('../core/parser');
+const Scope = require('../core/scope');
+const db = require('../core/db');
+const { toSystemPath } = require('../core/path');
+const config = require('./config');
+const options = config.session;
+
+if (!options.secret) {
+    options.secret = config.secret;
+}
+
+debug('init session store %o', options.store);
+
+if (options.store.$type == 'redis') { // https://www.npmjs.com/package/connect-redis
+    const RedisStore = require('connect-redis').default;
+    options.store = new RedisStore(Object.assign({
+        client: global.redisClient
+    }, options.store));
+} else if (options.store.$type == 'file') { // https://www.npmjs.com/package/session-file-store
+    const FileStore = require('session-file-store')(session);
+    options.store = new FileStore(options.store);
+} else if (options.store.$type == 'database') { // https://www.npmjs.com/package/connect-session-knex
+    const KnexStore = require('connect-session-knex')(session);
+
+    if (typeof options.store.knex == 'string') {
+        if (!global.db) global.db = {};
+
+        if (!global.db[options.store.knex]) {
+            const fs = require('fs-extra');
+            const action = fs.readJSONSync(`app/modules/connections/${options.store.knex}.json`);
+            const knex_options = Parser.parseValue(action.options, new Scope({ $_ENV: process.env }));
+
+            if (knex_options.connection && knex_options.connection.filename) {
+                knex_options.connection.filename = toSystemPath(knex_options.connection.filename);
+            }
+
+            if (knex_options.connection && knex_options.connection.ssl) {
+                if (knex_options.connection.ssl.key) {
+                    knex_options.connection.ssl.key = fs.readFileSync(toSystemPath(knex_options.connection.ssl.key));
+                }
+    
+                if (knex_options.connection.ssl.ca) {
+                    knex_options.connection.ssl.ca = fs.readFileSync(toSystemPath(knex_options.connection.ssl.ca));
+                }
+    
+                if (knex_options.connection.ssl.cert) {
+                    knex_options.connection.ssl.cert = fs.readFileSync(toSystemPath(knex_options.connection.ssl.cert));
+                }
+            }
+    
+            knex_options.useNullAsDefault = true;
+    
+            knex_options.postProcessResponse = function(result) {
+                if (Array.isArray(result)) {
+                    return result.map(row => {
+                        for (column in row) {
+                            if (row[column] && row[column].toJSON) {
+                                row[column] = row[column].toJSON();
+                            }
+                        }
+                        return row;
+                    });
+                } else {
+                    for (column in result) {
+                        if (result[column] && result[column].toJSON) {
+                            result[column] = result[column].toJSON();
+                        }
+                    }
+                    return result;
+                }
+            };
+
+            global.db[options.store.knex] = db(knex_options);
+        }
+
+        options.store.knex = global.db[options.store.knex];
+    }
+
+    if (options.store.ttl) {
+        options.cookie = { ...options.cookie, maxAge: options.store.ttl * 1000 };
+    }
+
+    options.store = new KnexStore(options.store);
+} else {
+    const MemoryStore = require('../core/memoryStore')(session);
+    if (options.store.ttl) {
+        options.store.ttl = options.store.ttl * 1000;
+    }
+    options.store = new MemoryStore(options.store);
+}
+
+module.exports = session(options);

lib/setup/sockets.js

@@ -0,0 +1,195 @@
+const fs = require('fs-extra');
+const { basename, extname } = require('path');
+const debug = require('debug')('server-connect:sockets');
+const { isEmpty } = require('./util');
+const config = require('./config');
+const cookieParser = require('cookie-parser');
+const { promisify } = require('util');
+
+module.exports = function (server, appSession) {
+    //if (isEmpty('app/sockets')) return null;
+
+    const io = require('socket.io')();
+
+    if (global.redisClient) {
+        const { createAdapter } = require('@socket.io/redis-streams-adapter');
+        io.adapter(createAdapter(global.redisClient));
+    }
+
+    // user hooks
+    if (fs.existsSync('extensions/server_connect/sockets')) {
+        const entries = fs.readdirSync('extensions/server_connect/sockets', { withFileTypes: true });
+
+        for (let entry of entries) {
+            if (entry.isFile() && extname(entry.name) == '.js') {
+                const hook = require(`../../extensions/server_connect/sockets/${entry.name}`);
+                if (hook.handler) hook.handler(io);
+                debug(`Custom sockets hook ${entry.name} loaded`);
+            }
+        }
+    }
+
+    // create socket connections for api endpoints
+    if (fs.existsSync('app/api')) {
+
+        io.of('/api').on('connection', async (socket) => {
+            socket.onAny(async (event, params, cb) => {
+                try {
+                    if (typeof cb == 'function' && global.redisClient && global.redisClient.isReady) {
+                        const cached = await global.redisClient.get('ws:' + event + ':' + JSON.stringify(params));
+                        if (cached) return cb(JSON.parse(cached));
+                    }
+
+                    const req = Object.assign({}, socket.handshake);
+                    const res = {
+                        statusCode: 200,
+                        getHeader: () => { },
+                        setHeader: () => { },
+                        sendStatus: (statusCode) => { res.statusCode = statusCode; },
+                        write: () => { },
+                        end: () => { }
+                    };
+
+                    cookieParser(config.secret)(req, res, () => {
+                        appSession(req, res, async () => {
+                            const App = require('../core/app');
+                            const app = new App(req, res);
+                            const action = await fs.readJSON(`app/api/${event}.json`);
+                            app.set('$_PARAM', params);
+                            app.set('$_GET', params); // fake query params
+                            app.socket = socket;
+                            await app.define(action, true);
+                            if (typeof cb == 'function') {
+                                cb({
+                                    status: res.statusCode,
+                                    data: res.statusCode == 200 ? app.data : null
+                                });
+
+                                if (global.redisClient && global.redisClient.isReady) {
+                                    let ttl = (action.settings && action.settings.options && action.settings.options.ttl) ? action.settings.options.ttl : null;
+
+                                    if (ttl && res.statusCode < 400) { // only cache valid response, not error response
+                                        global.redisClient.setEx('ws:' + event + ':' + JSON.stringify(params), ttl, JSON.stringify({
+                                            status: res.statusCode,
+                                            data: res.statusCode == 200 ? app.data : null
+                                        }));
+                                    }
+                                }
+                            }
+                        });
+                    });
+                } catch (e) {
+                    debug(`ERROR: ${e.message}`);
+                    console.error(e);
+                }
+            });
+        });
+    }
+
+    if (fs.existsSync('app/sockets')) {
+        parseSockets();
+    }
+
+    function parseSockets(namespace = '') {
+        const entries = fs.readdirSync('app/sockets' + namespace, { withFileTypes: true });
+
+        io.of(namespace || '/').on('connection', async (socket) => {
+            if (fs.existsSync(`app/sockets${namespace}/connect.json`)) {
+                try {
+                    const req = Object.assign({}, socket.handshake);
+                    const res = {
+                        statusCode: 200,
+                        getHeader: () => { },
+                        setHeader: () => { },
+                        sendStatus: (statusCode) => { res.statusCode = statusCode; },
+                        write: () => { },
+                        end: () => { }
+                    };
+
+                    cookieParser(config.secret)(req, res, () => {
+                        appSession(req, res, async () => {
+                            const App = require('../core/app');
+                            const app = new App(req, res);
+                            const action = await fs.readJSON(`app/sockets${namespace}/connect.json`);
+                            app.socket = socket;
+                            await app.define(action, true);
+                        });
+                    });
+                } catch (e) {
+                    debug(`ERROR: ${e.message}`);
+                    console.error(e);
+                }
+            }
+
+            if (fs.existsSync(`app/sockets${namespace}/disconnect.json`)) {
+                socket.on('disconnect', async (event) => {
+                    try {
+                        const req = Object.assign({}, socket.handshake);
+                        const res = {
+                            statusCode: 200,
+                            getHeader: () => { },
+                            setHeader: () => { },
+                            sendStatus: (statusCode) => { res.statusCode = statusCode; },
+                            write: () => { },
+                            end: () => { }
+                        };
+    
+                        cookieParser(config.secret)(req, res, () => {
+                            appSession(req, res, async () => {
+                                const App = require('../core/app');
+                                const app = new App(req, res);
+                                const action = await fs.readJSON(`app/sockets${namespace}/disconnect.json`);
+                                app.socket = socket;
+                                await app.define(action, true);
+                            });
+                        });
+                    } catch (e) {
+                        debug(`ERROR: ${e.message}`);
+                        console.error(e);
+                    }
+                });
+            }
+
+            socket.onAny(async (event, params, cb) => {
+                try {
+                    const req = Object.assign({}, socket.handshake);
+                    const res = {
+                        statusCode: 200,
+                        getHeader: () => { },
+                        setHeader: () => { },
+                        sendStatus: (statusCode) => { res.statusCode = statusCode; },
+                        write: () => { },
+                        end: () => { }
+                    };
+
+                    cookieParser(config.secret)(req, res, () => {
+                        appSession(req, res, async () => {
+                            const App = require('../core/app');
+                            const app = new App(req, res);
+                            const action = await fs.readJSON(`app/sockets${namespace}/${event}.json`);
+                            app.set('$_PARAM', params);
+                            app.socket = socket;
+                            await app.define(action, true);
+                            if (typeof cb == 'function') cb(app.data);
+                        });
+                    });
+                } catch (e) {
+                    debug(`ERROR: ${e.message}`);
+                    console.error(e);
+                }
+            });
+        });
+
+        for (let entry of entries) {
+            if (entry.isDirectory()) {
+                parseSockets(namespace + '/' + entry.name);
+            }
+        }
+    }
+
+    io.attach(server, {
+        cors: config.cors
+    });
+
+    return io;
+};

lib/setup/upload.js

@@ -0,0 +1,63 @@
+const config = require('./config');
+const debug = require('debug')('server-connect:setup:upload');
+const fs = require('fs-extra');
+const qs = require('qs');
+
+module.exports = function(app) {
+    const fileupload = require('express-fileupload');
+    const isEligibleRequest = require('express-fileupload/lib/isEligibleRequest');
+
+    // Make sure tmp folder exists and make it empty
+    fs.ensureDirSync(config.tmpFolder);
+    fs.emptyDirSync(config.tmpFolder);
+
+    // Always use tmp folder
+    app.use(fileupload(Object.assign({}, config.fileupload, {
+        useTempFiles: true,
+        tempFileDir: config.tmpFolder,
+        defParamCharset: 'utf8'
+    })));
+    
+    app.use((req, res, next) => {
+        if (!isEligibleRequest(req)) {
+            return next();
+        }
+
+        let encoded = qs.stringify(req.body);
+        if (req.files) {
+            for (let field in req.files) {
+                encoded += '&' + field + '=' + field;
+            }
+        }
+
+        req.body = qs.parse(encoded, {
+            arrayLimit: 10000,
+            parameterLimit: 10000,
+        });
+
+        // Cleanup
+        res.once('close', () => {
+            if (req.files) {
+                for (let field in req.files) {
+                    let file = req.files[field];
+                    
+                    if (Array.isArray(file)) {
+                        file.forEach((file) => {
+                            if (file.tempFilePath && fs.existsSync(file.tempFilePath)) {
+                                debug('delete %s', file.tempFilePath);
+                                fs.unlink(file.tempFilePath);
+                            }
+                        });
+                    } else if (file.tempFilePath && fs.existsSync(file.tempFilePath)) {
+                        debug('delete %s', file.tempFilePath);
+                        fs.unlink(file.tempFilePath);
+                    }
+                }
+            }
+        });
+
+        next();
+    });
+
+    debug('Upload middleware configured.');
+}

lib/setup/util.js

@@ -0,0 +1,12 @@
+const fs = require('fs-extra');
+
+exports.isEmpty = (path) => {
+    try {
+        let stat = fs.statSync(path);
+        if (!stat.isDirectory()) return true;
+        let items = fs.readdirSync(path);
+        return !items || !items.length;
+    } catch (e) {
+        return true;
+    }
+}

lib/setup/webhooks.js

@@ -0,0 +1,22 @@
+const fs = require('fs-extra');
+
+module.exports = function(app) {
+    app.all('/webhooks/:name', (req, res, next) => {
+        const name = req.params.name;
+
+        if (!/^[a-zA-Z0-9-_]+$/.test(name)) {
+            res.status(400).json({error: `Invalid webhook name.`});
+        } else if (fs.existsSync(`lib/webhooks/${name}.js`)) {
+            const webhook = require(`../webhooks/${name}`);
+            if (webhook.handler) {
+                webhook.handler(req, res, next);
+            } else {
+                res.status(400).json({error: `Webhook ${name} has no handler.`});
+            }
+        } else {
+            const webhook = require('../core/webhook');
+            const handler = webhook.createHandler(name);
+            handler(req, res, next);
+        }
+    });
+};

lib/validator/core.js

@@ -0,0 +1,243 @@
+const isValidString = (s) => typeof s == 'string' && s.length > 0;
+const getLength = (s) => s && s.length || 0;;
+const testRegexp = (re) => (v) => !isValidString(v) || re.test(v);
+const reEmail = /^(?!\.)((?!.*\.{2})[a-zA-Z0-9\u0080-\u00FF\u0100-\u017F\u0180-\u024F\u0250-\u02AF\u0300-\u036F\u0370-\u03FF\u0400-\u04FF\u0500-\u052F\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0700-\u074F\u0750-\u077F\u0780-\u07BF\u07C0-\u07FF\u0900-\u097F\u0980-\u09FF\u0A00-\u0A7F\u0A80-\u0AFF\u0B00-\u0B7F\u0B80-\u0BFF\u0C00-\u0C7F\u0C80-\u0CFF\u0D00-\u0D7F\u0D80-\u0DFF\u0E00-\u0E7F\u0E80-\u0EFF\u0F00-\u0FFF\u1000-\u109F\u10A0-\u10FF\u1100-\u11FF\u1200-\u137F\u1380-\u139F\u13A0-\u13FF\u1400-\u167F\u1680-\u169F\u16A0-\u16FF\u1700-\u171F\u1720-\u173F\u1740-\u175F\u1760-\u177F\u1780-\u17FF\u1800-\u18AF\u1900-\u194F\u1950-\u197F\u1980-\u19DF\u19E0-\u19FF\u1A00-\u1A1F\u1B00-\u1B7F\u1D00-\u1D7F\u1D80-\u1DBF\u1DC0-\u1DFF\u1E00-\u1EFF\u1F00-\u1FFFu20D0-\u20FF\u2100-\u214F\u2C00-\u2C5F\u2C60-\u2C7F\u2C80-\u2CFF\u2D00-\u2D2F\u2D30-\u2D7F\u2D80-\u2DDF\u2F00-\u2FDF\u2FF0-\u2FFF\u3040-\u309F\u30A0-\u30FF\u3100-\u312F\u3130-\u318F\u3190-\u319F\u31C0-\u31EF\u31F0-\u31FF\u3200-\u32FF\u3300-\u33FF\u3400-\u4DBF\u4DC0-\u4DFF\u4E00-\u9FFF\uA000-\uA48F\uA490-\uA4CF\uA700-\uA71F\uA800-\uA82F\uA840-\uA87F\uAC00-\uD7AF\uF900-\uFAFF\.!#$%&'*+-/=?^_`{|}~\-\d]+)@(?!\.)([a-zA-Z0-9\u0080-\u00FF\u0100-\u017F\u0180-\u024F\u0250-\u02AF\u0300-\u036F\u0370-\u03FF\u0400-\u04FF\u0500-\u052F\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0700-\u074F\u0750-\u077F\u0780-\u07BF\u07C0-\u07FF\u0900-\u097F\u0980-\u09FF\u0A00-\u0A7F\u0A80-\u0AFF\u0B00-\u0B7F\u0B80-\u0BFF\u0C00-\u0C7F\u0C80-\u0CFF\u0D00-\u0D7F\u0D80-\u0DFF\u0E00-\u0E7F\u0E80-\u0EFF\u0F00-\u0FFF\u1000-\u109F\u10A0-\u10FF\u1100-\u11FF\u1200-\u137F\u1380-\u139F\u13A0-\u13FF\u1400-\u167F\u1680-\u169F\u16A0-\u16FF\u1700-\u171F\u1720-\u173F\u1740-\u175F\u1760-\u177F\u1780-\u17FF\u1800-\u18AF\u1900-\u194F\u1950-\u197F\u1980-\u19DF\u19E0-\u19FF\u1A00-\u1A1F\u1B00-\u1B7F\u1D00-\u1D7F\u1D80-\u1DBF\u1DC0-\u1DFF\u1E00-\u1EFF\u1F00-\u1FFF\u20D0-\u20FF\u2100-\u214F\u2C00-\u2C5F\u2C60-\u2C7F\u2C80-\u2CFF\u2D00-\u2D2F\u2D30-\u2D7F\u2D80-\u2DDF\u2F00-\u2FDF\u2FF0-\u2FFF\u3040-\u309F\u30A0-\u30FF\u3100-\u312F\u3130-\u318F\u3190-\u319F\u31C0-\u31EF\u31F0-\u31FF\u3200-\u32FF\u3300-\u33FF\u3400-\u4DBF\u4DC0-\u4DFF\u4E00-\u9FFF\uA000-\uA48F\uA490-\uA4CF\uA700-\uA71F\uA800-\uA82F\uA840-\uA87F\uAC00-\uD7AF\uF900-\uFAFF\-\.\d]+)((\.([a-zA-Z\u0080-\u00FF\u0100-\u017F\u0180-\u024F\u0250-\u02AF\u0300-\u036F\u0370-\u03FF\u0400-\u04FF\u0500-\u052F\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0700-\u074F\u0750-\u077F\u0780-\u07BF\u07C0-\u07FF\u0900-\u097F\u0980-\u09FF\u0A00-\u0A7F\u0A80-\u0AFF\u0B00-\u0B7F\u0B80-\u0BFF\u0C00-\u0C7F\u0C80-\u0CFF\u0D00-\u0D7F\u0D80-\u0DFF\u0E00-\u0E7F\u0E80-\u0EFF\u0F00-\u0FFF\u1000-\u109F\u10A0-\u10FF\u1100-\u11FF\u1200-\u137F\u1380-\u139F\u13A0-\u13FF\u1400-\u167F\u1680-\u169F\u16A0-\u16FF\u1700-\u171F\u1720-\u173F\u1740-\u175F\u1760-\u177F\u1780-\u17FF\u1800-\u18AF\u1900-\u194F\u1950-\u197F\u1980-\u19DF\u19E0-\u19FF\u1A00-\u1A1F\u1B00-\u1B7F\u1D00-\u1D7F\u1D80-\u1DBF\u1DC0-\u1DFF\u1E00-\u1EFF\u1F00-\u1FFF\u20D0-\u20FF\u2100-\u214F\u2C00-\u2C5F\u2C60-\u2C7F\u2C80-\u2CFF\u2D00-\u2D2F\u2D30-\u2D7F\u2D80-\u2DDF\u2F00-\u2FDF\u2FF0-\u2FFF\u3040-\u309F\u30A0-\u30FF\u3100-\u312F\u3130-\u318F\u3190-\u319F\u31C0-\u31EF\u31F0-\u31FF\u3200-\u32FF\u3300-\u33FF\u3400-\u4DBF\u4DC0-\u4DFF\u4E00-\u9FFF\uA000-\uA48F\uA490-\uA4CF\uA700-\uA71F\uA800-\uA82F\uA840-\uA87F\uAC00-\uD7AF\uF900-\uFAFF]){2,63})+)$/i;
+const reUrl = /^(?:(?:(?:https?|ftp):)?\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})).?)(?::\d{2,5})?(?:[/?#]\S*)?$/i;
+const reDateTime = /^\d{4}[\/\-](0?[1-9]|1[012])[\/\-](0?[1-9]|[12][0-9]|3[01])[T ]([01][0-9]|2[0-4]):[0-5][0-9](:([0-5][0-9]|60))?Z?$/;
+const reDate = /^\d{4}[\/\-](0?[1-9]|1[012])[\/\-](0?[1-9]|[12][0-9]|3[01])$/;
+const reTime = /^([01][0-9]|2[0-4]):[0-5][0-9](:([0-5][0-9]|60))?$/;
+const reMonth = /^\d{4}-(0[1-9]|1[012])$/;
+const reWeek = /^\d{4}-W(0[1-9]|[1-4][0-9]|5[0-3])$/;
+const reColor = /^#[a-fA-F0-9]{6}$/;
+const reNumber = /^(?:-?\d+|-?\d{1,3}(?:,\d{3})+)?(?:\.\d+)?$/;
+const reDigits = /^\d+$/;
+const reAlphanumeric = /^\w+$/;
+const reBic = /^([A-Z]{6}[A-Z2-9][A-NP-Z1-2])(X{3}|[A-WY-Z0-9][A-Z0-9]{2})?$/;
+const reVat = /^((AT)?U[0-9]{8}|(BE)?0[0-9]{9}|(BG)?[0-9]{9,10}|(CY)?[0-9]{8}L|(CZ)?[0-9]{8,10}|(DE)?[0-9]{9}|(DK)?[0-9]{8}|(EE)?[0-9]{9}|(EL|GR)?[0-9]{9}|(ES)?[0-9A-Z][0-9]{7}[0-9A-Z]|(FI)?[0-9]{8}|(FR)?[0-9A-Z]{2}[0-9]{9}|(GB)?([0-9]{9}([0-9]{3})?|[A-Z]{2}[0-9]{3})|(HU)?[0-9]{8}|(IE)?[0-9]S[0-9]{5}L|(IT)?[0-9]{11}|(LT)?([0-9]{9}|[0-9]{12})|(LU)?[0-9]{8}|(LV)?[0-9]{11}|(MT)?[0-9]{8}|(NL)?[0-9]{9}B[0-9]{2}|(PL)?[0-9]{10}|(PT)?[0-9]{9}|(RO)?[0-9]{2,10}|(SE)?[0-9]{12}|(SI)?[0-9]{8}|(SK)?[0-9]{10})$/;
+const reInteger = /^-?\d+$/;
+const reIpv4 = /^(25[0-5]|2[0-4]\d|[01]?\d\d?)\.(25[0-5]|2[0-4]\d|[01]?\d\d?)\.(25[0-5]|2[0-4]\d|[01]?\d\d?)\.(25[0-5]|2[0-4]\d|[01]?\d\d?)$/;
+const reIpv6 = /^((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((\b((25[0-5])|(1\d{2})|(2[0-4]\d)|(\d{1,2}))\b)\.){3}(\b((25[0-5])|(1\d{2})|(2[0-4]\d)|(\d{1,2}))\b))|(([0-9A-Fa-f]{1,4}:){0,5}:((\b((25[0-5])|(1\d{2})|(2[0-4]\d)|(\d{1,2}))\b)\.){3}(\b((25[0-5])|(1\d{2})|(2[0-4]\d)|(\d{1,2}))\b))|(::([0-9A-Fa-f]{1,4}:){0,5}((\b((25[0-5])|(1\d{2})|(2[0-4]\d)|(\d{1,2}))\b)\.){3}(\b((25[0-5])|(1\d{2})|(2[0-4]\d)|(\d{1,2}))\b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))$/;
+const reLettersonly = /^[a-z]+$/i;
+const reUnicodelettersonly = /^[A-Za-z\xAA\xB5\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0-\u08B4\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09F0\u09F1\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C60\u0C61\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0\u0CE1\u0CF1\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D5F-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183\u2184\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005\u3006\u3031-\u3035\u303B\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FD5\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA7AD\uA7B0-\uA7B7\uA7F7-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA8FD\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uA9E0-\uA9E4\uA9E6-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB65\uAB70-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]+$/;
+const reLetterswithbasicpunc = /^[a-z\-.,()'"\s]+$/;
+const reNowhitespace = /^\S+$/;
+const ibanCountryPatterns = {
+    "AL": "\\d{8}[\\dA-Z]{16}",
+    "AD": "\\d{8}[\\dA-Z]{12}",
+    "AT": "\\d{16}",
+    "AZ": "[\\dA-Z]{4}\\d{20}",
+    "BE": "\\d{12}",
+    "BH": "[A-Z]{4}[\\dA-Z]{14}",
+    "BA": "\\d{16}",
+    "BR": "\\d{23}[A-Z][\\dA-Z]",
+    "BG": "[A-Z]{4}\\d{6}[\\dA-Z]{8}",
+    "CR": "\\d{17}",
+    "HR": "\\d{17}",
+    "CY": "\\d{8}[\\dA-Z]{16}",
+    "CZ": "\\d{20}",
+    "DK": "\\d{14}",
+    "DO": "[A-Z]{4}\\d{20}",
+    "EE": "\\d{16}",
+    "FO": "\\d{14}",
+    "FI": "\\d{14}",
+    "FR": "\\d{10}[\\dA-Z]{11}\\d{2}",
+    "GE": "[\\dA-Z]{2}\\d{16}",
+    "DE": "\\d{18}",
+    "GI": "[A-Z]{4}[\\dA-Z]{15}",
+    "GR": "\\d{7}[\\dA-Z]{16}",
+    "GL": "\\d{14}",
+    "GT": "[\\dA-Z]{4}[\\dA-Z]{20}",
+    "HU": "\\d{24}",
+    "IS": "\\d{22}",
+    "IE": "[\\dA-Z]{4}\\d{14}",
+    "IL": "\\d{19}",
+    "IT": "[A-Z]\\d{10}[\\dA-Z]{12}",
+    "KZ": "\\d{3}[\\dA-Z]{13}",
+    "KW": "[A-Z]{4}[\\dA-Z]{22}",
+    "LV": "[A-Z]{4}[\\dA-Z]{13}",
+    "LB": "\\d{4}[\\dA-Z]{20}",
+    "LI": "\\d{5}[\\dA-Z]{12}",
+    "LT": "\\d{16}",
+    "LU": "\\d{3}[\\dA-Z]{13}",
+    "MK": "\\d{3}[\\dA-Z]{10}\\d{2}",
+    "MT": "[A-Z]{4}\\d{5}[\\dA-Z]{18}",
+    "MR": "\\d{23}",
+    "MU": "[A-Z]{4}\\d{19}[A-Z]{3}",
+    "MC": "\\d{10}[\\dA-Z]{11}\\d{2}",
+    "MD": "[\\dA-Z]{2}\\d{18}",
+    "ME": "\\d{18}",
+    "NL": "[A-Z]{4}\\d{10}",
+    "NO": "\\d{11}",
+    "PK": "[\\dA-Z]{4}\\d{16}",
+    "PS": "[\\dA-Z]{4}\\d{21}",
+    "PL": "\\d{24}",
+    "PT": "\\d{21}",
+    "RO": "[A-Z]{4}[\\dA-Z]{16}",
+    "SM": "[A-Z]\\d{10}[\\dA-Z]{12}",
+    "SA": "\\d{2}[\\dA-Z]{18}",
+    "RS": "\\d{18}",
+    "SK": "\\d{20}",
+    "SI": "\\d{15}",
+    "ES": "\\d{20}",
+    "SE": "\\d{20}",
+    "CH": "\\d{5}[\\dA-Z]{12}",
+    "TN": "\\d{20}",
+    "TR": "\\d{5}[\\dA-Z]{17}",
+    "AE": "\\d{3}\\d{16}",
+    "GB": "[A-Z]{4}\\d{14}",
+    "VG": "[\\dA-Z]{4}\\d{16}"
+};
+
+module.exports = {
+
+    required: function(value) {
+        return value && value.length > 0;
+    },
+
+    email: testRegexp(reEmail),
+    url: testRegexp(reUrl),
+    datetime: testRegexp(reDateTime),
+    date: testRegexp(reDate),
+    time: testRegexp(reTime),
+    month: testRegexp(reMonth),
+    week: testRegexp(reWeek),
+    color: testRegexp(reColor),
+    number: testRegexp(reNumber),
+    digits: testRegexp(reDigits),
+    alphanumeric: testRegexp(reAlphanumeric),
+    bic: testRegexp(reBic),
+    vat: testRegexp(reVat),
+    integer: testRegexp(reInteger),
+    ipv4: testRegexp(reIpv4),
+    ipv6: testRegexp(reIpv6),
+    lettersonly: testRegexp(reLettersonly),
+    unicodelettersonly: testRegexp(reUnicodelettersonly),
+    letterswithbasicpunc: testRegexp(reLetterswithbasicpunc),
+    nowhitespace: testRegexp(reNowhitespace),
+
+    pattern: function(value, param) {
+        const re = new RegExp(`^(?:${param})$`);
+        return !isValidString(value) || re.test(value);
+    },    
+
+    creditcard: function(value) {
+        if (!isValidString(value)) {
+            return true;
+        }
+
+        if (/[^0-9 \-]+/.test(value)) {
+            return false;
+        }
+
+        value = value.replace(/\D/g, '');
+
+        if (value.length < 13 || value.length > 19) {
+            return false;
+        }
+
+        let check = 0, digit = 0, even = false;
+        for (let i = value.length - 1; i >= 0; i--) {
+            digit = parseInt(value.charAt(i), 10);
+            if (even && (digit *= 2) > 9) {
+                digit -= 9;
+            }
+            check += digit;
+            even = !even;
+        }
+
+        return (check % 10) === 0;
+    },
+
+    iban: function(value) {
+        if (!isValidString(value)) {
+            return true;
+        }
+
+        const iban = value.replace(/ /g, '').toUpperCase();
+        const country = iban.substr(0, 2);
+        const pattern = ibanCountryPatterns[country];
+
+        if (typeof patter !== 'undefined') {
+            const re = new RegExp(`^[A-Z]{2}\\d{2}${pattern}$`);
+            if (!re.test(iban)) {
+                return false;
+            }
+        }
+
+        let leadingZeroes = true, digits = '', rest = '';
+        
+        const lookup = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+        const check = iban.substr(4) + iban.substr(0, 4);
+
+        for (let i = 0; i < check.length; i++) {
+            const ch = check.charAt(i);
+            
+            if (ch !== '0') {
+                leadingZeroes = false;
+            }
+
+            if (!leadingZeroes) {
+                digits += lookup.indexOf(ch);
+            }
+        }
+
+        for (let i = 0; i < digits.length; i++) {
+            const ch = digits.charAt(i);
+            const op = rest + ch;
+            rest = op % 97;
+        }
+
+        return rest === 1;
+    },
+
+    minlength: function(value, param) {
+        const length = getLength(value);
+        return length == 0 || length >= param;
+    },
+
+    maxlength: function(value, param) {
+        const length = getLength(value);
+        return length == 0 || length <= param;
+    },
+
+    rangelength: function(value, param) {
+        const length = getLength(value);
+        return length == 0 || (length >= param['0'] && length <= param['1']);
+    },
+
+    minitems: function(value, param) {
+        const length = getLength(value);
+        return length == 0 || (Array.isArray(value) && length >= param);
+    },
+
+    maxitems: function(value, param) {
+        const length = getLength(value);
+        return length == 0 || (Array.isArray(value) && length <= param);
+    },
+
+    rangeitems: function(value, param) {
+        const length = getLength(value);
+        return length == 0 || (Array.isArray(value) && length >= param['0'] && length <= param['1']);
+    },
+
+    min: function(value, param) {
+        return value != null && value != '' && Number(value) >= Number(param);
+    },
+
+    max: function(value, param) {
+        return value != null && value != '' && Number(value) <= Number(param);
+    },
+
+    range: function(value, param) {
+        return value != null && value != '' && Number(value) >= Number(param['0']) && Number(value) <= Number(param['1']);
+    },
+
+    equalTo: function(value, param) {
+        return this.parse(`{{ $_POST.${param.replace(/\[([^\]]+)\]/g, '.$1')} }}`) == value;
+    },
+
+    notEqualTo: function(value, param) {
+        return this.parse(`{{ $_POST.${param.replace(/\[([^\]]+)\]/g, '.$1')} }}`) != value;
+    },
+
+};

lib/validator/db.js

@@ -0,0 +1,21 @@
+module.exports = {
+
+    exists: async function(value, options) {
+        if (!value) return true;
+
+        const db = this.getDbConnection(options.connection);
+        const results = await db.from(options.table).where(options.column, value).limit(1);
+
+        return results.length > 0;
+    },
+
+    notexists: async function(value, options) {
+        if (!value) return true;
+
+        const db = this.getDbConnection(options.connection);
+        const results = await db.from(options.table).where(options.column, value).limit(1);
+
+        return results.length == 0;
+    }
+
+};

lib/validator/index.js

@@ -0,0 +1,120 @@
+module.exports = {
+
+    async init(app, meta) {
+        const errors = {};
+
+        if (meta['$_GET'] && Array.isArray(meta['$_GET'])) {
+            await this.validateFields(app, meta['$_GET'], app.scope.data['$_GET'], errors);
+        }
+
+        if (meta['$_POST'] && Array.isArray(meta['$_POST'])) {
+            await this.validateFields(app, meta['$_POST'], app.scope.data['$_POST'], errors);
+        }
+
+        if (Object.keys(errors).length) {
+            app.res.status(400).json(errors);
+        }
+    },
+
+    async validateData(app, data, noError) {
+        const errors = {};
+
+        if (Array.isArray(data)) {
+            for (let item of data) {
+                for (let rule in item.rules) {
+                    const options = item.rules[rule];
+
+                    rule = this.getRule(rule);
+
+                    if (!await this.validateRule(app, rule, item.value, options)) {
+                        const t = item.fieldName ? 'form' : 'data';
+                        errors[t] = errors[t] || {};
+                        errors[t][item.fieldName || item.name] = this.errorMessage(rule, options);
+                    }
+                }
+            }
+        }
+
+        if (Object.keys(errors).length) {
+            if (noError) return false;
+            app.res.status(400).json(errors);
+        }
+
+        return true;
+    },
+
+    async validateFields(app, fields, parent, errors, fieldname) {
+        if (parent == null) return;
+
+        for (let field of fields) {
+            let value = parent[field.name];
+            let curFieldname = fieldname ? `${fieldname}[${field.name}]` : field.name;
+
+            if (field.type == 'array' && value == null) {
+                value = [];
+            }
+
+            if (field.options && field.options.rules) {
+                await this.validateField(app, field, value, errors, curFieldname);
+            }
+
+            if (field.type == 'object' && field.sub) {
+                await this.validateFields(app, field.sub, value, errors, curFieldname);
+            }
+
+            if (field.type == 'array' && field.sub && field.sub[0] && field.sub[0].sub) {
+                if (Array.isArray(value) && value.length) {
+                    for (let i = 0; i < value.length; i++) {
+                        await this.validateFields(app, field.sub[0].sub, value[i], errors, `${curFieldname}[${i}]`);
+                    }
+                } else {
+                    await this.validateFields(app, field.sub[0].sub, null, errors, `${curFieldname}[0]`);
+                }
+            }
+        }
+    },
+
+    async validateField(app, field, value, errors, fieldname) {
+        for (let rule in field.options.rules) {
+            const options = field.options.rules[rule];
+
+            rule = this.getRule(rule);
+
+            if (!await this.validateRule(app, rule, value, options)) {
+                const t = field.fieldName ? 'form' : 'data';
+                errors[t] = errors[t] || {};
+                errors[t][fieldname] = this.errorMessage(rule, options);
+            }
+        }
+    },
+
+    async validateRule(app, rule, value, options = {}) {
+        const module = require(`./${rule.module}`);
+        return module[rule.method].call(app, value, options.param);
+    },
+
+    errorMessage(rule, options = {}) {
+        let message = options.message;
+
+        if (!message) {
+            const { validator: messages } = require('../locale/en-US');
+            message = messages[rule.module][rule.method];
+        }
+
+        if (typeof options.param != 'object') {
+            options.param = { '0': options.param };
+        }
+
+        return message.replace(/{([^}]+)}/g, (m, i) => options.param[i]);
+    },
+
+    getRule(rule) {
+        const colon = rule.indexOf(':');
+
+        return {
+            module: colon > 0 ? rule.substr(0, colon) : 'core',
+            method: colon > 0 ? rule.substr(colon + 1) : rule
+        };
+    }
+
+};

lib/validator/unicode.js

@@ -0,0 +1,466 @@
+const unicode = {
+    'L': {
+        bmp: 'A-Za-z\xAA\xB5\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0-\u08B4\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09F0\u09F1\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C60\u0C61\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0\u0CE1\u0CF1\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D5F-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183\u2184\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005\u3006\u3031-\u3035\u303B\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FD5\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA7AD\uA7B0-\uA7B7\uA7F7-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA8FD\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uA9E0-\uA9E4\uA9E6-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB65\uAB70-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC',
+        astral: '\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD86D[\uDC00-\uDF34\uDF40-\uDFFF]|\uD869[\uDC00-\uDED6\uDF00-\uDFFF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2]|\uD83A[\uDC00-\uDCC4]|\uD801[\uDC00-\uDC9D\uDD00-\uDD27\uDD30-\uDD63\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDE80-\uDE9C\uDEA0-\uDED0\uDF00-\uDF1F\uDF30-\uDF40\uDF42-\uDF49\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF]|\uD80D[\uDC00-\uDC2E]|\uD87E[\uDC00-\uDE1D]|\uD81B[\uDF00-\uDF44\uDF50\uDF93-\uDF9F]|[\uD80C\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872][\uDC00-\uDFFF]|\uD805[\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE80-\uDEAA\uDF00-\uDF19]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF63-\uDF77\uDF7D-\uDF8F]|\uD809[\uDC80-\uDD43]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC60-\uDC76\uDC80-\uDC9E\uDCE0-\uDCF2\uDCF4\uDCF5\uDD00-\uDD15\uDD20-\uDD39\uDD80-\uDDB7\uDDBE\uDDBF\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE33\uDE60-\uDE7C\uDE80-\uDE9C\uDEC0-\uDEC7\uDEC9-\uDEE4\uDF00-\uDF35\uDF40-\uDF55\uDF60-\uDF72\uDF80-\uDF91]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB]|\uD804[\uDC03-\uDC37\uDC83-\uDCAF\uDCD0-\uDCE8\uDD03-\uDD26\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDDA\uDDDC\uDE00-\uDE11\uDE13-\uDE2B\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61]|\uD808[\uDC00-\uDF99]|\uD83B[\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD806[\uDCA0-\uDCDF\uDCFF\uDEC0-\uDEF8]|\uD811[\uDC00-\uDE46]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD82C[\uDC00\uDC01]|\uD873[\uDC00-\uDEA1]'
+    },
+    'M': {
+        bmp: '\u0300-\u036F\u0483-\u0489\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E4\u06E7\u06E8\u06EA-\u06ED\u0711\u0730-\u074A\u07A6-\u07B0\u07EB-\u07F3\u0816-\u0819\u081B-\u0823\u0825-\u0827\u0829-\u082D\u0859-\u085B\u08E3-\u0903\u093A-\u093C\u093E-\u094F\u0951-\u0957\u0962\u0963\u0981-\u0983\u09BC\u09BE-\u09C4\u09C7\u09C8\u09CB-\u09CD\u09D7\u09E2\u09E3\u0A01-\u0A03\u0A3C\u0A3E-\u0A42\u0A47\u0A48\u0A4B-\u0A4D\u0A51\u0A70\u0A71\u0A75\u0A81-\u0A83\u0ABC\u0ABE-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AE2\u0AE3\u0B01-\u0B03\u0B3C\u0B3E-\u0B44\u0B47\u0B48\u0B4B-\u0B4D\u0B56\u0B57\u0B62\u0B63\u0B82\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD7\u0C00-\u0C03\u0C3E-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55\u0C56\u0C62\u0C63\u0C81-\u0C83\u0CBC\u0CBE-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5\u0CD6\u0CE2\u0CE3\u0D01-\u0D03\u0D3E-\u0D44\u0D46-\u0D48\u0D4A-\u0D4D\u0D57\u0D62\u0D63\u0D82\u0D83\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2\u0DF3\u0E31\u0E34-\u0E3A\u0E47-\u0E4E\u0EB1\u0EB4-\u0EB9\u0EBB\u0EBC\u0EC8-\u0ECD\u0F18\u0F19\u0F35\u0F37\u0F39\u0F3E\u0F3F\u0F71-\u0F84\u0F86\u0F87\u0F8D-\u0F97\u0F99-\u0FBC\u0FC6\u102B-\u103E\u1056-\u1059\u105E-\u1060\u1062-\u1064\u1067-\u106D\u1071-\u1074\u1082-\u108D\u108F\u109A-\u109D\u135D-\u135F\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17B4-\u17D3\u17DD\u180B-\u180D\u18A9\u1920-\u192B\u1930-\u193B\u1A17-\u1A1B\u1A55-\u1A5E\u1A60-\u1A7C\u1A7F\u1AB0-\u1ABE\u1B00-\u1B04\u1B34-\u1B44\u1B6B-\u1B73\u1B80-\u1B82\u1BA1-\u1BAD\u1BE6-\u1BF3\u1C24-\u1C37\u1CD0-\u1CD2\u1CD4-\u1CE8\u1CED\u1CF2-\u1CF4\u1CF8\u1CF9\u1DC0-\u1DF5\u1DFC-\u1DFF\u20D0-\u20F0\u2CEF-\u2CF1\u2D7F\u2DE0-\u2DFF\u302A-\u302F\u3099\u309A\uA66F-\uA672\uA674-\uA67D\uA69E\uA69F\uA6F0\uA6F1\uA802\uA806\uA80B\uA823-\uA827\uA880\uA881\uA8B4-\uA8C4\uA8E0-\uA8F1\uA926-\uA92D\uA947-\uA953\uA980-\uA983\uA9B3-\uA9C0\uA9E5\uAA29-\uAA36\uAA43\uAA4C\uAA4D\uAA7B-\uAA7D\uAAB0\uAAB2-\uAAB4\uAAB7\uAAB8\uAABE\uAABF\uAAC1\uAAEB-\uAAEF\uAAF5\uAAF6\uABE3-\uABEA\uABEC\uABED\uFB1E\uFE00-\uFE0F\uFE20-\uFE2F',
+        astral: '\uD805[\uDCB0-\uDCC3\uDDAF-\uDDB5\uDDB8-\uDDC0\uDDDC\uDDDD\uDE30-\uDE40\uDEAB-\uDEB7\uDF1D-\uDF2B]|\uD834[\uDD65-\uDD69\uDD6D-\uDD72\uDD7B-\uDD82\uDD85-\uDD8B\uDDAA-\uDDAD\uDE42-\uDE44]|\uD804[\uDC00-\uDC02\uDC38-\uDC46\uDC7F-\uDC82\uDCB0-\uDCBA\uDD00-\uDD02\uDD27-\uDD34\uDD73\uDD80-\uDD82\uDDB3-\uDDC0\uDDCA-\uDDCC\uDE2C-\uDE37\uDEDF-\uDEEA\uDF00-\uDF03\uDF3C\uDF3E-\uDF44\uDF47\uDF48\uDF4B-\uDF4D\uDF57\uDF62\uDF63\uDF66-\uDF6C\uDF70-\uDF74]|\uD81B[\uDF51-\uDF7E\uDF8F-\uDF92]|\uD81A[\uDEF0-\uDEF4\uDF30-\uDF36]|\uD82F[\uDC9D\uDC9E]|\uD800[\uDDFD\uDEE0\uDF76-\uDF7A]|\uD836[\uDE00-\uDE36\uDE3B-\uDE6C\uDE75\uDE84\uDE9B-\uDE9F\uDEA1-\uDEAF]|\uD802[\uDE01-\uDE03\uDE05\uDE06\uDE0C-\uDE0F\uDE38-\uDE3A\uDE3F\uDEE5\uDEE6]|\uD83A[\uDCD0-\uDCD6]|\uDB40[\uDD00-\uDDEF]'
+    },
+    'N': {
+        bmp: '0-9\xB2\xB3\xB9\xBC-\xBE\u0660-\u0669\u06F0-\u06F9\u07C0-\u07C9\u0966-\u096F\u09E6-\u09EF\u09F4-\u09F9\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0B72-\u0B77\u0BE6-\u0BF2\u0C66-\u0C6F\u0C78-\u0C7E\u0CE6-\u0CEF\u0D66-\u0D75\u0DE6-\u0DEF\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F33\u1040-\u1049\u1090-\u1099\u1369-\u137C\u16EE-\u16F0\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1946-\u194F\u19D0-\u19DA\u1A80-\u1A89\u1A90-\u1A99\u1B50-\u1B59\u1BB0-\u1BB9\u1C40-\u1C49\u1C50-\u1C59\u2070\u2074-\u2079\u2080-\u2089\u2150-\u2182\u2185-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2CFD\u3007\u3021-\u3029\u3038-\u303A\u3192-\u3195\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\uA620-\uA629\uA6E6-\uA6EF\uA830-\uA835\uA8D0-\uA8D9\uA900-\uA909\uA9D0-\uA9D9\uA9F0-\uA9F9\uAA50-\uAA59\uABF0-\uABF9\uFF10-\uFF19',
+        astral: '\uD800[\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDEE1-\uDEFB\uDF20-\uDF23\uDF41\uDF4A\uDFD1-\uDFD5]|\uD801[\uDCA0-\uDCA9]|\uD803[\uDCFA-\uDCFF\uDE60-\uDE7E]|\uD835[\uDFCE-\uDFFF]|\uD83A[\uDCC7-\uDCCF]|\uD81A[\uDE60-\uDE69\uDF50-\uDF59\uDF5B-\uDF61]|\uD806[\uDCE0-\uDCF2]|\uD804[\uDC52-\uDC6F\uDCF0-\uDCF9\uDD36-\uDD3F\uDDD0-\uDDD9\uDDE1-\uDDF4\uDEF0-\uDEF9]|\uD834[\uDF60-\uDF71]|\uD83C[\uDD00-\uDD0C]|\uD809[\uDC00-\uDC6E]|\uD802[\uDC58-\uDC5F\uDC79-\uDC7F\uDCA7-\uDCAF\uDCFB-\uDCFF\uDD16-\uDD1B\uDDBC\uDDBD\uDDC0-\uDDCF\uDDD2-\uDDFF\uDE40-\uDE47\uDE7D\uDE7E\uDE9D-\uDE9F\uDEEB-\uDEEF\uDF58-\uDF5F\uDF78-\uDF7F\uDFA9-\uDFAF]|\uD805[\uDCD0-\uDCD9\uDE50-\uDE59\uDEC0-\uDEC9\uDF30-\uDF3B]'
+    },
+    'P': {
+        bmp: '\x21-\x23\x25-\\x2A\x2C-\x2F\x3A\x3B\\x3F\x40\\x5B-\\x5D\x5F\\x7B\x7D\xA1\xA7\xAB\xB6\xB7\xBB\xBF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E42\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65',
+        astral: '\uD802[\uDC57\uDD1F\uDD3F\uDE50-\uDE58\uDE7F\uDEF0-\uDEF6\uDF39-\uDF3F\uDF99-\uDF9C]|\uD809[\uDC70-\uDC74]|\uD805[\uDCC6\uDDC1-\uDDD7\uDE41-\uDE43\uDF3C-\uDF3E]|\uD836[\uDE87-\uDE8B]|\uD801\uDD6F|\uD82F\uDC9F|\uD804[\uDC47-\uDC4D\uDCBB\uDCBC\uDCBE-\uDCC1\uDD40-\uDD43\uDD74\uDD75\uDDC5-\uDDC9\uDDCD\uDDDB\uDDDD-\uDDDF\uDE38-\uDE3D\uDEA9]|\uD800[\uDD00-\uDD02\uDF9F\uDFD0]|\uD81A[\uDE6E\uDE6F\uDEF5\uDF37-\uDF3B\uDF44]'
+    },
+    'S': {
+        bmp: '\\x24\\x2B\x3C-\x3E\\x5E\x60\\x7C\x7E\xA2-\xA6\xA8\xA9\xAC\xAE-\xB1\xB4\xB8\xD7\xF7\u02C2-\u02C5\u02D2-\u02DF\u02E5-\u02EB\u02ED\u02EF-\u02FF\u0375\u0384\u0385\u03F6\u0482\u058D-\u058F\u0606-\u0608\u060B\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09F2\u09F3\u09FA\u09FB\u0AF1\u0B70\u0BF3-\u0BFA\u0C7F\u0D79\u0E3F\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u17DB\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u1FBD\u1FBF-\u1FC1\u1FCD-\u1FCF\u1FDD-\u1FDF\u1FED-\u1FEF\u1FFD\u1FFE\u2044\u2052\u207A-\u207C\u208A-\u208C\u20A0-\u20BE\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116-\u2118\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u2140-\u2144\u214A-\u214D\u214F\u218A\u218B\u2190-\u2307\u230C-\u2328\u232B-\u23FA\u2400-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u2767\u2794-\u27C4\u27C7-\u27E5\u27F0-\u2982\u2999-\u29D7\u29DC-\u29FB\u29FE-\u2B73\u2B76-\u2B95\u2B98-\u2BB9\u2BBD-\u2BC8\u2BCA-\u2BD1\u2BEC-\u2BEF\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u309B\u309C\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA700-\uA716\uA720\uA721\uA789\uA78A\uA828-\uA82B\uA836-\uA839\uAA77-\uAA79\uAB5B\uFB29\uFBB2-\uFBC1\uFDFC\uFDFD\uFE62\uFE64-\uFE66\uFE69\uFF04\uFF0B\uFF1C-\uFF1E\uFF3E\uFF40\uFF5C\uFF5E\uFFE0-\uFFE6\uFFE8-\uFFEE\uFFFC\uFFFD',
+        astral: '\uD83E[\uDC00-\uDC0B\uDC10-\uDC47\uDC50-\uDC59\uDC60-\uDC87\uDC90-\uDCAD\uDD10-\uDD18\uDD80-\uDD84\uDDC0]|\uD83C[\uDC00-\uDC2B\uDC30-\uDC93\uDCA0-\uDCAE\uDCB1-\uDCBF\uDCC1-\uDCCF\uDCD1-\uDCF5\uDD10-\uDD2E\uDD30-\uDD6B\uDD70-\uDD9A\uDDE6-\uDE02\uDE10-\uDE3A\uDE40-\uDE48\uDE50\uDE51\uDF00-\uDFFF]|\uD83D[\uDC00-\uDD79\uDD7B-\uDDA3\uDDA5-\uDED0\uDEE0-\uDEEC\uDEF0-\uDEF3\uDF00-\uDF73\uDF80-\uDFD4]|\uD835[\uDEC1\uDEDB\uDEFB\uDF15\uDF35\uDF4F\uDF6F\uDF89\uDFA9\uDFC3]|\uD800[\uDD37-\uDD3F\uDD79-\uDD89\uDD8C\uDD90-\uDD9B\uDDA0\uDDD0-\uDDFC]|\uD82F\uDC9C|\uD805\uDF3F|\uD802[\uDC77\uDC78\uDEC8]|\uD81A[\uDF3C-\uDF3F\uDF45]|\uD836[\uDC00-\uDDFF\uDE37-\uDE3A\uDE6D-\uDE74\uDE76-\uDE83\uDE85\uDE86]|\uD834[\uDC00-\uDCF5\uDD00-\uDD26\uDD29-\uDD64\uDD6A-\uDD6C\uDD83\uDD84\uDD8C-\uDDA9\uDDAE-\uDDE8\uDE00-\uDE41\uDE45\uDF00-\uDF56]|\uD83B[\uDEF0\uDEF1]'
+    },
+    'Z': {
+        bmp: '\x20\xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000'
+    },
+    'Ahom': {
+        astral: '\uD805[\uDF00-\uDF19\uDF1D-\uDF2B\uDF30-\uDF3F]'
+    },
+    'Anatolian_Hieroglyphs' :{
+        astral: '\uD811[\uDC00-\uDE46]'
+    },
+    'Arabic': {
+        bmp: '\u0600-\u0604\u0606-\u060B\u060D-\u061A\u061E\u0620-\u063F\u0641-\u064A\u0656-\u066F\u0671-\u06DC\u06DE-\u06FF\u0750-\u077F\u08A0-\u08B4\u08E3-\u08FF\uFB50-\uFBC1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFD\uFE70-\uFE74\uFE76-\uFEFC',
+        astral: '\uD803[\uDE60-\uDE7E]|\uD83B[\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB\uDEF0\uDEF1]'
+    },
+    'Armenian': {
+        bmp: '\u0531-\u0556\u0559-\u055F\u0561-\u0587\u058A\u058D-\u058F\uFB13-\uFB17'
+    },
+    'Avestan': {
+        astral: '\uD802[\uDF00-\uDF35\uDF39-\uDF3F]'
+    },
+    'Balinese': {
+        bmp: '\u1B00-\u1B4B\u1B50-\u1B7C'
+    },
+    'Bamum': {
+        bmp: '\uA6A0-\uA6F7',
+        astral: '\uD81A[\uDC00-\uDE38]'
+    },
+    'Bassa_Vah': {
+        astral: '\uD81A[\uDED0-\uDEED\uDEF0-\uDEF5]'
+    },
+    'Batak': {
+        bmp: '\u1BC0-\u1BF3\u1BFC-\u1BFF'
+    },
+    'Bengali': {
+        bmp: '\u0980-\u0983\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BC-\u09C4\u09C7\u09C8\u09CB-\u09CE\u09D7\u09DC\u09DD\u09DF-\u09E3\u09E6-\u09FB'
+    },
+    'Bopomofo': {
+        bmp: '\u02EA\u02EB\u3105-\u312D\u31A0-\u31BA'
+    },
+    'Brahmi': {
+        astral: '\uD804[\uDC00-\uDC4D\uDC52-\uDC6F\uDC7F]'
+    },
+    'Braille': {
+        bmp: '\u2800-\u28FF'
+    },
+    'Buginese': {
+        bmp: '\u1A00-\u1A1B\u1A1E\u1A1F'
+    },
+    'Buhid': {
+        bmp: '\u1740-\u1753'
+    },
+    'Canadian_Aboriginal': {
+        bmp: '\u1400-\u167F\u18B0-\u18F5'
+    },
+    'Carian': {
+        astral: '\uD800[\uDEA0-\uDED0]'
+    },
+    'Caucasian_Albanian': {
+        astral: '\uD801[\uDD30-\uDD63\uDD6F]'
+    },
+    'Chakma': {
+        astral: '\uD804[\uDD00-\uDD34\uDD36-\uDD43]'
+    },
+    'Cham': {
+        bmp: '\uAA00-\uAA36\uAA40-\uAA4D\uAA50-\uAA59\uAA5C-\uAA5F'
+    },
+    'Cherokee': {
+        bmp: '\u13A0-\u13F5\u13F8-\u13FD\uAB70-\uABBF'
+    },
+    'Common': {
+        bmp: '\0-\x40\\x5B-\x60\\x7B-\xA9\xAB-\xB9\xBB-\xBF\xD7\xF7\u02B9-\u02DF\u02E5-\u02E9\u02EC-\u02FF\u0374\u037E\u0385\u0387\u0589\u0605\u060C\u061B\u061C\u061F\u0640\u06DD\u0964\u0965\u0E3F\u0FD5-\u0FD8\u10FB\u16EB-\u16ED\u1735\u1736\u1802\u1803\u1805\u1CD3\u1CE1\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u2000-\u200B\u200E-\u2064\u2066-\u2070\u2074-\u207E\u2080-\u208E\u20A0-\u20BE\u2100-\u2125\u2127-\u2129\u212C-\u2131\u2133-\u214D\u214F-\u215F\u2189-\u218B\u2190-\u23FA\u2400-\u2426\u2440-\u244A\u2460-\u27FF\u2900-\u2B73\u2B76-\u2B95\u2B98-\u2BB9\u2BBD-\u2BC8\u2BCA-\u2BD1\u2BEC-\u2BEF\u2E00-\u2E42\u2FF0-\u2FFB\u3000-\u3004\u3006\u3008-\u3020\u3030-\u3037\u303C-\u303F\u309B\u309C\u30A0\u30FB\u30FC\u3190-\u319F\u31C0-\u31E3\u3220-\u325F\u327F-\u32CF\u3358-\u33FF\u4DC0-\u4DFF\uA700-\uA721\uA788-\uA78A\uA830-\uA839\uA92E\uA9CF\uAB5B\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE66\uFE68-\uFE6B\uFEFF\uFF01-\uFF20\uFF3B-\uFF40\uFF5B-\uFF65\uFF70\uFF9E\uFF9F\uFFE0-\uFFE6\uFFE8-\uFFEE\uFFF9-\uFFFD',
+        astral: '\uD83E[\uDC00-\uDC0B\uDC10-\uDC47\uDC50-\uDC59\uDC60-\uDC87\uDC90-\uDCAD\uDD10-\uDD18\uDD80-\uDD84\uDDC0]|\uD82F[\uDCA0-\uDCA3]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDFCB\uDFCE-\uDFFF]|\uDB40[\uDC01\uDC20-\uDC7F]|\uD83D[\uDC00-\uDD79\uDD7B-\uDDA3\uDDA5-\uDED0\uDEE0-\uDEEC\uDEF0-\uDEF3\uDF00-\uDF73\uDF80-\uDFD4]|\uD800[\uDD00-\uDD02\uDD07-\uDD33\uDD37-\uDD3F\uDD90-\uDD9B\uDDD0-\uDDFC\uDEE1-\uDEFB]|\uD834[\uDC00-\uDCF5\uDD00-\uDD26\uDD29-\uDD66\uDD6A-\uDD7A\uDD83\uDD84\uDD8C-\uDDA9\uDDAE-\uDDE8\uDF00-\uDF56\uDF60-\uDF71]|\uD83C[\uDC00-\uDC2B\uDC30-\uDC93\uDCA0-\uDCAE\uDCB1-\uDCBF\uDCC1-\uDCCF\uDCD1-\uDCF5\uDD00-\uDD0C\uDD10-\uDD2E\uDD30-\uDD6B\uDD70-\uDD9A\uDDE6-\uDDFF\uDE01\uDE02\uDE10-\uDE3A\uDE40-\uDE48\uDE50\uDE51\uDF00-\uDFFF]'
+    },
+    'Coptic': {
+        bmp: '\u03E2-\u03EF\u2C80-\u2CF3\u2CF9-\u2CFF'
+    },
+    'Cuneiform': {
+        astral: '\uD809[\uDC00-\uDC6E\uDC70-\uDC74\uDC80-\uDD43]|\uD808[\uDC00-\uDF99]'
+    },
+    'Cypriot': {
+        astral: '\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F]'
+    },
+    'Cyrillic': {
+        bmp: '\u0400-\u0484\u0487-\u052F\u1D2B\u1D78\u2DE0-\u2DFF\uA640-\uA69F\uFE2E\uFE2F'
+    },
+    'Deseret': {
+        astral: '\uD801[\uDC00-\uDC4F]'
+    },
+    'Devanagari': {
+        bmp: '\u0900-\u0950\u0953-\u0963\u0966-\u097F\uA8E0-\uA8FD'
+    },
+    'Duployan': {
+        astral: '\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99\uDC9C-\uDC9F]'
+    },
+    'Egyptian_Hieroglyphs': {
+        astral: '\uD80C[\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2E]'
+    },
+    'Elbasan': {
+        astral: '\uD801[\uDD00-\uDD27]'
+    },
+    'Ethiopic': {
+        bmp: '\u1200-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u135D-\u137C\u1380-\u1399\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E'
+    },
+    'Georgian': {
+        bmp: '\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u10FF\u2D00-\u2D25\u2D27\u2D2D'
+    },
+    'Glagolitic': {
+        bmp: '\u2C00-\u2C2E\u2C30-\u2C5E'
+    },
+    'Gothic': {
+        astral: '\uD800[\uDF30-\uDF4A]'
+    },
+    'Grantha': {
+        astral: '\uD804[\uDF00-\uDF03\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3C-\uDF44\uDF47\uDF48\uDF4B-\uDF4D\uDF50\uDF57\uDF5D-\uDF63\uDF66-\uDF6C\uDF70-\uDF74]'
+    },
+    'Greek': {
+        bmp: '\u0370-\u0373\u0375-\u0377\u037A-\u037D\u037F\u0384\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03E1\u03F0-\u03FF\u1D26-\u1D2A\u1D5D-\u1D61\u1D66-\u1D6A\u1DBF\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FC4\u1FC6-\u1FD3\u1FD6-\u1FDB\u1FDD-\u1FEF\u1FF2-\u1FF4\u1FF6-\u1FFE\u2126\uAB65',
+        astral: '\uD800[\uDD40-\uDD8C\uDDA0]|\uD834[\uDE00-\uDE45]'
+    },
+    'Gujarati': {
+        bmp: '\u0A81-\u0A83\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABC-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AD0\u0AE0-\u0AE3\u0AE6-\u0AF1\u0AF9'
+    },
+    'Gurmukhi': {
+        bmp: '\u0A01-\u0A03\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A3C\u0A3E-\u0A42\u0A47\u0A48\u0A4B-\u0A4D\u0A51\u0A59-\u0A5C\u0A5E\u0A66-\u0A75'
+    },
+    'Han': {
+        bmp: '\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u3005\u3007\u3021-\u3029\u3038-\u303B\u3400-\u4DB5\u4E00-\u9FD5\uF900-\uFA6D\uFA70-\uFAD9',
+        astral: '\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|[\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872][\uDC00-\uDFFF]|\uD86D[\uDC00-\uDF34\uDF40-\uDFFF]|\uD87E[\uDC00-\uDE1D]|\uD869[\uDC00-\uDED6\uDF00-\uDFFF]|\uD873[\uDC00-\uDEA1]'
+    },
+    'Hangul': {
+        bmp: '\u1100-\u11FF\u302E\u302F\u3131-\u318E\u3200-\u321E\u3260-\u327E\uA960-\uA97C\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uFFA0-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC'
+    },
+    'Hanunoo': {
+        bmp: '\u1720-\u1734'
+    },
+    'Hatran': {
+        astral: '\uD802[\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDCFF]'
+    },
+    'Hebrew': {
+        bmp: '\u0591-\u05C7\u05D0-\u05EA\u05F0-\u05F4\uFB1D-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFB4F'
+    },
+    'Hiragana': {
+        bmp: '\u3041-\u3096\u309D-\u309F',
+        astral: '\uD82C\uDC01|\uD83C\uDE00'
+    },
+    'Imperial_Aramaic': {
+        astral: '\uD802[\uDC40-\uDC55\uDC57-\uDC5F]'
+    },
+    'Inherited': {
+        bmp: '\u0300-\u036F\u0485\u0486\u064B-\u0655\u0670\u0951\u0952\u1AB0-\u1ABE\u1CD0-\u1CD2\u1CD4-\u1CE0\u1CE2-\u1CE8\u1CED\u1CF4\u1CF8\u1CF9\u1DC0-\u1DF5\u1DFC-\u1DFF\u200C\u200D\u20D0-\u20F0\u302A-\u302D\u3099\u309A\uFE00-\uFE0F\uFE20-\uFE2D',
+        astral: '\uD834[\uDD67-\uDD69\uDD7B-\uDD82\uDD85-\uDD8B\uDDAA-\uDDAD]|\uD800[\uDDFD\uDEE0]|\uDB40[\uDD00-\uDDEF]'
+    },
+    'Inscriptional_Pahlavi': {
+        astral: '\uD802[\uDF60-\uDF72\uDF78-\uDF7F]'
+    },
+    'Inscriptional_Parthian': {
+        astral: '\uD802[\uDF40-\uDF55\uDF58-\uDF5F]'
+    },
+    'Javanese': {
+        bmp: '\uA980-\uA9CD\uA9D0-\uA9D9\uA9DE\uA9DF'
+    },
+    'Kaithi': {
+        astral: '\uD804[\uDC80-\uDCC1]'
+    },
+    'Kannada': {
+        bmp: '\u0C81-\u0C83\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBC-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5\u0CD6\u0CDE\u0CE0-\u0CE3\u0CE6-\u0CEF\u0CF1\u0CF2'
+    },
+    'Katakana': {
+        bmp: '\u30A1-\u30FA\u30FD-\u30FF\u31F0-\u31FF\u32D0-\u32FE\u3300-\u3357\uFF66-\uFF6F\uFF71-\uFF9D',
+        astral: '\uD82C\uDC00'
+    },
+    'Kayah_Li': {
+        bmp: '\uA900-\uA92D\uA92F'
+    },
+    'Kharoshthi': {
+        astral: '\uD802[\uDE00-\uDE03\uDE05\uDE06\uDE0C-\uDE13\uDE15-\uDE17\uDE19-\uDE33\uDE38-\uDE3A\uDE3F-\uDE47\uDE50-\uDE58]'
+    },
+    'Khmer': {
+        bmp: '\u1780-\u17DD\u17E0-\u17E9\u17F0-\u17F9\u19E0-\u19FF'
+    },
+    'Khojki': {
+        astral: '\uD804[\uDE00-\uDE11\uDE13-\uDE3D]'
+    },
+    'Khudawadi': {
+        astral: '\uD804[\uDEB0-\uDEEA\uDEF0-\uDEF9]'
+    },
+    'Lao': {
+        bmp: '\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECD\u0ED0-\u0ED9\u0EDC-\u0EDF'
+    },
+    'Latin': {
+        bmp: 'A-Za-z\xAA\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02B8\u02E0-\u02E4\u1D00-\u1D25\u1D2C-\u1D5C\u1D62-\u1D65\u1D6B-\u1D77\u1D79-\u1DBE\u1E00-\u1EFF\u2071\u207F\u2090-\u209C\u212A\u212B\u2132\u214E\u2160-\u2188\u2C60-\u2C7F\uA722-\uA787\uA78B-\uA7AD\uA7B0-\uA7B7\uA7F7-\uA7FF\uAB30-\uAB5A\uAB5C-\uAB64\uFB00-\uFB06\uFF21-\uFF3A\uFF41-\uFF5A'
+    },
+    'Lepcha': {
+        bmp: '\u1C00-\u1C37\u1C3B-\u1C49\u1C4D-\u1C4F'
+    },
+    'Limbu': {
+        bmp: '\u1900-\u191E\u1920-\u192B\u1930-\u193B\u1940\u1944-\u194F'
+    },
+    'Linear_A': {
+        astral: '\uD801[\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67]'
+    },
+    'Linear_B': {
+        astral: '\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA]'
+    },
+    'Lisu': {
+        bmp: '\uA4D0-\uA4FF'
+    },
+    'Lycian': {
+        astral: '\uD800[\uDE80-\uDE9C]'
+    },
+    'Lydian': {
+        astral: '\uD802[\uDD20-\uDD39\uDD3F]'
+    },
+    'Mahajani': {
+        astral: '\uD804[\uDD50-\uDD76]'
+    },
+    'Malayalam': {
+        bmp: '\u0D01-\u0D03\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D-\u0D44\u0D46-\u0D48\u0D4A-\u0D4E\u0D57\u0D5F-\u0D63\u0D66-\u0D75\u0D79-\u0D7F'
+    },
+    'Mandaic': {
+        bmp: '\u0840-\u085B\u085E'
+    },
+    'Manichaean': {
+        astral: '\uD802[\uDEC0-\uDEE6\uDEEB-\uDEF6]'
+    },
+    'Meetei_Mayek': {
+        bmp: '\uAAE0-\uAAF6\uABC0-\uABED\uABF0-\uABF9'
+    },
+    'Mende_Kikakui': {
+        astral: '\uD83A[\uDC00-\uDCC4\uDCC7-\uDCD6]'
+    },
+    'Meroitic_Cursive': {
+        astral: '\uD802[\uDDA0-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDDFF]'
+    },
+    'Meroitic_Hieroglyphs': {
+        astral: '\uD802[\uDD80-\uDD9F]'
+    },
+    'Miao': {
+        astral: '\uD81B[\uDF00-\uDF44\uDF50-\uDF7E\uDF8F-\uDF9F]'
+    },
+    'Modi': {
+        astral: '\uD805[\uDE00-\uDE44\uDE50-\uDE59]'
+    },
+    'Mongolian': {
+        bmp: '\u1800\u1801\u1804\u1806-\u180E\u1810-\u1819\u1820-\u1877\u1880-\u18AA'
+    },
+    'Mro': {
+        astral: '\uD81A[\uDE40-\uDE5E\uDE60-\uDE69\uDE6E\uDE6F]'
+    },
+    'Multani': {
+        astral: '\uD804[\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA9]'
+    },
+    'Myanmar': {
+        bmp: '\u1000-\u109F\uA9E0-\uA9FE\uAA60-\uAA7F'
+    },
+    'Nabataean': {
+        astral: '\uD802[\uDC80-\uDC9E\uDCA7-\uDCAF]'
+    },
+    'New_Tai_Lue': {
+        bmp: '\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u19DE\u19DF'
+    },
+    'Nko': {
+        bmp: '\u07C0-\u07FA'
+    },
+    'Ogham': {
+        bmp: '\u1680-\u169C'
+    },
+    'Ol_Chiki': {
+        bmp: '\u1C50-\u1C7F'
+    },
+    'Old_Hungarian': {
+        astral: '\uD803[\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDCFF]'
+    },
+    'Old_Italic': {
+        astral: '\uD800[\uDF00-\uDF23]'
+    },
+    'Old_North_Arabian': {
+        astral: '\uD802[\uDE80-\uDE9F]'
+    },
+    'Old_Permic': {
+        astral: '\uD800[\uDF50-\uDF7A]'
+    },
+    'Old_Persian': {
+        astral: '\uD800[\uDFA0-\uDFC3\uDFC8-\uDFD5]'
+    },
+    'Old_South_Arabian': {
+        astral: '\uD802[\uDE60-\uDE7F]'
+    },
+    'Old_Turkic': {
+        astral: '\uD803[\uDC00-\uDC48]'
+    },
+    'Oriya': {
+        bmp: '\u0B01-\u0B03\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3C-\u0B44\u0B47\u0B48\u0B4B-\u0B4D\u0B56\u0B57\u0B5C\u0B5D\u0B5F-\u0B63\u0B66-\u0B77'
+    },
+    'Osmanya': {
+        astral: '\uD801[\uDC80-\uDC9D\uDCA0-\uDCA9]'
+    },
+    'Pahawh_Hmong': {
+        astral: '\uD81A[\uDF00-\uDF45\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]'
+    },
+    'Palmyrene': {
+        astral: '\uD802[\uDC60-\uDC7F]'
+    },
+    'Pau_Cin_Hau': {
+        astral: '\uD806[\uDEC0-\uDEF8]'
+    },
+    'Phags_Pa': {
+        bmp: '\uA840-\uA877'
+    },
+    'Phoenician': {
+        astral: '\uD802[\uDD00-\uDD1B\uDD1F]'
+    },
+    'Psalter_Pahlavi': {
+        astral: '\uD802[\uDF80-\uDF91\uDF99-\uDF9C\uDFA9-\uDFAF]'
+    },
+    'Rejang': {
+        bmp: '\uA930-\uA953\uA95F'
+    },
+    'Runic': {
+        bmp: '\u16A0-\u16EA\u16EE-\u16F8'
+    },
+    'Samaritan': {
+        bmp: '\u0800-\u082D\u0830-\u083E'
+    },
+    'Saurashtra': {
+        bmp: '\uA880-\uA8C4\uA8CE-\uA8D9'
+    },
+    'Sharada': {
+        astral: '\uD804[\uDD80-\uDDCD\uDDD0-\uDDDF]'
+    },
+    'Shavian': {
+        astral: '\uD801[\uDC50-\uDC7F]'
+    },
+    'Siddham': {
+        astral: '\uD805[\uDD80-\uDDB5\uDDB8-\uDDDD]'
+    },
+    'SignWriting': {
+        astral: '\uD836[\uDC00-\uDE8B\uDE9B-\uDE9F\uDEA1-\uDEAF]'
+    },
+    'Sinhala': {
+        bmp: '\u0D82\u0D83\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DE6-\u0DEF\u0DF2-\u0DF4',
+        astral: '\uD804[\uDDE1-\uDDF4]'
+    },
+    'Sora_Sompeng': {
+        astral: '\uD804[\uDCD0-\uDCE8\uDCF0-\uDCF9]'
+    },
+    'Sundanese': {
+        bmp: '\u1B80-\u1BBF\u1CC0-\u1CC7'
+    },
+    'Syloti_Nagri': {
+        bmp: '\uA800-\uA82B'
+    },
+    'Syriac': {
+        bmp: '\u0700-\u070D\u070F-\u074A\u074D-\u074F'
+    },
+    'Tagalog': {
+        bmp: '\u1700-\u170C\u170E-\u1714'
+    },
+    'Tagbanwa': {
+        bmp: '\u1760-\u176C\u176E-\u1770\u1772\u1773'
+    },
+    'Tai_Le': {
+        bmp: '\u1950-\u196D\u1970-\u1974'
+    },
+    'Tai_Tham': {
+        bmp: '\u1A20-\u1A5E\u1A60-\u1A7C\u1A7F-\u1A89\u1A90-\u1A99\u1AA0-\u1AAD'
+    },
+    'Tai_Viet': {
+        bmp: '\uAA80-\uAAC2\uAADB-\uAADF'
+    },
+    'Takri': {
+        astral: '\uD805[\uDE80-\uDEB7\uDEC0-\uDEC9]'
+    },
+    'Tamil': {
+        bmp: '\u0B82\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD0\u0BD7\u0BE6-\u0BFA'
+    },
+    'Telugu': {
+        bmp: '\u0C00-\u0C03\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55\u0C56\u0C58-\u0C5A\u0C60-\u0C63\u0C66-\u0C6F\u0C78-\u0C7F'
+    },
+    'Thaana': {
+        bmp: '\u0780-\u07B1'
+    },
+    'Thai': {
+        bmp: '\u0E01-\u0E3A\u0E40-\u0E5B'
+    },
+    'Tibetan': {
+        bmp: '\u0F00-\u0F47\u0F49-\u0F6C\u0F71-\u0F97\u0F99-\u0FBC\u0FBE-\u0FCC\u0FCE-\u0FD4\u0FD9\u0FDA'
+    },
+    'Tifinagh': {
+        bmp: '\u2D30-\u2D67\u2D6F\u2D70\u2D7F'
+    },
+    'Tirhuta': {
+        astral: '\uD805[\uDC80-\uDCC7\uDCD0-\uDCD9]'
+    },
+    'Ugaritic': {
+        astral: '\uD800[\uDF80-\uDF9D\uDF9F]'
+    },
+    'Vai': {
+        bmp: '\uA500-\uA62B'
+    },
+    'Warang_Citi': {
+        astral: '\uD806[\uDCA0-\uDCF2\uDCFF]'
+    },
+    'Yi': {
+        bmp: '\uA000-\uA48C\uA490-\uA4C6'
+    }
+};
+
+const isValidString = (s) => typeof s == 'string' && s.length > 0;
+const buildRegexp = (re) => new RegExp(re.replace(/\\p\{([^}]+)\}/g, function(a, b) {
+    if (!unicode[b]) throw Error('Invalid expression');
+    var i = unicode[b], c = '';
+    if (i.bmp) {
+        c = '[' + i.bmp + ']' + (i.astral ? '|' : '');
+    }
+    if (i.astral) {
+        c += i.astral;
+    }
+    return '(?:' + c + ')';
+}));
+
+module.exports = {
+
+    unicodelettersonly: function(value, param) {
+        const arr = ['L'];
+        
+        for (let key in param) {
+            arr.push(key);
+        }
+
+        return !isValidString(value) || (buildRegexp('^(\\p{' + arr.join('}|\\p{') + '})+$')).test(value)
+    },
+
+    unicodescript: function(value, param) {
+        const arr = param.scripts.slice(0);
+        
+        for (let key in param) {
+            if (key != 'scripts') {
+                arr.push(key);
+            }
+        }
+
+        return !isValidString(value) || (buildRegexp('^(\\p{' + arr.join('}|\\p{') + '})+$')).test(value)
+    },
+
+};

lib/validator/upload.js

@@ -0,0 +1,82 @@
+// IMPROVE: Improve this with the actual file field being checked, for now we check all uploaded files
+function getFiles(app, value) {
+    const files = [];
+
+    if (app.req.files) {
+        for (let field in app.req.files) {
+            if (Array.isArray(app.req.files[field])) {
+                for (let file of app.req.files[field]) {
+                    if (!file.truncated) {
+                        files.push(file);
+                    }
+                }
+            } else {
+                let file = app.req.files[field];
+                if (!file.truncated) {
+                    files.push(file);
+                }
+            }
+        }
+    }
+
+    return files;
+}
+
+module.exports = {
+
+    accept: function(value, param) {
+        const files = getFiles(this, value);
+        const allowed = param.replace(/\s/g, '').split(',');
+
+        if (!files.length) {
+            return true;
+        }
+
+        for (let file of files) {
+            if (!allowed.some(allow => {
+                if (allow[0] == '.') {
+                    const re = new RegExp(`\\${allow}$`, 'i');
+                    if (re.test(file.name)) {
+                        return true;
+                    }
+                } else if (/(audio|video|image)\/\*/i.test(allow)) {
+                    const re = new RegExp(`^${allow.replace('*', '.*')}$`, 'i');
+                    if (re.test(file.mimetype)) {
+                        return true;
+                    }
+                } else if (allow.toLowerCase() == file.mimetype.toLowerCase()) {
+                    return true;
+                }
+            })) {
+                return false;
+            }
+        }
+
+        return true;
+    },
+
+    minsize: function(value, param) {
+        return !getFiles(this, value).some(file => file.size < param);
+    },
+
+    maxsize: function(value, param) {
+        return !getFiles(this, value).some(file => file.size > param);
+    },
+
+    mintotalsize: function(value, param) {
+        return getFiles(this, value).reduce((size, file) => size + file.size, 0) >= param;
+    },
+    
+    maxtotalsize: function(value, param) {
+        return getFiles(this, value).reduce((size, file) => size + file.size, 0) <= param;
+    },
+
+    minfiles: function(value, param) {
+        return getFiles(this, value).length >= param;
+    },
+
+    maxfiles: function(value, param) {
+        return getFiles(this, value).length <= param;
+    },
+
+};

lib/webhooks/stripe.js

@@ -0,0 +1,22 @@
+const webhook = require('../core/webhook');
+const config = require('../setup/config');
+const fs = require('fs-extra')
+
+if (fs.existsSync('app/webhooks/stripe')) {
+    const stripe = require('stripe')(config.stripe.secretKey);
+    const endpointSecret = config.stripe.endpointSecret;
+
+    exports.handler = webhook.createHandler('stripe', (req, res, next) => {
+        const sig = req.headers['stripe-signature'];
+
+        try {
+            stripe.webhooks.constructEvent(req.rawBody, sig, endpointSecret);
+        } catch (err) {
+            res.status(400).send(`Webhook Error: ${err.message}`);
+            return false;
+        }
+
+        // return the action name to execute
+        return req.body.type;
+    });
+}

package.json

@@ -0,0 +1,52 @@
+{
+  "name": "ertfast_tiller",
+  "version": "2.0.0",
+  "private": true,
+  "description": "",
+  "main": "index.js",
+  "engines": {
+    "node": ">=16.9.0"
+  },
+  "scripts": {
+    "start": "node ./index.js"
+  },
+  "author": "Wappler",
+  "license": "ISC",
+  "dependencies": {
+    "archiver": "^7.0.1",
+    "compression": "^1.7.4",
+    "connect-session-knex": "^4.0.0",
+    "cookie-parser": "^1.4.6",
+    "cors": "^2.8.5",
+    "debug": "^4.3.2",
+    "dotenv": "^16.0.3",
+    "ejs": "^3.1.6",
+    "express": "^4.17.1",
+    "express-end": "0.0.8",
+    "express-fileupload": "^1.2.1",
+    "express-session": "^1.17.2",
+    "follow-redirects": "^1.14.5",
+    "fs-extra": "^11.2.0",
+    "http-proxy": "^1.18.1",
+    "knex": "^3.0.1",
+    "mime-types": "^2.1.34",
+    "node-schedule": "^2.0.0",
+    "nodemon": "^3.0.1",
+    "pdf-lib": "^1.17.1",
+    "qs": "^6.10.1",
+    "session-file-store": "^1.5.0",
+    "socket.io": "^4.7.5",
+    "unzipper": "^0.12.1",
+    "uuid": "^10.0.0"
+  },
+  "nodemonConfig": {
+    "watch": [
+      "app",
+      "lib",
+      "views",
+      "extensions",
+      "tmp/**/restart.txt"
+    ],
+    "ext": "ejs,js,json"
+  }
+}