TEST: Started working with S3 buckets for file storage

app/api/s3control.json

@@ -0,0 +1,61 @@
+[
+  {
+    "name": "listFiles",
+    "module": "s3",
+    "action": "listBuckets",
+    "options": {
+      "provider": "s3"
+    },
+    "output": true,
+    "outputType": "object",
+    "meta": [
+      {
+        "name": "Buckets",
+        "type": "array",
+        "sub": [
+          {
+            "name": "Name",
+            "type": "text"
+          },
+          {
+            "name": "CreationDate",
+            "type": "date"
+          }
+        ]
+      },
+      {
+        "name": "Owner",
+        "type": "object",
+        "sub": [
+          {
+            "name": "DisplayName",
+            "type": "text"
+          },
+          {
+            "name": "ID",
+            "type": "text"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "name": "s3b",
+    "module": "core",
+    "action": "setvalue",
+    "options": {
+      "key": "bucket",
+      "value": "{{listFiles.Buckets[0].Name}}"
+    },
+    "meta": [],
+    "outputType": "text"
+  },
+  {
+    "name": "api",
+    "module": "api",
+    "action": "send",
+    "options": {},
+    "output": true,
+    "collapsed": true
+  }
+]

lib/core/app.js

@@ -50,6 +50,7 @@ function App(req = {}, res = {}) {
 
   this.set({
     $_ERROR: null,
+    $_EXCEPTION: null,
     //$_SERVER: process.env,
     $_ENV: process.env,
     $_GET: req.query,
@@ -497,6 +498,8 @@ App.prototype = {
   },
 
   getDbConnection: function (name) {
+    if (this.trx[name]) return this.trx[name];
+
     if (config.db[name]) {
       return this.setDbConnection(name, config.db[name]);
     }
@@ -516,8 +519,6 @@ App.prototype = {
       name = JSON.stringify(this.parse(options));
     }
 
-    if (this.trx[name]) return this.trx[name];
-
     return this.setDbConnection(name, options);
   },
 
@@ -654,7 +655,8 @@ App.prototype = {
 
     if (this.error !== false) {
       if (actions.catch) {
-        this.scope.set('$_ERROR', this.error.message);
+        this.scope.set('$_ERROR', this.error.message || this.error);
+        this.scope.set('$_EXCEPTION', this.error);
         this.error = false;
         await this._exec(actions.catch, true);
       } else {

lib/errors/httpError.js

@@ -0,0 +1,17 @@
+class HttpError extends Error {
+  name = 'HttpError';
+  url = "";
+  status = 0;
+  statusText = "";
+  body = "";
+
+  constructor(options) {
+    super(`Request "${options.url || ''}" responded with ${options.status || ''} ${options.statusText || ''}`);
+    this.name = "HttpError";
+    this.status = options.status;
+    this.statusText = options.statusText;
+    this.body = options.body;
+  }
+}
+
+module.exports = HttpError;

lib/modules/api.js

@@ -2,6 +2,7 @@ const { http, https } = require('follow-redirects');
 const querystring = require('querystring');
 const zlib = require('zlib');
 const pkg = require('../../package.json');
+const HttpError = require('../errors/httpError');
 
 module.exports = {
 
@@ -68,6 +69,10 @@ module.exports = {
             const req = (Url.protocol == 'https:' ? https : http).request(Url, opts, res => {
                 let body = '';
 
+                if (res.statusCode == 204 || res.headers['content-length'] == 0) {
+                    return resolve({ status: res.statusCode, headers: res.headers, data: '' });
+                }
+
                 let output = res;
                 if (res.headers['content-encoding'] == 'br') {
                     output = res.pipe(zlib.createBrotliDecompress());
@@ -82,11 +87,7 @@ module.exports = {
                 output.setEncoding('utf8');
                 output.on('data', chunk => body += chunk);
                 output.on('end', () => {
-                    if (res.statusCode >= 400) {
+                    if (passErrors && res.statusCode >= 400) {
-                        if (throwErrors) {
-                            return reject(res.statusCode + ' ' + body);
-                        }
-
                         if (passErrors) {
                             this.res.status(res.statusCode).send(body);
                             return resolve();
@@ -105,6 +106,15 @@ module.exports = {
                         }
                     }
 
+                    if (throwErrors && res.statusCode >= 400) {
+                        return reject(new HttpError({
+                            url: url,
+                            status: res.statusCode,
+                            statusText: res.statusMessage,
+                            body: body
+                        }));
+                    }
+
                     resolve({
                         status: res.statusCode,
                         headers: res.headers,

lib/modules/core.js

@@ -218,7 +218,8 @@ module.exports = {
         try {
             await this.exec(options.try, true);
         } catch (error) {
-            this.scope.set('$_ERROR', error.message);
+            this.scope.set('$_ERROR', error.message || error);
+            this.scope.set('$_EXCEPTION', error);
             this.error = false;
             if (options.catch) {
                 await this.exec(options.catch, true);

lib/modules/image.js

@@ -154,7 +154,7 @@ module.exports = {
         if (!sharp) throw new Error(`image.save: instance "${options.instance} doesn't exist.`);
 
         let path = toSystemPath(this.parseRequired(options.path, 'string', 'image.save: path is required.'));
-        let format = this.parseOptional(options.format, 'string', 'jpeg').toLowerCase();
+        let format = this.parseOptional(options.format, 'string', 'auto').toLowerCase();
         let template = this.parseOptional(options.template, 'string', '{name}{ext}');
         let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
         let createPath = this.parseOptional(options.createPath, 'boolean', true);

lib/server.js

@@ -7,8 +7,10 @@ process.on('uncaughtException', (e) => {
     console.error(e);
 });
 
+const debug = require('debug');
+debug.log = console.log.bind(console);
+
 const config = require('./setup/config');
-const debug = require('debug')('server-connect:server');
 const secure = require('./setup/secure');
 const routes = require('./setup/routes');
 const sockets = require('./setup/sockets');
@@ -26,6 +28,12 @@ app.set('trust proxy', true);
 app.set('view engine', 'ejs');
 app.set('view options', { root: 'views', async: true });
 
+app.set('json replacer', (key, value) => {
+    if (value instanceof Set) return [...value];
+    if (value instanceof Error) return value.toString();
+    return value;
+});
+
 app.disable('x-powered-by')
 
 if (config.compression) {
@@ -74,6 +82,7 @@ module.exports = {
             // if user has a custom 404 page, redirect to it
             if (req.accepts('html') && req.url != '/404' && app.get('has404')) {
                 //res.redirect(303, '/404');
+                res.status(404);
                 req.url = '/404';
                 app.handle(req, res);
             } else {
@@ -85,10 +94,11 @@ module.exports = {
         });
         
         app.use((err, req, res, next) => {
-            debug(`Got error? %O`, err);
+            console.error(err);
             // if user has a custom 500 page, redirect to it
             if (req.accepts('html') && req.url != '/500' && app.get('has500')) {
                 //res.redirect(303, '/500');
+                res.status(500);
                 req.url = '/500';
                 app.handle(req, res);
             } else {

lib/setup/redis.js

@@ -2,8 +2,46 @@ const config = require('./config');
 
 if (config.redis) {
     const Redis = require('ioredis');
-    //global.redisClient = redis.createClient(config.redis === true ? 'redis://redis' : config.redis);
+    const debug = require('debug')('redis');
-    global.redisClient = new Redis(config.redis === true ? 'redis://redis' : config.redis);
+
+    global.redisClient = new Redis(config.redis === true ? 'redis://redis' : config.redis, {
+        retryStrategy: function(times) {
+            var delay = Math.min(times * 50, 2000);
+            return delay;
+        },
+
+        reconnectOnError: function(err) {
+            if (err.message.includes('READONLY')) {
+                return true;
+            }
+            if (err.message.includes('ECONNRESET')) {
+                return true;
+            }
+            return false;
+        }
+    });
+
+    
+    global.redisClient.on('connect', () => {
+        debug('Redis connected successfully.');
+    });
+
+    global.redisClient.on('ready', () => {
+        debug('Redis is ready to use.');
+    });
+
+    global.redisClient.on('error', (err) => {
+        debug('Got a Redis error');
+        console.error(err);
+    });
+
+    global.redisClient.on('reconnecting', (delay) => {
+        debug(`Reconnecting to Redis in ${delay}ms...`);
+    });
+
+    global.redisClient.on('end', () => {
+        debug('Redis connection has been closed.');
+    });
 }
 
 module.exports = global.redisClient;

lib/setup/routes.js

@@ -263,14 +263,20 @@ module.exports = async function (app) {
 
                         app[isPrivate ? 'privateRateLimiter' : 'rateLimiter'].consume(key, points).then(rateLimiterRes => {
                             const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
-                            res.set('RateLimit-Policy', `${config.rateLimit.points};w=${config.rateLimit.duration}`);
+                            const { points, duration } = isPrivate ? config.rateLimit.private : config.rateLimit;
-                            res.set('RateLimit', `limit=${config.rateLimit.points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+
+                            res.set('RateLimit-Policy', `${points};w=${duration}`);
+                            res.set('RateLimit', `limit=${points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+
                             next();
                         }).catch(rateLimiterRes => {
                             const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
-                            res.set('RateLimit-Policy', `${config.rateLimit.points};w=${config.rateLimit.duration}`);
+                            const { points, duration } = isPrivate ? config.rateLimit.private : config.rateLimit;
-                            res.set('RateLimit', `limit=${config.rateLimit.points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+
+                            res.set('RateLimit-Policy', `${points};w=${duration}`);
+                            res.set('RateLimit', `limit=${points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
                             res.set('Retry-After', reset);
+
                             if (req.is('json')) {
                                 res.status(429).json({ error: 'Too Many Requests' });
                             } else {

views/buckets.ejs

@@ -1,4 +1,17 @@
-<!-- Wappler include head-page="layouts/main" fontawesome_5="cdn" bootstrap5="local" is="dmx-app" id="buckets" appConnect="local" -->
+<!-- Wappler include head-page="layouts/main" fontawesome_5="cdn" bootstrap5="local" is="dmx-app" id="buckets" appConnect="local" components="{dmxS3Upload:{},dmxNotifications:{},dmxBootstrap5TableGenerator:{}}" -->
+<div id="s3upload1" is="dmx-s3-upload" url="/api/s3control" accept="image/*" class="text-center border">
+    <p dmx-show="!file">Select file</p>
+    <p dmx-show="file">{{file.name}}</p>
+    <p dmx-hide="state.uploading">
+        <button class="btn btn-primary" dmx-on:click.stop="s3upload1.select()" dmx-show="state.idle">Browse</button>
+        <button class="btn btn-primary" dmx-on:click.stop="s3upload1.upload()" dmx-show="state.ready">Upload</button>
+        <button class="btn btn-danger" dmx-on:click.stop="s3upload1.reset()" dmx-show="state.done">Reset</button>
+    </p>
+    <p dmx-show="state.uploading">
+        Uploading {{uploadProgress.percent}}%
+        <button class="btn btn-danger" dmx-on:click.stop="s3upload1.abort()">Abort</button>
+    </p>
+</div>
 <div class="container wappler-block p-3">
 
     <div class="progress mb-5">
@@ -29,6 +42,54 @@
     </div>
     <button id="btn2" class="btn">Button</button>
 </div>
+<div class="container">
+    <table class="table">
+        <thead>
+            <tr>
+                <th>Name</th>
+                <th>Creation date</th>
+            </tr>
+        </thead>
+        <tbody is="dmx-repeat" dmx-generator="bs5table" dmx-bind:repeat="s3upload1.data.listFiles.Buckets" id="tableRepeat1">
+            <tr>
+                <td dmx-text="Name"></td>
+                <td dmx-text="CreationDate"></td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+<dmx-notifications id="notifies1"></dmx-notifications>
+<div class="container">
+    <script>
+        // An efficient JavaScript program to remove all
+      // spaces from a string
+      let str = '2JAxQ9pH Mhy87Sr5NCJ I Be9qj Nd p9eY3vW7BKqN Msxk='
+      // Function to remove all spaces
+      // from a given string
+      function removeSpaces(str) {
+        // To keep track of non-space
+        // character count
+        var count = 0;
+
+        // Traverse the given string. If current
+        // character is not space, then place
+        // it at index 'count++'
+        for (var i = 0; i < str.length; i++)
+          if (str[i] !== " ") str[count++] = str[i]; 
+         // here count is
+        // incremented
+
+        return count;
+      }
+
+      // Driver code
+      var str = "g eeks for ge eeks ".split("");
+      var i = removeSpaces(str);
+      document.write(str.join("").substring(0, i));
+      
+    </script>
+
+</div>
 
 <!-- DO00RXH3C276N9Y9PQBG -->
 <!-- lMI2BSf8dS+ZmWkyvsq9gTTjScr1SLEsd0OpsZZLAkc -->

views/index0.ejs

@@ -1,4 +1,17 @@
 <!-- Wappler include head-page="layouts/main" fontawesome_5="cdn" bootstrap5="local" is="dmx-app" id="index" appConnect="local" components="{dmxBootstrap5Navigation:{},dmxAnimateCSS:{},dmxStateManagement:{},dmxDatastore:{},dmxBootstrap5Modal:{},dmxFormatter:{},dmxBootstrap5TableGenerator:{},dmxBootstrap5Toasts:{},dmxBootbox5:{},dmxBrowser:{},dmxBootstrap5Tooltips:{},dmxValidator:{},dmxS3Upload:{},dmxDatePicker:{},dmxMasonry:{},dmxBootstrap5Popovers:{},dmxPouchDB:{},dmxLazyLoad:{}}" jquery_slim_35="cdn" moment_2="cdn" -->
+<div id="s3upload1" is="dmx-s3-upload" url="" accept="image/*" class="text-center border">
+  <p dmx-show="!file">Select file</p>
+  <p dmx-show="file">{{file.name}}</p>
+  <p dmx-hide="state.uploading">
+    <button class="btn btn-primary" dmx-on:click.stop="s3upload1.select()" dmx-show="state.idle">Browse</button>
+    <button class="btn btn-primary" dmx-on:click.stop="s3upload1.upload()" dmx-show="state.ready">Upload</button>
+    <button class="btn btn-danger" dmx-on:click.stop="s3upload1.reset()" dmx-show="state.done">Reset</button>
+  </p>
+  <p dmx-show="state.uploading">
+    Uploading {{uploadProgress.percent}}%
+    <button class="btn btn-danger" dmx-on:click.stop="s3upload1.abort()">Abort</button>
+  </p>
+</div>
 
 
 

views/layouts/main.ejs

@@ -77,6 +77,7 @@
   <script src="https://cdn.jsdelivr.net/npm/pouchdb@8.0.1/dist/pouchdb.min.js" defer></script>
   <script src="https://cdn.jsdelivr.net/npm/pouchdb@8.0.1/dist/pouchdb.find.min.js" defer></script>
   <script src="https://cdn.jsdelivr.net/npm/pouchdb@8.0.1/dist/pouchdb.indexeddb.min.js" defer></script>
+  <script src="/dmxAppConnect/dmxPouchDB/dmxPouchDB.js" defer></script>
 </head>
 
 <body is="dmx-app" id="main">