TEST: Started working with S3 buckets for file storage

2025-03-10 21:59:59 -05:00 · 2025-03-10 21:59:59 -05:00 · 66ad3a4b5d
parent c7933c2ec4
commit 66ad3a4b5d
18 changed files with 499 additions and 279 deletions
--- a/app/api/s3control.json
+++ b/app/api/s3control.json
@ -0,0 +1,61 @@
+[
+  {
+    "name": "listFiles",
+    "module": "s3",
+    "action": "listBuckets",
+    "options": {
+      "provider": "s3"
+    },
+    "output": true,
+    "outputType": "object",
+    "meta": [
+      {
+        "name": "Buckets",
+        "type": "array",
+        "sub": [
+          {
+            "name": "Name",
+            "type": "text"
+          },
+          {
+            "name": "CreationDate",
+            "type": "date"
+          }
+        ]
+      },
+      {
+        "name": "Owner",
+        "type": "object",
+        "sub": [
+          {
+            "name": "DisplayName",
+            "type": "text"
+          },
+          {
+            "name": "ID",
+            "type": "text"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "name": "s3b",
+    "module": "core",
+    "action": "setvalue",
+    "options": {
+      "key": "bucket",
+      "value": "{{listFiles.Buckets[0].Name}}"
+    },
+    "meta": [],
+    "outputType": "text"
+  },
+  {
+    "name": "api",
+    "module": "api",
+    "action": "send",
+    "options": {},
+    "output": true,
+    "collapsed": true
+  }
+]
--- a/lib/core/app.js
+++ b/lib/core/app.js
@ -50,6 +50,7 @@ function App(req = {}, res = {}) {

  this.set({
    $_ERROR: null,
+    $_EXCEPTION: null,
    //$_SERVER: process.env,
    $_ENV: process.env,
    $_GET: req.query,
@ -497,6 +498,8 @@ App.prototype = {
  },

  getDbConnection: function (name) {
+    if (this.trx[name]) return this.trx[name];
+
    if (config.db[name]) {
      return this.setDbConnection(name, config.db[name]);
    }
@ -516,8 +519,6 @@ App.prototype = {
      name = JSON.stringify(this.parse(options));
    }

-    if (this.trx[name]) return this.trx[name];
-
    return this.setDbConnection(name, options);
  },

@ -654,7 +655,8 @@ App.prototype = {

    if (this.error !== false) {
      if (actions.catch) {
-        this.scope.set('$_ERROR', this.error.message);
+        this.scope.set('$_ERROR', this.error.message || this.error);
+        this.scope.set('$_EXCEPTION', this.error);
        this.error = false;
        await this._exec(actions.catch, true);
      } else {
--- a/lib/errors/httpError.js
+++ b/lib/errors/httpError.js
@ -0,0 +1,17 @@
+class HttpError extends Error {
+  name = 'HttpError';
+  url = "";
+  status = 0;
+  statusText = "";
+  body = "";
+
+  constructor(options) {
+    super(`Request "${options.url || ''}" responded with ${options.status || ''} ${options.statusText || ''}`);
+    this.name = "HttpError";
+    this.status = options.status;
+    this.statusText = options.statusText;
+    this.body = options.body;
+  }
+}
+
+module.exports = HttpError;
--- a/lib/modules/api.js
+++ b/lib/modules/api.js
@ -2,6 +2,7 @@ const { http, https } = require('follow-redirects');
 const querystring = require('querystring');
 const zlib = require('zlib');
 const pkg = require('../../package.json');
+const HttpError = require('../errors/httpError');

 module.exports = {

@ -68,6 +69,10 @@ module.exports = {
            const req = (Url.protocol == 'https:' ? https : http).request(Url, opts, res => {
                let body = '';

+                if (res.statusCode == 204 || res.headers['content-length'] == 0) {
+                    return resolve({ status: res.statusCode, headers: res.headers, data: '' });
+                }
+
                let output = res;
                if (res.headers['content-encoding'] == 'br') {
                    output = res.pipe(zlib.createBrotliDecompress());
@ -82,11 +87,7 @@ module.exports = {
                output.setEncoding('utf8');
                output.on('data', chunk => body += chunk);
                output.on('end', () => {
-                    if (res.statusCode >= 400) {
-                        if (throwErrors) {
-                            return reject(res.statusCode + ' ' + body);
-                        }
-
+                    if (passErrors && res.statusCode >= 400) {
                        if (passErrors) {
                            this.res.status(res.statusCode).send(body);
                            return resolve();
@ -105,6 +106,15 @@ module.exports = {
                        }
                    }

+                    if (throwErrors && res.statusCode >= 400) {
+                        return reject(new HttpError({
+                            url: url,
+                            status: res.statusCode,
+                            statusText: res.statusMessage,
+                            body: body
+                        }));
+                    }
+
                    resolve({
                        status: res.statusCode,
                        headers: res.headers,
--- a/lib/modules/core.js
+++ b/lib/modules/core.js
@ -218,7 +218,8 @@ module.exports = {
        try {
            await this.exec(options.try, true);
        } catch (error) {
-            this.scope.set('$_ERROR', error.message);
+            this.scope.set('$_ERROR', error.message || error);
+            this.scope.set('$_EXCEPTION', error);
            this.error = false;
            if (options.catch) {
                await this.exec(options.catch, true);
--- a/lib/modules/image.js
+++ b/lib/modules/image.js
@ -154,7 +154,7 @@ module.exports = {
        if (!sharp) throw new Error(`image.save: instance "${options.instance} doesn't exist.`);

        let path = toSystemPath(this.parseRequired(options.path, 'string', 'image.save: path is required.'));
-        let format = this.parseOptional(options.format, 'string', 'jpeg').toLowerCase();
+        let format = this.parseOptional(options.format, 'string', 'auto').toLowerCase();
        let template = this.parseOptional(options.template, 'string', '{name}{ext}');
        let overwrite = this.parseOptional(options.overwrite, 'boolean', false);
        let createPath = this.parseOptional(options.createPath, 'boolean', true);
--- a/lib/server.js
+++ b/lib/server.js
@ -7,8 +7,10 @@ process.on('uncaughtException', (e) => {
    console.error(e);
 });

+const debug = require('debug');
+debug.log = console.log.bind(console);
+
 const config = require('./setup/config');
-const debug = require('debug')('server-connect:server');
 const secure = require('./setup/secure');
 const routes = require('./setup/routes');
 const sockets = require('./setup/sockets');
@ -26,6 +28,12 @@ app.set('trust proxy', true);
 app.set('view engine', 'ejs');
 app.set('view options', { root: 'views', async: true });

+app.set('json replacer', (key, value) => {
+    if (value instanceof Set) return [...value];
+    if (value instanceof Error) return value.toString();
+    return value;
+});
+
 app.disable('x-powered-by')

 if (config.compression) {
@ -74,6 +82,7 @@ module.exports = {
            // if user has a custom 404 page, redirect to it
            if (req.accepts('html') && req.url != '/404' && app.get('has404')) {
                //res.redirect(303, '/404');
+                res.status(404);
                req.url = '/404';
                app.handle(req, res);
            } else {
@ -85,10 +94,11 @@ module.exports = {
        });
        
        app.use((err, req, res, next) => {
-            debug(`Got error? %O`, err);
+            console.error(err);
            // if user has a custom 500 page, redirect to it
            if (req.accepts('html') && req.url != '/500' && app.get('has500')) {
                //res.redirect(303, '/500');
+                res.status(500);
                req.url = '/500';
                app.handle(req, res);
            } else {
--- a/lib/setup/redis.js
+++ b/lib/setup/redis.js
@ -2,8 +2,46 @@ const config = require('./config');

 if (config.redis) {
    const Redis = require('ioredis');
-    //global.redisClient = redis.createClient(config.redis === true ? 'redis://redis' : config.redis);
-    global.redisClient = new Redis(config.redis === true ? 'redis://redis' : config.redis);
+    const debug = require('debug')('redis');
+
+    global.redisClient = new Redis(config.redis === true ? 'redis://redis' : config.redis, {
+        retryStrategy: function(times) {
+            var delay = Math.min(times * 50, 2000);
+            return delay;
+        },
+
+        reconnectOnError: function(err) {
+            if (err.message.includes('READONLY')) {
+                return true;
+            }
+            if (err.message.includes('ECONNRESET')) {
+                return true;
+            }
+            return false;
+        }
+    });
+
+    
+    global.redisClient.on('connect', () => {
+        debug('Redis connected successfully.');
+    });
+
+    global.redisClient.on('ready', () => {
+        debug('Redis is ready to use.');
+    });
+
+    global.redisClient.on('error', (err) => {
+        debug('Got a Redis error');
+        console.error(err);
+    });
+
+    global.redisClient.on('reconnecting', (delay) => {
+        debug(`Reconnecting to Redis in ${delay}ms...`);
+    });
+
+    global.redisClient.on('end', () => {
+        debug('Redis connection has been closed.');
+    });
 }

 module.exports = global.redisClient;
--- a/lib/setup/routes.js
+++ b/lib/setup/routes.js
@ -263,14 +263,20 @@ module.exports = async function (app) {

                        app[isPrivate ? 'privateRateLimiter' : 'rateLimiter'].consume(key, points).then(rateLimiterRes => {
                            const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
-                            res.set('RateLimit-Policy', `${config.rateLimit.points};w=${config.rateLimit.duration}`);
-                            res.set('RateLimit', `limit=${config.rateLimit.points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+                            const { points, duration } = isPrivate ? config.rateLimit.private : config.rateLimit;
+
+                            res.set('RateLimit-Policy', `${points};w=${duration}`);
+                            res.set('RateLimit', `limit=${points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+
                            next();
                        }).catch(rateLimiterRes => {
                            const reset = Math.ceil(rateLimiterRes.msBeforeNext / 1000);
-                            res.set('RateLimit-Policy', `${config.rateLimit.points};w=${config.rateLimit.duration}`);
-                            res.set('RateLimit', `limit=${config.rateLimit.points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
+                            const { points, duration } = isPrivate ? config.rateLimit.private : config.rateLimit;
+
+                            res.set('RateLimit-Policy', `${points};w=${duration}`);
+                            res.set('RateLimit', `limit=${points}, remaining=${rateLimiterRes.remainingPoints}, reset=${reset}`);
                            res.set('Retry-After', reset);
+
                            if (req.is('json')) {
                                res.status(429).json({ error: 'Too Many Requests' });
                            } else {
--- a/public/dmxAppConnect/dmxAppConnect.js
+++ b/public/dmxAppConnect/dmxAppConnect.js
--- a/public/dmxAppConnect/dmxAppConnect.js.map
+++ b/public/dmxAppConnect/dmxAppConnect.js.map
--- a/public/dmxAppConnect/dmxRouting/dmxRouting.js
+++ b/public/dmxAppConnect/dmxRouting/dmxRouting.js
--- a/public/dmxAppConnect/dmxRouting/dmxRouting.js.map
+++ b/public/dmxAppConnect/dmxRouting/dmxRouting.js.map
--- a/public/dmxAppConnect/dmxS3Upload/dmxS3Upload.js
+++ b/public/dmxAppConnect/dmxS3Upload/dmxS3Upload.js
--- a/public/dmxAppConnect/dmxS3Upload/dmxS3Upload.js.map
+++ b/public/dmxAppConnect/dmxS3Upload/dmxS3Upload.js.map
--- a/views/buckets.ejs
+++ b/views/buckets.ejs
@ -1,4 +1,17 @@
-<!-- Wappler include head-page="layouts/main" fontawesome_5="cdn" bootstrap5="local" is="dmx-app" id="buckets" appConnect="local" -->
+<!-- Wappler include head-page="layouts/main" fontawesome_5="cdn" bootstrap5="local" is="dmx-app" id="buckets" appConnect="local" components="{dmxS3Upload:{},dmxNotifications:{},dmxBootstrap5TableGenerator:{}}" -->
+<div id="s3upload1" is="dmx-s3-upload" url="/api/s3control" accept="image/*" class="text-center border">
+    <p dmx-show="!file">Select file</p>
+    <p dmx-show="file">{{file.name}}</p>
+    <p dmx-hide="state.uploading">
+        <button class="btn btn-primary" dmx-on:click.stop="s3upload1.select()" dmx-show="state.idle">Browse</button>
+        <button class="btn btn-primary" dmx-on:click.stop="s3upload1.upload()" dmx-show="state.ready">Upload</button>
+        <button class="btn btn-danger" dmx-on:click.stop="s3upload1.reset()" dmx-show="state.done">Reset</button>
+    </p>
+    <p dmx-show="state.uploading">
+        Uploading {{uploadProgress.percent}}%
+        <button class="btn btn-danger" dmx-on:click.stop="s3upload1.abort()">Abort</button>
+    </p>
+</div>
 <div class="container wappler-block p-3">

    <div class="progress mb-5">
@ -29,6 +42,54 @@
    </div>
    <button id="btn2" class="btn">Button</button>
 </div>
+<div class="container">
+    <table class="table">
+        <thead>
+            <tr>
+                <th>Name</th>
+                <th>Creation date</th>
+            </tr>
+        </thead>
+        <tbody is="dmx-repeat" dmx-generator="bs5table" dmx-bind:repeat="s3upload1.data.listFiles.Buckets" id="tableRepeat1">
+            <tr>
+                <td dmx-text="Name"></td>
+                <td dmx-text="CreationDate"></td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+<dmx-notifications id="notifies1"></dmx-notifications>
+<div class="container">
+    <script>
+        // An efficient JavaScript program to remove all
+      // spaces from a string
+      let str = '2JAxQ9pH Mhy87Sr5NCJ I Be9qj Nd p9eY3vW7BKqN Msxk='
+      // Function to remove all spaces
+      // from a given string
+      function removeSpaces(str) {
+        // To keep track of non-space
+        // character count
+        var count = 0;
+
+        // Traverse the given string. If current
+        // character is not space, then place
+        // it at index 'count++'
+        for (var i = 0; i < str.length; i++)
+          if (str[i] !== " ") str[count++] = str[i]; 
+         // here count is
+        // incremented
+
+        return count;
+      }
+
+      // Driver code
+      var str = "g eeks for ge eeks ".split("");
+      var i = removeSpaces(str);
+      document.write(str.join("").substring(0, i));
+      
+    </script>
+
+</div>

 <!-- DO00RXH3C276N9Y9PQBG -->
 <!-- lMI2BSf8dS+ZmWkyvsq9gTTjScr1SLEsd0OpsZZLAkc -->
--- a/views/index0.ejs
+++ b/views/index0.ejs
@ -1,4 +1,17 @@
 <!-- Wappler include head-page="layouts/main" fontawesome_5="cdn" bootstrap5="local" is="dmx-app" id="index" appConnect="local" components="{dmxBootstrap5Navigation:{},dmxAnimateCSS:{},dmxStateManagement:{},dmxDatastore:{},dmxBootstrap5Modal:{},dmxFormatter:{},dmxBootstrap5TableGenerator:{},dmxBootstrap5Toasts:{},dmxBootbox5:{},dmxBrowser:{},dmxBootstrap5Tooltips:{},dmxValidator:{},dmxS3Upload:{},dmxDatePicker:{},dmxMasonry:{},dmxBootstrap5Popovers:{},dmxPouchDB:{},dmxLazyLoad:{}}" jquery_slim_35="cdn" moment_2="cdn" -->
+<div id="s3upload1" is="dmx-s3-upload" url="" accept="image/*" class="text-center border">
+  <p dmx-show="!file">Select file</p>
+  <p dmx-show="file">{{file.name}}</p>
+  <p dmx-hide="state.uploading">
+    <button class="btn btn-primary" dmx-on:click.stop="s3upload1.select()" dmx-show="state.idle">Browse</button>
+    <button class="btn btn-primary" dmx-on:click.stop="s3upload1.upload()" dmx-show="state.ready">Upload</button>
+    <button class="btn btn-danger" dmx-on:click.stop="s3upload1.reset()" dmx-show="state.done">Reset</button>
+  </p>
+  <p dmx-show="state.uploading">
+    Uploading {{uploadProgress.percent}}%
+    <button class="btn btn-danger" dmx-on:click.stop="s3upload1.abort()">Abort</button>
+  </p>
+</div>



--- a/views/layouts/main.ejs
+++ b/views/layouts/main.ejs
@ -77,6 +77,7 @@
  <script src="https://cdn.jsdelivr.net/npm/pouchdb@8.0.1/dist/pouchdb.min.js" defer></script>
  <script src="https://cdn.jsdelivr.net/npm/pouchdb@8.0.1/dist/pouchdb.find.min.js" defer></script>
  <script src="https://cdn.jsdelivr.net/npm/pouchdb@8.0.1/dist/pouchdb.indexeddb.min.js" defer></script>
+  <script src="/dmxAppConnect/dmxPouchDB/dmxPouchDB.js" defer></script>
 </head>

 <body is="dmx-app" id="main">