Merge pull request #19 from which-ecosystem/s3-reuploads

S3 reuploads

7 files changed, 146 insertions, 10 deletions

diff --git a/.gitignore b/.gitignore
index 479880f..c775cf1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 /node_modules
-/.idea
\ No newline at end of file
+/.idea
+/tmp
+.env
diff --git a/hooks/fetchImages.ts b/hooks/fetchImages.ts
new file mode 100644
index 0000000..44aac6c
--- /dev/null
+++ b/hooks/fetchImages.ts
@@ -0,0 +1,31 @@
+import { HookContext } from '@feathersjs/feathers';
+import Bluebird from 'bluebird';
+import _ from 'lodash';
+
+
+export default (paths: string[]) => async (context: HookContext): Promise<HookContext> => {
+  const {
+    service,
+    app,
+    result,
+    params: { user }
+  } = context;
+
+  const fileService = app.service('files');
+  const model = service.Model;
+
+  Bluebird.map(paths, async (path: string) => {
+    const url = _.get(result, path);
+
+    // If image is not from our s3, fetch it!
+    if (!fileService.isS3url(url)) {
+      const filePath = await fileService.downloadFile(url);
+      const s3Path = fileService.generateS3Path(user?.username);
+      const s3Url = await fileService.uploadFileToS3(filePath, s3Path);
+      return model.findOneAndUpdate({ _id: result._id }, { [path]: s3Url });
+    }
+    return url;
+  });
+  return context;
+};
+
diff --git a/package-lock.json b/package-lock.json
index cbc2d25..12eb4b0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -170,11 +170,18 @@
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/@types/aws-sdk/-/aws-sdk-2.7.0.tgz",
       "integrity": "sha1-g1iLPRTr3KHUzl4CM4dXdWjOgvM=",
-      "dev": true,
       "requires": {
         "aws-sdk": "*"
       }
     },
+    "@types/axios": {
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz",
+      "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=",
+      "requires": {
+        "axios": "*"
+      }
+    },
     "@types/bluebird": {
       "version": "3.5.32",
       "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz",
@@ -336,8 +343,7 @@
     "@types/uuid": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.0.1.tgz",
-      "integrity": "sha512-2kE8rEFgJpbBAPw5JghccEevQb0XVU0tewF/8h7wPQTeCtoJ6h8qmBIwuzUVm2MutmzC/cpCkwxudixoNYDp1A==",
-      "dev": true
+      "integrity": "sha512-2kE8rEFgJpbBAPw5JghccEevQb0XVU0tewF/8h7wPQTeCtoJ6h8qmBIwuzUVm2MutmzC/cpCkwxudixoNYDp1A=="
     },
     "@typescript-eslint/eslint-plugin": {
       "version": "3.2.0",
@@ -554,6 +560,14 @@
         }
       }
     },
+    "axios": {
+      "version": "0.19.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz",
+      "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==",
+      "requires": {
+        "follow-redirects": "1.5.10"
+      }
+    },
     "backo2": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz",
@@ -1534,6 +1548,29 @@
       "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==",
       "dev": true
     },
+    "follow-redirects": {
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
+      "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
+      "requires": {
+        "debug": "=3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+        }
+      }
+    },
     "forwarded": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
diff --git a/package.json b/package.json
index 3d15a0b..272965e 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
     "@feathersjs/socketio": "^4.5.4",
     "@feathersjs/transport-commons": "^4.5.3",
     "@types/aws-sdk": "^2.6.1",
+    "@types/axios": "^0.14.0",
     "@types/bluebird": "^3.5.32",
     "@types/cors": "^2.8.6",
     "@types/lodash": "^4.14.155",
@@ -29,6 +30,7 @@
     "@typescript-eslint/eslint-plugin": "^3.2.0",
     "@typescript-eslint/parser": "^3.2.0",
     "aws-sdk": "^2.6.1",
+    "axios": "^0.19.2",
     "bluebird": "^3.7.2",
     "cors": "^2.8.5",
     "feathers-hooks-common": "^5.0.3",
diff --git a/services/files/files.class.ts b/services/files/files.class.ts
index e2f9df3..f2a0960 100644
--- a/services/files/files.class.ts
+++ b/services/files/files.class.ts
@@ -1,27 +1,86 @@
 import { Application } from '@feathersjs/express';
 import { Params } from '@feathersjs/feathers';
 import { v4 } from 'uuid';
+import axios from 'axios';
+import fs from 'fs';
 
 // Use require to avoid bug
 // https://stackoverflow.com/questions/62611373/heroku-crashes-when-importing-aws-sdk
+// TODO: use import statement
+// eslint-disable-next-line
 const S3 = require('aws-sdk/clients/s3');
 
 
 export default class Files {
-  app!: Application;
-  s3!: any;
-  bucket!: string;
+  public app!: Application;
+
+  private s3!: typeof S3;
+
+  private bucket!: string;
 
   async find(params: Params): Promise<string> {
+    const path = this.generateS3Path(params.user?.username);
+    return this.getUploadUrl(path);
+  }
+
+  public isS3url(url: string): boolean {
+    return url.startsWith(`https://${this.bucket}.s3`);
+  }
+
+  public generateS3Path(prefix = '', ext = 'png'): string {
+    const key = v4();
+    const fileName = `${key}.${ext}`;
+    return prefix ? `${prefix}/${fileName}` : fileName;
+  }
+
+  async getUploadUrl(path: string): Promise<string> {
     // Return signed upload URL
     return this.s3.getSignedUrl('putObject', {
       Bucket: this.bucket,
-      Key: `${params.user?.username}/${v4()}.png`,
+      Key: path,
       ContentType: 'image/*',
-      Expires: 300,
+      Expires: 300
     });
   }
 
+  async getDownloadUrl(path: string): Promise<string> {
+    return this.getUploadUrl(path).then((url: string) => {
+      const queryIndex = url.indexOf('?');
+      return url.slice(0, queryIndex);
+    });
+  }
+
+  private createTmpDir() {
+    if (!fs.existsSync('tmp')) fs.mkdirSync('tmp');
+  }
+
+  async downloadFile(url: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      this.createTmpDir();
+      const filePath = `tmp/${v4()}`;
+      const fileStream = fs.createWriteStream(filePath);
+      axios.get(url, { responseType: 'stream' })
+        .then(response => {
+          response.data.pipe(fileStream)
+            .on('error', reject)
+            .on('close', () => resolve(filePath));
+        })
+        .catch(error => reject(error));
+    });
+  }
+
+  async uploadFileToS3(filePath: string, s3Path: string): Promise<string> {
+    const fileStream = fs.createReadStream(filePath);
+    await this.s3.upload({
+      Bucket: this.bucket,
+      Key: s3Path,
+      Body: fileStream,
+      ContentType: 'image/png'
+    }).promise();
+    fs.unlinkSync(filePath);
+    return this.getDownloadUrl(s3Path);
+  }
+
   setup(app: Application): void {
     this.app = app;
     this.s3 = new S3({
diff --git a/services/polls/polls.hooks.ts b/services/polls/polls.hooks.ts
index 35eae29..7a5b1da 100644
--- a/services/polls/polls.hooks.ts
+++ b/services/polls/polls.hooks.ts
@@ -8,6 +8,7 @@ import { PollSchema } from '../../models/polls/poll.schema';
 import VoteModel from '../../models/votes/vote.model';
 import sortByDate from '../../hooks/sortByDate';
 import signAuthority from '../../hooks/signAuthority';
+import fetchImages from '../../hooks/fetchImages';
 
 
 const convertPoll = async (context: HookContext): Promise<HookContext> => {
@@ -53,7 +54,8 @@ export default {
     patch: disallow('external')
   },
   after: {
-    all: convertPoll
+    all: convertPoll,
+    create: fetchImages(['contents.left.url', 'contents.right.url'])
   }
 };
 
diff --git a/services/users/users.hooks.ts b/services/users/users.hooks.ts
index 29f1074..ddfc47f 100644
--- a/services/users/users.hooks.ts
+++ b/services/users/users.hooks.ts
@@ -4,6 +4,7 @@ import { discard, disallow } from 'feathers-hooks-common';
 import { HookContext } from '@feathersjs/feathers';
 import { NotAuthenticated } from '@feathersjs/errors';
 import requireAuth from '../../hooks/requireAuth';
+import fetchImages from '../../hooks/fetchImages';
 
 const hashPassword = hooks.hashPassword('password');
 
@@ -24,6 +25,8 @@ const compareUser = async (context: HookContext): Promise<HookContext> => {
 export default {
   after: {
     all: hooks.protect('password'),
+    create: fetchImages(['avatarUrl']),
+    patch: fetchImages(['avatarUrl']),
     get: discard('password') // Protect password from local get's
   },
   before: {