From 6d50d2ef3a64b8fe5037861cda8daab40f9a7213 Mon Sep 17 00:00:00 2001 From: eug-vs Date: Wed, 12 Aug 2020 18:26:04 +0300 Subject: chore: install axios --- package-lock.json | 43 ++++++++++++++++++++++++++++++++++++++++--- package.json | 2 ++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index cbc2d25..12eb4b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -170,11 +170,18 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/@types/aws-sdk/-/aws-sdk-2.7.0.tgz", "integrity": "sha1-g1iLPRTr3KHUzl4CM4dXdWjOgvM=", - "dev": true, "requires": { "aws-sdk": "*" } }, + "@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "requires": { + "axios": "*" + } + }, "@types/bluebird": { "version": "3.5.32", "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz", @@ -336,8 +343,7 @@ "@types/uuid": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.0.1.tgz", - "integrity": "sha512-2kE8rEFgJpbBAPw5JghccEevQb0XVU0tewF/8h7wPQTeCtoJ6h8qmBIwuzUVm2MutmzC/cpCkwxudixoNYDp1A==", - "dev": true + "integrity": "sha512-2kE8rEFgJpbBAPw5JghccEevQb0XVU0tewF/8h7wPQTeCtoJ6h8qmBIwuzUVm2MutmzC/cpCkwxudixoNYDp1A==" }, "@typescript-eslint/eslint-plugin": { "version": "3.2.0", @@ -554,6 +560,14 @@ } } }, + "axios": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "requires": { + "follow-redirects": "1.5.10" + } + }, "backo2": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", @@ -1534,6 +1548,29 @@ "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, + "follow-redirects": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "requires": { + "debug": "=3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", diff --git a/package.json b/package.json index 3d15a0b..272965e 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "@feathersjs/socketio": "^4.5.4", "@feathersjs/transport-commons": "^4.5.3", "@types/aws-sdk": "^2.6.1", + "@types/axios": "^0.14.0", "@types/bluebird": "^3.5.32", "@types/cors": "^2.8.6", "@types/lodash": "^4.14.155", @@ -29,6 +30,7 @@ "@typescript-eslint/eslint-plugin": "^3.2.0", "@typescript-eslint/parser": "^3.2.0", "aws-sdk": "^2.6.1", + "axios": "^0.19.2", "bluebird": "^3.7.2", "cors": "^2.8.5", "feathers-hooks-common": "^5.0.3", -- cgit v1.2.3 From f08e2a37677e711cad7397c532670913645f010a Mon Sep 17 00:00:00 2001 From: eug-vs Date: Wed, 12 Aug 2020 18:35:10 +0300 Subject: feat: File service logic --- services/files/files.class.ts | 59 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/services/files/files.class.ts b/services/files/files.class.ts index e2f9df3..7e4bcbf 100644 --- a/services/files/files.class.ts +++ b/services/files/files.class.ts @@ -1,6 +1,10 @@ import { Application } from '@feathersjs/express'; import { Params } from '@feathersjs/feathers'; import { v4 } from 'uuid'; +import axios from 'axios'; +import fs from 'fs'; +import path from 'path'; +import { User } from 'which-types'; // Use require to avoid bug // https://stackoverflow.com/questions/62611373/heroku-crashes-when-importing-aws-sdk @@ -13,15 +17,68 @@ export default class Files { bucket!: string; async find(params: Params): Promise { + const path = this.generateS3Path(params.user?.username); + return this.getUploadUrl(path); + } + + public isS3url(url: string): boolean { + return url.startsWith('https://${this.bucket}.s3'); + } + + public generateS3Path(prefix='', ext='png'): string { + const key = v4(); + const fileName = `${key}.${ext}`; + return prefix ? `${prefix}/${fileName}` : fileName; + } + + async getUploadUrl(path: string): Promise { // Return signed upload URL return this.s3.getSignedUrl('putObject', { Bucket: this.bucket, - Key: `${params.user?.username}/${v4()}.png`, + Key: path, ContentType: 'image/*', Expires: 300, }); } + async getDownloadUrl(path: string): Promise { + return this.getUploadUrl(path).then((url: string) => { + const queryIndex = url.indexOf('?'); + return url.slice(0, queryIndex); + }) + } + + private createTmpDir() { + if (!fs.existsSync('tmp')) fs.mkdirSync('tmp'); + } + + async downloadFile(url: string): Promise { + return new Promise(async (resolve, reject) => { + this.createTmpDir(); + const filePath = `tmp/${v4()}`; + const fileStream = fs.createWriteStream(filePath); + const response = await axios.get(url, { responseType: 'stream' }) + response.data.pipe(fileStream) + .on('error', reject) + .on('close', () => resolve(filePath)); + }); + } + + async uploadFileToS3(filePath: string, s3Path: string) { + const fileStream = fs.createReadStream(filePath); + const request = this.s3.upload({ + Bucket: this.bucket, + Key: s3Path, + Body: fileStream, + ContentType: 'image/png' + }); + request.on('httpUploadProgress', progress => { + console.log('progress', progress); + }) + await request.promise(); + return this.getDownloadUrl(s3Path); + } + setup(app: Application): void { this.app = app; this.s3 = new S3({ -- cgit v1.2.3 From 4b2397d68b62d9f5682d1fd9f4fb2082ac4aa260 Mon Sep 17 00:00:00 2001 From: eug-vs Date: Wed, 12 Aug 2020 18:42:29 +0300 Subject: feat: impelemnt fetchImages hook --- hooks/fetchImages.ts | 25 +++++++++++++++++++++++++ services/polls/polls.hooks.ts | 4 +++- services/users/users.hooks.ts | 3 +++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 hooks/fetchImages.ts diff --git a/hooks/fetchImages.ts b/hooks/fetchImages.ts new file mode 100644 index 0000000..d6e7d27 --- /dev/null +++ b/hooks/fetchImages.ts @@ -0,0 +1,25 @@ +import { HookContext } from '@feathersjs/feathers'; +import Bluebird from 'bluebird'; +import _ from 'lodash'; + + +export default (paths: string[]) => async (context: HookContext): Promise => { + const { service, app, result, params: { user } } = context; + const fileService = app.service('files'); + const model = service.Model; + + Bluebird.map(paths, async (path: string) => { + const url = _.get(result, path); + + // If image is not from our s3, fetch it! + if (!fileService.isS3url(url)) { + const filePath = await fileService.downloadFile(url); + const s3Path = fileService.generateS3Path(user?.username); + const s3Url = await fileService.uploadFileToS3(filePath, s3Path); + return model.findOneAndUpdate({ _id: result._id }, { [path]: s3Url }); + } + return url; + }); + return context; +}; + diff --git a/services/polls/polls.hooks.ts b/services/polls/polls.hooks.ts index 35eae29..7a5b1da 100644 --- a/services/polls/polls.hooks.ts +++ b/services/polls/polls.hooks.ts @@ -8,6 +8,7 @@ import { PollSchema } from '../../models/polls/poll.schema'; import VoteModel from '../../models/votes/vote.model'; import sortByDate from '../../hooks/sortByDate'; import signAuthority from '../../hooks/signAuthority'; +import fetchImages from '../../hooks/fetchImages'; const convertPoll = async (context: HookContext): Promise => { @@ -53,7 +54,8 @@ export default { patch: disallow('external') }, after: { - all: convertPoll + all: convertPoll, + create: fetchImages(['contents.left.url', 'contents.right.url']) } }; diff --git a/services/users/users.hooks.ts b/services/users/users.hooks.ts index 29f1074..ddfc47f 100644 --- a/services/users/users.hooks.ts +++ b/services/users/users.hooks.ts @@ -4,6 +4,7 @@ import { discard, disallow } from 'feathers-hooks-common'; import { HookContext } from '@feathersjs/feathers'; import { NotAuthenticated } from '@feathersjs/errors'; import requireAuth from '../../hooks/requireAuth'; +import fetchImages from '../../hooks/fetchImages'; const hashPassword = hooks.hashPassword('password'); @@ -24,6 +25,8 @@ const compareUser = async (context: HookContext): Promise => { export default { after: { all: hooks.protect('password'), + create: fetchImages(['avatarUrl']), + patch: fetchImages(['avatarUrl']), get: discard('password') // Protect password from local get's }, before: { -- cgit v1.2.3 From 8f4ae4fd89cab5ba4f9e8d2750bc8589ce997ff1 Mon Sep 17 00:00:00 2001 From: eug-vs Date: Wed, 12 Aug 2020 19:02:38 +0300 Subject: fix: delete files after upload --- .gitignore | 4 +++- hooks/fetchImages.ts | 8 +++++++- services/files/files.class.ts | 44 ++++++++++++++++++++++--------------------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 479880f..c775cf1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /node_modules -/.idea \ No newline at end of file +/.idea +/tmp +.env diff --git a/hooks/fetchImages.ts b/hooks/fetchImages.ts index d6e7d27..44aac6c 100644 --- a/hooks/fetchImages.ts +++ b/hooks/fetchImages.ts @@ -4,7 +4,13 @@ import _ from 'lodash'; export default (paths: string[]) => async (context: HookContext): Promise => { - const { service, app, result, params: { user } } = context; + const { + service, + app, + result, + params: { user } + } = context; + const fileService = app.service('files'); const model = service.Model; diff --git a/services/files/files.class.ts b/services/files/files.class.ts index 7e4bcbf..f2a0960 100644 --- a/services/files/files.class.ts +++ b/services/files/files.class.ts @@ -3,18 +3,20 @@ import { Params } from '@feathersjs/feathers'; import { v4 } from 'uuid'; import axios from 'axios'; import fs from 'fs'; -import path from 'path'; -import { User } from 'which-types'; // Use require to avoid bug // https://stackoverflow.com/questions/62611373/heroku-crashes-when-importing-aws-sdk +// TODO: use import statement +// eslint-disable-next-line const S3 = require('aws-sdk/clients/s3'); export default class Files { - app!: Application; - s3!: any; - bucket!: string; + public app!: Application; + + private s3!: typeof S3; + + private bucket!: string; async find(params: Params): Promise { const path = this.generateS3Path(params.user?.username); @@ -22,10 +24,10 @@ export default class Files { } public isS3url(url: string): boolean { - return url.startsWith('https://${this.bucket}.s3'); + return url.startsWith(`https://${this.bucket}.s3`); } - public generateS3Path(prefix='', ext='png'): string { + public generateS3Path(prefix = '', ext = 'png'): string { const key = v4(); const fileName = `${key}.${ext}`; return prefix ? `${prefix}/${fileName}` : fileName; @@ -37,7 +39,7 @@ export default class Files { Bucket: this.bucket, Key: path, ContentType: 'image/*', - Expires: 300, + Expires: 300 }); } @@ -45,7 +47,7 @@ export default class Files { return this.getUploadUrl(path).then((url: string) => { const queryIndex = url.indexOf('?'); return url.slice(0, queryIndex); - }) + }); } private createTmpDir() { @@ -53,29 +55,29 @@ export default class Files { } async downloadFile(url: string): Promise { - return new Promise(async (resolve, reject) => { + return new Promise((resolve, reject) => { this.createTmpDir(); const filePath = `tmp/${v4()}`; const fileStream = fs.createWriteStream(filePath); - const response = await axios.get(url, { responseType: 'stream' }) - response.data.pipe(fileStream) - .on('error', reject) - .on('close', () => resolve(filePath)); + axios.get(url, { responseType: 'stream' }) + .then(response => { + response.data.pipe(fileStream) + .on('error', reject) + .on('close', () => resolve(filePath)); + }) + .catch(error => reject(error)); }); } - async uploadFileToS3(filePath: string, s3Path: string) { + async uploadFileToS3(filePath: string, s3Path: string): Promise { const fileStream = fs.createReadStream(filePath); - const request = this.s3.upload({ + await this.s3.upload({ Bucket: this.bucket, Key: s3Path, Body: fileStream, ContentType: 'image/png' - }); - request.on('httpUploadProgress', progress => { - console.log('progress', progress); - }) - await request.promise(); + }).promise(); + fs.unlinkSync(filePath); return this.getDownloadUrl(s3Path); } -- cgit v1.2.3